diff options
Diffstat (limited to 'src')
13 files changed, 493 insertions, 64 deletions
diff --git a/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java b/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java index 6900a19..53c8cdc 100644 --- a/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java +++ b/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java @@ -53,7 +53,10 @@ public class DatabaseQuerier { ResultSet rs = prep.executeQuery(); while (rs.next()) { - stocks.add(rs.getString(1)); + String ticker = validateTicker(rs.getString(1)); + if (!ticker.equals("")) { + stocks.add(ticker); + } } rs.close(); @@ -94,6 +97,10 @@ public class DatabaseQuerier { public List<Trade> getTradeByStock(String stock, int isBuy, Instant startDate, Instant endDate) throws SQLException { List<Trade> trades = new ArrayList<>(); + /* + if (isValidStock(stock)) { + return trades; + }*/ PreparedStatement prep = conn.prepareStatement( "SELECT * FROM trades WHERE (stock_name = ? AND is_buy = ?) " @@ -133,13 +140,15 @@ public class DatabaseQuerier { ResultSet rs = prep.executeQuery(); while (rs.next()) { - trades.addFirst(new Trade(rs.getInt("trade_id"), - rs.getString("stock_name"), - rs.getDouble("trade_timestamp"), - rs.getInt("is_buy"), - rs.getInt("number_of_shares"), - new Holder(rs.getInt("holder_id"), rs.getString("holder_name")), - rs.getDouble("share_price"))); + String ticker = rs.getString("stock_name"); + //if (isValidTicker(ticker)) { + trades.addFirst(new Trade(rs.getInt("trade_id"), ticker, + rs.getDouble("trade_timestamp"), + rs.getInt("is_buy"), + rs.getInt("number_of_shares"), + new Holder(rs.getInt("holder_id"), rs.getString("holder_name")), + rs.getDouble("share_price"))); + //} } prep.close(); } catch (SQLException e) { @@ -147,4 +156,17 @@ public class DatabaseQuerier { } return trades; } + + private String validateTicker(String ticker) { + //this is cleaning some improperly formatted tickers + ticker = ticker.replaceAll("[^a-zA-Z0-9]", "").toUpperCase(); + if(ticker.contains("[0-9]") || + ticker.length() > 5 || + ticker.length() < 2 || + ticker.contains("NONE")) { + return ""; + } + + return ticker; + } } diff --git a/src/main/java/edu/brown/cs/student/term/Main.java b/src/main/java/edu/brown/cs/student/term/Main.java index 62c4ccf..df95ec0 100644 --- a/src/main/java/edu/brown/cs/student/term/Main.java +++ b/src/main/java/edu/brown/cs/student/term/Main.java @@ -67,7 +67,7 @@ public final class Main { OptionParser parser = new OptionParser(); parser.accepts("gui"); parser.accepts("port").withRequiredArg().ofType(Integer.class) - .defaultsTo(DEFAULT_PORT); + .defaultsTo(DEFAULT_PORT); parser.accepts("debug"); OptionSet options = parser.parse(args); @@ -78,12 +78,12 @@ public final class Main { setConnection.run(new String[] {"data/trades.sqlite3"}); } - if (!options.has("debug")) { + /*if (!options.has("debug")) { System.setErr(new PrintStream(new OutputStream() { public void write(int b) { } })); - } + }*/ HashMap<String, Command> commandHashMap = new HashMap<>(); @@ -102,7 +102,7 @@ public final class Main { config.setDirectoryForTemplateLoading(templates); } catch (IOException ioe) { System.out.printf("ERROR: Unable use %s for template loading.%n", - templates); + templates); System.exit(1); } return new FreeMarkerEngine(config); @@ -114,25 +114,26 @@ public final class Main { Spark.exception(Exception.class, new ExceptionPrinter()); Spark.options("/*", - (request, response) -> { - - String accessControlRequestHeaders = request - .headers("Access-Control-Request-Headers"); - if (accessControlRequestHeaders != null) { - response.header("Access-Control-Allow-Headers", - accessControlRequestHeaders); - } - - String accessControlRequestMethod = request - .headers("Access-Control-Request-Method"); - if (accessControlRequestMethod != null) { - response.header("Access-Control-Allow-Methods", - accessControlRequestMethod); - } - - return "OK"; - }); + (request, response) -> { + + String accessControlRequestHeaders = request + .headers("Access-Control-Request-Headers"); + if (accessControlRequestHeaders != null) { + response.header("Access-Control-Allow-Headers", + accessControlRequestHeaders); + } + + String accessControlRequestMethod = request + .headers("Access-Control-Request-Method"); + if (accessControlRequestMethod != null) { + response.header("Access-Control-Allow-Methods", + accessControlRequestMethod); + } + + return "OK"; + }); Spark.before((request, response) -> response.header("Access-Control-Allow-Origin", "*")); + //TODO: Add system testing for all of our end points Spark.post("/data", new SuspicionRankHandler()); Spark.post("/profit", new ProfitQueryHandler()); Spark.post("/trade-lookup", new TradeQueryHandler()); @@ -151,6 +152,7 @@ public final class Main { //String str = request.body(); //xmlLinks = new JSONObject(str); //this is all the filedAt times and xml files try { + System.err.println("LOG: Call to /data from frontend"); DatabaseQuerier db = SetupCommand.getDq(); SuspicionRanker ranker = new SuspicionRanker(db); @@ -160,7 +162,9 @@ public final class Main { long endMilli = data.getLong("end"); Instant start = Instant.ofEpochMilli(startMilli); Instant end = Instant.ofEpochMilli(endMilli); + System.err.println("LOG: Call to ranker.getSusscore in " + getClass()); List<Holder> suspiciousHolders = ranker.getSuspicionScoreList(start, end); + System.err.println("LOG: Making map " + getClass()); Map<String, Object> variables = ImmutableMap.of("holders", suspiciousHolders); return GSON.toJson(variables); } catch (Exception e) { @@ -177,9 +181,9 @@ public final class Main { String person = req.getString("person"); Date startPeriod = new Date(req.getLong("startTime")); Date endPeriod = new Date(req.getLong("endTime")); - + ProfitCalculation profit = - new ProfitCalculation(DatabaseQuerier.getConn(), person, startPeriod, endPeriod); + new ProfitCalculation(DatabaseQuerier.getConn(), person, startPeriod, endPeriod); List<StockHolding> holdings = profit.getHoldingsList(); double gains = profit.calculateGains(); double sp500PercentGain = profit.compareToSP500(); @@ -193,7 +197,6 @@ public final class Main { res.put("SP500", (1 + sp500PercentGain) * profit.getMoneyInput()); res.put("percentSP500", 100 * sp500PercentGain); return GSON.toJson(res); - } } diff --git a/src/main/java/edu/brown/cs/student/term/hub/HubSearch.java b/src/main/java/edu/brown/cs/student/term/hub/HubSearch.java index 86b883f..d4c48b9 100644 --- a/src/main/java/edu/brown/cs/student/term/hub/HubSearch.java +++ b/src/main/java/edu/brown/cs/student/term/hub/HubSearch.java @@ -22,9 +22,11 @@ public class HubSearch { double[] weights = new double[numHolders]; double[] rank = new double[numHolders]; double[] rankPrime = new double[numHolders]; + System.err.println(numHolders + "\t" + getDigits(numHolders)); Arrays.fill(rankPrime, 1.0 / numHolders); - while(!withinDistance(rank, rankPrime)){ + double thresh = Math.pow(.1, getDigits(numHolders)); + while(!withinDistance(rank, rankPrime, thresh)){ rank = Arrays.copyOf(rankPrime, rankPrime.length); //calculating hub rank for ith holder for(int i = 0; i < numHolders; i++){ @@ -65,7 +67,7 @@ public class HubSearch { if(peopleFollowed.contains(leader)){ //constructs the leader to follower links as we go for use later on - leader.addFollower(follower); + leader.addFollower(new Holder(follower.getId(), follower.getName())); return ((damp / numHolders) + (1 - damp) * (1.0 / numberFollowed)); } else if(numberFollowed == 0){ return ((damp / numHolders) + (1 - damp) * (1.0 / numHolders)); @@ -74,12 +76,25 @@ public class HubSearch { } } - private boolean withinDistance(double[] r, double[] rPrime){ + private boolean withinDistance(double[] r, double[] rPrime, double threshold){ double sum = 0.0; for(int i = 0; i < r.length; i++){ sum += Math.pow((r[i] - rPrime[i]), 2); } - return sum <= 0.001; + System.err.println(sum + "\t" + !(sum <= 0.0000001)); + + return sum <= 0.001*threshold; + } + + private int getDigits(int numFollowers) { + int count = 1; + int temp = numFollowers; + while(temp >= 10) { + count++; + temp /= 10; + System.out.println(temp); + } + return count; } } diff --git a/src/main/java/edu/brown/cs/student/term/hub/LinkMapper.java b/src/main/java/edu/brown/cs/student/term/hub/LinkMapper.java index b490ea1..31e2625 100644 --- a/src/main/java/edu/brown/cs/student/term/hub/LinkMapper.java +++ b/src/main/java/edu/brown/cs/student/term/hub/LinkMapper.java @@ -52,7 +52,6 @@ public class LinkMapper { } else { System.out.println("ERROR: No database loaded in yet"); } - return followerToLeaders; } diff --git a/src/main/java/edu/brown/cs/student/term/hub/SuspicionRanker.java b/src/main/java/edu/brown/cs/student/term/hub/SuspicionRanker.java index 74628aa..3283f5c 100644 --- a/src/main/java/edu/brown/cs/student/term/hub/SuspicionRanker.java +++ b/src/main/java/edu/brown/cs/student/term/hub/SuspicionRanker.java @@ -38,6 +38,12 @@ public class SuspicionRanker { } } + /** + * + * @param start + * @param end + * @return + */ public List<Holder> getSuspicionScoreList(Instant start, Instant end) { PriorityQueue<Holder> orderedSuspicion = new PriorityQueue<>(new SuspicionComparator()); List<Holder> suspicionList = new ArrayList<>(); @@ -46,6 +52,8 @@ public class SuspicionRanker { HubSearch hub = new HubSearch(lm); Map<Holder, Double> holderToHubScore = hub.runHubSearch(start, end); + /* + ProfitCalculation pc = new ProfitCalculation(DatabaseQuerier.getConn(), "", new Date(start.toEpochMilli()), new Date(end.toEpochMilli())); @@ -60,6 +68,7 @@ public class SuspicionRanker { double profitMax = getMaxOfMap(profitMap); /*if all of our values are negative, we need to flip sides so that the * biggest loser doesn't end up being the most suspicious person*/ + /* if(profitMax <= 0) { profitMax = Math.abs(getMinOfMap(profitMap)); } @@ -67,17 +76,20 @@ public class SuspicionRanker { /*if both the min we found and max we found are 0, then we have the special case where all the values are 0, in which case we need to avoid dividing by 0*/ + /* if(profitMax == 0){ profitMax = 1; } + */ + double hubMax = getMaxOfMap(holderToHubScore); for (Holder guy : holderToHubScore.keySet()) { - double normalizedProfitScore = profitMap.get(guy.getId()) / profitMax; + //double normalizedProfitScore = profitMap.get(guy.getId()) / profitMax; double normalizedHubScore = holderToHubScore.get(guy) / hubMax; - double suspicionScore = normalizedHubScore * 0.6 + normalizedProfitScore * 0.4; + double suspicionScore = normalizedHubScore; //* 0.6 + normalizedProfitScore * 0.4; guy.setSuspicionScore(suspicionScore); orderedSuspicion.add(guy); } diff --git a/src/main/java/edu/brown/cs/student/term/parsing/FilingFeed.java b/src/main/java/edu/brown/cs/student/term/parsing/FilingFeed.java new file mode 100644 index 0000000..b5a6acf --- /dev/null +++ b/src/main/java/edu/brown/cs/student/term/parsing/FilingFeed.java @@ -0,0 +1,60 @@ +package edu.brown.cs.student.term.parsing; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents the filing from the Edgar rss feed. + */ +public class FilingFeed { + private final List<String> filings; + + /** + * Constructor that takes the parsed document and extracts the url. + * @param document The document of the rss feed. + */ + public FilingFeed(Document document) { + // Init array + filings = new ArrayList<>(); + + // Get all entries + NodeList entries = document.getElementsByTagName("entry"); + for (int i = 0; i < entries.getLength(); i++) { + // Assertion allows the cast to be ok :) + assert entries.item(i).getNodeType() == Node.ELEMENT_NODE; + Element entry = (Element) entries.item(i); + + NodeList link = entry.getElementsByTagName("link"); + String linkUrl = link.item(0).getAttributes().getNamedItem("href").getNodeValue(); + + filings.add(getXmlUrl(linkUrl)); + } + } + + /** + * Turns the local url into a publicly hosted one. + * @param filingUrl The local url of the .txt to the filing. + * @return The publicly hosted version of the url. + */ + private String getXmlUrl(String filingUrl) { + String url = filingUrl.replace("-index.htm", ".txt"); + if (!url.contains("https://www.sec.gov/")) { + url = "https://www.sec.gov" + url; + } + return url; + } + + /** + * Accessor that returns the url to the txt format of the filings. + * @return The list of publicly hosted urls to each filing. + */ + public List<String> getFilings() { + return filings; + } + +} diff --git a/src/main/java/edu/brown/cs/student/term/parsing/TxtXmlParser.java b/src/main/java/edu/brown/cs/student/term/parsing/TxtXmlParser.java new file mode 100644 index 0000000..2e30fa7 --- /dev/null +++ b/src/main/java/edu/brown/cs/student/term/parsing/TxtXmlParser.java @@ -0,0 +1,103 @@ +package edu.brown.cs.student.term.parsing; + +import org.w3c.dom.Document; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.net.URL; +import java.net.URLConnection; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.Instant; + +/** + * Class that parses the XML contained within a publicly held txt file. + */ +public class TxtXmlParser extends XmlParser { + public final static SimpleDateFormat TIMECONVERTER = new SimpleDateFormat("yyyyMMddHHmmss"); + + private long timestamp; + + public TxtXmlParser() { + super(); + timestamp = -1; + } + + /** + * Method used to parse the xml file. + * + * @param pathToXml The path to the xml text file. + * @return The tree structure parsed as an xml doc. + */ + @Override + public Document parse(String pathToXml) { + try { + System.err.println("LOG: To make class for url: " + pathToXml + " in parse() of " + getClass()); + URL url = new URL(pathToXml); + System.err.println("LOG: To establish urlConnection in parse() of " + getClass()); + URLConnection conn = url.openConnection(); + conn.addRequestProperty("User-Agent", "Chrome"); + System.err.println("LOG: Making bufferedReader for url: " + pathToXml + " in " + getClass()); + BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream())); + + StringBuilder xmlParts = new StringBuilder(); + + boolean isXml = false; + String line; + while ((line = br.readLine()) != null) { + // Get timestamp + if (line.startsWith("<ACCEPTANCE-DATETIME>")) { + String datetime = line.replaceAll("<ACCEPTANCE-DATETIME>", ""); + // TODO: check for errors + this.timestamp = formatTimestamp(datetime); + } + + // For xml + if (line.equals("</XML>")) { + break; + } + if (isXml) { + xmlParts.append(line); + } + if (line.equals("<XML>")) { + isXml = true; + } + } + System.err.println("LOG: Calling builder.parse() after extracting xml parts from: " + pathToXml + " in " + getClass()); + + InputSource xmlLines = new InputSource(new StringReader(xmlParts.toString())); + return builder.parse(xmlLines); + } catch (SAXException e) { + System.err.println("INTERNAL: SAX " + getClass() + " : " + e.getClass()); + } catch (IOException e) { + e.printStackTrace(); + System.err.println("INTERNAL: IO " + getClass() + " : " + e.getClass()); + } + return null; + } + + public long formatTimestamp(String datetime) { + long timestamp = -1; + try { + timestamp = TIMECONVERTER.parse(datetime).toInstant().toEpochMilli(); + } catch (ParseException e) { + e.printStackTrace(); + } + return timestamp; + } + + /** + * Returns the timestamp then resets it to -1. + * @return The timestamp as a number (long). -1 if not assigned. + */ + public long getTimestamp() { + long temp = timestamp; + // Set to -1 for next one... + timestamp = -1; + return temp; + } +} diff --git a/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java b/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java index c89c31d..21cd7c5 100644 --- a/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java +++ b/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java @@ -6,8 +6,10 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.net.URL; import java.net.URLConnection; +import java.time.Instant; public class UrlXmlParser extends XmlParser{ + public UrlXmlParser() { super(); } diff --git a/src/main/java/edu/brown/cs/student/term/profit/ProfitCalculation.java b/src/main/java/edu/brown/cs/student/term/profit/ProfitCalculation.java index 18b79be..0ef87c3 100644 --- a/src/main/java/edu/brown/cs/student/term/profit/ProfitCalculation.java +++ b/src/main/java/edu/brown/cs/student/term/profit/ProfitCalculation.java @@ -24,6 +24,7 @@ import java.util.LinkedList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.*; public class ProfitCalculation { private Connection conn; @@ -78,6 +79,20 @@ public class ProfitCalculation { /** * This method fills the maps of sell and buy orders with lists of oldest - new trades. */ + + private String validateTicker(String ticker) { + //this is cleaning some improperly formatted tickers + ticker = ticker.replaceAll("[^a-zA-Z0-9]", "").toUpperCase(); + if(ticker.contains("[0-9]") || + ticker.length() > 5 || + ticker.length() < 2 || + ticker.contains("NONE")) { + return ""; + } + + return ticker; + } + private void organizeOrders() { //get a list of trades for a person to consider try { @@ -93,6 +108,10 @@ public class ProfitCalculation { while (rs.next()) { String ticker = rs.getString("stock_name"); + ticker = validateTicker(ticker); + if(ticker.equals("")){ + continue; + } int shares = rs.getInt("number_of_shares"); double price = rs.getDouble("share_price"); OrderTuple order = new OrderTuple(shares, price, rs.getDate("trade_timestamp")); diff --git a/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java b/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java index 54f9fc0..00ba3ad 100644 --- a/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java +++ b/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java @@ -2,8 +2,11 @@ package edu.brown.cs.student.term.repl.commands; import edu.brown.cs.student.term.DatabaseQuerier; import edu.brown.cs.student.term.Main; +import edu.brown.cs.student.term.parsing.FilingFeed; import edu.brown.cs.student.term.parsing.Transaction; +import edu.brown.cs.student.term.parsing.TxtXmlParser; import edu.brown.cs.student.term.parsing.UrlXmlParser; +import edu.brown.cs.student.term.parsing.XmlParser; import edu.brown.cs.student.term.repl.Command; import edu.brown.cs.student.term.trade.Trade; import org.json.JSONArray; @@ -15,10 +18,13 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.time.Instant; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; public class LoadCommand implements Command { private Connection conn; - private final static UrlXmlParser URL_XML_PARSER = new UrlXmlParser(); + private final static XmlParser URL_XML_PARSER = new UrlXmlParser(); + private final static TxtXmlParser TXT_XML_PARSER = new TxtXmlParser(); /** * Main run method for every command. @@ -27,46 +33,145 @@ public class LoadCommand implements Command { */ @Override public String run(String[] args) { - // TODO: add log comments + // param checking + if (args.length != 1 && args.length !=2 && args.length !=3) { + return "ERROR: Incorrect number of arguments for load command"; + } + + int numFilings; + try { + numFilings = Integer.parseInt(args[0]); + if (numFilings <=0) { + return "ERROR: Please input an positive integer for number of filings."; + } + } catch (NumberFormatException e) { + return "ERROR: Please input an integer for number of filings."; + } + + int shift = 0; + try { + if (args.length == 2) { + shift = Integer.parseInt(args[1]); + if (shift <=0) { + return "ERROR: Please input an positive integer for the count shift."; + } + } + } catch (NumberFormatException e) { + return "ERROR: Please input an integer for the shift."; + } + + String filingDate = null; + if (args.length == 3) { + filingDate = args[2]; + System.out.println("WARNING: The archive version of the command make take " + + "a long time if a broad query param is inputted."); + } + + System.err.println("LOG: Entered .run() of " + getClass()); - // TODO: call to api for urls to call through the urlxmlparser from reagan - if (Main.xmlLinks == null) { - return "ERROR: Please load xml links from frontend."; + //List<String> filingUrls = getFilings(numFilings); + getFilings(numFilings, shift, filingDate); + + //loadFilings(filingUrls); + + return "Finished loading " + numFilings + " filings."; + } + + private void timeout() { + // System.out.println("timeout 100 mil"); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); } + } - conn = DatabaseQuerier.getConn(); - JSONArray data = Main.xmlLinks.getJSONArray("data"); - for(int i =0; i < data.length(); i++) { - JSONObject link = data.optJSONObject(i); - String timestamp = link.getString("timestamp"); - String url = link.getString("url"); + /** + * Parses the urls to filings and loads them into the setup DB. + * @param urls The list of urls to parsable Edgar txt files. + */ + public void loadFilings(List<String> urls) { + if (urls.isEmpty()) { + System.err.println("WARNING: No filings loaded."); + return; + } + + conn = DatabaseQuerier.getConn(); + for(String url : urls) { try { System.err.println("LOG: Calling loadTransactionIntoDB() in " + getClass()); - loadTransactionIntoDB(timestamp, url); + loadTransactionIntoDB(url); } catch (SQLException throwables) { System.err.println("INTERNAL: SQLException in .run() of " + getClass()); //throwables.printStackTrace(); } } + } + + /** + * Makes a request to the public Edgar url and parses it's rss feed. + * @param numFilings The number of filings to parse. + */ + private void getFilings(int numFilings, int shift, String filingDate) { + int counter = 0; - return "Loaded?"; + while (100*counter <= (numFilings - shift)) { + timeout(); + + String queryUrl = + (filingDate != null) ? + "https://www.sec.gov/cgi-bin/srch-edgar?" + + "text=form-type%3D4+and+(filing-date%3D" + filingDate + ")" + + "&start=" + (100*counter++ + shift) + + "&count=" + 100 + + "&first=2020" + + "&last=2021" + + "&output=atom" + : + "https://www.sec.gov/cgi-bin/browse-edgar?" + + "action=getcurrent" + + "&CIK=" + + "&type=4" + + "&company=" + + "&dateb=" + + "&owner=only" + + "&start=" + (100*counter++ + shift) + + "&count=" + 100 + + "&output=atom"; + + System.err.println("LOG: Requesting filings with url: " + queryUrl); + Document document = URL_XML_PARSER.parse(queryUrl); + if (document == null) { + System.err.println("WARNING: Document was null " + queryUrl + " in getFilings(): " + getClass()); + continue; + } + + FilingFeed filingFeed = new FilingFeed(document); + loadFilings(filingFeed.getFilings()); + + if (counter%10 == 0) { + System.out.println("PROGRESS: " + counter*100 + "/" + numFilings); + } + } + // TODO: make params more adjustable } + /** * Loads a whole transaction, which can have multiple trades, into the DB. * @param url The url to the public xml file. * @throws SQLException If the prep statement fails or db doesn't exist, throws. */ - private void loadTransactionIntoDB(String timestamp, String url) throws SQLException { + private void loadTransactionIntoDB(String url) throws SQLException { System.err.println("LOG: Parsing XML into transaction in loadTransactionIntoDB(). URL: " + url); // TODO: check if this is right @julia // TODO: add parse error handling... - ZonedDateTime zonedDateTime = ZonedDateTime.parse(timestamp); - Instant instant = zonedDateTime.toInstant(); - - Document document = URL_XML_PARSER.parse(url); - if (document == null) { + // timeout to reduce the too many requests + timeout(); + Document document = TXT_XML_PARSER.parse(url); + long timestamp = TXT_XML_PARSER.getTimestamp(); + if (document == null || timestamp == -1) { System.err.println("WARNING: URL " + url + " failed to parse... continuing."); return; } @@ -77,7 +182,7 @@ public class LoadCommand implements Command { for(Trade trade : helper.getTrades()) { System.err.println("LOG: Loading a trade into DB -> " + trade); - loadTradeIntoDB(instant, trade); + loadTradeIntoDB(timestamp, trade, url); System.err.println("LOG: Loaded that trade."); } } catch (Exception e) { @@ -90,7 +195,7 @@ public class LoadCommand implements Command { * @param trade The trade to be loaded. * @throws SQLException If the prep statement fails or db doesn't exist, throws. */ - private void loadTradeIntoDB(Instant instant, Trade trade) throws SQLException { + private void loadTradeIntoDB(long timestamp, Trade trade, String url) throws SQLException { // current table schema that is used... // TODO: make this TABLE with this SCHEMA if doesn't exist. /* @@ -103,23 +208,25 @@ public class LoadCommand implements Command { number_of_shares INTEGER, holder_id INTEGER, share_price NUMERIC, + filing_url TEXT UNIQUE (trade_timestamp, is_buy, number_of_shares, holder_id, share_price)); */ System.err.println("LOG: Setting prepared statement on " + conn); PreparedStatement prep = conn.prepareStatement( "INSERT INTO trades (stock_name, holder_name, trade_timestamp, is_buy, " + - "number_of_shares, holder_id, share_price) " + - "VALUES (?, ?, ?, ?, ?, ?, ?)"); + "number_of_shares, holder_id, share_price, filing_url) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); prep.setString(1, trade.getStock()); prep.setString(2, trade.getHolder().getName()); // TODO: update with timestamp @julia - prep.setLong(3, instant.toEpochMilli()); + prep.setLong(3, timestamp); prep.setInt(4, trade.isBuy() ? 1 : 0); prep.setInt(5, trade.getNumShares()); prep.setInt(6, trade.getHolder().getId()); prep.setDouble(7, trade.getPrice()); + prep.setString(8, url); System.err.println("LOG: Inserted values into prep statement."); prep.execute(); diff --git a/src/test/java/edu/brown/cs/student/FilingTest.java b/src/test/java/edu/brown/cs/student/FilingTest.java new file mode 100644 index 0000000..a9b21d3 --- /dev/null +++ b/src/test/java/edu/brown/cs/student/FilingTest.java @@ -0,0 +1,82 @@ +package edu.brown.cs.student; + +import edu.brown.cs.student.term.parsing.LocalXmlParser; +import edu.brown.cs.student.term.parsing.Transaction; +import edu.brown.cs.student.term.parsing.TxtXmlParser; +import edu.brown.cs.student.term.parsing.UrlXmlParser; +import edu.brown.cs.student.term.parsing.XmlParser; +import edu.brown.cs.student.term.trade.Trade; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.print.Doc; + +import static org.junit.Assert.*; + +public class FilingTest { + private XmlParser _xmlParser, _txtXmlParser; + + @Before + public void setUp() { + _xmlParser = new UrlXmlParser(); + _txtXmlParser = new TxtXmlParser(); + } + + @After + public void tearDown() { + _xmlParser = null; + _txtXmlParser = null; + } + + @Test + public void seeWorks(){ + setUp(); + + String url = "https://www.sec.gov/cgi-bin/browse-edgar?" + + "action=getcurrent" + + "&CIK=" + + "&type=4" + + "&company=" + + "&dateb=" + + "&owner=only" + + "&start=0" + + "&count=10" + + "&output=atom"; + + Document doc = _xmlParser.parse(url); + assertNotNull(doc); + NodeList entries = doc.getElementsByTagName("entry"); + assertNotEquals(entries.getLength(), 0); + assertEquals(entries.item(0).getNodeType(), Node.ELEMENT_NODE); + for (int i = 0; i < entries.getLength(); i++) { + Element entry = (Element) entries.item(i); + NodeList link = entry.getElementsByTagName("link"); + assertEquals(link.getLength(), 1); + String linkUrl = link.item(0).getAttributes().getNamedItem("href").getNodeValue(); + System.out.println(linkUrl); + + NodeList updated = entry.getElementsByTagName("updated"); + assertEquals(link.getLength(), 1); + System.out.println(updated.item(0).getTextContent()); + } + + tearDown(); + } + + @Test + public void xmlUrlFromFilingUrl(){ + setUp(); + + String url = "https://www.sec.gov/Archives/edgar/data/1597341/000141588921001958/0001415889-21-001958.txt"; + Document doc = _txtXmlParser.parse(url); + assertNotNull(doc); + tearDown(); + } + + +} diff --git a/src/test/java/edu/brown/cs/student/HubRankTest.java b/src/test/java/edu/brown/cs/student/HubRankTest.java index 07fd282..cbef9ee 100644 --- a/src/test/java/edu/brown/cs/student/HubRankTest.java +++ b/src/test/java/edu/brown/cs/student/HubRankTest.java @@ -126,4 +126,6 @@ public class HubRankTest { tearDown(); } + + //TODO: Test special case where all the hub ranks should be the same } diff --git a/src/test/java/edu/brown/cs/student/SuspicionRankerTest.java b/src/test/java/edu/brown/cs/student/SuspicionRankerTest.java index 3f8ea3d..e004e46 100644 --- a/src/test/java/edu/brown/cs/student/SuspicionRankerTest.java +++ b/src/test/java/edu/brown/cs/student/SuspicionRankerTest.java @@ -82,6 +82,9 @@ public class SuspicionRankerTest { tearDown(); } + //TODO: Test special case for all the profit = 0 + //TODO: Test special case for all the profit is negative + } |