diff options
Diffstat (limited to 'src')
6 files changed, 149 insertions, 42 deletions
diff --git a/src/main/java/edu/brown/cs/student/term/Main.java b/src/main/java/edu/brown/cs/student/term/Main.java index 6dce755..2a75bd5 100644 --- a/src/main/java/edu/brown/cs/student/term/Main.java +++ b/src/main/java/edu/brown/cs/student/term/Main.java @@ -2,6 +2,7 @@ package edu.brown.cs.student.term; import edu.brown.cs.student.term.repl.Command; import edu.brown.cs.student.term.repl.REPL; +import edu.brown.cs.student.term.repl.commands.LoadCommand; import edu.brown.cs.student.term.repl.commands.SetupCommand; import joptsimple.OptionParser; import joptsimple.OptionSet; @@ -70,6 +71,8 @@ import java.sql.Statement; * The Main class of our project. This is where execution begins. */ public final class Main { + // TODO: fix temproary solution + public static JSONObject xmlLinks = null; private static final int DEFAULT_PORT = 4567; @@ -100,9 +103,10 @@ public final class Main { if (options.has("gui")) { runSparkServer((int) options.valueOf("port")); } - + HashMap<String, Command> commandHashMap = new HashMap<>(); commandHashMap.put("setup", new SetupCommand()); + commandHashMap.put("load", new LoadCommand()); /** add commands to map here! */ REPL repl = new REPL(commandHashMap); repl.runREPL(); @@ -153,12 +157,12 @@ public final class Main { private static class DataHandler implements Route { - @Override public Object handle(Request request, Response response) throws Exception { String str = request.body(); - JSONObject json = new JSONObject(str); //this is all the filedAt times and xml files - System.out.println(json); + xmlLinks = new JSONObject(str); //this is all the filedAt times and xml files + + return "replace"; } } diff --git a/src/main/java/edu/brown/cs/student/term/parsing/Transaction.java b/src/main/java/edu/brown/cs/student/term/parsing/Transaction.java index abf8405..1f502b1 100644 --- a/src/main/java/edu/brown/cs/student/term/parsing/Transaction.java +++ b/src/main/java/edu/brown/cs/student/term/parsing/Transaction.java @@ -27,6 +27,7 @@ public class Transaction { */ public Transaction(Document document) { // TODO: add log comments + System.err.println("LOG: Entered constructor for: " + getClass()); doc = document; trades = new ArrayList<>(); @@ -37,6 +38,7 @@ public class Transaction { // There are two types of transactions within the xml - derivative and non derivative. // We only look at non-derivative NodeList nonDerivative = document.getElementsByTagName("nonDerivativeTransaction"); + System.err.println("LOG: Called processTransaction in constructor for: " + getClass()); processTransactions(nonDerivative); } @@ -47,18 +49,22 @@ public class Transaction { private void processTransactions(NodeList tradesAsNodes) { int numTrades = tradesAsNodes.getLength(); for(int i = 0; i < numTrades; i++) { - NodeList tradeValues = getValueList(tradesAsNodes.item(i)); + // Convert to an element to extract specific info. + // I know there is a cast, but this how it is done and is checked with the assertion. + Node trade = tradesAsNodes.item(i); + assert trade.getNodeType() == Node.ELEMENT_NODE; + Element tradeValues = (Element) trade; // This stages the params for the trade. // TODO: update with real timestamp double ts = getDate(tradeValues).hashCode(); - int isBuy = getIsBuy(tradeValues) ? 1 : 0; + int isBuy = getIsBuy(tradeValues); int numShares = getNumShares(tradeValues); // TODO: update id? unique for each holder? Holder holder = new Holder(id, personName); double pricePerShare = getPriceShares(tradeValues); - if (pricePerShare != 0) { + if (pricePerShare != 0 && numShares != -1 && pricePerShare != -1 && isBuy != -1) { trades.add(new Trade(id, ticker, ts, isBuy, numShares, holder, pricePerShare)); } } @@ -97,16 +103,21 @@ public class Transaction { return idNode.item(0).getTextContent(); } - /** - * Extracts the trade data of one trade within a transaction. - * @param trade One trade from the transaction as a node. - * @return A nodelist with each element holding specific info of the trade. - */ - public NodeList getValueList(Node trade) { - // Data of trade in an array of values - assert trade.getNodeType() == Node.ELEMENT_NODE; - Element tradeElement = (Element) trade; - return tradeElement.getElementsByTagName("value"); + + public String getValueFromNode(NodeList dataHolder) { + // Convert to an element to extract the value form. + // I know there is a cast, but this how it is done and is checked with the assertion. + Node dataNode = dataHolder.item(0); + assert dataNode.getNodeType() == Node.ELEMENT_NODE; + Element data = (Element) dataNode; + + // Only one field within the value holding the value... + NodeList value = data.getElementsByTagName("value"); + if (value.getLength() == 0) { + // If market trade, some values don't have fields... + return ""; + } + return value.item(0).getTextContent(); } /** @@ -114,8 +125,9 @@ public class Transaction { * @param values The value array representing the trade. * @return The date in mm-dd-yyyy format as a string. */ - public String getDate(NodeList values) { - return values.item(1).getTextContent(); + public String getDate(Element values) { + NodeList date = values.getElementsByTagName("transactionDate"); + return getValueFromNode(date); } /** @@ -123,9 +135,17 @@ public class Transaction { * @param values The value array representing the trade. * @return The number of shares. */ - public int getNumShares(NodeList values) { - // TODO: this might have to be double - return Integer.parseInt(values.item(2).getTextContent()); + public int getNumShares(Element values) { + NodeList numShares = values.getElementsByTagName("transactionShares"); + String num = getValueFromNode(numShares); + num = num.equals("") ? "<empty>" : num; + try { + // Some ints have training zeros, so need to parse Double then truncate. + return (int) Double.parseDouble(num); + } catch (NumberFormatException e) { + System.err.println("WARNING: Num shares in XML bad format: " + num); + return -1; + } } /** @@ -133,8 +153,16 @@ public class Transaction { * @param values The value array representing the trade. * @return The price of each share. */ - public double getPriceShares(NodeList values) { - return Double.parseDouble(values.item(3).getTextContent()); + public double getPriceShares(Element values) { + NodeList priceShares = values.getElementsByTagName("transactionPricePerShare"); + String num = getValueFromNode(priceShares); + num = num.equals("") ? "<empty>" : num; + try { + return Double.parseDouble(num); + } catch (NumberFormatException e) { + System.err.println("WARNING: Price shares in XML bad format: " + num); + return -1; + } } /** @@ -142,8 +170,17 @@ public class Transaction { * @param values The value array representing the trade. * @return True if they were bought (acquired). False, if sold. */ - public boolean getIsBuy(NodeList values) { - return values.item(4).getTextContent().equals("A"); + public int getIsBuy(Element values) { + NodeList isBuy = values.getElementsByTagName("transactionAcquiredDisposedCode"); + String value = getValueFromNode(isBuy); + if (value.equals("A")) { + return 1; + } else if (value.equals("D")) { + return 0; + } else { + System.err.println("WARNING: Acquire code not recognized: " + value); + return -1; + } } /** diff --git a/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java b/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java index 440b898..c89c31d 100644 --- a/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java +++ b/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java @@ -21,15 +21,17 @@ public class UrlXmlParser extends XmlParser{ @Override public Document parse(String pathToXml) { try { - System.err.println("LOG: To make url class in parse() of " + getClass()); + System.err.println("LOG: To make class for url: " + pathToXml + " in parse() of " + getClass()); URL url = new URL(pathToXml); System.err.println("LOG: To establish urlConnection in parse() of " + getClass()); URLConnection conn = url.openConnection(); - System.err.println("LOG: Calling builder.parse() in " + getClass()); + conn.addRequestProperty("User-Agent", "Chrome"); + System.err.println("LOG: Calling builder.parse() for url: " + pathToXml + " in " + getClass()); return builder.parse(conn.getInputStream()); } catch (SAXException e) { System.err.println("INTERNAL: SAX " + getClass() + " : " + e.getClass()); } catch (IOException e) { + e.printStackTrace(); System.err.println("INTERNAL: IO " + getClass() + " : " + e.getClass()); } return null; diff --git a/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java b/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java index 09d98d2..54f9fc0 100644 --- a/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java +++ b/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java @@ -1,17 +1,23 @@ package edu.brown.cs.student.term.repl.commands; import edu.brown.cs.student.term.DatabaseQuerier; +import edu.brown.cs.student.term.Main; import edu.brown.cs.student.term.parsing.Transaction; import edu.brown.cs.student.term.parsing.UrlXmlParser; import edu.brown.cs.student.term.repl.Command; import edu.brown.cs.student.term.trade.Trade; +import org.json.JSONArray; +import org.json.JSONObject; +import org.w3c.dom.Document; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.time.Instant; +import java.time.ZonedDateTime; public class LoadCommand implements Command { - private final static Connection CONN = DatabaseQuerier.getConn(); + private Connection conn; private final static UrlXmlParser URL_XML_PARSER = new UrlXmlParser(); /** @@ -24,16 +30,26 @@ public class LoadCommand implements Command { // TODO: add log comments System.err.println("LOG: Entered .run() of " + getClass()); // TODO: call to api for urls to call through the urlxmlparser from reagan - String[] urls = new String[1]; - for (String url : urls) { + if (Main.xmlLinks == null) { + return "ERROR: Please load xml links from frontend."; + } + + conn = DatabaseQuerier.getConn(); + JSONArray data = Main.xmlLinks.getJSONArray("data"); + for(int i =0; i < data.length(); i++) { + JSONObject link = data.optJSONObject(i); + + String timestamp = link.getString("timestamp"); + String url = link.getString("url"); try { System.err.println("LOG: Calling loadTransactionIntoDB() in " + getClass()); - loadTransactionIntoDB(url); + loadTransactionIntoDB(timestamp, url); } catch (SQLException throwables) { System.err.println("INTERNAL: SQLException in .run() of " + getClass()); //throwables.printStackTrace(); } } + return "Loaded?"; } @@ -42,13 +58,30 @@ public class LoadCommand implements Command { * @param url The url to the public xml file. * @throws SQLException If the prep statement fails or db doesn't exist, throws. */ - private static void loadTransactionIntoDB(String url) throws SQLException { + private void loadTransactionIntoDB(String timestamp, String url) throws SQLException { System.err.println("LOG: Parsing XML into transaction in loadTransactionIntoDB(). URL: " + url); - Transaction helper = new Transaction(URL_XML_PARSER.parse(url)); - for(Trade trade : helper.getTrades()) { - System.err.println("LOG: Loading a trade into DB -> " + trade); - loadTradeIntoDB(trade); - System.err.println("LOG: Loaded that trade."); + // TODO: check if this is right @julia + // TODO: add parse error handling... + ZonedDateTime zonedDateTime = ZonedDateTime.parse(timestamp); + Instant instant = zonedDateTime.toInstant(); + + Document document = URL_XML_PARSER.parse(url); + if (document == null) { + System.err.println("WARNING: URL " + url + " failed to parse... continuing."); + return; + } + + try { + // Constructing below is throwing null pointer - not sure why... + Transaction helper = new Transaction(document); + + for(Trade trade : helper.getTrades()) { + System.err.println("LOG: Loading a trade into DB -> " + trade); + loadTradeIntoDB(instant, trade); + System.err.println("LOG: Loaded that trade."); + } + } catch (Exception e) { + System.err.println("WARNING: URL " + url + " failed to enter DB on " + e.getMessage()); } } @@ -57,8 +90,24 @@ public class LoadCommand implements Command { * @param trade The trade to be loaded. * @throws SQLException If the prep statement fails or db doesn't exist, throws. */ - private static void loadTradeIntoDB(Trade trade) throws SQLException { - PreparedStatement prep = CONN.prepareStatement( + private void loadTradeIntoDB(Instant instant, Trade trade) throws SQLException { + // current table schema that is used... + // TODO: make this TABLE with this SCHEMA if doesn't exist. + /* + CREATE TABLE IF NOT EXISTS trades ( + trade_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + stock_name TEXT, + holder_name TEXT, + trade_timestamp INTEGER, + is_buy INTEGER, + number_of_shares INTEGER, + holder_id INTEGER, + share_price NUMERIC, + UNIQUE (trade_timestamp, is_buy, number_of_shares, holder_id, share_price)); + */ + + System.err.println("LOG: Setting prepared statement on " + conn); + PreparedStatement prep = conn.prepareStatement( "INSERT INTO trades (stock_name, holder_name, trade_timestamp, is_buy, " + "number_of_shares, holder_id, share_price) " + "VALUES (?, ?, ?, ?, ?, ?, ?)"); @@ -66,13 +115,15 @@ public class LoadCommand implements Command { prep.setString(1, trade.getStock()); prep.setString(2, trade.getHolder().getName()); // TODO: update with timestamp @julia - prep.setDouble(3, trade.getTimestamp()); + prep.setLong(3, instant.toEpochMilli()); prep.setInt(4, trade.isBuy() ? 1 : 0); prep.setInt(5, trade.getNumShares()); prep.setInt(6, trade.getHolder().getId()); prep.setDouble(7, trade.getPrice()); + System.err.println("LOG: Inserted values into prep statement."); prep.execute(); + System.err.println("LOG: Executed prep statement."); // TODO: close connection - when should we do this } }
\ No newline at end of file diff --git a/src/test/java/edu/brown/cs/student/TransactionTest.java b/src/test/java/edu/brown/cs/student/TransactionTest.java index e8b95bc..faf24fb 100644 --- a/src/test/java/edu/brown/cs/student/TransactionTest.java +++ b/src/test/java/edu/brown/cs/student/TransactionTest.java @@ -95,4 +95,13 @@ public class TransactionTest { tearDown(); } + @Test + public void badFormats(){ + setUp(); + // TODO: ensure 0 price trades aren't there, market trades aren't there, .00 on end of ints, etc... + // no price -> https://www.sec.gov/Archives/edgar/data/1654595/000110465921048974/tm2112683-1_4seq1.xml + // market form -> https://www.sec.gov/Archives/edgar/data/39899/000121465921004094/marketforms-52237.xml + tearDown(); + } + } diff --git a/src/test/java/edu/brown/cs/student/XmlParserTest.java b/src/test/java/edu/brown/cs/student/XmlParserTest.java index e6a112f..eebccbb 100644 --- a/src/test/java/edu/brown/cs/student/XmlParserTest.java +++ b/src/test/java/edu/brown/cs/student/XmlParserTest.java @@ -48,9 +48,13 @@ public class XmlParserTest { Document doc = _urlXmlParser.parse("https://www.sec.gov/Archives/edgar/data/1517006/000110465921046242/tm2112036-4_4seq1.xml"); assertNotNull(doc); - // Id of person assertEquals(getIdFromDoc(doc), "0001561844"); + + Document doc2 = + _urlXmlParser.parse("https://www.sec.gov/Archives/edgar/data/1254015/000120919121026318/doc4.xml"); + assertNotNull(doc2); + assertEquals(getIdFromDoc(doc2), "0001254015"); tearDown(); } |