aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--data/trades.sqlite3bin0 -> 36864 bytes
-rw-r--r--frontend/src/SECAPIData.js38
-rw-r--r--mock_trades.sqlite30
-rw-r--r--src/main/java/edu/brown/cs/student/term/Main.java12
-rw-r--r--src/main/java/edu/brown/cs/student/term/parsing/Transaction.java81
-rw-r--r--src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java6
-rw-r--r--src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java77
-rw-r--r--src/test/java/edu/brown/cs/student/TransactionTest.java9
-rw-r--r--src/test/java/edu/brown/cs/student/XmlParserTest.java6
9 files changed, 175 insertions, 54 deletions
diff --git a/data/trades.sqlite3 b/data/trades.sqlite3
new file mode 100644
index 0000000..1fbf57a
--- /dev/null
+++ b/data/trades.sqlite3
Binary files differ
diff --git a/frontend/src/SECAPIData.js b/frontend/src/SECAPIData.js
index 3e670d3..b1b4939 100644
--- a/frontend/src/SECAPIData.js
+++ b/frontend/src/SECAPIData.js
@@ -1,15 +1,21 @@
-import React, {useState, useRef} from 'react';
+import React, {useState, useEffect} from 'react';
import Button from './Button';
function SECAPIData() {
-
const [dataToBackend, setDataToBackend] = useState([]);
const [displayData, setDisplayData] = useState("");
const requestData = () => {
+ // End early in debug to avoid too many requests.
+ if (dataToBackend.length !== 0) {
+ sendToBackend();
+ return;
+ }
+
+ console.log("Makig request...");
let date = new Date()
let today = date.toISOString().slice(0, 10);
@@ -33,25 +39,30 @@ function SECAPIData() {
})
.then(res => res.json())
.then(data => {
- let list = new Array();
- data.filings.forEach(function(filing) {
- if (filing.ticker == "") {
- list.push(filing.filedAt);
- list.push(filing.documentFormatFiles[1].documentUrl);
+ let list = [];
+ data.filings.forEach(filing => {
+ if (filing.ticker === "") {
+ // TODO: why are there repitions of urls.
+ list.push({
+ timestamp: filing.filedAt,
+ url: filing.documentFormatFiles[1].documentUrl
+ });
}
})
setDataToBackend(list);
- console.log(list);
})
- .catch(function (error) {
+ .catch(error => {
console.log(error);
});
+ }
+
+ const sendToBackend = () => {
+ console.log(dataToBackend);
-
fetch("http://localhost:4567/data", {
method: "POST",
body: JSON.stringify({
- "data" : {dataToBackend}
+ "data" : dataToBackend
}),
headers: {
"Content-Type": "application/json",
@@ -66,7 +77,10 @@ function SECAPIData() {
.catch(function (error) {
console.log(error);
});
- }
+ }
+
+ // This hook is autocalled once the setDataToBackend takes effect.
+ useEffect(() => sendToBackend(), [dataToBackend]);
return (
<div>
diff --git a/mock_trades.sqlite3 b/mock_trades.sqlite3
deleted file mode 100644
index e69de29..0000000
--- a/mock_trades.sqlite3
+++ /dev/null
diff --git a/src/main/java/edu/brown/cs/student/term/Main.java b/src/main/java/edu/brown/cs/student/term/Main.java
index 6dce755..2a75bd5 100644
--- a/src/main/java/edu/brown/cs/student/term/Main.java
+++ b/src/main/java/edu/brown/cs/student/term/Main.java
@@ -2,6 +2,7 @@ package edu.brown.cs.student.term;
import edu.brown.cs.student.term.repl.Command;
import edu.brown.cs.student.term.repl.REPL;
+import edu.brown.cs.student.term.repl.commands.LoadCommand;
import edu.brown.cs.student.term.repl.commands.SetupCommand;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
@@ -70,6 +71,8 @@ import java.sql.Statement;
* The Main class of our project. This is where execution begins.
*/
public final class Main {
+ // TODO: fix temproary solution
+ public static JSONObject xmlLinks = null;
private static final int DEFAULT_PORT = 4567;
@@ -100,9 +103,10 @@ public final class Main {
if (options.has("gui")) {
runSparkServer((int) options.valueOf("port"));
}
-
+
HashMap<String, Command> commandHashMap = new HashMap<>();
commandHashMap.put("setup", new SetupCommand());
+ commandHashMap.put("load", new LoadCommand());
/** add commands to map here! */
REPL repl = new REPL(commandHashMap);
repl.runREPL();
@@ -153,12 +157,12 @@ public final class Main {
private static class DataHandler implements Route {
-
@Override
public Object handle(Request request, Response response) throws Exception {
String str = request.body();
- JSONObject json = new JSONObject(str); //this is all the filedAt times and xml files
- System.out.println(json);
+ xmlLinks = new JSONObject(str); //this is all the filedAt times and xml files
+
+
return "replace";
}
}
diff --git a/src/main/java/edu/brown/cs/student/term/parsing/Transaction.java b/src/main/java/edu/brown/cs/student/term/parsing/Transaction.java
index abf8405..1f502b1 100644
--- a/src/main/java/edu/brown/cs/student/term/parsing/Transaction.java
+++ b/src/main/java/edu/brown/cs/student/term/parsing/Transaction.java
@@ -27,6 +27,7 @@ public class Transaction {
*/
public Transaction(Document document) {
// TODO: add log comments
+ System.err.println("LOG: Entered constructor for: " + getClass());
doc = document;
trades = new ArrayList<>();
@@ -37,6 +38,7 @@ public class Transaction {
// There are two types of transactions within the xml - derivative and non derivative.
// We only look at non-derivative
NodeList nonDerivative = document.getElementsByTagName("nonDerivativeTransaction");
+ System.err.println("LOG: Called processTransaction in constructor for: " + getClass());
processTransactions(nonDerivative);
}
@@ -47,18 +49,22 @@ public class Transaction {
private void processTransactions(NodeList tradesAsNodes) {
int numTrades = tradesAsNodes.getLength();
for(int i = 0; i < numTrades; i++) {
- NodeList tradeValues = getValueList(tradesAsNodes.item(i));
+ // Convert to an element to extract specific info.
+ // I know there is a cast, but this how it is done and is checked with the assertion.
+ Node trade = tradesAsNodes.item(i);
+ assert trade.getNodeType() == Node.ELEMENT_NODE;
+ Element tradeValues = (Element) trade;
// This stages the params for the trade.
// TODO: update with real timestamp
double ts = getDate(tradeValues).hashCode();
- int isBuy = getIsBuy(tradeValues) ? 1 : 0;
+ int isBuy = getIsBuy(tradeValues);
int numShares = getNumShares(tradeValues);
// TODO: update id? unique for each holder?
Holder holder = new Holder(id, personName);
double pricePerShare = getPriceShares(tradeValues);
- if (pricePerShare != 0) {
+ if (pricePerShare != 0 && numShares != -1 && pricePerShare != -1 && isBuy != -1) {
trades.add(new Trade(id, ticker, ts, isBuy, numShares, holder, pricePerShare));
}
}
@@ -97,16 +103,21 @@ public class Transaction {
return idNode.item(0).getTextContent();
}
- /**
- * Extracts the trade data of one trade within a transaction.
- * @param trade One trade from the transaction as a node.
- * @return A nodelist with each element holding specific info of the trade.
- */
- public NodeList getValueList(Node trade) {
- // Data of trade in an array of values
- assert trade.getNodeType() == Node.ELEMENT_NODE;
- Element tradeElement = (Element) trade;
- return tradeElement.getElementsByTagName("value");
+
+ public String getValueFromNode(NodeList dataHolder) {
+ // Convert to an element to extract the value form.
+ // I know there is a cast, but this how it is done and is checked with the assertion.
+ Node dataNode = dataHolder.item(0);
+ assert dataNode.getNodeType() == Node.ELEMENT_NODE;
+ Element data = (Element) dataNode;
+
+ // Only one field within the value holding the value...
+ NodeList value = data.getElementsByTagName("value");
+ if (value.getLength() == 0) {
+ // If market trade, some values don't have fields...
+ return "";
+ }
+ return value.item(0).getTextContent();
}
/**
@@ -114,8 +125,9 @@ public class Transaction {
* @param values The value array representing the trade.
* @return The date in mm-dd-yyyy format as a string.
*/
- public String getDate(NodeList values) {
- return values.item(1).getTextContent();
+ public String getDate(Element values) {
+ NodeList date = values.getElementsByTagName("transactionDate");
+ return getValueFromNode(date);
}
/**
@@ -123,9 +135,17 @@ public class Transaction {
* @param values The value array representing the trade.
* @return The number of shares.
*/
- public int getNumShares(NodeList values) {
- // TODO: this might have to be double
- return Integer.parseInt(values.item(2).getTextContent());
+ public int getNumShares(Element values) {
+ NodeList numShares = values.getElementsByTagName("transactionShares");
+ String num = getValueFromNode(numShares);
+ num = num.equals("") ? "<empty>" : num;
+ try {
+ // Some ints have training zeros, so need to parse Double then truncate.
+ return (int) Double.parseDouble(num);
+ } catch (NumberFormatException e) {
+ System.err.println("WARNING: Num shares in XML bad format: " + num);
+ return -1;
+ }
}
/**
@@ -133,8 +153,16 @@ public class Transaction {
* @param values The value array representing the trade.
* @return The price of each share.
*/
- public double getPriceShares(NodeList values) {
- return Double.parseDouble(values.item(3).getTextContent());
+ public double getPriceShares(Element values) {
+ NodeList priceShares = values.getElementsByTagName("transactionPricePerShare");
+ String num = getValueFromNode(priceShares);
+ num = num.equals("") ? "<empty>" : num;
+ try {
+ return Double.parseDouble(num);
+ } catch (NumberFormatException e) {
+ System.err.println("WARNING: Price shares in XML bad format: " + num);
+ return -1;
+ }
}
/**
@@ -142,8 +170,17 @@ public class Transaction {
* @param values The value array representing the trade.
* @return True if they were bought (acquired). False, if sold.
*/
- public boolean getIsBuy(NodeList values) {
- return values.item(4).getTextContent().equals("A");
+ public int getIsBuy(Element values) {
+ NodeList isBuy = values.getElementsByTagName("transactionAcquiredDisposedCode");
+ String value = getValueFromNode(isBuy);
+ if (value.equals("A")) {
+ return 1;
+ } else if (value.equals("D")) {
+ return 0;
+ } else {
+ System.err.println("WARNING: Acquire code not recognized: " + value);
+ return -1;
+ }
}
/**
diff --git a/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java b/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java
index 440b898..c89c31d 100644
--- a/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java
+++ b/src/main/java/edu/brown/cs/student/term/parsing/UrlXmlParser.java
@@ -21,15 +21,17 @@ public class UrlXmlParser extends XmlParser{
@Override
public Document parse(String pathToXml) {
try {
- System.err.println("LOG: To make url class in parse() of " + getClass());
+ System.err.println("LOG: To make class for url: " + pathToXml + " in parse() of " + getClass());
URL url = new URL(pathToXml);
System.err.println("LOG: To establish urlConnection in parse() of " + getClass());
URLConnection conn = url.openConnection();
- System.err.println("LOG: Calling builder.parse() in " + getClass());
+ conn.addRequestProperty("User-Agent", "Chrome");
+ System.err.println("LOG: Calling builder.parse() for url: " + pathToXml + " in " + getClass());
return builder.parse(conn.getInputStream());
} catch (SAXException e) {
System.err.println("INTERNAL: SAX " + getClass() + " : " + e.getClass());
} catch (IOException e) {
+ e.printStackTrace();
System.err.println("INTERNAL: IO " + getClass() + " : " + e.getClass());
}
return null;
diff --git a/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java b/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java
index 09d98d2..54f9fc0 100644
--- a/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java
+++ b/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java
@@ -1,17 +1,23 @@
package edu.brown.cs.student.term.repl.commands;
import edu.brown.cs.student.term.DatabaseQuerier;
+import edu.brown.cs.student.term.Main;
import edu.brown.cs.student.term.parsing.Transaction;
import edu.brown.cs.student.term.parsing.UrlXmlParser;
import edu.brown.cs.student.term.repl.Command;
import edu.brown.cs.student.term.trade.Trade;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.w3c.dom.Document;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
+import java.time.Instant;
+import java.time.ZonedDateTime;
public class LoadCommand implements Command {
- private final static Connection CONN = DatabaseQuerier.getConn();
+ private Connection conn;
private final static UrlXmlParser URL_XML_PARSER = new UrlXmlParser();
/**
@@ -24,16 +30,26 @@ public class LoadCommand implements Command {
// TODO: add log comments
System.err.println("LOG: Entered .run() of " + getClass());
// TODO: call to api for urls to call through the urlxmlparser from reagan
- String[] urls = new String[1];
- for (String url : urls) {
+ if (Main.xmlLinks == null) {
+ return "ERROR: Please load xml links from frontend.";
+ }
+
+ conn = DatabaseQuerier.getConn();
+ JSONArray data = Main.xmlLinks.getJSONArray("data");
+ for(int i =0; i < data.length(); i++) {
+ JSONObject link = data.optJSONObject(i);
+
+ String timestamp = link.getString("timestamp");
+ String url = link.getString("url");
try {
System.err.println("LOG: Calling loadTransactionIntoDB() in " + getClass());
- loadTransactionIntoDB(url);
+ loadTransactionIntoDB(timestamp, url);
} catch (SQLException throwables) {
System.err.println("INTERNAL: SQLException in .run() of " + getClass());
//throwables.printStackTrace();
}
}
+
return "Loaded?";
}
@@ -42,13 +58,30 @@ public class LoadCommand implements Command {
* @param url The url to the public xml file.
* @throws SQLException If the prep statement fails or db doesn't exist, throws.
*/
- private static void loadTransactionIntoDB(String url) throws SQLException {
+ private void loadTransactionIntoDB(String timestamp, String url) throws SQLException {
System.err.println("LOG: Parsing XML into transaction in loadTransactionIntoDB(). URL: " + url);
- Transaction helper = new Transaction(URL_XML_PARSER.parse(url));
- for(Trade trade : helper.getTrades()) {
- System.err.println("LOG: Loading a trade into DB -> " + trade);
- loadTradeIntoDB(trade);
- System.err.println("LOG: Loaded that trade.");
+ // TODO: check if this is right @julia
+ // TODO: add parse error handling...
+ ZonedDateTime zonedDateTime = ZonedDateTime.parse(timestamp);
+ Instant instant = zonedDateTime.toInstant();
+
+ Document document = URL_XML_PARSER.parse(url);
+ if (document == null) {
+ System.err.println("WARNING: URL " + url + " failed to parse... continuing.");
+ return;
+ }
+
+ try {
+ // Constructing below is throwing null pointer - not sure why...
+ Transaction helper = new Transaction(document);
+
+ for(Trade trade : helper.getTrades()) {
+ System.err.println("LOG: Loading a trade into DB -> " + trade);
+ loadTradeIntoDB(instant, trade);
+ System.err.println("LOG: Loaded that trade.");
+ }
+ } catch (Exception e) {
+ System.err.println("WARNING: URL " + url + " failed to enter DB on " + e.getMessage());
}
}
@@ -57,8 +90,24 @@ public class LoadCommand implements Command {
* @param trade The trade to be loaded.
* @throws SQLException If the prep statement fails or db doesn't exist, throws.
*/
- private static void loadTradeIntoDB(Trade trade) throws SQLException {
- PreparedStatement prep = CONN.prepareStatement(
+ private void loadTradeIntoDB(Instant instant, Trade trade) throws SQLException {
+ // current table schema that is used...
+ // TODO: make this TABLE with this SCHEMA if doesn't exist.
+ /*
+ CREATE TABLE IF NOT EXISTS trades (
+ trade_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+ stock_name TEXT,
+ holder_name TEXT,
+ trade_timestamp INTEGER,
+ is_buy INTEGER,
+ number_of_shares INTEGER,
+ holder_id INTEGER,
+ share_price NUMERIC,
+ UNIQUE (trade_timestamp, is_buy, number_of_shares, holder_id, share_price));
+ */
+
+ System.err.println("LOG: Setting prepared statement on " + conn);
+ PreparedStatement prep = conn.prepareStatement(
"INSERT INTO trades (stock_name, holder_name, trade_timestamp, is_buy, " +
"number_of_shares, holder_id, share_price) " +
"VALUES (?, ?, ?, ?, ?, ?, ?)");
@@ -66,13 +115,15 @@ public class LoadCommand implements Command {
prep.setString(1, trade.getStock());
prep.setString(2, trade.getHolder().getName());
// TODO: update with timestamp @julia
- prep.setDouble(3, trade.getTimestamp());
+ prep.setLong(3, instant.toEpochMilli());
prep.setInt(4, trade.isBuy() ? 1 : 0);
prep.setInt(5, trade.getNumShares());
prep.setInt(6, trade.getHolder().getId());
prep.setDouble(7, trade.getPrice());
+ System.err.println("LOG: Inserted values into prep statement.");
prep.execute();
+ System.err.println("LOG: Executed prep statement.");
// TODO: close connection - when should we do this
}
} \ No newline at end of file
diff --git a/src/test/java/edu/brown/cs/student/TransactionTest.java b/src/test/java/edu/brown/cs/student/TransactionTest.java
index e8b95bc..faf24fb 100644
--- a/src/test/java/edu/brown/cs/student/TransactionTest.java
+++ b/src/test/java/edu/brown/cs/student/TransactionTest.java
@@ -95,4 +95,13 @@ public class TransactionTest {
tearDown();
}
+ @Test
+ public void badFormats(){
+ setUp();
+ // TODO: ensure 0 price trades aren't there, market trades aren't there, .00 on end of ints, etc...
+ // no price -> https://www.sec.gov/Archives/edgar/data/1654595/000110465921048974/tm2112683-1_4seq1.xml
+ // market form -> https://www.sec.gov/Archives/edgar/data/39899/000121465921004094/marketforms-52237.xml
+ tearDown();
+ }
+
}
diff --git a/src/test/java/edu/brown/cs/student/XmlParserTest.java b/src/test/java/edu/brown/cs/student/XmlParserTest.java
index e6a112f..eebccbb 100644
--- a/src/test/java/edu/brown/cs/student/XmlParserTest.java
+++ b/src/test/java/edu/brown/cs/student/XmlParserTest.java
@@ -48,9 +48,13 @@ public class XmlParserTest {
Document doc =
_urlXmlParser.parse("https://www.sec.gov/Archives/edgar/data/1517006/000110465921046242/tm2112036-4_4seq1.xml");
assertNotNull(doc);
-
// Id of person
assertEquals(getIdFromDoc(doc), "0001561844");
+
+ Document doc2 =
+ _urlXmlParser.parse("https://www.sec.gov/Archives/edgar/data/1254015/000120919121026318/doc4.xml");
+ assertNotNull(doc2);
+ assertEquals(getIdFromDoc(doc2), "0001254015");
tearDown();
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 12:51:11 +0000