From e00c435a929afeac1965e04b8f6585696a915e77 Mon Sep 17 00:00:00 2001 From: Michael Foiani Date: Sun, 18 Apr 2021 00:53:51 -0400 Subject: Working build that scales with the large database. We need to figure out the profit calculation on the large scale, though. --- react-frontend/src/App.js | 5 +++++ react-frontend/src/components/Visualization.js | 15 ++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/react-frontend/src/App.js b/react-frontend/src/App.js index 34632eb..c9451a9 100644 --- a/react-frontend/src/App.js +++ b/react-frontend/src/App.js @@ -31,6 +31,10 @@ function App() { const toEpochMilli = date => Date.parse(date); const getGraphData = () => { + console.log({ + start: toEpochMilli(dates.start), + end: toEpochMilli(dates.end) + }); fetch("http://localhost:4567/data", { method: "POST", body: JSON.stringify({ @@ -46,6 +50,7 @@ function App() { .then(data => { setData(data.holders); setHasLoaded(true); + console.log(data.holders); }) .catch(err => console.log(err)); diff --git a/react-frontend/src/components/Visualization.js b/react-frontend/src/components/Visualization.js index 91082e9..2016129 100644 --- a/react-frontend/src/components/Visualization.js +++ b/react-frontend/src/components/Visualization.js @@ -15,8 +15,7 @@ function Visualization(props) { const options = { edges: { color: "#ffffff" - }, - autoResize: true + } }; const events = { select: () => event => props.setSelected(event.nodes[0]) @@ -29,11 +28,13 @@ function Visualization(props) { const getNodes = () => { let nodes = []; props.data.forEach(hub => { - nodes.push({ - id: hub.id, - label: hub.name, - size: hub.suspicionScore * 10 - }); + if (hub.followers) { + nodes.push({ + id: hub.id, + label: hub.name, + size: hub.suspicionScore * 10 + }); + } }); return nodes; } -- cgit v1.2.3-70-g09d2 From 6b5037e4e03cd44f79bb1d3460e0e45c6d7b6541 Mon Sep 17 00:00:00 2001 From: Michael Foiani Date: Sun, 18 Apr 2021 01:02:42 -0400 Subject: Small bug fixes to ticker symbols. --- .../edu/brown/cs/student/term/DatabaseQuerier.java | 24 ++++++++++++++-------- src/main/java/edu/brown/cs/student/term/Main.java | 4 +++- .../edu/brown/cs/student/term/hub/HubSearch.java | 23 +++++++++++++++++---- .../edu/brown/cs/student/term/hub/LinkMapper.java | 1 - .../brown/cs/student/term/hub/SuspicionRanker.java | 10 +++++++-- .../cs/student/term/profit/ProfitCalculation.java | 1 + 6 files changed, 47 insertions(+), 16 deletions(-) diff --git a/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java b/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java index 6900a19..4ff02f9 100644 --- a/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java +++ b/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java @@ -53,7 +53,10 @@ public class DatabaseQuerier { ResultSet rs = prep.executeQuery(); while (rs.next()) { - stocks.add(rs.getString(1)); + String ticker = rs.getString(1); + if (isValidTicker(ticker)) { + stocks.add(ticker); + } } rs.close(); @@ -94,6 +97,9 @@ public class DatabaseQuerier { public List getTradeByStock(String stock, int isBuy, Instant startDate, Instant endDate) throws SQLException { List trades = new ArrayList<>(); + if (isValidStock(stock)) { + return trades; + } PreparedStatement prep = conn.prepareStatement( "SELECT * FROM trades WHERE (stock_name = ? AND is_buy = ?) " @@ -133,13 +139,15 @@ public class DatabaseQuerier { ResultSet rs = prep.executeQuery(); while (rs.next()) { - trades.addFirst(new Trade(rs.getInt("trade_id"), - rs.getString("stock_name"), - rs.getDouble("trade_timestamp"), - rs.getInt("is_buy"), - rs.getInt("number_of_shares"), - new Holder(rs.getInt("holder_id"), rs.getString("holder_name")), - rs.getDouble("share_price"))); + String ticker = rs.getString("stock_name"); + if (isValidTicker(ticker)) { + trades.addFirst(new Trade(rs.getInt("trade_id"), ticker, + rs.getDouble("trade_timestamp"), + rs.getInt("is_buy"), + rs.getInt("number_of_shares"), + new Holder(rs.getInt("holder_id"), rs.getString("holder_name")), + rs.getDouble("share_price"))); + } } prep.close(); } catch (SQLException e) { diff --git a/src/main/java/edu/brown/cs/student/term/Main.java b/src/main/java/edu/brown/cs/student/term/Main.java index 7ee9874..ee3bec1 100644 --- a/src/main/java/edu/brown/cs/student/term/Main.java +++ b/src/main/java/edu/brown/cs/student/term/Main.java @@ -152,6 +152,7 @@ public final class Main { //String str = request.body(); //xmlLinks = new JSONObject(str); //this is all the filedAt times and xml files try { + System.err.println("LOG: Call to /data from frontend"); DatabaseQuerier db = SetupCommand.getDq(); SuspicionRanker ranker = new SuspicionRanker(db); @@ -161,7 +162,9 @@ public final class Main { long endMilli = data.getLong("end"); Instant start = Instant.ofEpochMilli(startMilli); Instant end = Instant.ofEpochMilli(endMilli); + System.err.println("LOG: Call to ranker.getSusscore in " + getClass()); List suspiciousHolders = ranker.getSuspicionScoreList(start, end); + System.err.println("LOG: Making map " + getClass()); Map variables = ImmutableMap.of("holders", suspiciousHolders); return GSON.toJson(variables); } catch (Exception e) { @@ -194,7 +197,6 @@ public final class Main { res.put("SP500", (1 + sp500PercentGain) * profit.getMoneyInput()); res.put("percentSP500", 100 * sp500PercentGain); return GSON.toJson(res); - } } diff --git a/src/main/java/edu/brown/cs/student/term/hub/HubSearch.java b/src/main/java/edu/brown/cs/student/term/hub/HubSearch.java index 86b883f..d4c48b9 100644 --- a/src/main/java/edu/brown/cs/student/term/hub/HubSearch.java +++ b/src/main/java/edu/brown/cs/student/term/hub/HubSearch.java @@ -22,9 +22,11 @@ public class HubSearch { double[] weights = new double[numHolders]; double[] rank = new double[numHolders]; double[] rankPrime = new double[numHolders]; + System.err.println(numHolders + "\t" + getDigits(numHolders)); Arrays.fill(rankPrime, 1.0 / numHolders); - while(!withinDistance(rank, rankPrime)){ + double thresh = Math.pow(.1, getDigits(numHolders)); + while(!withinDistance(rank, rankPrime, thresh)){ rank = Arrays.copyOf(rankPrime, rankPrime.length); //calculating hub rank for ith holder for(int i = 0; i < numHolders; i++){ @@ -65,7 +67,7 @@ public class HubSearch { if(peopleFollowed.contains(leader)){ //constructs the leader to follower links as we go for use later on - leader.addFollower(follower); + leader.addFollower(new Holder(follower.getId(), follower.getName())); return ((damp / numHolders) + (1 - damp) * (1.0 / numberFollowed)); } else if(numberFollowed == 0){ return ((damp / numHolders) + (1 - damp) * (1.0 / numHolders)); @@ -74,12 +76,25 @@ public class HubSearch { } } - private boolean withinDistance(double[] r, double[] rPrime){ + private boolean withinDistance(double[] r, double[] rPrime, double threshold){ double sum = 0.0; for(int i = 0; i < r.length; i++){ sum += Math.pow((r[i] - rPrime[i]), 2); } - return sum <= 0.001; + System.err.println(sum + "\t" + !(sum <= 0.0000001)); + + return sum <= 0.001*threshold; + } + + private int getDigits(int numFollowers) { + int count = 1; + int temp = numFollowers; + while(temp >= 10) { + count++; + temp /= 10; + System.out.println(temp); + } + return count; } } diff --git a/src/main/java/edu/brown/cs/student/term/hub/LinkMapper.java b/src/main/java/edu/brown/cs/student/term/hub/LinkMapper.java index b490ea1..31e2625 100644 --- a/src/main/java/edu/brown/cs/student/term/hub/LinkMapper.java +++ b/src/main/java/edu/brown/cs/student/term/hub/LinkMapper.java @@ -52,7 +52,6 @@ public class LinkMapper { } else { System.out.println("ERROR: No database loaded in yet"); } - return followerToLeaders; } diff --git a/src/main/java/edu/brown/cs/student/term/hub/SuspicionRanker.java b/src/main/java/edu/brown/cs/student/term/hub/SuspicionRanker.java index fbe05f7..3283f5c 100644 --- a/src/main/java/edu/brown/cs/student/term/hub/SuspicionRanker.java +++ b/src/main/java/edu/brown/cs/student/term/hub/SuspicionRanker.java @@ -52,6 +52,8 @@ public class SuspicionRanker { HubSearch hub = new HubSearch(lm); Map holderToHubScore = hub.runHubSearch(start, end); + /* + ProfitCalculation pc = new ProfitCalculation(DatabaseQuerier.getConn(), "", new Date(start.toEpochMilli()), new Date(end.toEpochMilli())); @@ -66,6 +68,7 @@ public class SuspicionRanker { double profitMax = getMaxOfMap(profitMap); /*if all of our values are negative, we need to flip sides so that the * biggest loser doesn't end up being the most suspicious person*/ + /* if(profitMax <= 0) { profitMax = Math.abs(getMinOfMap(profitMap)); } @@ -73,17 +76,20 @@ public class SuspicionRanker { /*if both the min we found and max we found are 0, then we have the special case where all the values are 0, in which case we need to avoid dividing by 0*/ + /* if(profitMax == 0){ profitMax = 1; } + */ + double hubMax = getMaxOfMap(holderToHubScore); for (Holder guy : holderToHubScore.keySet()) { - double normalizedProfitScore = profitMap.get(guy.getId()) / profitMax; + //double normalizedProfitScore = profitMap.get(guy.getId()) / profitMax; double normalizedHubScore = holderToHubScore.get(guy) / hubMax; - double suspicionScore = normalizedHubScore * 0.6 + normalizedProfitScore * 0.4; + double suspicionScore = normalizedHubScore; //* 0.6 + normalizedProfitScore * 0.4; guy.setSuspicionScore(suspicionScore); orderedSuspicion.add(guy); } diff --git a/src/main/java/edu/brown/cs/student/term/profit/ProfitCalculation.java b/src/main/java/edu/brown/cs/student/term/profit/ProfitCalculation.java index 7f08460..15f31cc 100644 --- a/src/main/java/edu/brown/cs/student/term/profit/ProfitCalculation.java +++ b/src/main/java/edu/brown/cs/student/term/profit/ProfitCalculation.java @@ -243,6 +243,7 @@ public class ProfitCalculation { return currentStockPrices.get(ticker); } else { String PRICE_URL = BASE_URL + "/last/stocks/" + ticker; + System.out.println("LOG: Making call to " + PRICE_URL + " in " + getClass()); HttpClient client = HttpClient.newHttpClient(); HttpRequest request = HttpRequest.newBuilder() -- cgit v1.2.3-70-g09d2 From 4cfdb31be3697565e5c4ae95cdc4b60161bd4e84 Mon Sep 17 00:00:00 2001 From: Michael Foiani Date: Sun, 18 Apr 2021 01:31:20 -0400 Subject: Have a good working canvas now by limiting the results shown. --- react-frontend/src/App.js | 8 +++++--- react-frontend/src/components/HubList.js | 4 ++-- react-frontend/src/components/Visualization.js | 2 +- .../edu/brown/cs/student/term/DatabaseQuerier.java | 24 +++++++++++++++++----- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/react-frontend/src/App.js b/react-frontend/src/App.js index c9451a9..0a6e6c1 100644 --- a/react-frontend/src/App.js +++ b/react-frontend/src/App.js @@ -48,9 +48,11 @@ function App() { }) .then(res => res.json()) .then(data => { - setData(data.holders); + //TODO: optimize this + const sliced = data.holders.slice(0, 500); + console.log(sliced); + setData(sliced); setHasLoaded(true); - console.log(data.holders); }) .catch(err => console.log(err)); @@ -59,7 +61,7 @@ function App() { // Hooks to update data on init and switching of data - useEffect(() => getGraphData(), []); + //useEffect(() => getGraphData(), []); useEffect(() => { setIsChanging(true); getGraphData(); diff --git a/react-frontend/src/components/HubList.js b/react-frontend/src/components/HubList.js index 5736e89..0df3020 100644 --- a/react-frontend/src/components/HubList.js +++ b/react-frontend/src/components/HubList.js @@ -22,8 +22,8 @@ function HubList(props) { const updateHubItems = () => { // sort and create the elemnts let hubs = []; - const sorted = props.data.sort((a, b) => b.suspicionScore - a.suspicionScore); - sorted.forEach(hub => hubs.push( + //const sorted = props.data.sort((a, b) => b.suspicionScore - a.suspicionScore); + props.data.forEach(hub => hubs.push( )); diff --git a/react-frontend/src/components/Visualization.js b/react-frontend/src/components/Visualization.js index 2016129..1975e86 100644 --- a/react-frontend/src/components/Visualization.js +++ b/react-frontend/src/components/Visualization.js @@ -32,7 +32,7 @@ function Visualization(props) { nodes.push({ id: hub.id, label: hub.name, - size: hub.suspicionScore * 10 + size: hub.suspicionScore }); } }); diff --git a/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java b/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java index 4ff02f9..53c8cdc 100644 --- a/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java +++ b/src/main/java/edu/brown/cs/student/term/DatabaseQuerier.java @@ -53,8 +53,8 @@ public class DatabaseQuerier { ResultSet rs = prep.executeQuery(); while (rs.next()) { - String ticker = rs.getString(1); - if (isValidTicker(ticker)) { + String ticker = validateTicker(rs.getString(1)); + if (!ticker.equals("")) { stocks.add(ticker); } } @@ -97,9 +97,10 @@ public class DatabaseQuerier { public List getTradeByStock(String stock, int isBuy, Instant startDate, Instant endDate) throws SQLException { List trades = new ArrayList<>(); + /* if (isValidStock(stock)) { return trades; - } + }*/ PreparedStatement prep = conn.prepareStatement( "SELECT * FROM trades WHERE (stock_name = ? AND is_buy = ?) " @@ -140,14 +141,14 @@ public class DatabaseQuerier { while (rs.next()) { String ticker = rs.getString("stock_name"); - if (isValidTicker(ticker)) { + //if (isValidTicker(ticker)) { trades.addFirst(new Trade(rs.getInt("trade_id"), ticker, rs.getDouble("trade_timestamp"), rs.getInt("is_buy"), rs.getInt("number_of_shares"), new Holder(rs.getInt("holder_id"), rs.getString("holder_name")), rs.getDouble("share_price"))); - } + //} } prep.close(); } catch (SQLException e) { @@ -155,4 +156,17 @@ public class DatabaseQuerier { } return trades; } + + private String validateTicker(String ticker) { + //this is cleaning some improperly formatted tickers + ticker = ticker.replaceAll("[^a-zA-Z0-9]", "").toUpperCase(); + if(ticker.contains("[0-9]") || + ticker.length() > 5 || + ticker.length() < 2 || + ticker.contains("NONE")) { + return ""; + } + + return ticker; + } } -- cgit v1.2.3-70-g09d2