1 files changed, 26 insertions, 17 deletions

diff --git a/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java b/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java
index a267322..b1e8cb4 100644
--- a/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java
+++ b/src/main/java/edu/brown/cs/student/term/repl/commands/LoadCommand.java
@@ -18,6 +18,8 @@ import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.time.Instant;
 import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.List;
 
 public class LoadCommand implements Command {
   private Connection conn;
@@ -45,13 +47,14 @@ public class LoadCommand implements Command {
 
 
     System.err.println("LOG: Entered .run() of " + getClass());
-    FilingFeed filings = getFilings(numFilings);
-    if (filings.getFilings().isEmpty()) {
+    List<FilingFeed.Filing> allFilings = getFilings(numFilings);
+
+    if (allFilings.isEmpty()) {
       System.err.println("WARNING: No filings loaded.");
     }
 
     conn = DatabaseQuerier.getConn();
-    for(FilingFeed.Filing filing : filings.getFilings()) {
+    for(FilingFeed.Filing filing : allFilings) {
       try {
         System.err.println("LOG: Calling loadTransactionIntoDB() in " + getClass());
         loadTransactionIntoDB(filing.getTimestamp(), filing.getXmlUrl());
@@ -64,21 +67,27 @@ public class LoadCommand implements Command {
     return "Loaded?";
   }
 
-  private FilingFeed getFilings(int numFilings) {
+  private List<FilingFeed.Filing> getFilings(int numFilings) {
+    List<FilingFeed.Filing> all = new ArrayList<>();
+    int counter = 0;
+    while (100*counter <= numFilings) {
+      String queryUrl = "https://www.sec.gov/cgi-bin/browse-edgar?" +
+          "action=getcurrent" +
+          "&CIK=" +
+          "&type=4" +
+          "&company=" +
+          "&dateb=" +
+          "&owner=only" +
+          "&start=" + (100*counter++) +
+          "&count=" + 100 +
+          "&output=atom";
+      System.out.println("LOG: Requesting filings with url: " + queryUrl);
+      Document document = URL_XML_PARSER.parse(queryUrl);
+      FilingFeed filingFeed = new FilingFeed(document);
+      all.addAll(filingFeed.getFilings());
+    }
     // TODO: make params more adjustable
-    String queryUrl = "https://www.sec.gov/cgi-bin/browse-edgar?" +
-        "action=getcurrent" +
-        "&CIK=" +
-        "&type=4" +
-        "&company=" +
-        "&dateb=" +
-        "&owner=only" +
-        "&start=0" +
-        "&count=" + numFilings +
-        "&output=atom";
-
-    Document document = URL_XML_PARSER.parse(queryUrl);
-    return new FilingFeed(document);
+    return all;
   }
 
   /**