Merge branch 'master' of https://github.com/browngraphicslab/Dash-Web

5 files changed, 72 insertions, 21 deletions

diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts
index 3c32a94c2..268239481 100644
--- a/src/server/GarbageCollector.ts
+++ b/src/server/GarbageCollector.ts
@@ -2,6 +2,7 @@ import { Database } from './database';
 
 import * as path from 'path';
 import * as fs from 'fs';
+import { Search } from './Search';
 
 function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) {
     for (const key in doc) {
@@ -60,7 +61,7 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) {
 
 async function GarbageCollect() {
     // await new Promise(res => setTimeout(res, 3000));
-    const cursor = await Database.Instance.query({}, 'users');
+    const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users');
     const users = await cursor.toArray();
     const ids: string[] = users.map(user => user.userDocumentId);
     const visited = new Set<string>();
@@ -90,12 +91,15 @@ async function GarbageCollect() {
 
     cursor.close();
 
-    const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } });
-    const toDelete = (await toDeleteCursor.toArray()).map(doc => doc._id);
+    const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }, { _id: 1 });
+    const toDelete: string[] = (await toDeleteCursor.toArray()).map(doc => doc._id);
     toDeleteCursor.close();
     const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments");
     console.log(`${result.deletedCount} documents deleted`);
 
+    await Search.Instance.deleteDocuments(toDelete);
+    console.log("Cleared search documents");
+
     const folder = "./src/server/public/files/";
     fs.readdir(folder, (_, fileList) => {
         const filesToDelete = fileList.filter(file => {
diff --git a/src/server/Search.ts b/src/server/Search.ts
index e2a74b737..8591f8857 100644
--- a/src/server/Search.ts
+++ b/src/server/Search.ts
@@ -18,6 +18,18 @@ export class Search {
         }
     }
 
+    public async updateDocuments(documents: any[]) {
+        try {
+            const res = await rp.post(this.url + "dash/update", {
+                headers: { 'content-type': 'application/json' },
+                body: JSON.stringify(documents)
+            });
+            return res;
+        } catch (e) {
+            // console.warn("Search error: " + e + document);
+        }
+    }
+
     public async search(query: string, start: number = 0) {
         try {
             const searchResults = JSON.parse(await rp.get(this.url + "dash/select", {
@@ -47,4 +59,25 @@ export class Search {
             });
         } catch { }
     }
+
+    public deleteDocuments(docs: string[]) {
+        const promises: rp.RequestPromise[] = [];
+        const nToDelete = 1000;
+        let index = 0;
+        while (index < docs.length) {
+            const count = Math.min(docs.length - index, nToDelete);
+            const deleteIds = docs.slice(index, index + count);
+            index += count;
+            promises.push(rp.post(this.url + "dash/update", {
+                body: {
+                    delete: {
+                        query: deleteIds.map(id => `id:"${id}"`).join(" ")
+                    }
+                },
+                json: true
+            }));
+        }
+
+        return Promise.all(promises);
+    }
 }
\ No newline at end of file
diff --git a/src/server/database.ts b/src/server/database.ts
index a17447629..29a8ffafa 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -126,12 +126,16 @@ export class Database {
         }
     }
 
-    public query(query: any, collectionName = "newDocuments"): Promise<mongodb.Cursor> {
+    public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise<mongodb.Cursor> {
         if (this.db) {
-            return Promise.resolve<mongodb.Cursor>(this.db.collection(collectionName).find(query));
+            let cursor = this.db.collection(collectionName).find(query);
+            if (projection) {
+                cursor = cursor.project(projection);
+            }
+            return Promise.resolve<mongodb.Cursor>(cursor);
         } else {
             return new Promise<mongodb.Cursor>(res => {
-                this.onConnect.push(() => res(this.query(query, collectionName)));
+                this.onConnect.push(() => res(this.query(query, projection, collectionName)));
             });
         }
     }
diff --git a/src/server/index.ts b/src/server/index.ts
index bf946fc9f..c9ec11bfa 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -218,7 +218,7 @@ addSecureRoute(
 addSecureRoute(
     Method.GET,
     async (_, res) => {
-        const cursor = await Database.Instance.query({}, "users");
+        const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users");
         const results = await cursor.toArray();
         res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId })));
     },
diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts
index f5de00978..906b795f1 100644
--- a/src/server/updateSearch.ts
+++ b/src/server/updateSearch.ts
@@ -7,7 +7,7 @@ const suffixMap: { [type: string]: (string | [string, string | ((json: any) => a
     "number": "_n",
     "string": "_t",
     "boolean": "_b",
-    "image": ["_t", "url"],
+    // "image": ["_t", "url"],
     "video": ["_t", "url"],
     "pdf": ["_t", "url"],
     "audio": ["_t", "url"],
@@ -67,7 +67,7 @@ async function update() {
         if ((numDocs % 50) === 0) {
             console.log("updateDoc " + numDocs);
         }
-        console.log("doc " + numDocs);
+        // console.log("doc " + numDocs);
         if (doc.__type !== "Doc") {
             return;
         }
@@ -88,22 +88,32 @@ async function update() {
         }
         if (dynfield) {
             updates.push(update);
-            console.log(updates.length);
+            // console.log(updates.length);
         }
     }
     await cursor.forEach(updateDoc);
-    for (let i = 0; i < updates.length; i++) {
-        console.log(i);
-        const result = await Search.Instance.updateDocument(updates[i]);
-        try {
-            console.log(JSON.parse(result).responseHeader.status);
-        } catch {
-            console.log("Error:");
-            console.log(updates[i]);
-            console.log(result);
-            console.log("\n");
-        }
+    console.log(`Updating ${updates.length} documents`);
+    const result = await Search.Instance.updateDocuments(updates);
+    try {
+        console.log(JSON.parse(result).responseHeader.status);
+    } catch {
+        console.log("Error:");
+        // console.log(updates[i]);
+        console.log(result);
+        console.log("\n");
     }
+    // for (let i = 0; i < updates.length; i++) {
+    //     console.log(i);
+    //     const result = await Search.Instance.updateDocument(updates[i]);
+    // try {
+    //     console.log(JSON.parse(result).responseHeader.status);
+    // } catch {
+    //     console.log("Error:");
+    //     console.log(updates[i]);
+    //     console.log(result);
+    //     console.log("\n");
+    // }
+    // }
     // await Promise.all(updates.map(update => {
     //     return limit(() => Search.Instance.updateDocument(update));
     // }));