aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorbob <bcz@cs.brown.edu>2019-07-09 14:18:28 -0400
committerbob <bcz@cs.brown.edu>2019-07-09 14:18:28 -0400
commitfb1e2078dc0e0cdc33f6037cd3c3e18ffded413e (patch)
treeddd2fb40e4e344b3c3b1e0f7057b62566a92084c /src
parent724d03b8d05b5ef7ef944c648aae8f43dde625be (diff)
parentce39600f0c7b9013c7c8ec8c74eb54d2d5e4c50e (diff)
Merge branch 'master' of https://github.com/browngraphicslab/Dash-Web
Diffstat (limited to 'src')
-rw-r--r--src/server/GarbageCollector.ts10
-rw-r--r--src/server/Search.ts33
-rw-r--r--src/server/database.ts10
-rw-r--r--src/server/index.ts2
-rw-r--r--src/server/updateSearch.ts38
5 files changed, 72 insertions, 21 deletions
diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts
index 3c32a94c2..268239481 100644
--- a/src/server/GarbageCollector.ts
+++ b/src/server/GarbageCollector.ts
@@ -2,6 +2,7 @@ import { Database } from './database';
import * as path from 'path';
import * as fs from 'fs';
+import { Search } from './Search';
function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) {
for (const key in doc) {
@@ -60,7 +61,7 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) {
async function GarbageCollect() {
// await new Promise(res => setTimeout(res, 3000));
- const cursor = await Database.Instance.query({}, 'users');
+ const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users');
const users = await cursor.toArray();
const ids: string[] = users.map(user => user.userDocumentId);
const visited = new Set<string>();
@@ -90,12 +91,15 @@ async function GarbageCollect() {
cursor.close();
- const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } });
- const toDelete = (await toDeleteCursor.toArray()).map(doc => doc._id);
+ const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }, { _id: 1 });
+ const toDelete: string[] = (await toDeleteCursor.toArray()).map(doc => doc._id);
toDeleteCursor.close();
const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments");
console.log(`${result.deletedCount} documents deleted`);
+ await Search.Instance.deleteDocuments(toDelete);
+ console.log("Cleared search documents");
+
const folder = "./src/server/public/files/";
fs.readdir(folder, (_, fileList) => {
const filesToDelete = fileList.filter(file => {
diff --git a/src/server/Search.ts b/src/server/Search.ts
index e2a74b737..8591f8857 100644
--- a/src/server/Search.ts
+++ b/src/server/Search.ts
@@ -18,6 +18,18 @@ export class Search {
}
}
+ public async updateDocuments(documents: any[]) {
+ try {
+ const res = await rp.post(this.url + "dash/update", {
+ headers: { 'content-type': 'application/json' },
+ body: JSON.stringify(documents)
+ });
+ return res;
+ } catch (e) {
+ // console.warn("Search error: " + e + document);
+ }
+ }
+
public async search(query: string, start: number = 0) {
try {
const searchResults = JSON.parse(await rp.get(this.url + "dash/select", {
@@ -47,4 +59,25 @@ export class Search {
});
} catch { }
}
+
+ public deleteDocuments(docs: string[]) {
+ const promises: rp.RequestPromise[] = [];
+ const nToDelete = 1000;
+ let index = 0;
+ while (index < docs.length) {
+ const count = Math.min(docs.length - index, nToDelete);
+ const deleteIds = docs.slice(index, index + count);
+ index += count;
+ promises.push(rp.post(this.url + "dash/update", {
+ body: {
+ delete: {
+ query: deleteIds.map(id => `id:"${id}"`).join(" ")
+ }
+ },
+ json: true
+ }));
+ }
+
+ return Promise.all(promises);
+ }
} \ No newline at end of file
diff --git a/src/server/database.ts b/src/server/database.ts
index a17447629..29a8ffafa 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -126,12 +126,16 @@ export class Database {
}
}
- public query(query: any, collectionName = "newDocuments"): Promise<mongodb.Cursor> {
+ public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise<mongodb.Cursor> {
if (this.db) {
- return Promise.resolve<mongodb.Cursor>(this.db.collection(collectionName).find(query));
+ let cursor = this.db.collection(collectionName).find(query);
+ if (projection) {
+ cursor = cursor.project(projection);
+ }
+ return Promise.resolve<mongodb.Cursor>(cursor);
} else {
return new Promise<mongodb.Cursor>(res => {
- this.onConnect.push(() => res(this.query(query, collectionName)));
+ this.onConnect.push(() => res(this.query(query, projection, collectionName)));
});
}
}
diff --git a/src/server/index.ts b/src/server/index.ts
index bf946fc9f..c9ec11bfa 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -218,7 +218,7 @@ addSecureRoute(
addSecureRoute(
Method.GET,
async (_, res) => {
- const cursor = await Database.Instance.query({}, "users");
+ const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users");
const results = await cursor.toArray();
res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId })));
},
diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts
index f5de00978..906b795f1 100644
--- a/src/server/updateSearch.ts
+++ b/src/server/updateSearch.ts
@@ -7,7 +7,7 @@ const suffixMap: { [type: string]: (string | [string, string | ((json: any) => a
"number": "_n",
"string": "_t",
"boolean": "_b",
- "image": ["_t", "url"],
+ // "image": ["_t", "url"],
"video": ["_t", "url"],
"pdf": ["_t", "url"],
"audio": ["_t", "url"],
@@ -67,7 +67,7 @@ async function update() {
if ((numDocs % 50) === 0) {
console.log("updateDoc " + numDocs);
}
- console.log("doc " + numDocs);
+ // console.log("doc " + numDocs);
if (doc.__type !== "Doc") {
return;
}
@@ -88,22 +88,32 @@ async function update() {
}
if (dynfield) {
updates.push(update);
- console.log(updates.length);
+ // console.log(updates.length);
}
}
await cursor.forEach(updateDoc);
- for (let i = 0; i < updates.length; i++) {
- console.log(i);
- const result = await Search.Instance.updateDocument(updates[i]);
- try {
- console.log(JSON.parse(result).responseHeader.status);
- } catch {
- console.log("Error:");
- console.log(updates[i]);
- console.log(result);
- console.log("\n");
- }
+ console.log(`Updating ${updates.length} documents`);
+ const result = await Search.Instance.updateDocuments(updates);
+ try {
+ console.log(JSON.parse(result).responseHeader.status);
+ } catch {
+ console.log("Error:");
+ // console.log(updates[i]);
+ console.log(result);
+ console.log("\n");
}
+ // for (let i = 0; i < updates.length; i++) {
+ // console.log(i);
+ // const result = await Search.Instance.updateDocument(updates[i]);
+ // try {
+ // console.log(JSON.parse(result).responseHeader.status);
+ // } catch {
+ // console.log("Error:");
+ // console.log(updates[i]);
+ // console.log(result);
+ // console.log("\n");
+ // }
+ // }
// await Promise.all(updates.map(update => {
// return limit(() => Search.Instance.updateDocument(update));
// }));