Added update search script

Added debug current file launch config
Made database not ignore requests before it is initialized.

4 files changed, 196 insertions, 54 deletions

diff --git a/.vscode/launch.json b/.vscode/launch.json
index 452acc823..d245be2ad 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -34,6 +34,20 @@
         {
             "type": "node",
             "request": "launch",
+            "name": "Current TS File",
+            "runtimeExecutable": "npx",
+            "runtimeArgs": [
+                "ts-node-dev",
+                "--nolazy",
+                "--inspect",
+                "--",
+                "${relativeFile}"
+            ],
+            "port": 9229
+        },
+        {
+            "type": "node",
+            "request": "launch",
             "name": "Mocha Tests",
             "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha",
             "args": [
@@ -65,15 +79,5 @@
             "internalConsoleOptions": "openOnSessionStart",
             "protocol": "inspector"
         },
-        {
-            "type": "node",
-            "request": "launch",
-            "name": "Launch via NPM",
-            "runtimeExecutable": "npm",
-            "runtimeArgs": [
-                "start"
-            ],
-            "port": 9229
-        }
     ]
 }
\ No newline at end of file
diff --git a/package.json b/package.json
index 40402b44c..790535728 100644
--- a/package.json
+++ b/package.json
@@ -133,6 +133,7 @@
         "nodemon": "^1.18.10",
         "normalize.css": "^8.0.1",
         "npm": "^6.9.0",
+        "p-limit": "^2.2.0",
         "passport": "^0.4.0",
         "passport-local": "^1.0.0",
         "prosemirror-commands": "^1.0.7",
diff --git a/src/server/database.ts b/src/server/database.ts
index 69005d2d3..70b3efced 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -8,9 +8,13 @@ export class Database {
     private url = 'mongodb://localhost:27017/Dash';
     private currentWrites: { [id: string]: Promise<void> } = {};
     private db?: mongodb.Db;
+    private onConnect: (() => void)[] = [];
 
     constructor() {
-        this.MongoClient.connect(this.url, (err, client) => this.db = client.db());
+        this.MongoClient.connect(this.url, (err, client) => {
+            this.db = client.db();
+            this.onConnect.forEach(fn => fn());
+        });
     }
 
     public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) {
@@ -32,66 +36,98 @@ export class Database {
             };
             newProm = prom ? prom.then(run) : run();
             this.currentWrites[id] = newProm;
+        } else {
+            this.onConnect.push(() => this.update(id, value, callback, upsert, collectionName));
         }
     }
 
     public delete(id: string, collectionName = Database.DocumentsCollection) {
-        this.db && this.db.collection(collectionName).remove({ id: id });
+        if (this.db) {
+            this.db.collection(collectionName).remove({ id: id });
+        } else {
+            this.onConnect.push(() => this.delete(id, collectionName));
+        }
     }
 
     public deleteAll(collectionName = Database.DocumentsCollection): Promise<any> {
-        return new Promise(res =>
-            this.db && this.db.collection(collectionName).deleteMany({}, res));
+        return new Promise(res => {
+            if (this.db) {
+                this.db.collection(collectionName).deleteMany({}, res);
+            } else {
+                this.onConnect.push(() => this.db && this.db.collection(collectionName).deleteMany({}, res));
+            }
+        });
     }
 
     public insert(value: any, collectionName = Database.DocumentsCollection) {
-        if (!this.db) { return; }
-        if ("id" in value) {
-            value._id = value.id;
-            delete value.id;
-        }
-        const id = value._id;
-        const collection = this.db.collection(collectionName);
-        const prom = this.currentWrites[id];
-        let newProm: Promise<void>;
-        const run = (): Promise<void> => {
-            return new Promise<void>(resolve => {
-                collection.insertOne(value, (err, res) => {
-                    if (this.currentWrites[id] === newProm) {
-                        delete this.currentWrites[id];
-                    }
-                    resolve();
+        if (this.db) {
+            if ("id" in value) {
+                value._id = value.id;
+                delete value.id;
+            }
+            const id = value._id;
+            const collection = this.db.collection(collectionName);
+            const prom = this.currentWrites[id];
+            let newProm: Promise<void>;
+            const run = (): Promise<void> => {
+                return new Promise<void>(resolve => {
+                    collection.insertOne(value, (err, res) => {
+                        if (this.currentWrites[id] === newProm) {
+                            delete this.currentWrites[id];
+                        }
+                        resolve();
+                    });
                 });
-            });
-        };
-        newProm = prom ? prom.then(run) : run();
-        this.currentWrites[id] = newProm;
+            };
+            newProm = prom ? prom.then(run) : run();
+            this.currentWrites[id] = newProm;
+        } else {
+            this.onConnect.push(() => this.insert(value, collectionName));
+        }
     }
 
     public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = Database.DocumentsCollection) {
-        this.db && this.db.collection(collectionName).findOne({ _id: id }, (err, result) => {
-            if (result) {
-                result.id = result._id;
-                delete result._id;
-                fn(result);
-            } else {
-                fn(undefined);
-            }
-        });
+        if (this.db) {
+            this.db.collection(collectionName).findOne({ _id: id }, (err, result) => {
+                if (result) {
+                    result.id = result._id;
+                    delete result._id;
+                    fn(result);
+                } else {
+                    fn(undefined);
+                }
+            });
+        } else {
+            this.onConnect.push(() => this.getDocument(id, fn, collectionName));
+        }
     }
 
     public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = Database.DocumentsCollection) {
-        this.db && this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => {
-            if (err) {
-                console.log(err.message);
-                console.log(err.errmsg);
-            }
-            fn(docs.map(doc => {
-                doc.id = doc._id;
-                delete doc._id;
-                return doc;
-            }));
-        });
+        if (this.db) {
+            this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => {
+                if (err) {
+                    console.log(err.message);
+                    console.log(err.errmsg);
+                }
+                fn(docs.map(doc => {
+                    doc.id = doc._id;
+                    delete doc._id;
+                    return doc;
+                }));
+            });
+        } else {
+            this.onConnect.push(() => this.getDocuments(ids, fn, collectionName));
+        }
+    }
+
+    public query(query: any): Promise<mongodb.Cursor> {
+        if (this.db) {
+            return Promise.resolve<mongodb.Cursor>(this.db.collection('newDocuments').find(query));
+        } else {
+            return new Promise<mongodb.Cursor>(res => {
+                this.onConnect.push(() => res(this.query(query)));
+            });
+        }
     }
 
     public print() {
diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts
new file mode 100644
index 000000000..da33e3747
--- /dev/null
+++ b/src/server/updateSearch.ts
@@ -0,0 +1,101 @@
+import { Database } from "./database";
+import { Cursor } from "mongodb";
+import { Search } from "./Search";
+import pLimit from 'p-limit';
+
+const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
+    "number": "_n",
+    "string": "_t",
+    // "boolean": "_b",
+    "image": ["_t", "url"],
+    "video": ["_t", "url"],
+    "pdf": ["_t", "url"],
+    "audio": ["_t", "url"],
+    "web": ["_t", "url"],
+    "date": ["_d", value => new Date(value.date).toISOString()],
+    "proxy": ["_i", "fieldId"],
+    "list": ["_l", list => {
+        const results = [];
+        for (const value of list.fields) {
+            const term = ToSearchTerm(value);
+            if (term) {
+                results.push(term.value);
+            }
+        }
+        return results.length ? results : null;
+    }]
+};
+
+function ToSearchTerm(val: any): { suffix: string, value: any } | undefined {
+    if (val === null || val === undefined) {
+        return;
+    }
+    const type = val.__type || typeof val;
+    let suffix = suffixMap[type];
+    if (!suffix) {
+        return;
+    }
+
+    if (Array.isArray(suffix)) {
+        const accessor = suffix[1];
+        if (typeof accessor === "function") {
+            val = accessor(val);
+        } else {
+            val = val[accessor];
+        }
+        suffix = suffix[0];
+    }
+
+    return { suffix, value: val };
+}
+
+function getSuffix(value: string | [string, any]): string {
+    return typeof value === "string" ? value : value[0];
+}
+
+const limit = pLimit(5);
+async function update() {
+    // await new Promise(res => setTimeout(res, 5));
+    console.log("update");
+    await Search.Instance.clear();
+    const cursor = await Database.Instance.query({});
+    console.log("Cleared");
+    const updates: any[] = [];
+    let numDocs = 0;
+    function updateDoc(doc: any) {
+        numDocs++;
+        if ((numDocs % 50) === 0) {
+            console.log("updateDoc " + numDocs);
+        }
+        console.log("doc " + numDocs);
+        if (doc.__type !== "Doc") {
+            return;
+        }
+        const fields = doc.fields;
+        if (!fields) {
+            return;
+        }
+        const update: any = { id: doc._id };
+        let dynfield = false;
+        for (const key in fields) {
+            const value = fields[key];
+            const term = ToSearchTerm(value);
+            if (term !== undefined) {
+                let { suffix, value } = term;
+                update[key + suffix] = value;
+                dynfield = true;
+            }
+        }
+        if (dynfield) {
+            updates.push(update);
+            console.log(updates.length);
+        }
+    }
+    await cursor.forEach(updateDoc);
+    await Promise.all(updates.map(update => {
+        return limit(() => Search.Instance.updateDocument(update));
+    }));
+    cursor.close();
+}
+
+update();
\ No newline at end of file