Merge branch 'master' of https://github.com/browngraphicslab/Dash-Web into cont_menu_ui

4 files changed, 226 insertions, 48 deletions

diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts
index fdf5b6a5c..c4af5cdaa 100644
--- a/src/server/RouteStore.ts
+++ b/src/server/RouteStore.ts
@@ -11,6 +11,7 @@ export enum RouteStore {
     // UPLOAD AND STATIC FILE SERVING
     public = "/public",
     upload = "/upload",
+    dataUriToImage = "/uploadURI",
     images = "/images",
 
     // USER AND WORKSPACES
diff --git a/src/server/database.ts b/src/server/database.ts
index 69005d2d3..70b3efced 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -8,9 +8,13 @@ export class Database {
     private url = 'mongodb://localhost:27017/Dash';
     private currentWrites: { [id: string]: Promise<void> } = {};
     private db?: mongodb.Db;
+    private onConnect: (() => void)[] = [];
 
     constructor() {
-        this.MongoClient.connect(this.url, (err, client) => this.db = client.db());
+        this.MongoClient.connect(this.url, (err, client) => {
+            this.db = client.db();
+            this.onConnect.forEach(fn => fn());
+        });
     }
 
     public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) {
@@ -32,66 +36,98 @@ export class Database {
             };
             newProm = prom ? prom.then(run) : run();
             this.currentWrites[id] = newProm;
+        } else {
+            this.onConnect.push(() => this.update(id, value, callback, upsert, collectionName));
         }
     }
 
     public delete(id: string, collectionName = Database.DocumentsCollection) {
-        this.db && this.db.collection(collectionName).remove({ id: id });
+        if (this.db) {
+            this.db.collection(collectionName).remove({ id: id });
+        } else {
+            this.onConnect.push(() => this.delete(id, collectionName));
+        }
     }
 
     public deleteAll(collectionName = Database.DocumentsCollection): Promise<any> {
-        return new Promise(res =>
-            this.db && this.db.collection(collectionName).deleteMany({}, res));
+        return new Promise(res => {
+            if (this.db) {
+                this.db.collection(collectionName).deleteMany({}, res);
+            } else {
+                this.onConnect.push(() => this.db && this.db.collection(collectionName).deleteMany({}, res));
+            }
+        });
     }
 
     public insert(value: any, collectionName = Database.DocumentsCollection) {
-        if (!this.db) { return; }
-        if ("id" in value) {
-            value._id = value.id;
-            delete value.id;
-        }
-        const id = value._id;
-        const collection = this.db.collection(collectionName);
-        const prom = this.currentWrites[id];
-        let newProm: Promise<void>;
-        const run = (): Promise<void> => {
-            return new Promise<void>(resolve => {
-                collection.insertOne(value, (err, res) => {
-                    if (this.currentWrites[id] === newProm) {
-                        delete this.currentWrites[id];
-                    }
-                    resolve();
+        if (this.db) {
+            if ("id" in value) {
+                value._id = value.id;
+                delete value.id;
+            }
+            const id = value._id;
+            const collection = this.db.collection(collectionName);
+            const prom = this.currentWrites[id];
+            let newProm: Promise<void>;
+            const run = (): Promise<void> => {
+                return new Promise<void>(resolve => {
+                    collection.insertOne(value, (err, res) => {
+                        if (this.currentWrites[id] === newProm) {
+                            delete this.currentWrites[id];
+                        }
+                        resolve();
+                    });
                 });
-            });
-        };
-        newProm = prom ? prom.then(run) : run();
-        this.currentWrites[id] = newProm;
+            };
+            newProm = prom ? prom.then(run) : run();
+            this.currentWrites[id] = newProm;
+        } else {
+            this.onConnect.push(() => this.insert(value, collectionName));
+        }
     }
 
     public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = Database.DocumentsCollection) {
-        this.db && this.db.collection(collectionName).findOne({ _id: id }, (err, result) => {
-            if (result) {
-                result.id = result._id;
-                delete result._id;
-                fn(result);
-            } else {
-                fn(undefined);
-            }
-        });
+        if (this.db) {
+            this.db.collection(collectionName).findOne({ _id: id }, (err, result) => {
+                if (result) {
+                    result.id = result._id;
+                    delete result._id;
+                    fn(result);
+                } else {
+                    fn(undefined);
+                }
+            });
+        } else {
+            this.onConnect.push(() => this.getDocument(id, fn, collectionName));
+        }
     }
 
     public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = Database.DocumentsCollection) {
-        this.db && this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => {
-            if (err) {
-                console.log(err.message);
-                console.log(err.errmsg);
-            }
-            fn(docs.map(doc => {
-                doc.id = doc._id;
-                delete doc._id;
-                return doc;
-            }));
-        });
+        if (this.db) {
+            this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => {
+                if (err) {
+                    console.log(err.message);
+                    console.log(err.errmsg);
+                }
+                fn(docs.map(doc => {
+                    doc.id = doc._id;
+                    delete doc._id;
+                    return doc;
+                }));
+            });
+        } else {
+            this.onConnect.push(() => this.getDocuments(ids, fn, collectionName));
+        }
+    }
+
+    public query(query: any): Promise<mongodb.Cursor> {
+        if (this.db) {
+            return Promise.resolve<mongodb.Cursor>(this.db.collection('newDocuments').find(query));
+        } else {
+            return new Promise<mongodb.Cursor>(res => {
+                this.onConnect.push(() => res(this.query(query)));
+            });
+        }
     }
 
     public print() {
diff --git a/src/server/index.ts b/src/server/index.ts
index 83c0bc222..deb3c1bd6 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -7,6 +7,7 @@ import * as expressValidator from 'express-validator';
 import * as formidable from 'formidable';
 import * as fs from 'fs';
 import * as sharp from 'sharp';
+const imageDataUri = require('image-data-uri');
 import * as mobileDetect from 'mobile-detect';
 import { ObservableMap } from 'mobx';
 import * as passport from 'passport';
@@ -59,7 +60,7 @@ app.use(session({
 
 app.use(flash());
 app.use(expressFlash());
-app.use(bodyParser.json());
+app.use(bodyParser.json({ limit: "10mb" }));
 app.use(bodyParser.urlencoded({ extended: true }));
 app.use(expressValidator());
 app.use(passport.initialize());
@@ -216,6 +217,45 @@ addSecureRoute(
     RouteStore.upload
 );
 
+addSecureRoute(
+    Method.POST,
+    (user, res, req) => {
+        const uri = req.body.uri;
+        const filename = req.body.name;
+        if (!uri || !filename) {
+            res.status(401).send("incorrect parameters specified");
+            return;
+        }
+        imageDataUri.outputFile(uri, uploadDir + filename).then((savedName: string) => {
+            const ext = path.extname(savedName);
+            let resizers = [
+                { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" },
+                { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" },
+                { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" },
+            ];
+            let isImage = false;
+            if (pngTypes.includes(ext)) {
+                resizers.forEach(element => {
+                    element.resizer = element.resizer.png();
+                });
+                isImage = true;
+            } else if (jpgTypes.includes(ext)) {
+                resizers.forEach(element => {
+                    element.resizer = element.resizer.jpeg();
+                });
+                isImage = true;
+            }
+            if (isImage) {
+                resizers.forEach(resizer => {
+                    fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + filename + resizer.suffix + ext));
+                });
+            }
+            res.send("/files/" + filename + ext);
+        });
+    },
+    undefined,
+    RouteStore.dataUriToImage
+);
 // AUTHENTICATION
 
 // Sign Up
@@ -327,7 +367,7 @@ const suffixMap: { [type: string]: (string | [string, string | ((json: any) => a
     "number": "_n",
     "string": "_t",
     // "boolean": "_b",
-    "image": ["_t", "url"],
+    // "image": ["_t", "url"],
     "video": ["_t", "url"],
     "pdf": ["_t", "url"],
     "audio": ["_t", "url"],
@@ -390,8 +430,8 @@ function UpdateField(socket: Socket, diff: Diff) {
         Object.values(suffixMap).forEach(suf => update[key + getSuffix(suf)] = { set: null });
         let term = ToSearchTerm(val);
         if (term !== undefined) {
-            // let { suffix, value } = term;
-            // update[key + suffix] = { set: value };
+            let { suffix, value } = term;
+            update[key + suffix] = { set: value };
         }
     }
     if (dynfield) {
diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts
new file mode 100644
index 000000000..de1fd25e1
--- /dev/null
+++ b/src/server/updateSearch.ts
@@ -0,0 +1,101 @@
+import { Database } from "./database";
+import { Cursor } from "mongodb";
+import { Search } from "./Search";
+import pLimit from 'p-limit';
+
+const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
+    "number": "_n",
+    "string": "_t",
+    // "boolean": "_b",
+    // "image": ["_t", "url"],
+    "video": ["_t", "url"],
+    "pdf": ["_t", "url"],
+    "audio": ["_t", "url"],
+    "web": ["_t", "url"],
+    "date": ["_d", value => new Date(value.date).toISOString()],
+    "proxy": ["_i", "fieldId"],
+    "list": ["_l", list => {
+        const results = [];
+        for (const value of list.fields) {
+            const term = ToSearchTerm(value);
+            if (term) {
+                results.push(term.value);
+            }
+        }
+        return results.length ? results : null;
+    }]
+};
+
+function ToSearchTerm(val: any): { suffix: string, value: any } | undefined {
+    if (val === null || val === undefined) {
+        return;
+    }
+    const type = val.__type || typeof val;
+    let suffix = suffixMap[type];
+    if (!suffix) {
+        return;
+    }
+
+    if (Array.isArray(suffix)) {
+        const accessor = suffix[1];
+        if (typeof accessor === "function") {
+            val = accessor(val);
+        } else {
+            val = val[accessor];
+        }
+        suffix = suffix[0];
+    }
+
+    return { suffix, value: val };
+}
+
+function getSuffix(value: string | [string, any]): string {
+    return typeof value === "string" ? value : value[0];
+}
+
+const limit = pLimit(5);
+async function update() {
+    // await new Promise(res => setTimeout(res, 5));
+    console.log("update");
+    await Search.Instance.clear();
+    const cursor = await Database.Instance.query({});
+    console.log("Cleared");
+    const updates: any[] = [];
+    let numDocs = 0;
+    function updateDoc(doc: any) {
+        numDocs++;
+        if ((numDocs % 50) === 0) {
+            console.log("updateDoc " + numDocs);
+        }
+        console.log("doc " + numDocs);
+        if (doc.__type !== "Doc") {
+            return;
+        }
+        const fields = doc.fields;
+        if (!fields) {
+            return;
+        }
+        const update: any = { id: doc._id };
+        let dynfield = false;
+        for (const key in fields) {
+            const value = fields[key];
+            const term = ToSearchTerm(value);
+            if (term !== undefined) {
+                let { suffix, value } = term;
+                update[key + suffix] = value;
+                dynfield = true;
+            }
+        }
+        if (dynfield) {
+            updates.push(update);
+            console.log(updates.length);
+        }
+    }
+    await cursor.forEach(updateDoc);
+    await Promise.all(updates.map(update => {
+        return limit(() => Search.Instance.updateDocument(update));
+    }));
+    cursor.close();
+}
+
+update();
\ No newline at end of file