merge with master

1 files changed, 243 insertions, 38 deletions

diff --git a/src/server/index.ts b/src/server/index.ts
index 51fafbdc3..10a84c823 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -25,8 +25,9 @@ import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLo
 import { DashUserModel } from './authentication/models/user_model';
 import { Client } from './Client';
 import { Database } from './database';
-import { MessageStore, Transferable, Types, Diff, Message } from "./Message";
+import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message";
 import { RouteStore } from './RouteStore';
+import v4 = require('uuid/v4');
 const app = express();
 const config = require('../../webpack.config');
 import { createCanvas, loadImage, Canvas } from "canvas";
@@ -39,12 +40,20 @@ import c = require("crypto");
 import { Search } from './Search';
 import { debug } from 'util';
 import _ = require('lodash');
+import * as Archiver from 'archiver';
+import * as AdmZip from 'adm-zip';
+import * as YoutubeApi from './youtubeApi/youtubeApiSample.js';
 import { Response } from 'express-serve-static-core';
+import { DocComponent } from '../client/views/DocComponent';
 const MongoStore = require('connect-mongo')(session);
 const mongoose = require('mongoose');
 const probe = require("probe-image-size");
+var SolrNode = require('solr-node');
+var shell = require('shelljs');
 
 const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest));
+let youtubeApiKey: string;
+YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey);
 
 const release = process.env.RELEASE === "true";
 if (process.env.RELEASE === "true") {
@@ -110,7 +119,7 @@ function addSecureRoute(method: Method,
         if (req.user) {
             handler(req.user, res, req);
         } else {
-            req.session!.target = `${req.headers.host}${req.originalUrl}`;
+            req.session!.target = req.originalUrl;
             onRejection(res, req);
         }
     };
@@ -139,7 +148,18 @@ app.get("/pull", (req, res) =>
         res.redirect("/");
     }));
 
+app.get("/version", (req, res) => {
+    exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout, stderr) => {
+        if (err) {
+            res.send(err.message);
+            return;
+        }
+        res.send(stdout);
+    });
+});
+
 // SEARCH
+const solrURL = "http://localhost:8983/solr/#/dash";
 
 // GETTERS
 
@@ -167,6 +187,186 @@ function msToTime(duration: number) {
     return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds;
 }
 
+async function getDocs(id: string) {
+    const files = new Set<string>();
+    const docs: { [id: string]: any } = {};
+    const fn = (doc: any): string[] => {
+        const id = doc.id;
+        if (typeof id === "string" && id.endsWith("Proto")) {
+            //Skip protos
+            return [];
+        }
+        const ids: string[] = [];
+        for (const key in doc.fields) {
+            if (!doc.fields.hasOwnProperty(key)) {
+                continue;
+            }
+            const field = doc.fields[key];
+            if (field === undefined || field === null) {
+                continue;
+            }
+
+            if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+                ids.push(field.fieldId);
+            } else if (field.__type === "script" || field.__type === "computed") {
+                if (field.captures) {
+                    ids.push(field.captures.fieldId);
+                }
+            } else if (field.__type === "list") {
+                ids.push(...fn(field));
+            } else if (typeof field === "string") {
+                const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g;
+                let match: string[] | null;
+                while ((match = re.exec(field)) !== null) {
+                    ids.push(match[1]);
+                }
+            } else if (field.__type === "RichTextField") {
+                const re = /"href"\s*:\s*"(.*?)"/g;
+                let match: string[] | null;
+                while ((match = re.exec(field.Data)) !== null) {
+                    const urlString = match[1];
+                    const split = new URL(urlString).pathname.split("doc/");
+                    if (split.length > 1) {
+                        ids.push(split[split.length - 1]);
+                    }
+                }
+                const re2 = /"src"\s*:\s*"(.*?)"/g;
+                while ((match = re2.exec(field.Data)) !== null) {
+                    const urlString = match[1];
+                    const pathname = new URL(urlString).pathname;
+                    files.add(pathname);
+                }
+            } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) {
+                const url = new URL(field.url);
+                const pathname = url.pathname;
+                files.add(pathname);
+            }
+        }
+
+        if (doc.id) {
+            docs[doc.id] = doc;
+        }
+        return ids;
+    };
+    await Database.Instance.visit([id], fn);
+    return { id, docs, files };
+}
+app.get("/serializeDoc/:docId", async (req, res) => {
+    const { docs, files } = await getDocs(req.params.docId);
+    res.send({ docs, files: Array.from(files) });
+});
+
+app.get("/downloadId/:docId", async (req, res) => {
+    res.set('Content-disposition', `attachment;`);
+    res.set('Content-Type', "application/zip");
+    const { id, docs, files } = await getDocs(req.params.docId);
+    const docString = JSON.stringify({ id, docs });
+    const zip = Archiver('zip');
+    zip.pipe(res);
+    zip.append(docString, { name: "doc.json" });
+    files.forEach(val => {
+        zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) });
+    });
+    zip.finalize();
+});
+
+app.post("/uploadDoc", (req, res) => {
+    let form = new formidable.IncomingForm();
+    form.keepExtensions = true;
+    // let path = req.body.path;
+    const ids: { [id: string]: string } = {};
+    let remap = true;
+    const getId = (id: string): string => {
+        if (!remap) return id;
+        if (id.endsWith("Proto")) return id;
+        if (id in ids) {
+            return ids[id];
+        } else {
+            return ids[id] = v4();
+        }
+    };
+    const mapFn = (doc: any) => {
+        if (doc.id) {
+            doc.id = getId(doc.id);
+        }
+        for (const key in doc.fields) {
+            if (!doc.fields.hasOwnProperty(key)) {
+                continue;
+            }
+            const field = doc.fields[key];
+            if (field === undefined || field === null) {
+                continue;
+            }
+
+            if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+                field.fieldId = getId(field.fieldId);
+            } else if (field.__type === "script" || field.__type === "computed") {
+                if (field.captures) {
+                    field.captures.fieldId = getId(field.captures.fieldId);
+                }
+            } else if (field.__type === "list") {
+                mapFn(field);
+            } else if (typeof field === "string") {
+                const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g;
+                doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => {
+                    return `${p1}${getId(p2)}"`;
+                });
+            } else if (field.__type === "RichTextField") {
+                const re = /("href"\s*:\s*")(.*?)"/g;
+                field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => {
+                    return `${p1}${getId(p2)}"`;
+                });
+            }
+        }
+    };
+    form.parse(req, async (err, fields, files) => {
+        remap = fields.remap !== "false";
+        let id: string = "";
+        try {
+            for (const name in files) {
+                const path_2 = files[name].path;
+                const zip = new AdmZip(path_2);
+                zip.getEntries().forEach(entry => {
+                    if (!entry.entryName.startsWith("files/")) return;
+                    let dirname = path.dirname(entry.entryName) + "/";
+                    let extname = path.extname(entry.entryName);
+                    let basename = path.basename(entry.entryName).split(".")[0];
+                    // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false);
+                    // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false);
+                    // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false);
+                    // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false);
+                    zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false);
+                    dirname = "/" + dirname;
+
+                    fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname));
+                    fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname));
+                    fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname));
+                    fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname));
+                });
+                const json = zip.getEntry("doc.json");
+                let docs: any;
+                try {
+                    let data = JSON.parse(json.getData().toString("utf8"));
+                    docs = data.docs;
+                    id = data.id;
+                    docs = Object.keys(docs).map(key => docs[key]);
+                    docs.forEach(mapFn);
+                    await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => {
+                        err && console.log(err);
+                        res();
+                    }, true, "newDocuments"))));
+                } catch (e) { console.log(e); }
+                fs.unlink(path_2, () => { });
+            }
+            if (id) {
+                res.send(JSON.stringify(getId(id)));
+            } else {
+                res.send(JSON.stringify("error"));
+            }
+        } catch (e) { console.log(e); }
+    });
+});
+
 app.get("/whosOnline", (req, res) => {
     let users: any = { active: {}, inactive: {} };
     const now = Date.now();
@@ -179,7 +379,6 @@ app.get("/whosOnline", (req, res) => {
 
     res.send(users);
 });
-
 app.get("/thumbnail/:filename", (req, res) => {
     let filename = req.params.filename;
     let noExt = filename.substring(0, filename.length - ".png".length);
@@ -285,6 +484,17 @@ addSecureRoute(
     RouteStore.getCurrUser
 );
 
+const ServicesApiKeyMap = new Map<string, string | undefined>([
+    ["face", process.env.FACE],
+    ["vision", process.env.VISION],
+    ["handwriting", process.env.HANDWRITING]
+]);
+
+addSecureRoute(Method.GET, (user, res, req) => {
+    let service = req.params.requestedservice;
+    res.send(ServicesApiKeyMap.get(service));
+}, undefined, `${RouteStore.cognitiveServices}/:requestedservice`);
+
 class NodeCanvasFactory {
     create = (width: number, height: number) => {
         var canvas = createCanvas(width, height);
@@ -345,38 +555,6 @@ app.post(
                     });
                     isImage = true;
                 }
-                else if (pdfTypes.includes(ext)) {
-                    // Pdfjs.getDocument(uploadDir + file).promise
-                    //     .then((pdf: Pdfjs.PDFDocumentProxy) => {
-                    //         let numPages = pdf.numPages;
-                    //         let factory = new NodeCanvasFactory();
-                    //         for (let pageNum = 0; pageNum < numPages; pageNum++) {
-                    //             console.log(pageNum);
-                    //             pdf.getPage(pageNum + 1).then((page: Pdfjs.PDFPageProxy) => {
-                    //                 console.log("reading " + pageNum);
-                    //                 let viewport = page.getViewport(1);
-                    //                 let canvasAndContext = factory.create(viewport.width, viewport.height);
-                    //                 let renderContext = {
-                    //                     canvasContext: canvasAndContext.context,
-                    //                     viewport: viewport,
-                    //                     canvasFactory: factory
-                    //                 }
-                    //                 console.log("read " + pageNum);
-
-                    //                 page.render(renderContext).promise
-                    //                     .then(() => {
-                    //                         console.log("saving " + pageNum);
-                    //                         let stream = canvasAndContext.canvas.createPNGStream();
-                    //                         let out = fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + `-${pageNum + 1}.PNG`);
-                    //                         stream.pipe(out);
-                    //                         out.on("finish", () => console.log(`Success! Saved to ${uploadDir + file.substring(0, file.length - ext.length) + `-${pageNum + 1}.PNG`}`));
-                    //                     }, (reason: string) => {
-                    //                         console.error(reason + ` ${pageNum}`);
-                    //                     });
-                    //             });
-                    //         }
-                    //     });
-                }
                 if (isImage) {
                     resizers.forEach(resizer => {
                         fs.createReadStream(uploadDir + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + resizer.suffix + ext));
@@ -449,8 +627,22 @@ app.post(RouteStore.forgot, postForgot);
 app.get(RouteStore.reset, getReset);
 app.post(RouteStore.reset, postReset);
 
-app.use(RouteStore.corsProxy, (req, res) =>
-    req.pipe(request(req.url.substring(1))).pipe(res));
+const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/;
+app.use(RouteStore.corsProxy, (req, res) => {
+    req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => {
+        const headers = Object.keys(res.headers);
+        headers.forEach(headerName => {
+            const header = res.headers[headerName];
+            if (Array.isArray(header)) {
+                res.headers[headerName] = header.filter(h => !headerCharRegex.test(h));
+            } else if (header) {
+                if (headerCharRegex.test(header as any)) {
+                    delete res.headers[headerName];
+                }
+            }
+        });
+    }).pipe(res);
+});
 
 app.get(RouteStore.delete, (req, res) => {
     if (release) {
@@ -505,6 +697,7 @@ server.on("connection", function (socket: Socket) {
     }
 
     Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField);
+    Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery);
     Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff));
     Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id));
     Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids));
@@ -559,6 +752,17 @@ function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => v
     Database.Instance.getDocuments(ids, callback, "newDocuments");
 }
 
+function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) {
+    switch (query.type) {
+        case YoutubeQueryType.Channels:
+            YoutubeApi.authorizedGetChannel(youtubeApiKey);
+            break;
+        case YoutubeQueryType.SearchVideo:
+            YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback);
+        case YoutubeQueryType.VideoDetails:
+            YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback);
+    }
+}
 
 const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
     "number": "_n",
@@ -658,4 +862,5 @@ function CreateField(newValue: any) {
 }
 
 server.listen(serverPort);
-console.log(`listening on port ${serverPort}`);
\ No newline at end of file
+console.log(`listening on port ${serverPort}`);
+