Added document export and import

5 files changed, 198 insertions, 32 deletions

diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 8dac785e1..cbab14976 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -5,7 +5,7 @@ import { Id } from "../../../../new_fields/FieldSymbols";
 import { InkField, StrokeData } from "../../../../new_fields/InkField";
 import { createSchema, makeInterface } from "../../../../new_fields/Schema";
 import { BoolCast, Cast, FieldValue, NumCast, StrCast } from "../../../../new_fields/Types";
-import { emptyFunction, returnOne } from "../../../../Utils";
+import { emptyFunction, returnOne, Utils } from "../../../../Utils";
 import { DocumentManager } from "../../../util/DocumentManager";
 import { DragManager } from "../../../util/DragManager";
 import { HistoryUtil } from "../../../util/History";
@@ -34,12 +34,14 @@ import { CompileScript } from "../../../util/Scripting";
 import { CognitiveServices } from "../../../cognitive_services/CognitiveServices";
 import { library } from "@fortawesome/fontawesome-svg-core";
 import { faEye } from "@fortawesome/free-regular-svg-icons";
-import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass } from "@fortawesome/free-solid-svg-icons";
+import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload } from "@fortawesome/free-solid-svg-icons";
 import { undo } from "prosemirror-history";
 import { number } from "prop-types";
 import { ContextMenu } from "../../ContextMenu";
+import { RouteStore } from "../../../../server/RouteStore";
+import { DocServer } from "../../../DocServer";
 
-library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass);
+library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload);
 
 export const panZoomSchema = createSchema({
     panX: "number",
@@ -516,7 +518,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY));
     }
 
-    onContextMenu = () => {
+    onContextMenu = (e: React.MouseEvent) => {
         let layoutItems: ContextMenuProps[] = [];
         layoutItems.push({
             description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`,
@@ -561,6 +563,35 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                 CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData);
             }, icon: "paint-brush"
         });
+        ContextMenu.Instance.addItem({
+            description: "Import document", icon: "upload", event: () => {
+                const input = document.createElement("input");
+                input.type = "file";
+                input.accept = ".zip";
+                input.onchange = async _e => {
+                    const files = input.files;
+                    if (!files) return;
+                    const file = files[0];
+                    let formData = new FormData();
+                    formData.append('file', file);
+                    formData.append('remap', "true");
+                    const upload = Utils.prepend("/uploadDoc");
+                    const response = await fetch(upload, { method: "POST", body: formData });
+                    const json = await response.json();
+                    if (json === "error") {
+                        return;
+                    }
+                    const doc = await DocServer.GetRefField(json);
+                    if (!doc || !(doc instanceof Doc)) {
+                        return;
+                    }
+                    const [x, y] = this.props.ScreenToLocalTransform().transformPoint(e.pageX, e.pageY);
+                    doc.x = x, doc.y = y;
+                    this.addDocument(doc, false);
+                };
+                input.click();
+            }
+        });
     }
 
 
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 4b5cf3a43..58e2443c2 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -45,6 +45,7 @@ const JsxParser = require('react-jsx-parser').default; //TODO Why does this need
 
 library.add(fa.faTrash);
 library.add(fa.faShare);
+library.add(fa.faDownload);
 library.add(fa.faExpandArrowsAlt);
 library.add(fa.faCompressArrowsAlt);
 library.add(fa.faLayerGroup);
@@ -597,6 +598,15 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
             copies.push({ description: "Copy ID", event: () => Utils.CopyText(this.props.Document[Id]), icon: "fingerprint" });
             cm.addItem({ description: "Copy...", subitems: copies, icon: "copy" });
         }
+        cm.addItem({
+            description: "Download document", icon: "download", event: () => {
+                const a = document.createElement("a");
+                const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`);
+                a.href = url;
+                a.download = `DocExport-${this.props.Document[Id]}.zip`;
+                a.click();
+            }
+        });
         cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" });
         type User = { email: string, userDocumentId: string };
         let usersMenu: ContextMenuProps[] = [];
diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py
index 1ff0e3b31..8ff7cb223 100644
--- a/src/scraping/buxton/scraper.py
+++ b/src/scraping/buxton/scraper.py
@@ -236,6 +236,8 @@ def parse_document(file_name: str):
         view_guids.append(write_image(pure_name, image))
         copyfile(dir_path + "/" + image, dir_path +
                  "/" + image.replace(".", "_o.", 1))
+        copyfile(dir_path + "/" + image, dir_path +
+                 "/" + image)
         os.rename(dir_path + "/" + image, dir_path +
                   "/" + image.replace(".", "_m.", 1))
     print(f"extracted {count} images...")
diff --git a/src/server/database.ts b/src/server/database.ts
index acb6ce751..a7254fb0c 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -17,7 +17,7 @@ export class Database {
         });
     }
 
-    public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) {
+    public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) {
         if (this.db) {
             let collection = this.db.collection(collectionName);
             const prom = this.currentWrites[id];
@@ -30,7 +30,7 @@ export class Database {
                                 delete this.currentWrites[id];
                             }
                             resolve();
-                            callback();
+                            callback(err, res);
                         });
                 });
             };
@@ -41,6 +41,30 @@ export class Database {
         }
     }
 
+    public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) {
+        if (this.db) {
+            let collection = this.db.collection(collectionName);
+            const prom = this.currentWrites[id];
+            let newProm: Promise<void>;
+            const run = (): Promise<void> => {
+                return new Promise<void>(resolve => {
+                    collection.replaceOne({ _id: id }, value, { upsert }
+                        , (err, res) => {
+                            if (this.currentWrites[id] === newProm) {
+                                delete this.currentWrites[id];
+                            }
+                            resolve();
+                            callback(err, res);
+                        });
+                });
+            };
+            newProm = prom ? prom.then(run) : run();
+            this.currentWrites[id] = newProm;
+        } else {
+            this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName));
+        }
+    }
+
     public delete(query: any, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
     public delete(id: string, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
     public delete(id: any, collectionName = Database.DocumentsCollection) {
@@ -126,7 +150,7 @@ export class Database {
         }
     }
 
-    public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments") {
+    public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise<void> {
         if (this.db) {
             const visited = new Set<string>();
             while (ids.length) {
@@ -145,7 +169,12 @@ export class Database {
             }
 
         } else {
-            this.onConnect.push(() => this.visit(ids, fn, collectionName));
+            return new Promise(res => {
+                this.onConnect.push(() => {
+                    this.visit(ids, fn, collectionName);
+                    res();
+                });
+            });
         }
     }
 
diff --git a/src/server/index.ts b/src/server/index.ts
index 230c574cf..2fa5132d0 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -27,6 +27,7 @@ import { Client } from './Client';
 import { Database } from './database';
 import { MessageStore, Transferable, Types, Diff, Message } from "./Message";
 import { RouteStore } from './RouteStore';
+import v4 = require('uuid/v4');
 const app = express();
 const config = require('../../webpack.config');
 import { createCanvas, loadImage, Canvas } from "canvas";
@@ -39,7 +40,10 @@ import c = require("crypto");
 import { Search } from './Search';
 import { debug } from 'util';
 import _ = require('lodash');
+import * as Archiver from 'archiver';
+import * as AdmZip from 'adm-zip';
 import { Response } from 'express-serve-static-core';
+import { DocComponent } from '../client/views/DocComponent';
 const MongoStore = require('connect-mongo')(session);
 const mongoose = require('mongoose');
 const probe = require("probe-image-size");
@@ -177,16 +181,21 @@ function msToTime(duration: number) {
     return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds;
 }
 
-app.get("/serializeDoc/:docId", async (req, res) => {
-    const files: { [name: string]: string[] } = {};
+async function getDocs(id: string) {
+    const files = new Set<string>();
     const docs: { [id: string]: any } = {};
     const fn = (doc: any): string[] => {
+        const id = doc.id;
+        if (typeof id === "string" && id.endsWith("Proto")) {
+            //Skip protos
+            return [];
+        }
         const ids: string[] = [];
-        for (const key in doc) {
-            if (!doc.hasOwnProperty(key)) {
+        for (const key in doc.fields) {
+            if (!doc.fields.hasOwnProperty(key)) {
                 continue;
             }
-            const field = doc[key];
+            const field = doc.fields[key];
             if (field === undefined || field === null) {
                 continue;
             }
@@ -194,7 +203,7 @@ app.get("/serializeDoc/:docId", async (req, res) => {
             if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
                 ids.push(field.fieldId);
             } else if (field.__type === "list") {
-                ids.push(...fn(field.fields));
+                ids.push(...fn(field));
             } else if (typeof field === "string") {
                 const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g;
                 let match: string[] | null;
@@ -215,35 +224,120 @@ app.get("/serializeDoc/:docId", async (req, res) => {
                 while ((match = re2.exec(field.Data)) !== null) {
                     const urlString = match[1];
                     const pathname = new URL(urlString).pathname;
-                    const ext = path.extname(pathname);
-                    const fileName = path.basename(pathname, ext);
-                    let exts = files[fileName];
-                    if (!exts) {
-                        files[fileName] = exts = [];
-                    }
-                    exts.push(ext);
+                    files.add(pathname);
                 }
             } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) {
                 const url = new URL(field.url);
                 const pathname = url.pathname;
-                const ext = path.extname(pathname);
-                const fileName = path.basename(pathname, ext);
-                let exts = files[fileName];
-                if (!exts) {
-                    files[fileName] = exts = [];
-                }
-                exts.push(ext);
+                files.add(pathname);
             }
         }
 
-        docs[doc.id] = doc;
+        if (doc.id) {
+            docs[doc.id] = doc;
+        }
         return ids;
     };
-    Database.Instance.visit([req.params.docId], fn);
+    await Database.Instance.visit([id], fn);
+    return { id, docs, files };
+}
+app.get("/serializeDoc/:docId", async (req, res) => {
+    const { docs, files } = await getDocs(req.params.docId);
+    res.send({ docs, files: Array.from(files) });
 });
 
-app.get("/downloadId/:docId", (req, res) => {
-    res.download(`/serializeDoc/${req.params.docId}`, `DocumentExport.zip`);
+app.get("/downloadId/:docId", async (req, res) => {
+    res.set('Content-disposition', `attachment;`);
+    res.set('Content-Type', "application/zip");
+    const { id, docs, files } = await getDocs(req.params.docId);
+    const docString = JSON.stringify({ id, docs });
+    const zip = Archiver('zip');
+    zip.pipe(res);
+    zip.append(docString, { name: "doc.json" });
+    files.forEach(val => {
+        zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) });
+    });
+    zip.finalize();
+});
+
+app.post("/uploadDoc", (req, res) => {
+    let form = new formidable.IncomingForm();
+    form.keepExtensions = true;
+    // let path = req.body.path;
+    const ids: { [id: string]: string } = {};
+    let remap = true;
+    const getId = (id: string): string => {
+        if (!remap) return id;
+        if (id.endsWith("Proto")) return id;
+        if (id in ids) {
+            return ids[id];
+        } else {
+            return ids[id] = v4();
+        }
+    };
+    const mapFn = (doc: any) => {
+        if (doc.id) {
+            doc.id = getId(doc.id);
+        }
+        for (const key in doc.fields) {
+            if (!doc.fields.hasOwnProperty(key)) {
+                continue;
+            }
+            const field = doc.fields[key];
+            if (field === undefined || field === null) {
+                continue;
+            }
+
+            if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+                field.fieldId = getId(field.fieldId);
+            } else if (field.__type === "list") {
+                mapFn(field);
+            } else if (typeof field === "string") {
+                const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g;
+                doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => {
+                    return `${p1}${getId(p2)}"`;
+                });
+            } else if (field.__type === "RichTextField") {
+                const re = /("href"\s*:\s*")(.*?)"/g;
+                field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => {
+                    return `${p1}${getId(p2)}"`;
+                });
+            }
+        }
+    };
+    form.parse(req, async (err, fields, files) => {
+        remap = fields.remap !== "false";
+        let id: string = "";
+        try {
+            for (const name in files) {
+                const path = files[name].path;
+                const zip = new AdmZip(path);
+                zip.getEntries().forEach(entry => {
+                    if (!entry.name.startsWith("files/")) return;
+                    zip.extractEntryTo(entry.name, __dirname + RouteStore.public, true, false);
+                });
+                const json = zip.getEntry("doc.json");
+                let docs: any;
+                try {
+                    let data = JSON.parse(json.getData().toString("utf8"));
+                    docs = data.docs;
+                    id = data.id;
+                    docs = Object.keys(docs).map(key => docs[key]);
+                    docs.forEach(mapFn);
+                    await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => {
+                        err && console.log(err);
+                        res();
+                    }, true, "newDocuments"))));
+                } catch (e) { console.log(e); }
+                fs.unlink(path, () => { });
+            }
+            if (id) {
+                res.send(JSON.stringify(getId(id)));
+            } else {
+                res.send(JSON.stringify("error"));
+            }
+        } catch (e) { console.log(e); }
+    });
 });
 
 app.get("/whosOnline", (req, res) => {