Merge branch 'master' of https://github.com/browngraphicslab/Dash-Web

16 files changed, 488 insertions, 133 deletions

diff --git a/src/client/apis/youtube/YoutubeBox.scss b/src/client/apis/youtube/YoutubeBox.scss
index 1fc91a9ae..eabdbb1ac 100644
--- a/src/client/apis/youtube/YoutubeBox.scss
+++ b/src/client/apis/youtube/YoutubeBox.scss
@@ -1,124 +1,126 @@
-ul {
-    list-style-type: none;
-    padding-inline-start: 10px;
-}
-
-
-li {
-    margin: 4px;
-    display: inline-flex;
-}
-
-li:hover {
-    cursor: pointer;
-    opacity: 0.8;
-}
-
-.search_wrapper {
-    width: 100%;
-    display: inline-flex;
-    height: 175px;
-
-    .video_duration {
-        // margin: 0;
-        // padding: 0;
-        border: 0;
-        background: transparent;
-        display: inline-block;
-        position: relative;
-        bottom: 25px;
-        left: 85%;
+.youtubeBox-cont {
+    ul {
+        list-style-type: none;
+        padding-inline-start: 10px;
+    }
+
+
+    li {
         margin: 4px;
-        color: #FFFFFF;
-        background-color: rgba(0, 0, 0, 0.80);
-        padding: 2px 4px;
-        border-radius: 2px;
-        letter-spacing: .5px;
-        font-size: 1.2rem;
-        font-weight: 500;
-        line-height: 1.2rem;
+        display: inline-flex;
+    }
 
+    li:hover {
+        cursor: pointer;
+        opacity: 0.8;
     }
 
-    .textual_info {
-        font-family: Arial, Helvetica, sans-serif;
-
-        .videoTitle {
-            margin-left: 4px;
-            // display: inline-block;
-            color: #0D0D0D;
-            -webkit-line-clamp: 2;
-            display: block;
-            max-height: 4.8rem;
-            overflow: hidden;
-            font-size: 1.8rem;
-            font-weight: 400;
-            line-height: 2.4rem;
-            -webkit-box-orient: vertical;
-            text-overflow: ellipsis;
-            white-space: normal;
-            display: -webkit-box;
-        }
+    .search_wrapper {
+        width: 100%;
+        display: inline-flex;
+        height: 175px;
 
-        .channelName {
-            color:#606060;
-            margin-left: 4px;
-            font-size: 1.3rem;
-            font-weight: 400;
-            line-height: 1.8rem;
-            text-transform: none;
-            margin-top: 0px;
+        .video_duration {
+            // margin: 0;
+            // padding: 0;
+            border: 0;
+            background: transparent;
             display: inline-block;
-        }
+            position: relative;
+            bottom: 25px;
+            left: 85%;
+            margin: 4px;
+            color: #FFFFFF;
+            background-color: rgba(0, 0, 0, 0.80);
+            padding: 2px 4px;
+            border-radius: 2px;
+            letter-spacing: .5px;
+            font-size: 1.2rem;
+            font-weight: 500;
+            line-height: 1.2rem;
 
-        .video_description {
-            margin-left: 4px;
-            // font-size: 12px;
-            color: #606060;
-            padding-top: 8px;
-            margin-bottom: 8px;
-            display: block;
-            line-height: 1.8rem;
-            max-height: 4.2rem;
-            overflow: hidden;
-            font-size: 1.3rem;
-            font-weight: 400;
-            text-transform: none;
         }
 
-        .publish_time {
-            //display: inline-block;
-            margin-left: 8px;
-            padding: 0;
-            border: 0;
-            background: transparent;
-            color: #606060;
-            max-width: 100%;
-            line-height: 1.8rem;
-            max-height: 3.6rem;
-            overflow: hidden;
-            font-size: 1.3rem;
-            font-weight: 400;
-            text-transform: none;
-        }
+        .textual_info {
+            font-family: Arial, Helvetica, sans-serif;
+
+            .videoTitle {
+                margin-left: 4px;
+                // display: inline-block;
+                color: #0D0D0D;
+                -webkit-line-clamp: 2;
+                display: block;
+                max-height: 4.8rem;
+                overflow: hidden;
+                font-size: 1.8rem;
+                font-weight: 400;
+                line-height: 2.4rem;
+                -webkit-box-orient: vertical;
+                text-overflow: ellipsis;
+                white-space: normal;
+                display: -webkit-box;
+            }
+
+            .channelName {
+                color: #606060;
+                margin-left: 4px;
+                font-size: 1.3rem;
+                font-weight: 400;
+                line-height: 1.8rem;
+                text-transform: none;
+                margin-top: 0px;
+                display: inline-block;
+            }
+
+            .video_description {
+                margin-left: 4px;
+                // font-size: 12px;
+                color: #606060;
+                padding-top: 8px;
+                margin-bottom: 8px;
+                display: block;
+                line-height: 1.8rem;
+                max-height: 4.2rem;
+                overflow: hidden;
+                font-size: 1.3rem;
+                font-weight: 400;
+                text-transform: none;
+            }
+
+            .publish_time {
+                //display: inline-block;
+                margin-left: 8px;
+                padding: 0;
+                border: 0;
+                background: transparent;
+                color: #606060;
+                max-width: 100%;
+                line-height: 1.8rem;
+                max-height: 3.6rem;
+                overflow: hidden;
+                font-size: 1.3rem;
+                font-weight: 400;
+                text-transform: none;
+            }
+
+            .viewCount {
+
+                margin-left: 8px;
+                padding: 0;
+                border: 0;
+                background: transparent;
+                color: #606060;
+                max-width: 100%;
+                line-height: 1.8rem;
+                max-height: 3.6rem;
+                overflow: hidden;
+                font-size: 1.3rem;
+                font-weight: 400;
+                text-transform: none;
+            }
+
 
-        .viewCount {
 
-            margin-left: 8px;
-            padding: 0;
-            border: 0;
-            background: transparent;
-            color: #606060;
-            max-width: 100%;
-            line-height: 1.8rem;
-            max-height: 3.6rem;
-            overflow: hidden;
-            font-size: 1.3rem;
-            font-weight: 400;
-            text-transform: none;
         }
-        
-            
-        
     }
 }
\ No newline at end of file
diff --git a/src/client/apis/youtube/YoutubeBox.tsx b/src/client/apis/youtube/YoutubeBox.tsx
index 7f9a3ad70..dc142802c 100644
--- a/src/client/apis/youtube/YoutubeBox.tsx
+++ b/src/client/apis/youtube/YoutubeBox.tsx
@@ -343,7 +343,7 @@ export class YoutubeBox extends React.Component<FieldViewProps> {
 
     render() {
         let content =
-            <div style={{ width: "100%", height: "100%", position: "absolute" }} onWheel={this.onPostWheel} onPointerDown={this.onPostPointer} onPointerMove={this.onPostPointer} onPointerUp={this.onPostPointer}>
+            <div className="youtubeBox-cont" style={{ width: "100%", height: "100%", position: "absolute" }} onWheel={this.onPostWheel} onPointerDown={this.onPostPointer} onPointerMove={this.onPostPointer} onPointerUp={this.onPostPointer}>
                 <input type="text" placeholder="Search for a video" onKeyDown={this.onEnterKeyDown} style={{ height: 40, width: "100%", border: "1px solid black", padding: 5, textAlign: "center" }} ref={(e) => this.YoutubeSearchElement = e!} />
                 {this.renderSearchResultsOrVideo()}
             </div>;
diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts
index 076907f09..c118d91d3 100644
--- a/src/client/cognitive_services/CognitiveServices.ts
+++ b/src/client/cognitive_services/CognitiveServices.ts
@@ -1,15 +1,11 @@
 import * as request from "request-promise";
 import { Doc, Field, Opt } from "../../new_fields/Doc";
 import { Cast } from "../../new_fields/Types";
-import { ImageField } from "../../new_fields/URLField";
-import { List } from "../../new_fields/List";
 import { Docs } from "../documents/Documents";
 import { RouteStore } from "../../server/RouteStore";
 import { Utils } from "../../Utils";
-import { CompileScript } from "../util/Scripting";
-import { ComputedField } from "../../new_fields/ScriptField";
 import { InkData } from "../../new_fields/InkField";
-import { undoBatch, UndoManager } from "../util/UndoManager";
+import { UndoManager } from "../util/UndoManager";
 
 type APIManager<D> = { converter: BodyConverter<D>, requester: RequestExecutor, analyzer: AnalysisApplier };
 type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise<string>;
diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts
new file mode 100644
index 000000000..b58bdb6c7
--- /dev/null
+++ b/src/client/util/DictationManager.ts
@@ -0,0 +1,39 @@
+namespace CORE {
+    export interface IWindow extends Window {
+        webkitSpeechRecognition: any;
+    }
+}
+
+const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow;
+
+export default class DictationManager {
+    public static Instance = new DictationManager();
+    private isListening = false;
+    private recognizer: any;
+
+    constructor() {
+        this.recognizer = new webkitSpeechRecognition();
+        this.recognizer.interimResults = false;
+        this.recognizer.continuous = true;
+    }
+
+    finish = (handler: any, data: any) => {
+        handler(data);
+        this.isListening = false;
+        this.recognizer.stop();
+    }
+
+    listen = () => {
+        if (this.isListening) {
+            return undefined;
+        }
+        this.isListening = true;
+        this.recognizer.start();
+        return new Promise<string>((resolve, reject) => {
+            this.recognizer.onresult = (e: any) => this.finish(resolve, e.results[0][0].transcript);
+            this.recognizer.onerror = (e: any) => this.finish(reject, e);
+        });
+
+    }
+
+}
\ No newline at end of file
diff --git a/src/client/views/ContextMenu.tsx b/src/client/views/ContextMenu.tsx
index a608e448a..98025ac31 100644
--- a/src/client/views/ContextMenu.tsx
+++ b/src/client/views/ContextMenu.tsx
@@ -38,8 +38,12 @@ export class ContextMenu extends React.Component {
         this._items = [];
     }
 
-    findByDescription = (target: string) => {
-        return this._items.find(menuItem => menuItem.description === target);
+    findByDescription = (target: string, toLowerCase = false) => {
+        return this._items.find(menuItem => {
+            let reference = menuItem.description;
+            toLowerCase && (reference = reference.toLowerCase());
+            reference === target;
+        });
     }
 
     @action
diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts
index e31b44514..ea2e3e196 100644
--- a/src/client/views/GlobalKeyHandler.ts
+++ b/src/client/views/GlobalKeyHandler.ts
@@ -5,9 +5,13 @@ import { MainView } from "./MainView";
 import { DragManager } from "../util/DragManager";
 import { action } from "mobx";
 import { Doc } from "../../new_fields/Doc";
+import { CognitiveServices } from "../cognitive_services/CognitiveServices";
+import DictationManager from "../util/DictationManager";
+import { ContextMenu } from "./ContextMenu";
+import { ContextMenuProps } from "./ContextMenuItem";
 
 const modifiers = ["control", "meta", "shift", "alt"];
-type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo;
+type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo | Promise<KeyControlInfo>;
 type KeyControlInfo = {
     preventDefault: boolean,
     stopPropagation: boolean
@@ -25,9 +29,10 @@ export default class KeyManager {
         this.router.set(isMac ? "0001" : "0100", this.ctrl);
         this.router.set(isMac ? "0100" : "0010", this.alt);
         this.router.set(isMac ? "1001" : "1100", this.ctrl_shift);
+        this.router.set("1000", this.shift);
     }
 
-    public handle = (e: KeyboardEvent) => {
+    public handle = async (e: KeyboardEvent) => {
         let keyname = e.key.toLowerCase();
         this.handleGreedy(keyname);
 
@@ -43,7 +48,7 @@ export default class KeyManager {
             return;
         }
 
-        let control = handleConstrained(keyname, e);
+        let control = await handleConstrained(keyname, e);
 
         control.stopPropagation && e.stopPropagation();
         control.preventDefault && e.preventDefault();
@@ -95,6 +100,27 @@ export default class KeyManager {
         };
     });
 
+    private shift = async (keyname: string) => {
+        let stopPropagation = false;
+        let preventDefault = false;
+
+        switch (keyname) {
+            case " ":
+                let transcript = await DictationManager.Instance.listen();
+                console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`);
+                let command: ContextMenuProps | undefined;
+                transcript && (command = ContextMenu.Instance.findByDescription(transcript, true)) && "event" in command && command.event();
+                stopPropagation = true;
+                preventDefault = true;
+                break;
+        }
+
+        return {
+            stopPropagation: stopPropagation,
+            preventDefault: preventDefault
+        };
+    }
+
     private alt = action((keyname: string) => {
         let stopPropagation = true;
         let preventDefault = true;
diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx
index a15ed8f94..7386eaba6 100644
--- a/src/client/views/collections/CollectionSubView.tsx
+++ b/src/client/views/collections/CollectionSubView.tsx
@@ -66,6 +66,9 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) {
                     if (res.success) {
                         return res.result;
                     }
+                    else {
+                        console.log(res.error);
+                    }
                 });
             }
             return docs;
diff --git a/src/client/views/collections/CollectionViewChromes.scss b/src/client/views/collections/CollectionViewChromes.scss
index 989315194..1738ac227 100644
--- a/src/client/views/collections/CollectionViewChromes.scss
+++ b/src/client/views/collections/CollectionViewChromes.scss
@@ -37,9 +37,10 @@
             }
 
             .collectionViewBaseChrome-collapse {
-                transition: all .5s;
+                transition: all .5s, opacity 0.3s;
                 position: absolute;
                 width: 40px;
+                transform-origin: top left;
             }
 
             .collectionViewBaseChrome-viewSpecs {
diff --git a/src/client/views/collections/CollectionViewChromes.tsx b/src/client/views/collections/CollectionViewChromes.tsx
index 38aafd3cc..43e84cfda 100644
--- a/src/client/views/collections/CollectionViewChromes.tsx
+++ b/src/client/views/collections/CollectionViewChromes.tsx
@@ -142,7 +142,7 @@ export class CollectionViewBaseChrome extends React.Component<CollectionViewChro
             `return ${dateRestrictionScript} ${keyRestrictionScript.length ? "&&" : ""} ${keyRestrictionScript}` :
             `return ${keyRestrictionScript} ${dateRestrictionScript.length ? "&&" : ""} ${dateRestrictionScript}` :
             "return true";
-        let compiled = CompileScript(fullScript, { params: { doc: Doc.name } });
+        let compiled = CompileScript(fullScript, { params: { doc: Doc.name }, typecheck: false });
         if (compiled.compiled) {
             this.props.CollectionView.props.Document.viewSpecScript = new ScriptField(compiled);
         }
@@ -189,7 +189,12 @@ export class CollectionViewBaseChrome extends React.Component<CollectionViewChro
                 <div className="collectionViewChrome">
                     <div className="collectionViewBaseChrome">
                         <button className="collectionViewBaseChrome-collapse"
-                            style={{ marginTop: this._collapsed ? 60 : 0, transform: `rotate(${this._collapsed ? 180 : 0}deg)` }}
+                            style={{
+                                marginTop: this._collapsed ? 60 : 0,
+                                transform: `rotate(${this._collapsed ? 180 : 0}deg) scale(${this._collapsed ? 0.5 : 1}) translate(${this._collapsed ? "-100%, -100%" : "0, 0"})`,
+                                opacity: (this._collapsed && !this.props.CollectionView.props.isSelected()) ? 0 : 0.9,
+                                left: (this._collapsed ? 0 : "unset"),
+                            }}
                             title="Collapse collection chrome" onClick={this.toggleCollapse}>
                             <FontAwesomeIcon icon="caret-up" size="2x" />
                         </button>
diff --git a/src/client/views/collections/KeyRestrictionRow.tsx b/src/client/views/collections/KeyRestrictionRow.tsx
index 9baa250a6..1b59547d8 100644
--- a/src/client/views/collections/KeyRestrictionRow.tsx
+++ b/src/client/views/collections/KeyRestrictionRow.tsx
@@ -2,6 +2,7 @@ import * as React from "react";
 import { observable, runInAction } from "mobx";
 import { observer } from "mobx-react";
 import { PastelSchemaPalette } from "../../../new_fields/SchemaHeaderField";
+import { Doc } from "../../../new_fields/Doc";
 
 interface IKeyRestrictionProps {
     contains: boolean;
@@ -23,7 +24,10 @@ export default class KeyRestrictionRow extends React.Component<IKeyRestrictionPr
                 parsedValue = parsed;
                 type = "number";
             }
-            let scriptText = `${this._contains ? "" : "!"}((doc.${this._key} as ${type})${type === "string" ? ".includes" : "<="}(${parsedValue}))`;
+            let scriptText = `${this._contains ? "" : "!"}(((doc.${this._key} && (doc.${this._key} as ${type})${type === "string" ? ".includes" : "<="}(${parsedValue}))) ||
+                ((doc.data_ext && doc.data_ext.${this._key}) && (doc.data_ext.${this._key} as ${type})${type === "string" ? ".includes" : "<="}(${parsedValue}))))`;
+            // let doc = new Doc();
+            // ((doc.data_ext && doc.data_ext!.text) && (doc.data_ext!.text as string).includes("hello"));
             this.props.script(scriptText);
         }
         else {
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 8dac785e1..cbab14976 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -5,7 +5,7 @@ import { Id } from "../../../../new_fields/FieldSymbols";
 import { InkField, StrokeData } from "../../../../new_fields/InkField";
 import { createSchema, makeInterface } from "../../../../new_fields/Schema";
 import { BoolCast, Cast, FieldValue, NumCast, StrCast } from "../../../../new_fields/Types";
-import { emptyFunction, returnOne } from "../../../../Utils";
+import { emptyFunction, returnOne, Utils } from "../../../../Utils";
 import { DocumentManager } from "../../../util/DocumentManager";
 import { DragManager } from "../../../util/DragManager";
 import { HistoryUtil } from "../../../util/History";
@@ -34,12 +34,14 @@ import { CompileScript } from "../../../util/Scripting";
 import { CognitiveServices } from "../../../cognitive_services/CognitiveServices";
 import { library } from "@fortawesome/fontawesome-svg-core";
 import { faEye } from "@fortawesome/free-regular-svg-icons";
-import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass } from "@fortawesome/free-solid-svg-icons";
+import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload } from "@fortawesome/free-solid-svg-icons";
 import { undo } from "prosemirror-history";
 import { number } from "prop-types";
 import { ContextMenu } from "../../ContextMenu";
+import { RouteStore } from "../../../../server/RouteStore";
+import { DocServer } from "../../../DocServer";
 
-library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass);
+library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload);
 
 export const panZoomSchema = createSchema({
     panX: "number",
@@ -516,7 +518,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY));
     }
 
-    onContextMenu = () => {
+    onContextMenu = (e: React.MouseEvent) => {
         let layoutItems: ContextMenuProps[] = [];
         layoutItems.push({
             description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`,
@@ -561,6 +563,35 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                 CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData);
             }, icon: "paint-brush"
         });
+        ContextMenu.Instance.addItem({
+            description: "Import document", icon: "upload", event: () => {
+                const input = document.createElement("input");
+                input.type = "file";
+                input.accept = ".zip";
+                input.onchange = async _e => {
+                    const files = input.files;
+                    if (!files) return;
+                    const file = files[0];
+                    let formData = new FormData();
+                    formData.append('file', file);
+                    formData.append('remap', "true");
+                    const upload = Utils.prepend("/uploadDoc");
+                    const response = await fetch(upload, { method: "POST", body: formData });
+                    const json = await response.json();
+                    if (json === "error") {
+                        return;
+                    }
+                    const doc = await DocServer.GetRefField(json);
+                    if (!doc || !(doc instanceof Doc)) {
+                        return;
+                    }
+                    const [x, y] = this.props.ScreenToLocalTransform().transformPoint(e.pageX, e.pageY);
+                    doc.x = x, doc.y = y;
+                    this.addDocument(doc, false);
+                };
+                input.click();
+            }
+        });
     }
 
 
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 4b5cf3a43..51662274d 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -41,10 +41,13 @@ import { ClientUtils } from '../../util/ClientUtils';
 import { EditableView } from '../EditableView';
 import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-icons';
 import { DocumentDecorations } from '../DocumentDecorations';
+import { CognitiveServices } from '../../cognitive_services/CognitiveServices';
+import DictationManager from '../../util/DictationManager';
 const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this?
 
 library.add(fa.faTrash);
 library.add(fa.faShare);
+library.add(fa.faDownload);
 library.add(fa.faExpandArrowsAlt);
 library.add(fa.faCompressArrowsAlt);
 library.add(fa.faLayerGroup);
@@ -62,7 +65,7 @@ library.add(fa.faCrosshairs);
 library.add(fa.faDesktop);
 library.add(fa.faUnlock);
 library.add(fa.faLock);
-library.add(fa.faLaptopCode, fa.faMale, fa.faCopy, fa.faHandPointRight, fa.faCompass, fa.faSnowflake);
+library.add(fa.faLaptopCode, fa.faMale, fa.faCopy, fa.faHandPointRight, fa.faCompass, fa.faSnowflake, fa.faMicrophone);
 
 // const linkSchema = createSchema({
 //     title: "string",
@@ -535,6 +538,11 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
         this.props.Document.lockedPosition = BoolCast(this.props.Document.lockedPosition) ? undefined : true;
     }
 
+    listen = async () => {
+        let transcript = await DictationManager.Instance.listen();
+        transcript && (Doc.GetProto(this.props.Document).transcript = transcript);
+    }
+
     @action
     onContextMenu = async (e: React.MouseEvent): Promise<void> => {
         e.persist();
@@ -558,6 +566,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
         cm.addItem({ description: BoolCast(this.props.Document.ignoreAspect, false) || !this.props.Document.nativeWidth || !this.props.Document.nativeHeight ? "Freeze" : "Unfreeze", event: this.freezeNativeDimensions, icon: "snowflake" });
         cm.addItem({ description: "Pin to Presentation", event: () => PresentationView.Instance.PinDoc(this.props.Document), icon: "map-pin" });
         cm.addItem({ description: BoolCast(this.props.Document.lockedPosition) ? "Unlock Position" : "Lock Position", event: this.toggleLockPosition, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" });
+        cm.addItem({ description: "Transcribe Speech", event: this.listen, icon: "microphone" });
         let makes: ContextMenuProps[] = [];
         makes.push({ description: "Make Background", event: this.makeBackground, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" });
         makes.push({ description: this.props.Document.isButton ? "Remove Button" : "Make Button", event: this.makeBtnClicked, icon: "concierge-bell" });
@@ -597,6 +606,15 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
             copies.push({ description: "Copy ID", event: () => Utils.CopyText(this.props.Document[Id]), icon: "fingerprint" });
             cm.addItem({ description: "Copy...", subitems: copies, icon: "copy" });
         }
+        cm.addItem({
+            description: "Download document", icon: "download", event: () => {
+                const a = document.createElement("a");
+                const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`);
+                a.href = url;
+                a.download = `DocExport-${this.props.Document[Id]}.zip`;
+                a.click();
+            }
+        });
         cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" });
         type User = { email: string, userDocumentId: string };
         let usersMenu: ContextMenuProps[] = [];
diff --git a/src/new_fields/ObjectField.ts b/src/new_fields/ObjectField.ts
index 5f4a6f8fb..65ada91c0 100644
--- a/src/new_fields/ObjectField.ts
+++ b/src/new_fields/ObjectField.ts
@@ -1,6 +1,7 @@
 import { Doc } from "./Doc";
 import { RefField } from "./RefField";
 import { OnUpdate, Parent, Copy, ToScriptString } from "./FieldSymbols";
+import { Scripting } from "../client/util/Scripting";
 
 export abstract class ObjectField {
     protected [OnUpdate](diff?: any) { }
@@ -15,3 +16,5 @@ export namespace ObjectField {
         return field[Copy]();
     }
 }
+
+Scripting.addGlobal(ObjectField);
\ No newline at end of file
diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py
index 1ff0e3b31..f0f45d8f9 100644
--- a/src/scraping/buxton/scraper.py
+++ b/src/scraping/buxton/scraper.py
@@ -236,7 +236,7 @@ def parse_document(file_name: str):
         view_guids.append(write_image(pure_name, image))
         copyfile(dir_path + "/" + image, dir_path +
                  "/" + image.replace(".", "_o.", 1))
-        os.rename(dir_path + "/" + image, dir_path +
+        copyfile(dir_path + "/" + image, dir_path +
                   "/" + image.replace(".", "_m.", 1))
     print(f"extracted {count} images...")
 
diff --git a/src/server/database.ts b/src/server/database.ts
index 7f5331998..a7254fb0c 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -17,7 +17,7 @@ export class Database {
         });
     }
 
-    public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) {
+    public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) {
         if (this.db) {
             let collection = this.db.collection(collectionName);
             const prom = this.currentWrites[id];
@@ -30,7 +30,7 @@ export class Database {
                                 delete this.currentWrites[id];
                             }
                             resolve();
-                            callback();
+                            callback(err, res);
                         });
                 });
             };
@@ -41,6 +41,30 @@ export class Database {
         }
     }
 
+    public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) {
+        if (this.db) {
+            let collection = this.db.collection(collectionName);
+            const prom = this.currentWrites[id];
+            let newProm: Promise<void>;
+            const run = (): Promise<void> => {
+                return new Promise<void>(resolve => {
+                    collection.replaceOne({ _id: id }, value, { upsert }
+                        , (err, res) => {
+                            if (this.currentWrites[id] === newProm) {
+                                delete this.currentWrites[id];
+                            }
+                            resolve();
+                            callback(err, res);
+                        });
+                });
+            };
+            newProm = prom ? prom.then(run) : run();
+            this.currentWrites[id] = newProm;
+        } else {
+            this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName));
+        }
+    }
+
     public delete(query: any, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
     public delete(id: string, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
     public delete(id: any, collectionName = Database.DocumentsCollection) {
@@ -126,6 +150,34 @@ export class Database {
         }
     }
 
+    public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise<void> {
+        if (this.db) {
+            const visited = new Set<string>();
+            while (ids.length) {
+                const count = Math.min(ids.length, 1000);
+                const index = ids.length - count;
+                const fetchIds = ids.splice(index, count).filter(id => !visited.has(id));
+                if (!fetchIds.length) {
+                    continue;
+                }
+                const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments"));
+                for (const doc of docs) {
+                    const id = doc.id;
+                    visited.add(id);
+                    ids.push(...fn(doc));
+                }
+            }
+
+        } else {
+            return new Promise(res => {
+                this.onConnect.push(() => {
+                    this.visit(ids, fn, collectionName);
+                    res();
+                });
+            });
+        }
+    }
+
     public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise<mongodb.Cursor> {
         if (this.db) {
             let cursor = this.db.collection(collectionName).find(query);
diff --git a/src/server/index.ts b/src/server/index.ts
index 080b50ada..1912cf5c1 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -27,6 +27,7 @@ import { Client } from './Client';
 import { Database } from './database';
 import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message";
 import { RouteStore } from './RouteStore';
+import v4 = require('uuid/v4');
 const app = express();
 const config = require('../../webpack.config');
 import { createCanvas, loadImage, Canvas } from "canvas";
@@ -39,8 +40,11 @@ import c = require("crypto");
 import { Search } from './Search';
 import { debug } from 'util';
 import _ = require('lodash');
+import * as Archiver from 'archiver';
+import * as AdmZip from 'adm-zip';
 import * as YoutubeApi from './youtubeApi/youtubeApiSample.js';
 import { Response } from 'express-serve-static-core';
+import { DocComponent } from '../client/views/DocComponent';
 const MongoStore = require('connect-mongo')(session);
 const mongoose = require('mongoose');
 const probe = require("probe-image-size");
@@ -180,6 +184,173 @@ function msToTime(duration: number) {
     return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds;
 }
 
+async function getDocs(id: string) {
+    const files = new Set<string>();
+    const docs: { [id: string]: any } = {};
+    const fn = (doc: any): string[] => {
+        const id = doc.id;
+        if (typeof id === "string" && id.endsWith("Proto")) {
+            //Skip protos
+            return [];
+        }
+        const ids: string[] = [];
+        for (const key in doc.fields) {
+            if (!doc.fields.hasOwnProperty(key)) {
+                continue;
+            }
+            const field = doc.fields[key];
+            if (field === undefined || field === null) {
+                continue;
+            }
+
+            if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+                ids.push(field.fieldId);
+            } else if (field.__type === "script" || field.__type === "computed") {
+                if (field.captures) {
+                    ids.push(field.captures.fieldId);
+                }
+            } else if (field.__type === "list") {
+                ids.push(...fn(field));
+            } else if (typeof field === "string") {
+                const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g;
+                let match: string[] | null;
+                while ((match = re.exec(field)) !== null) {
+                    ids.push(match[1]);
+                }
+            } else if (field.__type === "RichTextField") {
+                const re = /"href"\s*:\s*"(.*?)"/g;
+                let match: string[] | null;
+                while ((match = re.exec(field.Data)) !== null) {
+                    const urlString = match[1];
+                    const split = new URL(urlString).pathname.split("doc/");
+                    if (split.length > 1) {
+                        ids.push(split[split.length - 1]);
+                    }
+                }
+                const re2 = /"src"\s*:\s*"(.*?)"/g;
+                while ((match = re2.exec(field.Data)) !== null) {
+                    const urlString = match[1];
+                    const pathname = new URL(urlString).pathname;
+                    files.add(pathname);
+                }
+            } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) {
+                const url = new URL(field.url);
+                const pathname = url.pathname;
+                files.add(pathname);
+            }
+        }
+
+        if (doc.id) {
+            docs[doc.id] = doc;
+        }
+        return ids;
+    };
+    await Database.Instance.visit([id], fn);
+    return { id, docs, files };
+}
+app.get("/serializeDoc/:docId", async (req, res) => {
+    const { docs, files } = await getDocs(req.params.docId);
+    res.send({ docs, files: Array.from(files) });
+});
+
+app.get("/downloadId/:docId", async (req, res) => {
+    res.set('Content-disposition', `attachment;`);
+    res.set('Content-Type', "application/zip");
+    const { id, docs, files } = await getDocs(req.params.docId);
+    const docString = JSON.stringify({ id, docs });
+    const zip = Archiver('zip');
+    zip.pipe(res);
+    zip.append(docString, { name: "doc.json" });
+    files.forEach(val => {
+        zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) });
+    });
+    zip.finalize();
+});
+
+app.post("/uploadDoc", (req, res) => {
+    let form = new formidable.IncomingForm();
+    form.keepExtensions = true;
+    // let path = req.body.path;
+    const ids: { [id: string]: string } = {};
+    let remap = true;
+    const getId = (id: string): string => {
+        if (!remap) return id;
+        if (id.endsWith("Proto")) return id;
+        if (id in ids) {
+            return ids[id];
+        } else {
+            return ids[id] = v4();
+        }
+    };
+    const mapFn = (doc: any) => {
+        if (doc.id) {
+            doc.id = getId(doc.id);
+        }
+        for (const key in doc.fields) {
+            if (!doc.fields.hasOwnProperty(key)) {
+                continue;
+            }
+            const field = doc.fields[key];
+            if (field === undefined || field === null) {
+                continue;
+            }
+
+            if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+                field.fieldId = getId(field.fieldId);
+            } else if (field.__type === "script" || field.__type === "computed") {
+                if (field.captures) {
+                    field.captures.fieldId = getId(field.captures.fieldId);
+                }
+            } else if (field.__type === "list") {
+                mapFn(field);
+            } else if (typeof field === "string") {
+                const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g;
+                doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => {
+                    return `${p1}${getId(p2)}"`;
+                });
+            } else if (field.__type === "RichTextField") {
+                const re = /("href"\s*:\s*")(.*?)"/g;
+                field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => {
+                    return `${p1}${getId(p2)}"`;
+                });
+            }
+        }
+    };
+    form.parse(req, async (err, fields, files) => {
+        remap = fields.remap !== "false";
+        let id: string = "";
+        try {
+            for (const name in files) {
+                const path = files[name].path;
+                const zip = new AdmZip(path);
+                zip.getEntries().forEach(entry => {
+                    if (!entry.name.startsWith("files/")) return;
+                    zip.extractEntryTo(entry.name, __dirname + RouteStore.public, true, false);
+                });
+                const json = zip.getEntry("doc.json");
+                let docs: any;
+                try {
+                    let data = JSON.parse(json.getData().toString("utf8"));
+                    docs = data.docs;
+                    id = data.id;
+                    docs = Object.keys(docs).map(key => docs[key]);
+                    docs.forEach(mapFn);
+                    await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => {
+                        err && console.log(err);
+                        res();
+                    }, true, "newDocuments"))));
+                } catch (e) { console.log(e); }
+                fs.unlink(path, () => { });
+            }
+            if (id) {
+                res.send(JSON.stringify(getId(id)));
+            } else {
+                res.send(JSON.stringify("error"));
+            }
+        } catch (e) { console.log(e); }
+    });
+});
+
 app.get("/whosOnline", (req, res) => {
     let users: any = { active: {}, inactive: {} };
     const now = Date.now();