Merge pull request #241 from browngraphicslab/speech-to-text

Speech to text overlay with registered commands on shift+space

4 files changed, 262 insertions, 53 deletions

diff --git a/src/client/views/collections/CollectionBaseView.tsx b/src/client/views/collections/CollectionBaseView.tsx
index 6801b94fd..b53e83eb1 100644
--- a/src/client/views/collections/CollectionBaseView.tsx
+++ b/src/client/views/collections/CollectionBaseView.tsx
@@ -22,6 +22,24 @@ export enum CollectionViewType {
     Masonry
 }
 
+export namespace CollectionViewType {
+
+    const stringMapping = new Map<string, CollectionViewType>([
+        ["invalid", CollectionViewType.Invalid],
+        ["freeform", CollectionViewType.Freeform],
+        ["schema", CollectionViewType.Schema],
+        ["docking", CollectionViewType.Docking],
+        ["tree", CollectionViewType.Tree],
+        ["stacking", CollectionViewType.Stacking],
+        ["masonry", CollectionViewType.Masonry]
+    ]);
+
+    export const valueOf = (value: string) => {
+        return stringMapping.get(value.toLowerCase());
+    };
+
+}
+
 export interface CollectionRenderProps {
     addDocument: (document: Doc, allowDuplicates?: boolean) => boolean;
     removeDocument: (document: Doc) => boolean;
diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx
index f59fee985..7a402798e 100644
--- a/src/client/views/collections/CollectionView.tsx
+++ b/src/client/views/collections/CollectionView.tsx
@@ -1,6 +1,6 @@
 import { library } from '@fortawesome/fontawesome-svg-core';
 import { faEye } from '@fortawesome/free-regular-svg-icons';
-import { faColumns, faEllipsisV, faFingerprint, faImage, faProjectDiagram, faSignature, faSquare, faTh, faThList, faTree } from '@fortawesome/free-solid-svg-icons';
+import { faColumns, faEllipsisV, faFingerprint, faImage, faProjectDiagram, faSignature, faSquare, faTh, faThList, faTree, faCopy } from '@fortawesome/free-solid-svg-icons';
 import { action, IReactionDisposer, observable, reaction, runInAction } from 'mobx';
 import { observer } from "mobx-react";
 import * as React from 'react';
@@ -20,7 +20,7 @@ import { CollectionTreeView } from "./CollectionTreeView";
 import { CollectionViewBaseChrome } from './CollectionViewChromes';
 export const COLLECTION_BORDER_WIDTH = 2;
 
-library.add(faTh, faTree, faSquare, faProjectDiagram, faSignature, faThList, faFingerprint, faColumns, faEllipsisV, faImage, faEye as any);
+library.add(faTh, faTree, faSquare, faProjectDiagram, faSignature, faThList, faFingerprint, faColumns, faEllipsisV, faImage, faEye as any, faCopy);
 
 @observer
 export class CollectionView extends React.Component<FieldViewProps> {
@@ -86,7 +86,12 @@ export class CollectionView extends React.Component<FieldViewProps> {
     onContextMenu = (e: React.MouseEvent): void => {
         if (!this.isAnnotationOverlay && !e.isPropagationStopped() && this.props.Document[Id] !== CurrentUserUtils.MainDocId) { // need to test this because GoldenLayout causes a parallel hierarchy in the React DOM for its children and the main document view7
             let subItems: ContextMenuProps[] = [];
-            subItems.push({ description: "Freeform", event: () => this.props.Document.viewType = CollectionViewType.Freeform, icon: "signature" });
+            subItems.push({
+                description: "Freeform", event: () => {
+                    this.props.Document.viewType = CollectionViewType.Freeform;
+                    delete this.props.Document.usePivotLayout;
+                }, icon: "signature"
+            });
             if (CollectionBaseView.InSafeMode()) {
                 ContextMenu.Instance.addItem({ description: "Test Freeform", event: () => this.props.Document.viewType = CollectionViewType.Invalid, icon: "project-diagram" });
             }
@@ -97,6 +102,7 @@ export class CollectionView extends React.Component<FieldViewProps> {
             switch (this.props.Document.viewType) {
                 case CollectionViewType.Freeform: {
                     subItems.push({ description: "Custom", icon: "fingerprint", event: CollectionFreeFormView.AddCustomLayout(this.props.Document, this.props.fieldKey) });
+                    subItems.push({ description: "Pivot", icon: "copy", event: () => this.props.Document.usePivotLayout = true });
                     break;
                 }
             }
diff --git a/src/client/views/collections/CollectionViewChromes.tsx b/src/client/views/collections/CollectionViewChromes.tsx
index 1b2561953..52c47e7e8 100644
--- a/src/client/views/collections/CollectionViewChromes.tsx
+++ b/src/client/views/collections/CollectionViewChromes.tsx
@@ -3,7 +3,7 @@ import { CollectionView } from "./CollectionView";
 import "./CollectionViewChromes.scss";
 import { CollectionViewType } from "./CollectionBaseView";
 import { undoBatch } from "../../util/UndoManager";
-import { action, observable, runInAction, computed, IObservable, IObservableValue } from "mobx";
+import { action, observable, runInAction, computed, IObservable, IObservableValue, reaction, autorun } from "mobx";
 import { observer } from "mobx-react";
 import { Doc, DocListCast } from "../../../new_fields/Doc";
 import { DocLike } from "../MetadataEntryMenu";
@@ -187,6 +187,36 @@ export class CollectionViewBaseChrome extends React.Component<CollectionViewChro
         }
     }
 
+    private get document() {
+        return this.props.CollectionView.props.Document;
+    }
+
+    private get pivotKey() {
+        return StrCast(this.document.pivotField);
+    }
+
+    private set pivotKey(value: string) {
+        this.document.pivotField = value;
+    }
+
+    @observable private pivotKeyDisplay = this.pivotKey;
+    getPivotInput = () => {
+        if (!this.document.usePivotLayout) {
+            return (null);
+        }
+        return (<input className="collectionViewBaseChrome-viewSpecsInput"
+            placeholder="PIVOT ON..."
+            value={this.pivotKeyDisplay}
+            onChange={action((e: React.ChangeEvent<HTMLInputElement>) => this.pivotKeyDisplay = e.currentTarget.value)}
+            onKeyPress={action((e: React.KeyboardEvent<HTMLInputElement>) => {
+                let value = e.currentTarget.value;
+                if (e.which === 13) {
+                    this.pivotKey = value;
+                    this.pivotKeyDisplay = "";
+                }
+            })} />);
+    }
+
     render() {
         return (
             <div className="collectionViewChrome-cont" style={{ top: this._collapsed ? -70 : 0 }}>
@@ -219,6 +249,7 @@ export class CollectionViewBaseChrome extends React.Component<CollectionViewChro
                                 value={this.filterValue ? this.filterValue.script.originalScript : ""}
                                 onChange={(e) => { }}
                                 onPointerDown={this.openViewSpecs} />
+                            {this.getPivotInput()}
                             <div className="collectionViewBaseChrome-viewSpecsMenu"
                                 onPointerDown={this.openViewSpecs}
                                 style={{
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 8322625f1..ba8dcff98 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -3,7 +3,7 @@ import { faEye } from "@fortawesome/free-regular-svg-icons";
 import { faCompass, faCompressArrowsAlt, faExpandArrowsAlt, faPaintBrush, faTable, faUpload, faChalkboard, faBraille } from "@fortawesome/free-solid-svg-icons";
 import { action, computed, observable } from "mobx";
 import { observer } from "mobx-react";
-import { Doc, DocListCastAsync, HeightSym, WidthSym } from "../../../../new_fields/Doc";
+import { Doc, DocListCastAsync, HeightSym, WidthSym, DocListCast, FieldResult, Field, Opt } from "../../../../new_fields/Doc";
 import { Id } from "../../../../new_fields/FieldSymbols";
 import { InkField, StrokeData } from "../../../../new_fields/InkField";
 import { createSchema, makeInterface } from "../../../../new_fields/Schema";
@@ -29,14 +29,16 @@ import { DocumentViewProps, positionSchema } from "../../nodes/DocumentView";
 import { pageSchema } from "../../nodes/ImageBox";
 import { OverlayElementOptions, OverlayView } from "../../OverlayView";
 import PDFMenu from "../../pdf/PDFMenu";
-import { ScriptBox } from "../../ScriptBox";
 import { CollectionSubView } from "../CollectionSubView";
+import { ScriptBox } from "../../ScriptBox";
 import { CollectionFreeFormLinksView } from "./CollectionFreeFormLinksView";
 import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCursors";
 import "./CollectionFreeFormView.scss";
 import { MarqueeView } from "./MarqueeView";
 import React = require("react");
 import { DocumentType, Docs } from "../../../documents/Documents";
+import { RouteStore } from "../../../../server/RouteStore";
+import { string, number, elementType } from "prop-types";
 
 library.add(faEye as any, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBraille, faChalkboard);
 
@@ -48,6 +50,139 @@ export const panZoomSchema = createSchema({
     arrangeInit: ScriptField,
 });
 
+export interface ViewDefBounds {
+    x: number;
+    y: number;
+    z?: number;
+    width: number;
+    height: number;
+}
+
+export interface ViewDefResult {
+    ele: JSX.Element;
+    bounds?: ViewDefBounds;
+}
+
+export namespace PivotView {
+
+    export interface PivotData {
+        type: string;
+        text: string;
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+        fontSize: number;
+    }
+
+    export const elements = (target: CollectionFreeFormView) => {
+        let collection = target.Document;
+        const field = StrCast(collection.pivotField) || "title";
+        const width = NumCast(collection.pivotWidth) || 200;
+
+        const groups = new Map<FieldResult<Field>, Doc[]>();
+
+        for (const doc of target.childDocs) {
+            const val = doc[field];
+            if (val === undefined) continue;
+
+            const l = groups.get(val);
+            if (l) {
+                l.push(doc);
+            } else {
+                groups.set(val, [doc]);
+            }
+
+        }
+
+        let minSize = Infinity;
+
+        groups.forEach((val, key) => {
+            minSize = Math.min(minSize, val.length);
+        });
+
+        const numCols = NumCast(collection.pivotNumColumns) || Math.ceil(Math.sqrt(minSize));
+        const fontSize = NumCast(collection.pivotFontSize);
+
+        const docMap = new Map<Doc, ViewDefBounds>();
+        const groupNames: PivotData[] = [];
+
+        let x = 0;
+        groups.forEach((val, key) => {
+            let y = 0;
+            let xCount = 0;
+            groupNames.push({
+                type: "text",
+                text: String(key),
+                x,
+                y: width + 50,
+                width: width * 1.25 * numCols,
+                height: 100, fontSize: fontSize
+            });
+            for (const doc of val) {
+                docMap.set(doc, {
+                    x: x + xCount * width * 1.25,
+                    y: -y,
+                    width,
+                    height: width
+                });
+                xCount++;
+                if (xCount >= numCols) {
+                    xCount = 0;
+                    y += width * 1.25;
+                }
+            }
+            x += width * 1.25 * (numCols + 1);
+        });
+
+        let elements = target.viewDefsToJSX(groupNames);
+        let curPage = FieldValue(target.Document.curPage, -1);
+
+        let docViews = target.childDocs.filter(doc => doc instanceof Doc).reduce((prev, doc) => {
+            var page = NumCast(doc.page, -1);
+            if ((Math.abs(Math.round(page) - Math.round(curPage)) < 3) || page === -1) {
+                let minim = BoolCast(doc.isMinimized);
+                if (minim === undefined || !minim) {
+                    let defaultPosition = (): ViewDefBounds => {
+                        return {
+                            x: NumCast(doc.x),
+                            y: NumCast(doc.y),
+                            z: NumCast(doc.z),
+                            width: NumCast(doc.width),
+                            height: NumCast(doc.height)
+                        };
+                    };
+                    const pos = docMap.get(doc) || defaultPosition();
+                    prev.push({
+                        ele: (
+                            <CollectionFreeFormDocumentView
+                                key={doc[Id]}
+                                x={pos.x}
+                                y={pos.y}
+                                width={pos.width}
+                                height={pos.height}
+                                {...target.getChildDocumentViewProps(doc)}
+                            />),
+                        bounds: {
+                            x: pos.x,
+                            y: pos.y,
+                            z: pos.z,
+                            width: NumCast(pos.width),
+                            height: NumCast(pos.height)
+                        }
+                    });
+                }
+            }
+            return prev;
+        }, elements);
+
+        target.resetSelectOnLoaded();
+
+        return docViews;
+    };
+
+}
+
 type PanZoomDocument = makeInterface<[typeof panZoomSchema, typeof positionSchema, typeof pageSchema]>;
 const PanZoomDocument = makeInterface(panZoomSchema, positionSchema, pageSchema);
 
@@ -488,9 +623,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
 
     getScale = () => this.Document.scale ? this.Document.scale : 1;
 
-
     getChildDocumentViewProps(childDocLayout: Doc): DocumentViewProps {
-        let self = this;
         let pair = Doc.GetLayoutDataDocPair(this.props.Document, this.props.DataDoc, this.props.fieldKey, childDocLayout);
         return {
             DataDoc: pair.data,
@@ -549,7 +682,19 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         return result.result === undefined ? { x: Cast(doc.x, "number"), y: Cast(doc.y, "number"), z: Cast(doc.z, "number"), width: Cast(doc.width, "number"), height: Cast(doc.height, "number") } : result.result;
     }
 
-    private viewDefToJSX(viewDef: any): { ele: JSX.Element, bounds?: { x: number, y: number, z?: number, width: number, height: number } } | undefined {
+    viewDefsToJSX = (views: any[]) => {
+        let elements: ViewDefResult[] = [];
+        if (Array.isArray(views)) {
+            elements = views.reduce<typeof elements>((prev, ele) => {
+                const jsx = this.viewDefToJSX(ele);
+                jsx && prev.push(jsx);
+                return prev;
+            }, elements);
+        }
+        return elements;
+    }
+
+    private viewDefToJSX(viewDef: any): Opt<ViewDefResult> {
         if (viewDef.type === "text") {
             const text = Cast(viewDef.text, "string");
             const x = Cast(viewDef.x, "number");
@@ -578,20 +723,14 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         const script = this.Document.arrangeScript;
         let state: any = undefined;
         const docs = this.childDocs;
-        let elements: { ele: JSX.Element, bounds?: { x: number, y: number, z?: number, width: number, height: number } }[] = [];
+        let elements: ViewDefResult[] = [];
         if (initScript) {
             const initResult = initScript.script.run({ docs, collection: this.Document });
             if (initResult.success) {
                 const result = initResult.result;
                 const { state: scriptState, views } = result;
                 state = scriptState;
-                if (Array.isArray(views)) {
-                    elements = views.reduce<typeof elements>((prev, ele) => {
-                        const jsx = this.viewDefToJSX(ele);
-                        jsx && prev.push(jsx);
-                        return prev;
-                    }, elements);
-                }
+                elements = this.viewDefsToJSX(views);
             }
         }
         let docviews = docs.filter(doc => doc instanceof Doc).reduce((prev, doc) => {
@@ -613,14 +752,17 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
             return prev;
         }, elements);
 
-        setTimeout(() => this._selectOnLoaded = "", 600);// bcz: surely there must be a better way ....
+        this.resetSelectOnLoaded();
 
         return docviews;
     }
 
+    resetSelectOnLoaded = () => setTimeout(() => this._selectOnLoaded = "", 600);// bcz: surely there must be a better way ....
+
     @computed.struct
     get views() {
-        return this.elements.filter(ele => ele.bounds && !ele.bounds.z).map(ele => ele.ele);
+        let source = this.Document.usePivotLayout === true ? PivotView.elements(this) : this.elements;
+        return source.filter(ele => ele.bounds && !ele.bounds.z).map(ele => ele.ele);
     }
     @computed.struct
     get overlayViews() {
@@ -633,11 +775,46 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY));
     }
 
+    fitToContainer = async () => this.props.Document.fitToBox = !this.fitToBox;
+
+    arrangeContents = async () => {
+        const docs = await DocListCastAsync(this.Document[this.props.fieldKey]);
+        UndoManager.RunInBatch(() => {
+            if (docs) {
+                let startX = this.Document.panX || 0;
+                let x = startX;
+                let y = this.Document.panY || 0;
+                let i = 0;
+                const width = Math.max(...docs.map(doc => NumCast(doc.width)));
+                const height = Math.max(...docs.map(doc => NumCast(doc.height)));
+                for (const doc of docs) {
+                    doc.x = x;
+                    doc.y = y;
+                    x += width + 20;
+                    if (++i === 6) {
+                        i = 0;
+                        x = startX;
+                        y += height + 20;
+                    }
+                }
+            }
+        }, "arrange contents");
+    }
+
+    analyzeStrokes = async () => {
+        let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField);
+        if (!data) {
+            return;
+        }
+        let relevantKeys = ["inkAnalysis", "handwriting"];
+        CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.fieldExtensionDoc, relevantKeys, data.inkData);
+    }
+
     onContextMenu = (e: React.MouseEvent) => {
         let layoutItems: ContextMenuProps[] = [];
         layoutItems.push({
             description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`,
-            event: async () => this.props.Document.fitToBox = !this.fitToBox,
+            event: this.fitToContainer,
             icon: !this.fitToBox ? "expand-arrows-alt" : "compress-arrows-alt"
         });
         layoutItems.push({
@@ -662,41 +839,18 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         });
         layoutItems.push({
             description: "Arrange contents in grid",
-            icon: "table",
-            event: async () => {
-                const docs = await DocListCastAsync(this.Document[this.props.fieldKey]);
-                UndoManager.RunInBatch(() => {
-                    if (docs) {
-                        let startX = this.Document.panX || 0;
-                        let x = startX;
-                        let y = this.Document.panY || 0;
-                        let i = 0;
-                        const width = Math.max(...docs.map(doc => NumCast(doc.width)));
-                        const height = Math.max(...docs.map(doc => NumCast(doc.height)));
-                        for (const doc of docs) {
-                            doc.x = x;
-                            doc.y = y;
-                            x += width + 20;
-                            if (++i === 6) {
-                                i = 0;
-                                x = startX;
-                                y += height + 20;
-                            }
-                        }
-                    }
-                }, "arrange contents");
-            }
+            event: this.arrangeContents,
+            icon: "table"
         });
-        ContextMenu.Instance.addItem({ description: "Layout...", subitems: layoutItems, icon: "compass" });
         ContextMenu.Instance.addItem({
-            description: "Analyze Strokes", event: async () => {
-                let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField);
-                if (!data) {
-                    return;
-                }
-                let relevantKeys = ["inkAnalysis", "handwriting"];
-                CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData);
-            }, icon: "paint-brush"
+            description: "Layout...",
+            subitems: layoutItems,
+            icon: "compass"
+        });
+        ContextMenu.Instance.addItem({
+            description: "Analyze Strokes",
+            event: this.analyzeStrokes,
+            icon: "paint-brush"
         });
         ContextMenu.Instance.addItem({
             description: "Import document", icon: "upload", event: () => {