more interesting speech commands and command manager pattern for DictationManager

6 files changed, 216 insertions, 40 deletions

diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts
index b58bdb6c7..60b25afc5 100644
--- a/src/client/util/DictationManager.ts
+++ b/src/client/util/DictationManager.ts
@@ -1,3 +1,10 @@
+import { string } from "prop-types";
+import { observable, action } from "mobx";
+import { SelectionManager } from "./SelectionManager";
+import { DocumentView } from "../views/nodes/DocumentView";
+import { UndoManager } from "./UndoManager";
+import * as converter from "words-to-numbers";
+
 namespace CORE {
     export interface IWindow extends Window {
         webkitSpeechRecognition: any;
@@ -5,9 +12,14 @@ namespace CORE {
 }
 
 const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow;
+export type Action = (target: DocumentView) => any | Promise<any>;
+export type DynamicAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise<any>;
+export type RegexEntry = { key: RegExp, value: DynamicAction };
 
 export default class DictationManager {
     public static Instance = new DictationManager();
+    private registeredCommands = new Map<string, Action>();
+    private registeredRegexes: RegexEntry[] = [];
     private isListening = false;
     private recognizer: any;
 
@@ -17,8 +29,16 @@ export default class DictationManager {
         this.recognizer.continuous = true;
     }
 
+    @observable public current = "";
+
+    @action
     finish = (handler: any, data: any) => {
+        this.current = data;
         handler(data);
+        this.stop();
+    }
+
+    stop = () => {
         this.isListening = false;
         this.recognizer.stop();
     }
@@ -36,4 +56,72 @@ export default class DictationManager {
 
     }
 
+    private sanitize = (title: string) => {
+        return title.replace("...", "").toLowerCase().trim();
+    }
+
+    public registerStatic = (keys: Array<string>, action: Action, overwrite = false) => {
+        let success = true;
+        keys.forEach(key => {
+            key = this.sanitize(key);
+            let existing = this.registeredCommands.get(key);
+            if (!existing || overwrite) {
+                this.registeredCommands.set(key, action);
+            } else {
+                success = false;
+            }
+        });
+        return success;
+    }
+
+    public interpretNumber = (number: string) => {
+        let initial = parseInt(number);
+        if (!isNaN(initial)) {
+            return initial;
+        }
+        let converted = converter.wordsToNumbers(number, { fuzzy: true });
+        if (converted === null) {
+            return NaN;
+        }
+        return typeof converted === "string" ? parseInt(converted) : converted;
+    }
+
+    public registerDynamic = (dynamicKey: RegExp, action: DynamicAction) => {
+        this.registeredRegexes.push({
+            key: dynamicKey,
+            value: action
+        });
+    }
+
+    public execute = async (phrase: string) => {
+        let target = SelectionManager.SelectedDocuments()[0];
+        if (!target) {
+            return;
+        }
+        let batch = UndoManager.StartBatch("Dictation Action");
+        phrase = this.sanitize(phrase);
+
+        let registeredAction = this.registeredCommands.get(phrase);
+        if (registeredAction) {
+            await registeredAction(target);
+            return true;
+        }
+
+        let success = false;
+        for (let entry of this.registeredRegexes) {
+            let regex = entry.key;
+            let registeredDynamicAction = entry.value;
+            let matches = regex.exec(phrase);
+            regex.lastIndex = 0;
+            if (matches !== null) {
+                await registeredDynamicAction(target, matches);
+                success = true;
+                break;
+            }
+        }
+        batch.end();
+
+        return success;
+    }
+
 }
\ No newline at end of file
diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts
index 373584b4e..1d3c77ec7 100644
--- a/src/client/views/GlobalKeyHandler.ts
+++ b/src/client/views/GlobalKeyHandler.ts
@@ -73,6 +73,7 @@ export default class KeyManager {
                 }
                 MainView.Instance.toggleColorPicker(true);
                 SelectionManager.DeselectAll();
+                DictationManager.Instance.stop();
                 break;
             case "delete":
             case "backspace":
@@ -106,10 +107,11 @@ export default class KeyManager {
 
         switch (keyname) {
             case " ":
-                let transcript = await DictationManager.Instance.listen();
+                console.log("Listening...");
+                let analyzer = DictationManager.Instance;
+                let transcript = await analyzer.listen();
                 console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`);
-                let command: ContextMenuProps | undefined;
-                transcript && (command = ContextMenu.Instance.findByDescription(transcript, true)) && "event" in command && command.event();
+                transcript && analyzer.execute(transcript);
         }
 
         return {
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 91c8fe57c..36ac96907 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -39,6 +39,7 @@ import { FilterBox } from './search/FilterBox';
 import { CollectionTreeView } from './collections/CollectionTreeView';
 import { ClientUtils } from '../util/ClientUtils';
 import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField';
+import DictationManager from '../util/DictationManager';
 
 @observer
 export class MainView extends React.Component {
@@ -459,6 +460,7 @@ export class MainView extends React.Component {
     render() {
         return (
             <div id="main-div">
+                <h1>{DictationManager.Instance.current}</h1>
                 <DocumentDecorations />
                 {this.mainContent}
                 <PreviewCursor />
diff --git a/src/client/views/collections/CollectionBaseView.tsx b/src/client/views/collections/CollectionBaseView.tsx
index c595a4c56..3f88ed98c 100644
--- a/src/client/views/collections/CollectionBaseView.tsx
+++ b/src/client/views/collections/CollectionBaseView.tsx
@@ -22,6 +22,24 @@ export enum CollectionViewType {
     Masonry
 }
 
+export namespace CollectionViewType {
+
+    const stringMapping = new Map<string, CollectionViewType>([
+        ["invalid", CollectionViewType.Invalid],
+        ["freeform", CollectionViewType.Freeform],
+        ["schema", CollectionViewType.Schema],
+        ["docking", CollectionViewType.Docking],
+        ["tree", CollectionViewType.Tree],
+        ["stacking", CollectionViewType.Stacking],
+        ["masonry", CollectionViewType.Masonry]
+    ]);
+
+    export const ValueOf = (value: string) => {
+        return stringMapping.get(value.toLowerCase());
+    };
+
+}
+
 export interface CollectionRenderProps {
     addDocument: (document: Doc, allowDuplicates?: boolean) => boolean;
     removeDocument: (document: Doc) => boolean;
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 8dac785e1..7856f3718 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -20,7 +20,7 @@ import { DocumentContentsView } from "../../nodes/DocumentContentsView";
 import { DocumentViewProps, positionSchema } from "../../nodes/DocumentView";
 import { pageSchema } from "../../nodes/ImageBox";
 import PDFMenu from "../../pdf/PDFMenu";
-import { CollectionSubView } from "../CollectionSubView";
+import { CollectionSubView, SubCollectionViewProps } from "../CollectionSubView";
 import { CollectionFreeFormLinksView } from "./CollectionFreeFormLinksView";
 import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCursors";
 import "./CollectionFreeFormView.scss";
@@ -38,6 +38,7 @@ import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsA
 import { undo } from "prosemirror-history";
 import { number } from "prop-types";
 import { ContextMenu } from "../../ContextMenu";
+import DictationManager from "../../../util/DictationManager";
 
 library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass);
 
@@ -121,11 +122,18 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         });
     }
 
+    constructor(props: SubCollectionViewProps) {
+        super(props);
+        let fixed = DictationManager.Instance.registerStatic;
+        fixed(["Unset Fit To Container", "Set Fit To Container"], this.fitToContainer);
+        fixed(["Arrange contents in grid"], this.arrangeContents);
+        fixed(["Analyze Strokes"], this.analyzeStrokes);
+    }
+
     @computed get fieldExtensionDoc() {
         return Doc.resolvedFieldDataDoc(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey, "true");
     }
 
-
     @undoBatch
     @action
     drop = (e: Event, de: DragManager.DropEvent) => {
@@ -516,50 +524,62 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY));
     }
 
+    fitToContainer = async () => this.props.Document.fitToBox = !this.fitToBox;
+
+    arrangeContents = async () => {
+        const docs = await DocListCastAsync(this.Document[this.props.fieldKey]);
+        UndoManager.RunInBatch(() => {
+            if (docs) {
+                let startX = this.Document.panX || 0;
+                let x = startX;
+                let y = this.Document.panY || 0;
+                let i = 0;
+                const width = Math.max(...docs.map(doc => NumCast(doc.width)));
+                const height = Math.max(...docs.map(doc => NumCast(doc.height)));
+                for (const doc of docs) {
+                    doc.x = x;
+                    doc.y = y;
+                    x += width + 20;
+                    if (++i === 6) {
+                        i = 0;
+                        x = startX;
+                        y += height + 20;
+                    }
+                }
+            }
+        }, "arrange contents");
+    }
+
+    analyzeStrokes = async () => {
+        let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField);
+        if (!data) {
+            return;
+        }
+        let relevantKeys = ["inkAnalysis", "handwriting"];
+        CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData);
+    }
+
     onContextMenu = () => {
         let layoutItems: ContextMenuProps[] = [];
         layoutItems.push({
             description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`,
-            event: async () => this.props.Document.fitToBox = !this.fitToBox,
+            event: this.fitToContainer,
             icon: !this.fitToBox ? "expand-arrows-alt" : "compress-arrows-alt"
         });
         layoutItems.push({
             description: "Arrange contents in grid",
-            icon: "table",
-            event: async () => {
-                const docs = await DocListCastAsync(this.Document[this.props.fieldKey]);
-                UndoManager.RunInBatch(() => {
-                    if (docs) {
-                        let startX = this.Document.panX || 0;
-                        let x = startX;
-                        let y = this.Document.panY || 0;
-                        let i = 0;
-                        const width = Math.max(...docs.map(doc => NumCast(doc.width)));
-                        const height = Math.max(...docs.map(doc => NumCast(doc.height)));
-                        for (const doc of docs) {
-                            doc.x = x;
-                            doc.y = y;
-                            x += width + 20;
-                            if (++i === 6) {
-                                i = 0;
-                                x = startX;
-                                y += height + 20;
-                            }
-                        }
-                    }
-                }, "arrange contents");
-            }
+            event: this.arrangeContents,
+            icon: "table"
         });
-        ContextMenu.Instance.addItem({ description: "Layout...", subitems: layoutItems, icon: "compass" });
         ContextMenu.Instance.addItem({
-            description: "Analyze Strokes", event: async () => {
-                let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField);
-                if (!data) {
-                    return;
-                }
-                let relevantKeys = ["inkAnalysis", "handwriting"];
-                CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData);
-            }, icon: "paint-brush"
+            description: "Layout...",
+            subitems: layoutItems,
+            icon: "compass"
+        });
+        ContextMenu.Instance.addItem({
+            description: "Analyze Strokes",
+            event: this.analyzeStrokes,
+            icon: "paint-brush"
         });
     }
 
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index dc56c1c8f..00416ca42 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -43,6 +43,7 @@ import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-i
 import { DocumentDecorations } from '../DocumentDecorations';
 import { CognitiveServices } from '../../cognitive_services/CognitiveServices';
 import DictationManager from '../../util/DictationManager';
+import { CollectionViewType } from '../collections/CollectionBaseView';
 const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this?
 
 library.add(fa.faTrash);
@@ -153,6 +154,43 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
 
     constructor(props: DocumentViewProps) {
         super(props);
+        let fixed = DictationManager.Instance.registerStatic;
+        let dynamic = DictationManager.Instance.registerDynamic;
+        fixed(["Open Fields"], DocumentView.OpenFieldsDictation);
+        fixed(["Clear"], DocumentView.ClearChildren);
+        dynamic(/create (\w+) documents of type (image|nested collection)/g, DocumentView.BuildLayout);
+        dynamic(/view as (freeform|stacking|masonry|schema|tree)/g, DocumentView.SetViewMode);
+    }
+
+    public static ClearChildren = (target: DocumentView) => {
+        Doc.GetProto(target.props.Document).data = new List();
+    }
+
+    public static BuildLayout = (target: DocumentView, matches: RegExpExecArray) => {
+        let count = DictationManager.Instance.interpretNumber(matches[1]);
+        let what = matches[2];
+        if (!("viewType" in target.props.Document)) {
+            return;
+        }
+        let dataDoc = Doc.GetProto(target.props.Document);
+        let fieldKey = "data";
+        for (let i = 0; i < count; i++) {
+            let created: Doc | undefined;
+            switch (what) {
+                case "image":
+                    created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg");
+                    break;
+                case "nested collection":
+                    created = Docs.Create.FreeformDocument([], {});
+                    break;
+            }
+            created && Doc.AddDocToList(dataDoc, fieldKey, created);
+        }
+    }
+
+    public static SetViewMode = (target: DocumentView, matches: RegExpExecArray) => {
+        let mode = CollectionViewType.ValueOf(matches[1]);
+        mode && (target.props.Document.viewType = mode);
     }
 
     _animateToIconDisposer?: IReactionDisposer;
@@ -413,7 +451,15 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
     deleteClicked = (): void => { SelectionManager.DeselectAll(); this.props.removeDocument && this.props.removeDocument(this.props.Document); }
 
     @undoBatch
-    fieldsClicked = (): void => { let kvp = Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 }); this.props.addDocTab(kvp, this.dataDoc, "onRight"); }
+    fieldsClicked = (): void => {
+        let kvp = Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 });
+        this.props.addDocTab(kvp, this.dataDoc, "onRight");
+    }
+
+    public static OpenFieldsDictation = (target: DocumentView) => {
+        let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 });
+        target.props.addDocTab(kvp, target.dataDoc, "onRight");
+    }
 
     @undoBatch
     makeBtnClicked = (): void => {