Merge branch 'master' into presentation_f

1 files changed, 349 insertions, 0 deletions

diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts
new file mode 100644
index 000000000..9c61fe125
--- /dev/null
+++ b/src/client/util/DictationManager.ts
@@ -0,0 +1,349 @@
+import { SelectionManager } from "./SelectionManager";
+import { DocumentView } from "../views/nodes/DocumentView";
+import { UndoManager } from "./UndoManager";
+import * as interpreter from "words-to-numbers";
+import { Doc } from "../../new_fields/Doc";
+import { List } from "../../new_fields/List";
+import { Docs, DocumentType } from "../documents/Documents";
+import { CollectionViewType } from "../views/collections/CollectionBaseView";
+import { Cast, CastCtor } from "../../new_fields/Types";
+import { listSpec } from "../../new_fields/Schema";
+import { AudioField, ImageField } from "../../new_fields/URLField";
+import { HistogramField } from "../northstar/dash-fields/HistogramField";
+import { MainView } from "../views/MainView";
+import { Utils } from "../../Utils";
+
+/**
+ * This namespace provides a singleton instance of a manager that
+ * handles the listening and text-conversion of user speech.
+ * 
+ * The basic manager functionality can be attained by the DictationManager.Controls namespace, which provide
+ * a simple recording operation that returns the interpreted text as a string.
+ * 
+ * Additionally, however, the DictationManager also exposes the ability to execute voice commands within Dash.
+ * It stores a default library of registered commands that can be triggered by listen()'ing for a phrase and then
+ * passing the results into the execute() function.
+ * 
+ * In addition to compile-time default commands, you can invoke DictationManager.Commands.Register(Independent|Dependent)
+ * to add new commands as classes or components are constructed.
+ */
+export namespace DictationManager {
+
+    /**
+     * Some type maneuvering to access Webkit's built-in
+     * speech recognizer.
+     */
+    namespace CORE {
+        export interface IWindow extends Window {
+            webkitSpeechRecognition: any;
+        }
+    }
+    const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow;
+    export const placeholder = "Listening...";
+
+    export namespace Controls {
+
+        const infringe = "unable to process: dictation manager still involved in previous session";
+        const intraSession = ". ";
+        const interSession = " ... ";
+
+        let isListening = false;
+        let isManuallyStopped = false;
+
+        let current: string | undefined = undefined;
+        let sessionResults: string[] = [];
+
+        const recognizer: SpeechRecognition = new webkitSpeechRecognition() || new SpeechRecognition();
+        recognizer.onstart = () => console.log("initiating speech recognition session...");
+
+        export type InterimResultHandler = (results: string) => any;
+        export type ContinuityArgs = { indefinite: boolean } | false;
+        export type DelimiterArgs = { inter: string, intra: string };
+        export type ListeningUIStatus = { interim: boolean } | false;
+
+        export interface ListeningOptions {
+            language: string;
+            continuous: ContinuityArgs;
+            delimiters: DelimiterArgs;
+            interimHandler: InterimResultHandler;
+            tryExecute: boolean;
+        }
+
+        export const listen = async (options?: Partial<ListeningOptions>) => {
+            let results: string | undefined;
+            let main = MainView.Instance;
+
+            main.dictationOverlayVisible = true;
+            main.isListening = { interim: false };
+
+            try {
+                results = await listenImpl(options);
+                if (results) {
+                    Utils.CopyText(results);
+                    main.isListening = false;
+                    let execute = options && options.tryExecute;
+                    main.dictatedPhrase = execute ? results.toLowerCase() : results;
+                    main.dictationSuccess = execute ? await DictationManager.Commands.execute(results) : true;
+                }
+            } catch (e) {
+                main.isListening = false;
+                main.dictatedPhrase = results = `dictation error: ${"error" in e ? e.error : "unknown error"}`;
+                main.dictationSuccess = false;
+            } finally {
+                main.initiateDictationFade();
+            }
+
+            return results;
+        };
+
+        const listenImpl = (options?: Partial<ListeningOptions>) => {
+            if (isListening) {
+                return infringe;
+            }
+            isListening = true;
+
+            let handler = options ? options.interimHandler : undefined;
+            let continuous = options ? options.continuous : undefined;
+            let indefinite = continuous && continuous.indefinite;
+            let language = options ? options.language : undefined;
+            let intra = options && options.delimiters ? options.delimiters.intra : undefined;
+            let inter = options && options.delimiters ? options.delimiters.inter : undefined;
+
+            recognizer.interimResults = handler !== undefined;
+            recognizer.continuous = continuous === undefined ? false : continuous !== false;
+            recognizer.lang = language === undefined ? "en-US" : language;
+
+            recognizer.start();
+
+            return new Promise<string>((resolve, reject) => {
+
+                recognizer.onerror = (e: SpeechRecognitionError) => {
+                    if (!(indefinite && e.error === "no-speech")) {
+                        recognizer.stop();
+                        reject(e);
+                    }
+                };
+
+                recognizer.onresult = (e: SpeechRecognitionEvent) => {
+                    current = synthesize(e, intra);
+                    handler && handler(current);
+                    isManuallyStopped && complete();
+                };
+
+                recognizer.onend = (e: Event) => {
+                    if (!indefinite || isManuallyStopped) {
+                        return complete();
+                    }
+
+                    if (current) {
+                        sessionResults.push(current);
+                        current = undefined;
+                    }
+                    recognizer.start();
+                };
+
+                let complete = () => {
+                    if (indefinite) {
+                        current && sessionResults.push(current);
+                        sessionResults.length && resolve(sessionResults.join(inter || interSession));
+                    } else {
+                        resolve(current);
+                    }
+                    reset();
+                };
+
+            });
+        };
+
+        export const stop = (salvageSession = true) => {
+            if (!isListening) {
+                return;
+            }
+            isManuallyStopped = true;
+            salvageSession ? recognizer.stop() : recognizer.abort();
+            let main = MainView.Instance;
+            if (main.dictationOverlayVisible) {
+                main.cancelDictationFade();
+                main.dictationOverlayVisible = false;
+                main.dictationSuccess = undefined;
+                setTimeout(() => main.dictatedPhrase = placeholder, 500);
+            }
+        };
+
+        const synthesize = (e: SpeechRecognitionEvent, delimiter?: string) => {
+            let results = e.results;
+            let transcripts: string[] = [];
+            for (let i = 0; i < results.length; i++) {
+                transcripts.push(results.item(i).item(0).transcript.trim());
+            }
+            return transcripts.join(delimiter || intraSession);
+        };
+
+        const reset = () => {
+            current = undefined;
+            sessionResults = [];
+            isListening = false;
+            isManuallyStopped = false;
+            recognizer.onresult = null;
+            recognizer.onerror = null;
+            recognizer.onend = null;
+        };
+
+    }
+
+    export namespace Commands {
+
+        export const dictationFadeDuration = 2000;
+
+        export type IndependentAction = (target: DocumentView) => any | Promise<any>;
+        export type IndependentEntry = { action: IndependentAction, restrictTo?: DocumentType[] };
+
+        export type DependentAction = (target: DocumentView, matches: RegExpExecArray) => any | Promise<any>;
+        export type DependentEntry = { expression: RegExp, action: DependentAction, restrictTo?: DocumentType[] };
+
+        export const RegisterIndependent = (key: string, value: IndependentEntry) => Independent.set(key, value);
+        export const RegisterDependent = (entry: DependentEntry) => Dependent.push(entry);
+
+        export const execute = async (phrase: string) => {
+            return UndoManager.RunInBatch(async () => {
+                let targets = SelectionManager.SelectedDocuments();
+                if (!targets || !targets.length) {
+                    return;
+                }
+
+                phrase = phrase.toLowerCase();
+                let entry = Independent.get(phrase);
+
+                if (entry) {
+                    let success = false;
+                    let restrictTo = entry.restrictTo;
+                    for (let target of targets) {
+                        if (!restrictTo || validate(target, restrictTo)) {
+                            await entry.action(target);
+                            success = true;
+                        }
+                    }
+                    return success;
+                }
+
+                for (let entry of Dependent) {
+                    let regex = entry.expression;
+                    let matches = regex.exec(phrase);
+                    regex.lastIndex = 0;
+                    if (matches !== null) {
+                        let success = false;
+                        let restrictTo = entry.restrictTo;
+                        for (let target of targets) {
+                            if (!restrictTo || validate(target, restrictTo)) {
+                                await entry.action(target, matches);
+                                success = true;
+                            }
+                        }
+                        return success;
+                    }
+                }
+
+                return false;
+            }, "Execute Command");
+        };
+
+        const ConstructorMap = new Map<DocumentType, CastCtor>([
+            [DocumentType.COL, listSpec(Doc)],
+            [DocumentType.AUDIO, AudioField],
+            [DocumentType.IMG, ImageField],
+            [DocumentType.HIST, HistogramField],
+            [DocumentType.IMPORT, listSpec(Doc)],
+            [DocumentType.TEXT, "string"]
+        ]);
+
+        const tryCast = (view: DocumentView, type: DocumentType) => {
+            let ctor = ConstructorMap.get(type);
+            if (!ctor) {
+                return false;
+            }
+            return Cast(Doc.GetProto(view.props.Document).data, ctor) !== undefined;
+        };
+
+        const validate = (target: DocumentView, types: DocumentType[]) => {
+            for (let type of types) {
+                if (tryCast(target, type)) {
+                    return true;
+                }
+            }
+            return false;
+        };
+
+        const interpretNumber = (number: string) => {
+            let initial = parseInt(number);
+            if (!isNaN(initial)) {
+                return initial;
+            }
+            let converted = interpreter.wordsToNumbers(number, { fuzzy: true });
+            if (converted === null) {
+                return NaN;
+            }
+            return typeof converted === "string" ? parseInt(converted) : converted;
+        };
+
+        const Independent = new Map<string, IndependentEntry>([
+
+            ["clear", {
+                action: (target: DocumentView) => Doc.GetProto(target.props.Document).data = new List(),
+                restrictTo: [DocumentType.COL]
+            }],
+
+            ["open fields", {
+                action: (target: DocumentView) => {
+                    let kvp = Docs.Create.KVPDocument(target.props.Document, { width: 300, height: 300 });
+                    target.props.addDocTab(kvp, target.dataDoc, "onRight");
+                }
+            }],
+
+            ["promote", {
+                action: (target: DocumentView) => {
+                    console.log(target);
+                },
+                restrictTo: [DocumentType.TEXT]
+            }]
+
+        ]);
+
+        const Dependent = new Array<DependentEntry>(
+
+            {
+                expression: /create (\w+) documents of type (image|nested collection)/g,
+                action: (target: DocumentView, matches: RegExpExecArray) => {
+                    let count = interpretNumber(matches[1]);
+                    let what = matches[2];
+                    let dataDoc = Doc.GetProto(target.props.Document);
+                    let fieldKey = "data";
+                    for (let i = 0; i < count; i++) {
+                        let created: Doc | undefined;
+                        switch (what) {
+                            case "image":
+                                created = Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg");
+                                break;
+                            case "nested collection":
+                                created = Docs.Create.FreeformDocument([], {});
+                                break;
+                        }
+                        created && Doc.AddDocToList(dataDoc, fieldKey, created);
+                    }
+                },
+                restrictTo: [DocumentType.COL]
+            },
+
+            {
+                expression: /view as (freeform|stacking|masonry|schema|tree)/g,
+                action: (target: DocumentView, matches: RegExpExecArray) => {
+                    let mode = CollectionViewType.valueOf(matches[1]);
+                    mode && (target.props.Document.viewType = mode);
+                },
+                restrictTo: [DocumentType.COL]
+            }
+
+        );
+
+    }
+
+}
\ No newline at end of file