Ink recognition and presentation view toggle

8 files changed, 124 insertions, 18 deletions

diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts
index d4085cf76..71e4276dc 100644
--- a/src/client/cognitive_services/CognitiveServices.ts
+++ b/src/client/cognitive_services/CognitiveServices.ts
@@ -8,10 +8,12 @@ import { RouteStore } from "../../server/RouteStore";
 import { Utils } from "../../Utils";
 import { CompileScript } from "../util/Scripting";
 import { ComputedField } from "../../new_fields/ScriptField";
+import { InkData } from "../../new_fields/InkField";
 
 export enum Services {
     ComputerVision = "vision",
-    Face = "face"
+    Face = "face",
+    Handwriting = "handwriting"
 }
 
 export enum Confidence {
@@ -132,4 +134,86 @@ export namespace CognitiveServices {
 
     }
 
+    export namespace Inking {
+
+        export interface AzureStrokeData {
+            id: number;
+            points: string;
+            language?: string;
+        }
+
+        export interface HandwritingUnit {
+            version: number;
+            language: string;
+            unit: string;
+            strokes: AzureStrokeData[];
+        }
+
+        export const analyze = async (inkData: InkData, target: Doc) => {
+            return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${Services.Handwriting}`)).then(async response => {
+                let apiKey = await response.text();
+                if (!apiKey) {
+                    return undefined;
+                }
+
+                let xhttp = new XMLHttpRequest();
+                let serverAddress = "https://api.cognitive.microsoft.com";
+                let endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize";
+
+                let results = await new Promise<any>((resolve, reject) => {
+                    let result: any;
+                    xhttp.onreadystatechange = function () {
+                        if (this.readyState === 4) {
+                            try {
+                                result = JSON.parse(xhttp.responseText);
+                            } catch (e) {
+                                return reject(e);
+                            }
+                            switch (this.status) {
+                                case 200:
+                                    return resolve(result);
+                                case 400:
+                                default:
+                                    return reject(result);
+                            }
+                        }
+                    };
+
+                    xhttp.open("PUT", endpoint, true);
+                    xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey);
+                    xhttp.setRequestHeader('Content-Type', 'application/json');
+                    xhttp.send(JSON.stringify(toHandwritingUnit(inkData)));
+                });
+
+                let recognizedText = results.recognitionUnits.map((unit: any) => unit.recognizedText);
+                let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1);
+                target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results.recognitionUnits, "Ink Analysis");
+                target.handwriting = individualWords.join(" ");
+            });
+        };
+
+        const toHandwritingUnit = (inkData: InkData): HandwritingUnit => {
+            let entries = inkData.entries(), next = entries.next();
+            let strokes: AzureStrokeData[] = [];
+            let id = 0;
+            while (!next.done) {
+                let entry = next.value;
+                let data = {
+                    id: id++,
+                    points: entry[1].pathData.map(point => `${point.x},${point.y}`).join(","),
+                    language: "en-US"
+                };
+                strokes.push(data);
+                next = entries.next();
+            }
+            return {
+                version: 1,
+                language: "en-US",
+                unit: "mm",
+                strokes: strokes
+            };
+        };
+
+    }
+
 }
\ No newline at end of file
diff --git a/src/client/views/InkingCanvas.tsx b/src/client/views/InkingCanvas.tsx
index 3e0d7b476..e48b45b56 100644
--- a/src/client/views/InkingCanvas.tsx
+++ b/src/client/views/InkingCanvas.tsx
@@ -10,6 +10,8 @@ import { undoBatch, UndoManager } from "../util/UndoManager";
 import { StrokeData, InkField, InkTool } from "../../new_fields/InkField";
 import { Doc } from "../../new_fields/Doc";
 import { Cast, PromiseValue, NumCast } from "../../new_fields/Types";
+import { CognitiveServices } from "../cognitive_services/CognitiveServices";
+import { ContextMenu } from "./ContextMenu";
 
 interface InkCanvasProps {
     getScreenTransform: () => Transform;
@@ -178,7 +180,7 @@ export class InkingCanvas extends React.Component<InkCanvasProps> {
     render() {
         let svgCanvasStyle = InkingControl.Instance.selectedTool !== InkTool.None ? "canSelect" : "noSelect";
         return (
-            <div className="inkingCanvas" >
+            <div className="inkingCanvas">
                 <div className={`inkingCanvas-${svgCanvasStyle}`} onPointerDown={this.onPointerDown} />
                 {this.props.children()}
                 {this.drawnPaths}
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 94a4835a1..53be0278e 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -394,6 +394,7 @@ export class MainView extends React.Component {
             <div id="add-options-content">
                 <ul id="add-options-list">
                     <li key="search"><button className="add-button round-button" title="Search" onClick={this.toggleSearch}><FontAwesomeIcon icon="search" size="sm" /></button></li>
+                    <li key="presentation"><button className="add-button round-button" title="Open Presentation View" onClick={() => PresentationView.Instance.toggle(undefined)}><FontAwesomeIcon icon="table" size="sm" /></button></li>
                     <li key="undo"><button className="add-button round-button" title="Undo" style={{ opacity: UndoManager.CanUndo() ? 1 : 0.5, transition: "0.4s ease all" }} onClick={() => UndoManager.Undo()}><FontAwesomeIcon icon="undo-alt" size="sm" /></button></li>
                     <li key="redo"><button className="add-button round-button" title="Redo" style={{ opacity: UndoManager.CanRedo() ? 1 : 0.5, transition: "0.4s ease all" }} onClick={() => UndoManager.Redo()}><FontAwesomeIcon icon="redo-alt" size="sm" /></button></li>
                     {btns.map(btn =>
@@ -444,7 +445,6 @@ export class MainView extends React.Component {
         this.isSearchVisible = !this.isSearchVisible;
     }
 
-
     render() {
         return (
             <div id="main-div">
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 703873681..3fd86e950 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -31,6 +31,7 @@ import { ScriptField } from "../../../../new_fields/ScriptField";
 import { OverlayView, OverlayElementOptions } from "../../OverlayView";
 import { ScriptBox } from "../../ScriptBox";
 import { CompileScript } from "../../../util/Scripting";
+import { CognitiveServices } from "../../../cognitive_services/CognitiveServices";
 
 
 export const panZoomSchema = createSchema({
@@ -51,6 +52,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
     private _lastY: number = 0;
     private get _pwidth() { return this.props.PanelWidth(); }
     private get _pheight() { return this.props.PanelHeight(); }
+    private inkKey = "ink";
 
     @computed get contentBounds() {
         let bounds = this.props.fitToBox && !NumCast(this.nativeWidth) ? Doc.ComputeContentBounds(DocListCast(this.props.Document.data)) : undefined;
@@ -478,6 +480,15 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
             }
         });
         ContextMenu.Instance.addItem({
+            description: "Analyze Strokes", event: async () => {
+                let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField);
+                if (!data) {
+                    return;
+                }
+                CognitiveServices.Inking.analyze(data.inkData, Doc.GetProto(this.props.Document));
+            }
+        });
+        ContextMenu.Instance.addItem({
             description: "Add freeform arrangement",
             event: () => {
                 let addOverlay = (key: "arrangeScript" | "arrangeInit", options: OverlayElementOptions, params?: Record<string, string>, requiredType?: string) => {
diff --git a/src/client/views/presentationview/PresentationView.tsx b/src/client/views/presentationview/PresentationView.tsx
index f80840f96..966ade3de 100644
--- a/src/client/views/presentationview/PresentationView.tsx
+++ b/src/client/views/presentationview/PresentationView.tsx
@@ -31,6 +31,8 @@ export interface PresViewProps {
     Documents: List<Doc>;
 }
 
+const expandedWidth = 400;
+
 @observer
 export class PresentationView extends React.Component<PresViewProps>  {
     public static Instance: PresentationView;
@@ -67,6 +69,14 @@ export class PresentationView extends React.Component<PresViewProps>  {
         PresentationView.Instance = this;
     }
 
+    toggle = (forcedValue: boolean | undefined) => {
+        if (forcedValue !== undefined) {
+            this.curPresentation.width = forcedValue ? expandedWidth : 0;
+        } else {
+            this.curPresentation.width = this.curPresentation.width === expandedWidth ? 0 : expandedWidth;
+        }
+    }
+
     //The first lifecycle function that gets called to set up the current presentation.
     async componentWillMount() {
         this.props.Documents.forEach(async (doc, index: number) => {
@@ -543,7 +553,7 @@ export class PresentationView extends React.Component<PresViewProps>  {
             this.curPresentation.data = new List([doc]);
         }
 
-        this.curPresentation.width = 400;
+        this.toggle(true);
     }
 
     //Function that sets the store of the children docs.
diff --git a/src/new_fields/FieldSymbols.ts b/src/new_fields/FieldSymbols.ts
index a436dcf2b..b5b3aa588 100644
--- a/src/new_fields/FieldSymbols.ts
+++ b/src/new_fields/FieldSymbols.ts
@@ -7,4 +7,4 @@ export const Id = Symbol("Id");
 export const OnUpdate = Symbol("OnUpdate");
 export const Parent = Symbol("Parent");
 export const Copy = Symbol("Copy");
-export const ToScriptString = Symbol("Copy");
\ No newline at end of file
+export const ToScriptString = Symbol("ToScriptString");
\ No newline at end of file
diff --git a/src/new_fields/InkField.ts b/src/new_fields/InkField.ts
index 39c6c8ce3..8f64c1c2e 100644
--- a/src/new_fields/InkField.ts
+++ b/src/new_fields/InkField.ts
@@ -19,6 +19,8 @@ export interface StrokeData {
     page: number;
 }
 
+export type InkData = Map<string, StrokeData>;
+
 const pointSchema = createSimpleSchema({
     x: true, y: true
 });
@@ -31,9 +33,9 @@ const strokeDataSchema = createSimpleSchema({
 @Deserializable("ink")
 export class InkField extends ObjectField {
     @serializable(map(object(strokeDataSchema)))
-    readonly inkData: Map<string, StrokeData>;
+    readonly inkData: InkData;
 
-    constructor(data?: Map<string, StrokeData>) {
+    constructor(data?: InkData) {
         super();
         this.inkData = data || new Map;
     }
diff --git a/src/server/index.ts b/src/server/index.ts
index 5b086a2cf..66c982adc 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -284,18 +284,15 @@ addSecureRoute(
     RouteStore.getCurrUser
 );
 
+const ServicesApiKeyMap = new Map<string, string | undefined>([
+    ["face", process.env.FACE],
+    ["vision", process.env.VISION],
+    ["handwriting", process.env.HANDWRITING]
+]);
+
 addSecureRoute(Method.GET, (user, res, req) => {
-    let requested = req.params.requestedservice;
-    switch (requested) {
-        case "face":
-            res.send(process.env.FACE);
-            break;
-        case "vision":
-            res.send(process.env.VISION);
-            break;
-        default:
-            res.send(undefined);
-    }
+    let service = req.params.requestedservice;
+    res.send(ServicesApiKeyMap.get(service));
 }, undefined, `${RouteStore.cognitiveServices}/:requestedservice`);
 
 class NodeCanvasFactory {