From cb95d4b43238538f59babcaa0b0d9dabb9b393b7 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 22:07:13 -0400 Subject: Ink recognition and presentation view toggle --- src/client/cognitive_services/CognitiveServices.ts | 86 +++++++++++++++++++++- src/client/views/InkingCanvas.tsx | 4 +- src/client/views/MainView.tsx | 2 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 11 +++ .../views/presentationview/PresentationView.tsx | 12 ++- 5 files changed, 111 insertions(+), 4 deletions(-) (limited to 'src/client') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d4085cf76..71e4276dc 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -8,10 +8,12 @@ import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { CompileScript } from "../util/Scripting"; import { ComputedField } from "../../new_fields/ScriptField"; +import { InkData } from "../../new_fields/InkField"; export enum Services { ComputerVision = "vision", - Face = "face" + Face = "face", + Handwriting = "handwriting" } export enum Confidence { @@ -132,4 +134,86 @@ export namespace CognitiveServices { } + export namespace Inking { + + export interface AzureStrokeData { + id: number; + points: string; + language?: string; + } + + export interface HandwritingUnit { + version: number; + language: string; + unit: string; + strokes: AzureStrokeData[]; + } + + export const analyze = async (inkData: InkData, target: Doc) => { + return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${Services.Handwriting}`)).then(async response => { + let apiKey = await response.text(); + if (!apiKey) { + return undefined; + } + + let xhttp = new XMLHttpRequest(); + let serverAddress = "https://api.cognitive.microsoft.com"; + let endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize"; + + let results = await new Promise((resolve, reject) => { + let result: any; + xhttp.onreadystatechange = function () { + if (this.readyState === 4) { + try { + result = JSON.parse(xhttp.responseText); + } catch (e) { + return reject(e); + } + switch (this.status) { + case 200: + return resolve(result); + case 400: + default: + return reject(result); + } + } + }; + + xhttp.open("PUT", endpoint, true); + xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); + xhttp.setRequestHeader('Content-Type', 'application/json'); + xhttp.send(JSON.stringify(toHandwritingUnit(inkData))); + }); + + let recognizedText = results.recognitionUnits.map((unit: any) => unit.recognizedText); + let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); + target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results.recognitionUnits, "Ink Analysis"); + target.handwriting = individualWords.join(" "); + }); + }; + + const toHandwritingUnit = (inkData: InkData): HandwritingUnit => { + let entries = inkData.entries(), next = entries.next(); + let strokes: AzureStrokeData[] = []; + let id = 0; + while (!next.done) { + let entry = next.value; + let data = { + id: id++, + points: entry[1].pathData.map(point => `${point.x},${point.y}`).join(","), + language: "en-US" + }; + strokes.push(data); + next = entries.next(); + } + return { + version: 1, + language: "en-US", + unit: "mm", + strokes: strokes + }; + }; + + } + } \ No newline at end of file diff --git a/src/client/views/InkingCanvas.tsx b/src/client/views/InkingCanvas.tsx index 3e0d7b476..e48b45b56 100644 --- a/src/client/views/InkingCanvas.tsx +++ b/src/client/views/InkingCanvas.tsx @@ -10,6 +10,8 @@ import { undoBatch, UndoManager } from "../util/UndoManager"; import { StrokeData, InkField, InkTool } from "../../new_fields/InkField"; import { Doc } from "../../new_fields/Doc"; import { Cast, PromiseValue, NumCast } from "../../new_fields/Types"; +import { CognitiveServices } from "../cognitive_services/CognitiveServices"; +import { ContextMenu } from "./ContextMenu"; interface InkCanvasProps { getScreenTransform: () => Transform; @@ -178,7 +180,7 @@ export class InkingCanvas extends React.Component { render() { let svgCanvasStyle = InkingControl.Instance.selectedTool !== InkTool.None ? "canSelect" : "noSelect"; return ( -
+
{this.props.children()} {this.drawnPaths} diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 94a4835a1..53be0278e 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -394,6 +394,7 @@ export class MainView extends React.Component {
    • +
    • {btns.map(btn => @@ -444,7 +445,6 @@ export class MainView extends React.Component { this.isSearchVisible = !this.isSearchVisible; } - render() { return (
    diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 703873681..3fd86e950 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -31,6 +31,7 @@ import { ScriptField } from "../../../../new_fields/ScriptField"; import { OverlayView, OverlayElementOptions } from "../../OverlayView"; import { ScriptBox } from "../../ScriptBox"; import { CompileScript } from "../../../util/Scripting"; +import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; export const panZoomSchema = createSchema({ @@ -51,6 +52,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { private _lastY: number = 0; private get _pwidth() { return this.props.PanelWidth(); } private get _pheight() { return this.props.PanelHeight(); } + private inkKey = "ink"; @computed get contentBounds() { let bounds = this.props.fitToBox && !NumCast(this.nativeWidth) ? Doc.ComputeContentBounds(DocListCast(this.props.Document.data)) : undefined; @@ -477,6 +479,15 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { }, "arrange contents"); } }); + ContextMenu.Instance.addItem({ + description: "Analyze Strokes", event: async () => { + let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField); + if (!data) { + return; + } + CognitiveServices.Inking.analyze(data.inkData, Doc.GetProto(this.props.Document)); + } + }); ContextMenu.Instance.addItem({ description: "Add freeform arrangement", event: () => { diff --git a/src/client/views/presentationview/PresentationView.tsx b/src/client/views/presentationview/PresentationView.tsx index f80840f96..966ade3de 100644 --- a/src/client/views/presentationview/PresentationView.tsx +++ b/src/client/views/presentationview/PresentationView.tsx @@ -31,6 +31,8 @@ export interface PresViewProps { Documents: List; } +const expandedWidth = 400; + @observer export class PresentationView extends React.Component { public static Instance: PresentationView; @@ -67,6 +69,14 @@ export class PresentationView extends React.Component { PresentationView.Instance = this; } + toggle = (forcedValue: boolean | undefined) => { + if (forcedValue !== undefined) { + this.curPresentation.width = forcedValue ? expandedWidth : 0; + } else { + this.curPresentation.width = this.curPresentation.width === expandedWidth ? 0 : expandedWidth; + } + } + //The first lifecycle function that gets called to set up the current presentation. async componentWillMount() { this.props.Documents.forEach(async (doc, index: number) => { @@ -543,7 +553,7 @@ export class PresentationView extends React.Component { this.curPresentation.data = new List([doc]); } - this.curPresentation.width = 400; + this.toggle(true); } //Function that sets the store of the children docs. -- cgit v1.2.3-70-g09d2