From 2899cbc887398688b82dd284ee482cdb136c6452 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 22 Jul 2019 00:38:50 -0400 Subject: json conversion tweaks --- src/new_fields/InkField.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/new_fields/InkField.ts') diff --git a/src/new_fields/InkField.ts b/src/new_fields/InkField.ts index 4e3b7abe0..39c6c8ce3 100644 --- a/src/new_fields/InkField.ts +++ b/src/new_fields/InkField.ts @@ -2,7 +2,7 @@ import { Deserializable } from "../client/util/SerializationHelper"; import { serializable, custom, createSimpleSchema, list, object, map } from "serializr"; import { ObjectField } from "./ObjectField"; import { Copy, ToScriptString } from "./FieldSymbols"; -import { deepCopy } from "../Utils"; +import { DeepCopy } from "../Utils"; export enum InkTool { None, @@ -39,7 +39,7 @@ export class InkField extends ObjectField { } [Copy]() { - return new InkField(deepCopy(this.inkData)); + return new InkField(DeepCopy(this.inkData)); } [ToScriptString]() { -- cgit v1.2.3-70-g09d2 From cb95d4b43238538f59babcaa0b0d9dabb9b393b7 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 22:07:13 -0400 Subject: Ink recognition and presentation view toggle --- src/client/cognitive_services/CognitiveServices.ts | 86 +++++++++++++++++++++- src/client/views/InkingCanvas.tsx | 4 +- src/client/views/MainView.tsx | 2 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 11 +++ .../views/presentationview/PresentationView.tsx | 12 ++- src/new_fields/FieldSymbols.ts | 2 +- src/new_fields/InkField.ts | 6 +- src/server/index.ts | 19 ++--- 8 files changed, 124 insertions(+), 18 deletions(-) (limited to 'src/new_fields/InkField.ts') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d4085cf76..71e4276dc 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -8,10 +8,12 @@ import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { CompileScript } from "../util/Scripting"; import { ComputedField } from "../../new_fields/ScriptField"; +import { InkData } from "../../new_fields/InkField"; export enum Services { ComputerVision = "vision", - Face = "face" + Face = "face", + Handwriting = "handwriting" } export enum Confidence { @@ -132,4 +134,86 @@ export namespace CognitiveServices { } + export namespace Inking { + + export interface AzureStrokeData { + id: number; + points: string; + language?: string; + } + + export interface HandwritingUnit { + version: number; + language: string; + unit: string; + strokes: AzureStrokeData[]; + } + + export const analyze = async (inkData: InkData, target: Doc) => { + return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${Services.Handwriting}`)).then(async response => { + let apiKey = await response.text(); + if (!apiKey) { + return undefined; + } + + let xhttp = new XMLHttpRequest(); + let serverAddress = "https://api.cognitive.microsoft.com"; + let endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize"; + + let results = await new Promise((resolve, reject) => { + let result: any; + xhttp.onreadystatechange = function () { + if (this.readyState === 4) { + try { + result = JSON.parse(xhttp.responseText); + } catch (e) { + return reject(e); + } + switch (this.status) { + case 200: + return resolve(result); + case 400: + default: + return reject(result); + } + } + }; + + xhttp.open("PUT", endpoint, true); + xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); + xhttp.setRequestHeader('Content-Type', 'application/json'); + xhttp.send(JSON.stringify(toHandwritingUnit(inkData))); + }); + + let recognizedText = results.recognitionUnits.map((unit: any) => unit.recognizedText); + let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); + target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results.recognitionUnits, "Ink Analysis"); + target.handwriting = individualWords.join(" "); + }); + }; + + const toHandwritingUnit = (inkData: InkData): HandwritingUnit => { + let entries = inkData.entries(), next = entries.next(); + let strokes: AzureStrokeData[] = []; + let id = 0; + while (!next.done) { + let entry = next.value; + let data = { + id: id++, + points: entry[1].pathData.map(point => `${point.x},${point.y}`).join(","), + language: "en-US" + }; + strokes.push(data); + next = entries.next(); + } + return { + version: 1, + language: "en-US", + unit: "mm", + strokes: strokes + }; + }; + + } + } \ No newline at end of file diff --git a/src/client/views/InkingCanvas.tsx b/src/client/views/InkingCanvas.tsx index 3e0d7b476..e48b45b56 100644 --- a/src/client/views/InkingCanvas.tsx +++ b/src/client/views/InkingCanvas.tsx @@ -10,6 +10,8 @@ import { undoBatch, UndoManager } from "../util/UndoManager"; import { StrokeData, InkField, InkTool } from "../../new_fields/InkField"; import { Doc } from "../../new_fields/Doc"; import { Cast, PromiseValue, NumCast } from "../../new_fields/Types"; +import { CognitiveServices } from "../cognitive_services/CognitiveServices"; +import { ContextMenu } from "./ContextMenu"; interface InkCanvasProps { getScreenTransform: () => Transform; @@ -178,7 +180,7 @@ export class InkingCanvas extends React.Component { render() { let svgCanvasStyle = InkingControl.Instance.selectedTool !== InkTool.None ? "canSelect" : "noSelect"; return ( -
+
{this.props.children()} {this.drawnPaths} diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 94a4835a1..53be0278e 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -394,6 +394,7 @@ export class MainView extends React.Component {
    • +
    • {btns.map(btn => @@ -444,7 +445,6 @@ export class MainView extends React.Component { this.isSearchVisible = !this.isSearchVisible; } - render() { return (
    diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 703873681..3fd86e950 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -31,6 +31,7 @@ import { ScriptField } from "../../../../new_fields/ScriptField"; import { OverlayView, OverlayElementOptions } from "../../OverlayView"; import { ScriptBox } from "../../ScriptBox"; import { CompileScript } from "../../../util/Scripting"; +import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; export const panZoomSchema = createSchema({ @@ -51,6 +52,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { private _lastY: number = 0; private get _pwidth() { return this.props.PanelWidth(); } private get _pheight() { return this.props.PanelHeight(); } + private inkKey = "ink"; @computed get contentBounds() { let bounds = this.props.fitToBox && !NumCast(this.nativeWidth) ? Doc.ComputeContentBounds(DocListCast(this.props.Document.data)) : undefined; @@ -477,6 +479,15 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { }, "arrange contents"); } }); + ContextMenu.Instance.addItem({ + description: "Analyze Strokes", event: async () => { + let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField); + if (!data) { + return; + } + CognitiveServices.Inking.analyze(data.inkData, Doc.GetProto(this.props.Document)); + } + }); ContextMenu.Instance.addItem({ description: "Add freeform arrangement", event: () => { diff --git a/src/client/views/presentationview/PresentationView.tsx b/src/client/views/presentationview/PresentationView.tsx index f80840f96..966ade3de 100644 --- a/src/client/views/presentationview/PresentationView.tsx +++ b/src/client/views/presentationview/PresentationView.tsx @@ -31,6 +31,8 @@ export interface PresViewProps { Documents: List; } +const expandedWidth = 400; + @observer export class PresentationView extends React.Component { public static Instance: PresentationView; @@ -67,6 +69,14 @@ export class PresentationView extends React.Component { PresentationView.Instance = this; } + toggle = (forcedValue: boolean | undefined) => { + if (forcedValue !== undefined) { + this.curPresentation.width = forcedValue ? expandedWidth : 0; + } else { + this.curPresentation.width = this.curPresentation.width === expandedWidth ? 0 : expandedWidth; + } + } + //The first lifecycle function that gets called to set up the current presentation. async componentWillMount() { this.props.Documents.forEach(async (doc, index: number) => { @@ -543,7 +553,7 @@ export class PresentationView extends React.Component { this.curPresentation.data = new List([doc]); } - this.curPresentation.width = 400; + this.toggle(true); } //Function that sets the store of the children docs. diff --git a/src/new_fields/FieldSymbols.ts b/src/new_fields/FieldSymbols.ts index a436dcf2b..b5b3aa588 100644 --- a/src/new_fields/FieldSymbols.ts +++ b/src/new_fields/FieldSymbols.ts @@ -7,4 +7,4 @@ export const Id = Symbol("Id"); export const OnUpdate = Symbol("OnUpdate"); export const Parent = Symbol("Parent"); export const Copy = Symbol("Copy"); -export const ToScriptString = Symbol("Copy"); \ No newline at end of file +export const ToScriptString = Symbol("ToScriptString"); \ No newline at end of file diff --git a/src/new_fields/InkField.ts b/src/new_fields/InkField.ts index 39c6c8ce3..8f64c1c2e 100644 --- a/src/new_fields/InkField.ts +++ b/src/new_fields/InkField.ts @@ -19,6 +19,8 @@ export interface StrokeData { page: number; } +export type InkData = Map; + const pointSchema = createSimpleSchema({ x: true, y: true }); @@ -31,9 +33,9 @@ const strokeDataSchema = createSimpleSchema({ @Deserializable("ink") export class InkField extends ObjectField { @serializable(map(object(strokeDataSchema))) - readonly inkData: Map; + readonly inkData: InkData; - constructor(data?: Map) { + constructor(data?: InkData) { super(); this.inkData = data || new Map; } diff --git a/src/server/index.ts b/src/server/index.ts index 5b086a2cf..66c982adc 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -284,18 +284,15 @@ addSecureRoute( RouteStore.getCurrUser ); +const ServicesApiKeyMap = new Map([ + ["face", process.env.FACE], + ["vision", process.env.VISION], + ["handwriting", process.env.HANDWRITING] +]); + addSecureRoute(Method.GET, (user, res, req) => { - let requested = req.params.requestedservice; - switch (requested) { - case "face": - res.send(process.env.FACE); - break; - case "vision": - res.send(process.env.VISION); - break; - default: - res.send(undefined); - } + let service = req.params.requestedservice; + res.send(ServicesApiKeyMap.get(service)); }, undefined, `${RouteStore.cognitiveServices}/:requestedservice`); class NodeCanvasFactory { -- cgit v1.2.3-70-g09d2