diff options
| -rw-r--r-- | src/client/cognitive_services/CognitiveServices.ts | 86 | ||||
| -rw-r--r-- | src/client/views/InkingCanvas.tsx | 4 | ||||
| -rw-r--r-- | src/client/views/MainView.tsx | 2 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx | 11 | ||||
| -rw-r--r-- | src/client/views/presentationview/PresentationView.tsx | 12 | ||||
| -rw-r--r-- | src/new_fields/FieldSymbols.ts | 2 | ||||
| -rw-r--r-- | src/new_fields/InkField.ts | 6 | ||||
| -rw-r--r-- | src/server/index.ts | 19 |
8 files changed, 124 insertions, 18 deletions
diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d4085cf76..71e4276dc 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -8,10 +8,12 @@ import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { CompileScript } from "../util/Scripting"; import { ComputedField } from "../../new_fields/ScriptField"; +import { InkData } from "../../new_fields/InkField"; export enum Services { ComputerVision = "vision", - Face = "face" + Face = "face", + Handwriting = "handwriting" } export enum Confidence { @@ -132,4 +134,86 @@ export namespace CognitiveServices { } + export namespace Inking { + + export interface AzureStrokeData { + id: number; + points: string; + language?: string; + } + + export interface HandwritingUnit { + version: number; + language: string; + unit: string; + strokes: AzureStrokeData[]; + } + + export const analyze = async (inkData: InkData, target: Doc) => { + return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${Services.Handwriting}`)).then(async response => { + let apiKey = await response.text(); + if (!apiKey) { + return undefined; + } + + let xhttp = new XMLHttpRequest(); + let serverAddress = "https://api.cognitive.microsoft.com"; + let endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize"; + + let results = await new Promise<any>((resolve, reject) => { + let result: any; + xhttp.onreadystatechange = function () { + if (this.readyState === 4) { + try { + result = JSON.parse(xhttp.responseText); + } catch (e) { + return reject(e); + } + switch (this.status) { + case 200: + return resolve(result); + case 400: + default: + return reject(result); + } + } + }; + + xhttp.open("PUT", endpoint, true); + xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); + xhttp.setRequestHeader('Content-Type', 'application/json'); + xhttp.send(JSON.stringify(toHandwritingUnit(inkData))); + }); + + let recognizedText = results.recognitionUnits.map((unit: any) => unit.recognizedText); + let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); + target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results.recognitionUnits, "Ink Analysis"); + target.handwriting = individualWords.join(" "); + }); + }; + + const toHandwritingUnit = (inkData: InkData): HandwritingUnit => { + let entries = inkData.entries(), next = entries.next(); + let strokes: AzureStrokeData[] = []; + let id = 0; + while (!next.done) { + let entry = next.value; + let data = { + id: id++, + points: entry[1].pathData.map(point => `${point.x},${point.y}`).join(","), + language: "en-US" + }; + strokes.push(data); + next = entries.next(); + } + return { + version: 1, + language: "en-US", + unit: "mm", + strokes: strokes + }; + }; + + } + }
\ No newline at end of file diff --git a/src/client/views/InkingCanvas.tsx b/src/client/views/InkingCanvas.tsx index 3e0d7b476..e48b45b56 100644 --- a/src/client/views/InkingCanvas.tsx +++ b/src/client/views/InkingCanvas.tsx @@ -10,6 +10,8 @@ import { undoBatch, UndoManager } from "../util/UndoManager"; import { StrokeData, InkField, InkTool } from "../../new_fields/InkField"; import { Doc } from "../../new_fields/Doc"; import { Cast, PromiseValue, NumCast } from "../../new_fields/Types"; +import { CognitiveServices } from "../cognitive_services/CognitiveServices"; +import { ContextMenu } from "./ContextMenu"; interface InkCanvasProps { getScreenTransform: () => Transform; @@ -178,7 +180,7 @@ export class InkingCanvas extends React.Component<InkCanvasProps> { render() { let svgCanvasStyle = InkingControl.Instance.selectedTool !== InkTool.None ? "canSelect" : "noSelect"; return ( - <div className="inkingCanvas" > + <div className="inkingCanvas"> <div className={`inkingCanvas-${svgCanvasStyle}`} onPointerDown={this.onPointerDown} /> {this.props.children()} {this.drawnPaths} diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 94a4835a1..53be0278e 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -394,6 +394,7 @@ export class MainView extends React.Component { <div id="add-options-content"> <ul id="add-options-list"> <li key="search"><button className="add-button round-button" title="Search" onClick={this.toggleSearch}><FontAwesomeIcon icon="search" size="sm" /></button></li> + <li key="presentation"><button className="add-button round-button" title="Open Presentation View" onClick={() => PresentationView.Instance.toggle(undefined)}><FontAwesomeIcon icon="table" size="sm" /></button></li> <li key="undo"><button className="add-button round-button" title="Undo" style={{ opacity: UndoManager.CanUndo() ? 1 : 0.5, transition: "0.4s ease all" }} onClick={() => UndoManager.Undo()}><FontAwesomeIcon icon="undo-alt" size="sm" /></button></li> <li key="redo"><button className="add-button round-button" title="Redo" style={{ opacity: UndoManager.CanRedo() ? 1 : 0.5, transition: "0.4s ease all" }} onClick={() => UndoManager.Redo()}><FontAwesomeIcon icon="redo-alt" size="sm" /></button></li> {btns.map(btn => @@ -444,7 +445,6 @@ export class MainView extends React.Component { this.isSearchVisible = !this.isSearchVisible; } - render() { return ( <div id="main-div"> diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 703873681..3fd86e950 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -31,6 +31,7 @@ import { ScriptField } from "../../../../new_fields/ScriptField"; import { OverlayView, OverlayElementOptions } from "../../OverlayView"; import { ScriptBox } from "../../ScriptBox"; import { CompileScript } from "../../../util/Scripting"; +import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; export const panZoomSchema = createSchema({ @@ -51,6 +52,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { private _lastY: number = 0; private get _pwidth() { return this.props.PanelWidth(); } private get _pheight() { return this.props.PanelHeight(); } + private inkKey = "ink"; @computed get contentBounds() { let bounds = this.props.fitToBox && !NumCast(this.nativeWidth) ? Doc.ComputeContentBounds(DocListCast(this.props.Document.data)) : undefined; @@ -478,6 +480,15 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { } }); ContextMenu.Instance.addItem({ + description: "Analyze Strokes", event: async () => { + let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField); + if (!data) { + return; + } + CognitiveServices.Inking.analyze(data.inkData, Doc.GetProto(this.props.Document)); + } + }); + ContextMenu.Instance.addItem({ description: "Add freeform arrangement", event: () => { let addOverlay = (key: "arrangeScript" | "arrangeInit", options: OverlayElementOptions, params?: Record<string, string>, requiredType?: string) => { diff --git a/src/client/views/presentationview/PresentationView.tsx b/src/client/views/presentationview/PresentationView.tsx index f80840f96..966ade3de 100644 --- a/src/client/views/presentationview/PresentationView.tsx +++ b/src/client/views/presentationview/PresentationView.tsx @@ -31,6 +31,8 @@ export interface PresViewProps { Documents: List<Doc>; } +const expandedWidth = 400; + @observer export class PresentationView extends React.Component<PresViewProps> { public static Instance: PresentationView; @@ -67,6 +69,14 @@ export class PresentationView extends React.Component<PresViewProps> { PresentationView.Instance = this; } + toggle = (forcedValue: boolean | undefined) => { + if (forcedValue !== undefined) { + this.curPresentation.width = forcedValue ? expandedWidth : 0; + } else { + this.curPresentation.width = this.curPresentation.width === expandedWidth ? 0 : expandedWidth; + } + } + //The first lifecycle function that gets called to set up the current presentation. async componentWillMount() { this.props.Documents.forEach(async (doc, index: number) => { @@ -543,7 +553,7 @@ export class PresentationView extends React.Component<PresViewProps> { this.curPresentation.data = new List([doc]); } - this.curPresentation.width = 400; + this.toggle(true); } //Function that sets the store of the children docs. diff --git a/src/new_fields/FieldSymbols.ts b/src/new_fields/FieldSymbols.ts index a436dcf2b..b5b3aa588 100644 --- a/src/new_fields/FieldSymbols.ts +++ b/src/new_fields/FieldSymbols.ts @@ -7,4 +7,4 @@ export const Id = Symbol("Id"); export const OnUpdate = Symbol("OnUpdate"); export const Parent = Symbol("Parent"); export const Copy = Symbol("Copy"); -export const ToScriptString = Symbol("Copy");
\ No newline at end of file +export const ToScriptString = Symbol("ToScriptString");
\ No newline at end of file diff --git a/src/new_fields/InkField.ts b/src/new_fields/InkField.ts index 39c6c8ce3..8f64c1c2e 100644 --- a/src/new_fields/InkField.ts +++ b/src/new_fields/InkField.ts @@ -19,6 +19,8 @@ export interface StrokeData { page: number; } +export type InkData = Map<string, StrokeData>; + const pointSchema = createSimpleSchema({ x: true, y: true }); @@ -31,9 +33,9 @@ const strokeDataSchema = createSimpleSchema({ @Deserializable("ink") export class InkField extends ObjectField { @serializable(map(object(strokeDataSchema))) - readonly inkData: Map<string, StrokeData>; + readonly inkData: InkData; - constructor(data?: Map<string, StrokeData>) { + constructor(data?: InkData) { super(); this.inkData = data || new Map; } diff --git a/src/server/index.ts b/src/server/index.ts index 5b086a2cf..66c982adc 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -284,18 +284,15 @@ addSecureRoute( RouteStore.getCurrUser ); +const ServicesApiKeyMap = new Map<string, string | undefined>([ + ["face", process.env.FACE], + ["vision", process.env.VISION], + ["handwriting", process.env.HANDWRITING] +]); + addSecureRoute(Method.GET, (user, res, req) => { - let requested = req.params.requestedservice; - switch (requested) { - case "face": - res.send(process.env.FACE); - break; - case "vision": - res.send(process.env.VISION); - break; - default: - res.send(undefined); - } + let service = req.params.requestedservice; + res.send(ServicesApiKeyMap.get(service)); }, undefined, `${RouteStore.cognitiveServices}/:requestedservice`); class NodeCanvasFactory { |