From cc712e50e41bbe720299b5ab5f2d316c4d4c218a Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 00:54:03 -0400 Subject: cognitive services, not broken so far --- src/client/cognitive_services/CognitiveServices.ts | 134 +++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 src/client/cognitive_services/CognitiveServices.ts (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts new file mode 100644 index 000000000..1a002e025 --- /dev/null +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -0,0 +1,134 @@ +import * as request from "request-promise"; +import { Doc, Field } from "../../new_fields/Doc"; +import { Cast } from "../../new_fields/Types"; +import { ImageField } from "../../new_fields/URLField"; +import { values } from "mobx"; +import { List } from "../../new_fields/List"; +import { Docs } from "../documents/Documents"; +import { Result } from "../northstar/model/idea/idea"; +import { RouteStore } from "../../server/RouteStore"; +import { Utils } from "../../Utils"; + +export enum Services { + ComputerVision, + Face +} + +export enum Confidence { + Yikes = 0.0, + Unlikely = 0.2, + Poor = 0.4, + Fair = 0.6, + Good = 0.8, + Excellent = 0.95 +} + +export type Tag = { name: string, confidence: number }; +export type Rectangle = { top: number, left: number, width: number, height: number }; +export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle }; +export type Converter = (results: any) => Field; + +/** + * A file that handles all interactions with Microsoft Azure's Cognitive + * Services APIs. These machine learning endpoints allow basic data analytics for + * various media types. + */ +export namespace CognitiveServices { + + export namespace Image { + + export const analyze = async (imageUrl: string, service: Services) => { + return fetch(Utils.prepend(RouteStore.cognitiveServices + service)).then(async response => { + let apiKey = await response.text(); + if (apiKey) { + return; + } + let uriBase; + let parameters; + + switch (service) { + case Services.Face: + uriBase = 'face/v1.0/detect'; + parameters = { + 'returnFaceId': 'true', + 'returnFaceLandmarks': 'false', + 'returnFaceAttributes': 'age,gender,headPose,smile,facialHair,glasses,' + + 'emotion,hair,makeup,occlusion,accessories,blur,exposure,noise' + }; + break; + case Services.ComputerVision: + uriBase = 'vision/v2.0/analyze'; + parameters = { + 'visualFeatures': 'Categories,Description,Color,Objects,Tags,Adult', + 'details': 'Celebrities,Landmarks', + 'language': 'en', + }; + break; + } + + const options = { + uri: 'https://eastus.api.cognitive.microsoft.com/' + uriBase, + qs: parameters, + body: `{"url": "${imageUrl}"}`, + headers: { + 'Content-Type': 'application/json', + 'Ocp-Apim-Subscription-Key': apiKey + } + }; + + let results: any; + try { + results = await request.post(options).then(response => JSON.parse(response)); + } catch (e) { + results = undefined; + } + return results; + }); + }; + + const analyzeDocument = async (target: Doc, service: Services, converter: Converter, storageKey: string) => { + let imageData = Cast(target.data, ImageField); + if (!imageData || await Cast(target[storageKey], Doc)) { + return; + } + let toStore: any; + let results = await analyze(imageData.url.href, service); + if (!results) { + toStore = "Cognitive Services could not process the given image URL."; + } else { + if (!results.length) { + toStore = converter(results); + } else { + toStore = results.length > 0 ? converter(results) : "Empty list returned."; + } + } + target[storageKey] = toStore; + }; + + export const generateMetadata = async (target: Doc, threshold: Confidence = Confidence.Excellent) => { + let converter = (results: any) => { + let tagDoc = new Doc; + results.tags.map((tag: Tag) => { + if (tag.confidence >= +threshold) { + tagDoc[tag.name] = tag.confidence; + } + }); + tagDoc.title = "Generated Tags"; + tagDoc.confidenceThreshold = threshold.toString(); + return tagDoc; + }; + analyzeDocument(target, Services.ComputerVision, converter, "generatedTags"); + }; + + export const extractFaces = async (target: Doc) => { + let converter = (results: any) => { + let faceDocs = new List(); + results.map((face: Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); + return faceDocs; + }; + analyzeDocument(target, Services.Face, converter, "faces"); + }; + + } + +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 6248a0a4afd154b84ebea1202a48bec0fd4b87de Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 01:10:09 -0400 Subject: fixed server compilation bug --- src/client/cognitive_services/CognitiveServices.ts | 4 +--- src/server/RouteStore.ts | 2 +- src/server/index.ts | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 1a002e025..2a3fe1d26 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -2,10 +2,8 @@ import * as request from "request-promise"; import { Doc, Field } from "../../new_fields/Doc"; import { Cast } from "../../new_fields/Types"; import { ImageField } from "../../new_fields/URLField"; -import { values } from "mobx"; import { List } from "../../new_fields/List"; import { Docs } from "../documents/Documents"; -import { Result } from "../northstar/model/idea/idea"; import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; @@ -38,7 +36,7 @@ export namespace CognitiveServices { export namespace Image { export const analyze = async (imageUrl: string, service: Services) => { - return fetch(Utils.prepend(RouteStore.cognitiveServices + service)).then(async response => { + return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { let apiKey = await response.text(); if (apiKey) { return; diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 96bd62e23..e30015e39 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -30,6 +30,6 @@ export enum RouteStore { reset = "/reset/:token", // APIS - cognitiveServices = "/cognitiveservices/" + cognitiveServices = "/cognitiveservices" } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 6b4e59bfc..d808e8158 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -284,6 +284,21 @@ addSecureRoute( RouteStore.getCurrUser ); +addSecureRoute(Method.GET, (user, res, req) => { + let requested = req.params.requestedservice; + console.log(requested); + switch (requested) { + case "Face": + res.send(process.env.FACE); + break; + case "ComputerVision": + res.send(process.env.VISION); + break; + default: + res.send(undefined); + } +}, undefined, `${RouteStore.cognitiveServices}/:requestedservice`); + class NodeCanvasFactory { create = (width: number, height: number) => { var canvas = createCanvas(width, height); -- cgit v1.2.3-70-g09d2 From 010c7978ce19392b4f02bdea29e734ac2abac01f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 01:33:14 -0400 Subject: further non-breaking cognitive services --- src/client/cognitive_services/CognitiveServices.ts | 8 ++++---- src/client/views/nodes/ImageBox.tsx | 18 +++++++++++++----- src/server/index.ts | 5 ++--- 3 files changed, 19 insertions(+), 12 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 2a3fe1d26..d525455cb 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -8,8 +8,8 @@ import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; export enum Services { - ComputerVision, - Face + ComputerVision = "vision", + Face = "face" } export enum Confidence { @@ -38,8 +38,8 @@ export namespace CognitiveServices { export const analyze = async (imageUrl: string, service: Services) => { return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { let apiKey = await response.text(); - if (apiKey) { - return; + if (!apiKey) { + return undefined; } let uriBase; let parameters; diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index c3ee1e823..f88150d66 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -25,6 +25,7 @@ import { Docs } from '../../documents/Documents'; import { DocServer } from '../../DocServer'; import { Font } from '@react-pdf/renderer'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { CognitiveServices } from '../../cognitive_services/CognitiveServices'; var requestImageSize = require('../../util/request-image-size'); var path = require('path'); const { Howl, Howler } = require('howler'); @@ -195,10 +196,10 @@ export class ImageBox extends DocComponent(ImageD let field = Cast(this.Document[this.props.fieldKey], ImageField); if (field) { let url = field.url.href; - let subitems: ContextMenuProps[] = []; - subitems.push({ description: "Copy path", event: () => Utils.CopyText(url), icon: "expand-arrows-alt" }); - subitems.push({ description: "Record 1sec audio", event: this.recordAudioAnnotation, icon: "expand-arrows-alt" }); - subitems.push({ + let funcs: ContextMenuProps[] = []; + funcs.push({ description: "Copy path", event: () => Utils.CopyText(url), icon: "expand-arrows-alt" }); + funcs.push({ description: "Record 1sec audio", event: this.recordAudioAnnotation, icon: "expand-arrows-alt" }); + funcs.push({ description: "Rotate", event: action(() => { let proto = Doc.GetProto(this.props.Document); let nw = this.props.Document.nativeWidth; @@ -212,7 +213,14 @@ export class ImageBox extends DocComponent(ImageD this.props.Document.height = w; }), icon: "expand-arrows-alt" }); - ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: subitems }); + + let modes: ContextMenuProps[] = []; + let dataDoc = Doc.GetProto(this.Document); + modes.push({ description: "Generate Tags", event: () => CognitiveServices.Image.generateMetadata(dataDoc), icon: "tag" }); + modes.push({ description: "Find Faces", event: () => CognitiveServices.Image.extractFaces(dataDoc), icon: "camera" }); + + ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: funcs }); + ContextMenu.Instance.addItem({ description: "Analyze...", subitems: modes }); } } diff --git a/src/server/index.ts b/src/server/index.ts index d808e8158..5b086a2cf 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -286,12 +286,11 @@ addSecureRoute( addSecureRoute(Method.GET, (user, res, req) => { let requested = req.params.requestedservice; - console.log(requested); switch (requested) { - case "Face": + case "face": res.send(process.env.FACE); break; - case "ComputerVision": + case "vision": res.send(process.env.VISION); break; default: -- cgit v1.2.3-70-g09d2 From 767901272eeea30137330e710cc8cf70fd34f1de Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 03:13:08 -0400 Subject: autotag based on confidence, allow concealment of computed fields that resolve to undefined with namespace constant ComputedField.undefined --- src/client/cognitive_services/CognitiveServices.ts | 11 ++++-- src/client/views/nodes/FaceRectangle.tsx | 29 ++++++++++++++ src/client/views/nodes/FaceRectangles.tsx | 46 ++++++++++++++++++++++ src/client/views/nodes/ImageBox.tsx | 2 + src/client/views/nodes/KeyValueBox.tsx | 2 +- src/new_fields/ScriptField.ts | 2 + 6 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 src/client/views/nodes/FaceRectangle.tsx create mode 100644 src/client/views/nodes/FaceRectangles.tsx (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d525455cb..04a21c837 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -6,6 +6,8 @@ import { List } from "../../new_fields/List"; import { Docs } from "../documents/Documents"; import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; +import { CompileScript } from "../util/Scripting"; +import { ComputedField } from "../../new_fields/ScriptField"; export enum Services { ComputerVision = "vision", @@ -107,12 +109,13 @@ export namespace CognitiveServices { let converter = (results: any) => { let tagDoc = new Doc; results.tags.map((tag: Tag) => { - if (tag.confidence >= +threshold) { - tagDoc[tag.name] = tag.confidence; - } + let sanitized = tag.name.replace(" ", "_"); + let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : ${ComputedField.undefined}`; + let computed = CompileScript(script, { params: { this: "Doc" } }); + computed.compiled && (tagDoc[sanitized] = new ComputedField(computed)); }); tagDoc.title = "Generated Tags"; - tagDoc.confidenceThreshold = threshold.toString(); + tagDoc.confidence = threshold; return tagDoc; }; analyzeDocument(target, Services.ComputerVision, converter, "generatedTags"); diff --git a/src/client/views/nodes/FaceRectangle.tsx b/src/client/views/nodes/FaceRectangle.tsx new file mode 100644 index 000000000..887efc0d5 --- /dev/null +++ b/src/client/views/nodes/FaceRectangle.tsx @@ -0,0 +1,29 @@ +import React = require("react"); +import { observer } from "mobx-react"; +import { observable, runInAction } from "mobx"; +import { RectangleTemplate } from "./FaceRectangles"; + +@observer +export default class FaceRectangle extends React.Component<{ rectangle: RectangleTemplate }> { + @observable private opacity = 0; + + componentDidMount() { + setTimeout(() => runInAction(() => this.opacity = 1), 500); + } + + render() { + let rectangle = this.props.rectangle; + return ( +
+ ); + } + +} \ No newline at end of file diff --git a/src/client/views/nodes/FaceRectangles.tsx b/src/client/views/nodes/FaceRectangles.tsx new file mode 100644 index 000000000..3570531b2 --- /dev/null +++ b/src/client/views/nodes/FaceRectangles.tsx @@ -0,0 +1,46 @@ +import React = require("react"); +import { Doc, DocListCast } from "../../../new_fields/Doc"; +import { Cast, NumCast } from "../../../new_fields/Types"; +import { observer } from "mobx-react"; +import { Id } from "../../../new_fields/FieldSymbols"; +import FaceRectangle from "./FaceRectangle"; + +interface FaceRectanglesProps { + document: Doc; + color: string; + backgroundColor: string; +} + +export interface RectangleTemplate { + id: string; + style: Partial; +} + +@observer +export default class FaceRectangles extends React.Component { + + render() { + let faces = DocListCast(Doc.GetProto(this.props.document).faces); + let templates: RectangleTemplate[] = faces.map(faceDoc => { + let rectangle = Cast(faceDoc.faceRectangle, Doc) as Doc; + let style = { + top: NumCast(rectangle.top), + left: NumCast(rectangle.left), + width: NumCast(rectangle.width), + height: NumCast(rectangle.height), + backgroundColor: `${this.props.backgroundColor}33`, + border: `solid 2px ${this.props.color}`, + } as React.CSSProperties; + return { + id: rectangle[Id], + style: style + }; + }); + return ( +
+ {templates.map(rectangle => )} +
+ ); + } + +} \ No newline at end of file diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index f88150d66..0f60bd0fb 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -26,6 +26,7 @@ import { DocServer } from '../../DocServer'; import { Font } from '@react-pdf/renderer'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { CognitiveServices } from '../../cognitive_services/CognitiveServices'; +import FaceRectangles from './FaceRectangles'; var requestImageSize = require('../../util/request-image-size'); var path = require('path'); const { Howl, Howler } = require('howler'); @@ -379,6 +380,7 @@ export class ImageBox extends DocComponent(ImageD style={{ color: [DocListCast(this.extensionDoc.audioAnnotations).length ? "blue" : "gray", "green", "red"][this._audioState] }} icon={faFileAudio} size="sm" />
{/* {this.lightbox(paths)} */} + ); } } \ No newline at end of file diff --git a/src/client/views/nodes/KeyValueBox.tsx b/src/client/views/nodes/KeyValueBox.tsx index c9dd9a64e..9fc0f2080 100644 --- a/src/client/views/nodes/KeyValueBox.tsx +++ b/src/client/views/nodes/KeyValueBox.tsx @@ -114,7 +114,7 @@ export class KeyValueBox extends React.Component { let protos = Doc.GetAllPrototypes(doc); for (const proto of protos) { Object.keys(proto).forEach(key => { - if (!(key in ids)) { + if (!(key in ids) && realDoc[key] !== ComputedField.undefined) { ids[key] = key; } }); diff --git a/src/new_fields/ScriptField.ts b/src/new_fields/ScriptField.ts index e8a1ea28a..e5ec34f57 100644 --- a/src/new_fields/ScriptField.ts +++ b/src/new_fields/ScriptField.ts @@ -107,6 +107,8 @@ export namespace ComputedField { useComputed = true; } + export const undefined = "__undefined"; + export function WithoutComputed(fn: () => T) { DisableComputedFields(); try { -- cgit v1.2.3-70-g09d2 From d880e4b2fcb4e7bab3ee25d63209b173efcf37c0 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 03:51:15 -0400 Subject: escaped name for computed field undefined --- src/client/cognitive_services/CognitiveServices.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 04a21c837..d4085cf76 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -110,7 +110,7 @@ export namespace CognitiveServices { let tagDoc = new Doc; results.tags.map((tag: Tag) => { let sanitized = tag.name.replace(" ", "_"); - let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : ${ComputedField.undefined}`; + let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`; let computed = CompileScript(script, { params: { this: "Doc" } }); computed.compiled && (tagDoc[sanitized] = new ComputedField(computed)); }); -- cgit v1.2.3-70-g09d2 From cb95d4b43238538f59babcaa0b0d9dabb9b393b7 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 22:07:13 -0400 Subject: Ink recognition and presentation view toggle --- src/client/cognitive_services/CognitiveServices.ts | 86 +++++++++++++++++++++- src/client/views/InkingCanvas.tsx | 4 +- src/client/views/MainView.tsx | 2 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 11 +++ .../views/presentationview/PresentationView.tsx | 12 ++- src/new_fields/FieldSymbols.ts | 2 +- src/new_fields/InkField.ts | 6 +- src/server/index.ts | 19 ++--- 8 files changed, 124 insertions(+), 18 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d4085cf76..71e4276dc 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -8,10 +8,12 @@ import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { CompileScript } from "../util/Scripting"; import { ComputedField } from "../../new_fields/ScriptField"; +import { InkData } from "../../new_fields/InkField"; export enum Services { ComputerVision = "vision", - Face = "face" + Face = "face", + Handwriting = "handwriting" } export enum Confidence { @@ -132,4 +134,86 @@ export namespace CognitiveServices { } + export namespace Inking { + + export interface AzureStrokeData { + id: number; + points: string; + language?: string; + } + + export interface HandwritingUnit { + version: number; + language: string; + unit: string; + strokes: AzureStrokeData[]; + } + + export const analyze = async (inkData: InkData, target: Doc) => { + return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${Services.Handwriting}`)).then(async response => { + let apiKey = await response.text(); + if (!apiKey) { + return undefined; + } + + let xhttp = new XMLHttpRequest(); + let serverAddress = "https://api.cognitive.microsoft.com"; + let endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize"; + + let results = await new Promise((resolve, reject) => { + let result: any; + xhttp.onreadystatechange = function () { + if (this.readyState === 4) { + try { + result = JSON.parse(xhttp.responseText); + } catch (e) { + return reject(e); + } + switch (this.status) { + case 200: + return resolve(result); + case 400: + default: + return reject(result); + } + } + }; + + xhttp.open("PUT", endpoint, true); + xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); + xhttp.setRequestHeader('Content-Type', 'application/json'); + xhttp.send(JSON.stringify(toHandwritingUnit(inkData))); + }); + + let recognizedText = results.recognitionUnits.map((unit: any) => unit.recognizedText); + let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); + target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results.recognitionUnits, "Ink Analysis"); + target.handwriting = individualWords.join(" "); + }); + }; + + const toHandwritingUnit = (inkData: InkData): HandwritingUnit => { + let entries = inkData.entries(), next = entries.next(); + let strokes: AzureStrokeData[] = []; + let id = 0; + while (!next.done) { + let entry = next.value; + let data = { + id: id++, + points: entry[1].pathData.map(point => `${point.x},${point.y}`).join(","), + language: "en-US" + }; + strokes.push(data); + next = entries.next(); + } + return { + version: 1, + language: "en-US", + unit: "mm", + strokes: strokes + }; + }; + + } + } \ No newline at end of file diff --git a/src/client/views/InkingCanvas.tsx b/src/client/views/InkingCanvas.tsx index 3e0d7b476..e48b45b56 100644 --- a/src/client/views/InkingCanvas.tsx +++ b/src/client/views/InkingCanvas.tsx @@ -10,6 +10,8 @@ import { undoBatch, UndoManager } from "../util/UndoManager"; import { StrokeData, InkField, InkTool } from "../../new_fields/InkField"; import { Doc } from "../../new_fields/Doc"; import { Cast, PromiseValue, NumCast } from "../../new_fields/Types"; +import { CognitiveServices } from "../cognitive_services/CognitiveServices"; +import { ContextMenu } from "./ContextMenu"; interface InkCanvasProps { getScreenTransform: () => Transform; @@ -178,7 +180,7 @@ export class InkingCanvas extends React.Component { render() { let svgCanvasStyle = InkingControl.Instance.selectedTool !== InkTool.None ? "canSelect" : "noSelect"; return ( -
+
{this.props.children()} {this.drawnPaths} diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 94a4835a1..53be0278e 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -394,6 +394,7 @@ export class MainView extends React.Component {
    • +
    • {btns.map(btn => @@ -444,7 +445,6 @@ export class MainView extends React.Component { this.isSearchVisible = !this.isSearchVisible; } - render() { return (
    diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 703873681..3fd86e950 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -31,6 +31,7 @@ import { ScriptField } from "../../../../new_fields/ScriptField"; import { OverlayView, OverlayElementOptions } from "../../OverlayView"; import { ScriptBox } from "../../ScriptBox"; import { CompileScript } from "../../../util/Scripting"; +import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; export const panZoomSchema = createSchema({ @@ -51,6 +52,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { private _lastY: number = 0; private get _pwidth() { return this.props.PanelWidth(); } private get _pheight() { return this.props.PanelHeight(); } + private inkKey = "ink"; @computed get contentBounds() { let bounds = this.props.fitToBox && !NumCast(this.nativeWidth) ? Doc.ComputeContentBounds(DocListCast(this.props.Document.data)) : undefined; @@ -477,6 +479,15 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { }, "arrange contents"); } }); + ContextMenu.Instance.addItem({ + description: "Analyze Strokes", event: async () => { + let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField); + if (!data) { + return; + } + CognitiveServices.Inking.analyze(data.inkData, Doc.GetProto(this.props.Document)); + } + }); ContextMenu.Instance.addItem({ description: "Add freeform arrangement", event: () => { diff --git a/src/client/views/presentationview/PresentationView.tsx b/src/client/views/presentationview/PresentationView.tsx index f80840f96..966ade3de 100644 --- a/src/client/views/presentationview/PresentationView.tsx +++ b/src/client/views/presentationview/PresentationView.tsx @@ -31,6 +31,8 @@ export interface PresViewProps { Documents: List; } +const expandedWidth = 400; + @observer export class PresentationView extends React.Component { public static Instance: PresentationView; @@ -67,6 +69,14 @@ export class PresentationView extends React.Component { PresentationView.Instance = this; } + toggle = (forcedValue: boolean | undefined) => { + if (forcedValue !== undefined) { + this.curPresentation.width = forcedValue ? expandedWidth : 0; + } else { + this.curPresentation.width = this.curPresentation.width === expandedWidth ? 0 : expandedWidth; + } + } + //The first lifecycle function that gets called to set up the current presentation. async componentWillMount() { this.props.Documents.forEach(async (doc, index: number) => { @@ -543,7 +553,7 @@ export class PresentationView extends React.Component { this.curPresentation.data = new List([doc]); } - this.curPresentation.width = 400; + this.toggle(true); } //Function that sets the store of the children docs. diff --git a/src/new_fields/FieldSymbols.ts b/src/new_fields/FieldSymbols.ts index a436dcf2b..b5b3aa588 100644 --- a/src/new_fields/FieldSymbols.ts +++ b/src/new_fields/FieldSymbols.ts @@ -7,4 +7,4 @@ export const Id = Symbol("Id"); export const OnUpdate = Symbol("OnUpdate"); export const Parent = Symbol("Parent"); export const Copy = Symbol("Copy"); -export const ToScriptString = Symbol("Copy"); \ No newline at end of file +export const ToScriptString = Symbol("ToScriptString"); \ No newline at end of file diff --git a/src/new_fields/InkField.ts b/src/new_fields/InkField.ts index 39c6c8ce3..8f64c1c2e 100644 --- a/src/new_fields/InkField.ts +++ b/src/new_fields/InkField.ts @@ -19,6 +19,8 @@ export interface StrokeData { page: number; } +export type InkData = Map; + const pointSchema = createSimpleSchema({ x: true, y: true }); @@ -31,9 +33,9 @@ const strokeDataSchema = createSimpleSchema({ @Deserializable("ink") export class InkField extends ObjectField { @serializable(map(object(strokeDataSchema))) - readonly inkData: Map; + readonly inkData: InkData; - constructor(data?: Map) { + constructor(data?: InkData) { super(); this.inkData = data || new Map; } diff --git a/src/server/index.ts b/src/server/index.ts index 5b086a2cf..66c982adc 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -284,18 +284,15 @@ addSecureRoute( RouteStore.getCurrUser ); +const ServicesApiKeyMap = new Map([ + ["face", process.env.FACE], + ["vision", process.env.VISION], + ["handwriting", process.env.HANDWRITING] +]); + addSecureRoute(Method.GET, (user, res, req) => { - let requested = req.params.requestedservice; - switch (requested) { - case "face": - res.send(process.env.FACE); - break; - case "vision": - res.send(process.env.VISION); - break; - default: - res.send(undefined); - } + let service = req.params.requestedservice; + res.send(ServicesApiKeyMap.get(service)); }, undefined, `${RouteStore.cognitiveServices}/:requestedservice`); class NodeCanvasFactory { -- cgit v1.2.3-70-g09d2 From 80493ba3d5e578a37f703b1c0f9e04879c43c134 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 24 Jul 2019 00:22:33 -0400 Subject: cleanup --- src/client/cognitive_services/CognitiveServices.ts | 31 +++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 71e4276dc..a778bd47b 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -160,8 +160,10 @@ export namespace CognitiveServices { let serverAddress = "https://api.cognitive.microsoft.com"; let endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize"; - let results = await new Promise((resolve, reject) => { + let requestExecutor = (resolve: any, reject: any) => { let result: any; + let body = format(inkData); + xhttp.onreadystatechange = function () { if (this.readyState === 4) { try { @@ -182,36 +184,35 @@ export namespace CognitiveServices { xhttp.open("PUT", endpoint, true); xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); xhttp.setRequestHeader('Content-Type', 'application/json'); - xhttp.send(JSON.stringify(toHandwritingUnit(inkData))); - }); + xhttp.send(body); + }; + + let results = (await new Promise(requestExecutor)).recognitionUnits; - let recognizedText = results.recognitionUnits.map((unit: any) => unit.recognizedText); + target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); + let recognizedText = results.map((item: any) => item.recognizedText); let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); - target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results.recognitionUnits, "Ink Analysis"); target.handwriting = individualWords.join(" "); }); }; - const toHandwritingUnit = (inkData: InkData): HandwritingUnit => { + const format = (inkData: InkData): string => { let entries = inkData.entries(), next = entries.next(); - let strokes: AzureStrokeData[] = []; - let id = 0; + let strokes: AzureStrokeData[] = [], id = 0; while (!next.done) { - let entry = next.value; - let data = { + strokes.push({ id: id++, - points: entry[1].pathData.map(point => `${point.x},${point.y}`).join(","), + points: next.value[1].pathData.map(point => `${point.x},${point.y}`).join(","), language: "en-US" - }; - strokes.push(data); + }); next = entries.next(); } - return { + return JSON.stringify({ version: 1, language: "en-US", unit: "mm", strokes: strokes - }; + }); }; } -- cgit v1.2.3-70-g09d2 From 99ebc483059f5637e0731c9d0b43ddd3d531f2e3 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 24 Jul 2019 10:43:55 -0400 Subject: wrapped presentation view toggle in an action, cognitive services cleanup --- src/client/cognitive_services/CognitiveServices.ts | 15 +++++++++------ src/client/views/presentationview/PresentationView.tsx | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index a778bd47b..dcd27f858 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -187,12 +187,15 @@ export namespace CognitiveServices { xhttp.send(body); }; - let results = (await new Promise(requestExecutor)).recognitionUnits; - - target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); - let recognizedText = results.map((item: any) => item.recognizedText); - let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); - target.handwriting = individualWords.join(" "); + let results = await new Promise(requestExecutor); + + if (results) { + results.recognitionUnits && (results = results.recognitionUnits); + target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); + let recognizedText = results.map((item: any) => item.recognizedText); + let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); + target.handwriting = individualWords.join(" "); + } }); }; diff --git a/src/client/views/presentationview/PresentationView.tsx b/src/client/views/presentationview/PresentationView.tsx index 966ade3de..b318f0321 100644 --- a/src/client/views/presentationview/PresentationView.tsx +++ b/src/client/views/presentationview/PresentationView.tsx @@ -69,6 +69,7 @@ export class PresentationView extends React.Component { PresentationView.Instance = this; } + @action toggle = (forcedValue: boolean | undefined) => { if (forcedValue !== undefined) { this.curPresentation.width = forcedValue ? expandedWidth : 0; -- cgit v1.2.3-70-g09d2