From cc712e50e41bbe720299b5ab5f2d316c4d4c218a Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 00:54:03 -0400 Subject: cognitive services, not broken so far --- src/client/cognitive_services/CognitiveServices.ts | 134 +++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 src/client/cognitive_services/CognitiveServices.ts (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts new file mode 100644 index 000000000..1a002e025 --- /dev/null +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -0,0 +1,134 @@ +import * as request from "request-promise"; +import { Doc, Field } from "../../new_fields/Doc"; +import { Cast } from "../../new_fields/Types"; +import { ImageField } from "../../new_fields/URLField"; +import { values } from "mobx"; +import { List } from "../../new_fields/List"; +import { Docs } from "../documents/Documents"; +import { Result } from "../northstar/model/idea/idea"; +import { RouteStore } from "../../server/RouteStore"; +import { Utils } from "../../Utils"; + +export enum Services { + ComputerVision, + Face +} + +export enum Confidence { + Yikes = 0.0, + Unlikely = 0.2, + Poor = 0.4, + Fair = 0.6, + Good = 0.8, + Excellent = 0.95 +} + +export type Tag = { name: string, confidence: number }; +export type Rectangle = { top: number, left: number, width: number, height: number }; +export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle }; +export type Converter = (results: any) => Field; + +/** + * A file that handles all interactions with Microsoft Azure's Cognitive + * Services APIs. These machine learning endpoints allow basic data analytics for + * various media types. + */ +export namespace CognitiveServices { + + export namespace Image { + + export const analyze = async (imageUrl: string, service: Services) => { + return fetch(Utils.prepend(RouteStore.cognitiveServices + service)).then(async response => { + let apiKey = await response.text(); + if (apiKey) { + return; + } + let uriBase; + let parameters; + + switch (service) { + case Services.Face: + uriBase = 'face/v1.0/detect'; + parameters = { + 'returnFaceId': 'true', + 'returnFaceLandmarks': 'false', + 'returnFaceAttributes': 'age,gender,headPose,smile,facialHair,glasses,' + + 'emotion,hair,makeup,occlusion,accessories,blur,exposure,noise' + }; + break; + case Services.ComputerVision: + uriBase = 'vision/v2.0/analyze'; + parameters = { + 'visualFeatures': 'Categories,Description,Color,Objects,Tags,Adult', + 'details': 'Celebrities,Landmarks', + 'language': 'en', + }; + break; + } + + const options = { + uri: 'https://eastus.api.cognitive.microsoft.com/' + uriBase, + qs: parameters, + body: `{"url": "${imageUrl}"}`, + headers: { + 'Content-Type': 'application/json', + 'Ocp-Apim-Subscription-Key': apiKey + } + }; + + let results: any; + try { + results = await request.post(options).then(response => JSON.parse(response)); + } catch (e) { + results = undefined; + } + return results; + }); + }; + + const analyzeDocument = async (target: Doc, service: Services, converter: Converter, storageKey: string) => { + let imageData = Cast(target.data, ImageField); + if (!imageData || await Cast(target[storageKey], Doc)) { + return; + } + let toStore: any; + let results = await analyze(imageData.url.href, service); + if (!results) { + toStore = "Cognitive Services could not process the given image URL."; + } else { + if (!results.length) { + toStore = converter(results); + } else { + toStore = results.length > 0 ? converter(results) : "Empty list returned."; + } + } + target[storageKey] = toStore; + }; + + export const generateMetadata = async (target: Doc, threshold: Confidence = Confidence.Excellent) => { + let converter = (results: any) => { + let tagDoc = new Doc; + results.tags.map((tag: Tag) => { + if (tag.confidence >= +threshold) { + tagDoc[tag.name] = tag.confidence; + } + }); + tagDoc.title = "Generated Tags"; + tagDoc.confidenceThreshold = threshold.toString(); + return tagDoc; + }; + analyzeDocument(target, Services.ComputerVision, converter, "generatedTags"); + }; + + export const extractFaces = async (target: Doc) => { + let converter = (results: any) => { + let faceDocs = new List(); + results.map((face: Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); + return faceDocs; + }; + analyzeDocument(target, Services.Face, converter, "faces"); + }; + + } + +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 6248a0a4afd154b84ebea1202a48bec0fd4b87de Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 01:10:09 -0400 Subject: fixed server compilation bug --- src/client/cognitive_services/CognitiveServices.ts | 4 +--- src/server/RouteStore.ts | 2 +- src/server/index.ts | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 1a002e025..2a3fe1d26 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -2,10 +2,8 @@ import * as request from "request-promise"; import { Doc, Field } from "../../new_fields/Doc"; import { Cast } from "../../new_fields/Types"; import { ImageField } from "../../new_fields/URLField"; -import { values } from "mobx"; import { List } from "../../new_fields/List"; import { Docs } from "../documents/Documents"; -import { Result } from "../northstar/model/idea/idea"; import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; @@ -38,7 +36,7 @@ export namespace CognitiveServices { export namespace Image { export const analyze = async (imageUrl: string, service: Services) => { - return fetch(Utils.prepend(RouteStore.cognitiveServices + service)).then(async response => { + return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { let apiKey = await response.text(); if (apiKey) { return; diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 96bd62e23..e30015e39 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -30,6 +30,6 @@ export enum RouteStore { reset = "/reset/:token", // APIS - cognitiveServices = "/cognitiveservices/" + cognitiveServices = "/cognitiveservices" } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 6b4e59bfc..d808e8158 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -284,6 +284,21 @@ addSecureRoute( RouteStore.getCurrUser ); +addSecureRoute(Method.GET, (user, res, req) => { + let requested = req.params.requestedservice; + console.log(requested); + switch (requested) { + case "Face": + res.send(process.env.FACE); + break; + case "ComputerVision": + res.send(process.env.VISION); + break; + default: + res.send(undefined); + } +}, undefined, `${RouteStore.cognitiveServices}/:requestedservice`); + class NodeCanvasFactory { create = (width: number, height: number) => { var canvas = createCanvas(width, height); -- cgit v1.2.3-70-g09d2 From 010c7978ce19392b4f02bdea29e734ac2abac01f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 01:33:14 -0400 Subject: further non-breaking cognitive services --- src/client/cognitive_services/CognitiveServices.ts | 8 ++++---- src/client/views/nodes/ImageBox.tsx | 18 +++++++++++++----- src/server/index.ts | 5 ++--- 3 files changed, 19 insertions(+), 12 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 2a3fe1d26..d525455cb 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -8,8 +8,8 @@ import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; export enum Services { - ComputerVision, - Face + ComputerVision = "vision", + Face = "face" } export enum Confidence { @@ -38,8 +38,8 @@ export namespace CognitiveServices { export const analyze = async (imageUrl: string, service: Services) => { return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { let apiKey = await response.text(); - if (apiKey) { - return; + if (!apiKey) { + return undefined; } let uriBase; let parameters; diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index c3ee1e823..f88150d66 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -25,6 +25,7 @@ import { Docs } from '../../documents/Documents'; import { DocServer } from '../../DocServer'; import { Font } from '@react-pdf/renderer'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { CognitiveServices } from '../../cognitive_services/CognitiveServices'; var requestImageSize = require('../../util/request-image-size'); var path = require('path'); const { Howl, Howler } = require('howler'); @@ -195,10 +196,10 @@ export class ImageBox extends DocComponent(ImageD let field = Cast(this.Document[this.props.fieldKey], ImageField); if (field) { let url = field.url.href; - let subitems: ContextMenuProps[] = []; - subitems.push({ description: "Copy path", event: () => Utils.CopyText(url), icon: "expand-arrows-alt" }); - subitems.push({ description: "Record 1sec audio", event: this.recordAudioAnnotation, icon: "expand-arrows-alt" }); - subitems.push({ + let funcs: ContextMenuProps[] = []; + funcs.push({ description: "Copy path", event: () => Utils.CopyText(url), icon: "expand-arrows-alt" }); + funcs.push({ description: "Record 1sec audio", event: this.recordAudioAnnotation, icon: "expand-arrows-alt" }); + funcs.push({ description: "Rotate", event: action(() => { let proto = Doc.GetProto(this.props.Document); let nw = this.props.Document.nativeWidth; @@ -212,7 +213,14 @@ export class ImageBox extends DocComponent(ImageD this.props.Document.height = w; }), icon: "expand-arrows-alt" }); - ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: subitems }); + + let modes: ContextMenuProps[] = []; + let dataDoc = Doc.GetProto(this.Document); + modes.push({ description: "Generate Tags", event: () => CognitiveServices.Image.generateMetadata(dataDoc), icon: "tag" }); + modes.push({ description: "Find Faces", event: () => CognitiveServices.Image.extractFaces(dataDoc), icon: "camera" }); + + ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: funcs }); + ContextMenu.Instance.addItem({ description: "Analyze...", subitems: modes }); } } diff --git a/src/server/index.ts b/src/server/index.ts index d808e8158..5b086a2cf 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -286,12 +286,11 @@ addSecureRoute( addSecureRoute(Method.GET, (user, res, req) => { let requested = req.params.requestedservice; - console.log(requested); switch (requested) { - case "Face": + case "face": res.send(process.env.FACE); break; - case "ComputerVision": + case "vision": res.send(process.env.VISION); break; default: -- cgit v1.2.3-70-g09d2 From 767901272eeea30137330e710cc8cf70fd34f1de Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 03:13:08 -0400 Subject: autotag based on confidence, allow concealment of computed fields that resolve to undefined with namespace constant ComputedField.undefined --- src/client/cognitive_services/CognitiveServices.ts | 11 ++++-- src/client/views/nodes/FaceRectangle.tsx | 29 ++++++++++++++ src/client/views/nodes/FaceRectangles.tsx | 46 ++++++++++++++++++++++ src/client/views/nodes/ImageBox.tsx | 2 + src/client/views/nodes/KeyValueBox.tsx | 2 +- src/new_fields/ScriptField.ts | 2 + 6 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 src/client/views/nodes/FaceRectangle.tsx create mode 100644 src/client/views/nodes/FaceRectangles.tsx (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d525455cb..04a21c837 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -6,6 +6,8 @@ import { List } from "../../new_fields/List"; import { Docs } from "../documents/Documents"; import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; +import { CompileScript } from "../util/Scripting"; +import { ComputedField } from "../../new_fields/ScriptField"; export enum Services { ComputerVision = "vision", @@ -107,12 +109,13 @@ export namespace CognitiveServices { let converter = (results: any) => { let tagDoc = new Doc; results.tags.map((tag: Tag) => { - if (tag.confidence >= +threshold) { - tagDoc[tag.name] = tag.confidence; - } + let sanitized = tag.name.replace(" ", "_"); + let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : ${ComputedField.undefined}`; + let computed = CompileScript(script, { params: { this: "Doc" } }); + computed.compiled && (tagDoc[sanitized] = new ComputedField(computed)); }); tagDoc.title = "Generated Tags"; - tagDoc.confidenceThreshold = threshold.toString(); + tagDoc.confidence = threshold; return tagDoc; }; analyzeDocument(target, Services.ComputerVision, converter, "generatedTags"); diff --git a/src/client/views/nodes/FaceRectangle.tsx b/src/client/views/nodes/FaceRectangle.tsx new file mode 100644 index 000000000..887efc0d5 --- /dev/null +++ b/src/client/views/nodes/FaceRectangle.tsx @@ -0,0 +1,29 @@ +import React = require("react"); +import { observer } from "mobx-react"; +import { observable, runInAction } from "mobx"; +import { RectangleTemplate } from "./FaceRectangles"; + +@observer +export default class FaceRectangle extends React.Component<{ rectangle: RectangleTemplate }> { + @observable private opacity = 0; + + componentDidMount() { + setTimeout(() => runInAction(() => this.opacity = 1), 500); + } + + render() { + let rectangle = this.props.rectangle; + return ( +
+ ); + } + +} \ No newline at end of file diff --git a/src/client/views/nodes/FaceRectangles.tsx b/src/client/views/nodes/FaceRectangles.tsx new file mode 100644 index 000000000..3570531b2 --- /dev/null +++ b/src/client/views/nodes/FaceRectangles.tsx @@ -0,0 +1,46 @@ +import React = require("react"); +import { Doc, DocListCast } from "../../../new_fields/Doc"; +import { Cast, NumCast } from "../../../new_fields/Types"; +import { observer } from "mobx-react"; +import { Id } from "../../../new_fields/FieldSymbols"; +import FaceRectangle from "./FaceRectangle"; + +interface FaceRectanglesProps { + document: Doc; + color: string; + backgroundColor: string; +} + +export interface RectangleTemplate { + id: string; + style: Partial; +} + +@observer +export default class FaceRectangles extends React.Component { + + render() { + let faces = DocListCast(Doc.GetProto(this.props.document).faces); + let templates: RectangleTemplate[] = faces.map(faceDoc => { + let rectangle = Cast(faceDoc.faceRectangle, Doc) as Doc; + let style = { + top: NumCast(rectangle.top), + left: NumCast(rectangle.left), + width: NumCast(rectangle.width), + height: NumCast(rectangle.height), + backgroundColor: `${this.props.backgroundColor}33`, + border: `solid 2px ${this.props.color}`, + } as React.CSSProperties; + return { + id: rectangle[Id], + style: style + }; + }); + return ( +
+ {templates.map(rectangle => )} +
+ ); + } + +} \ No newline at end of file diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index f88150d66..0f60bd0fb 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -26,6 +26,7 @@ import { DocServer } from '../../DocServer'; import { Font } from '@react-pdf/renderer'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { CognitiveServices } from '../../cognitive_services/CognitiveServices'; +import FaceRectangles from './FaceRectangles'; var requestImageSize = require('../../util/request-image-size'); var path = require('path'); const { Howl, Howler } = require('howler'); @@ -379,6 +380,7 @@ export class ImageBox extends DocComponent(ImageD style={{ color: [DocListCast(this.extensionDoc.audioAnnotations).length ? "blue" : "gray", "green", "red"][this._audioState] }} icon={faFileAudio} size="sm" />
{/* {this.lightbox(paths)} */} + ); } } \ No newline at end of file diff --git a/src/client/views/nodes/KeyValueBox.tsx b/src/client/views/nodes/KeyValueBox.tsx index c9dd9a64e..9fc0f2080 100644 --- a/src/client/views/nodes/KeyValueBox.tsx +++ b/src/client/views/nodes/KeyValueBox.tsx @@ -114,7 +114,7 @@ export class KeyValueBox extends React.Component { let protos = Doc.GetAllPrototypes(doc); for (const proto of protos) { Object.keys(proto).forEach(key => { - if (!(key in ids)) { + if (!(key in ids) && realDoc[key] !== ComputedField.undefined) { ids[key] = key; } }); diff --git a/src/new_fields/ScriptField.ts b/src/new_fields/ScriptField.ts index e8a1ea28a..e5ec34f57 100644 --- a/src/new_fields/ScriptField.ts +++ b/src/new_fields/ScriptField.ts @@ -107,6 +107,8 @@ export namespace ComputedField { useComputed = true; } + export const undefined = "__undefined"; + export function WithoutComputed(fn: () => T) { DisableComputedFields(); try { -- cgit v1.2.3-70-g09d2 From d880e4b2fcb4e7bab3ee25d63209b173efcf37c0 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 03:51:15 -0400 Subject: escaped name for computed field undefined --- src/client/cognitive_services/CognitiveServices.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 04a21c837..d4085cf76 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -110,7 +110,7 @@ export namespace CognitiveServices { let tagDoc = new Doc; results.tags.map((tag: Tag) => { let sanitized = tag.name.replace(" ", "_"); - let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : ${ComputedField.undefined}`; + let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`; let computed = CompileScript(script, { params: { this: "Doc" } }); computed.compiled && (tagDoc[sanitized] = new ComputedField(computed)); }); -- cgit v1.2.3-70-g09d2 From cb95d4b43238538f59babcaa0b0d9dabb9b393b7 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 23 Jul 2019 22:07:13 -0400 Subject: Ink recognition and presentation view toggle --- src/client/cognitive_services/CognitiveServices.ts | 86 +++++++++++++++++++++- src/client/views/InkingCanvas.tsx | 4 +- src/client/views/MainView.tsx | 2 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 11 +++ .../views/presentationview/PresentationView.tsx | 12 ++- src/new_fields/FieldSymbols.ts | 2 +- src/new_fields/InkField.ts | 6 +- src/server/index.ts | 19 ++--- 8 files changed, 124 insertions(+), 18 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d4085cf76..71e4276dc 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -8,10 +8,12 @@ import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { CompileScript } from "../util/Scripting"; import { ComputedField } from "../../new_fields/ScriptField"; +import { InkData } from "../../new_fields/InkField"; export enum Services { ComputerVision = "vision", - Face = "face" + Face = "face", + Handwriting = "handwriting" } export enum Confidence { @@ -132,4 +134,86 @@ export namespace CognitiveServices { } + export namespace Inking { + + export interface AzureStrokeData { + id: number; + points: string; + language?: string; + } + + export interface HandwritingUnit { + version: number; + language: string; + unit: string; + strokes: AzureStrokeData[]; + } + + export const analyze = async (inkData: InkData, target: Doc) => { + return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${Services.Handwriting}`)).then(async response => { + let apiKey = await response.text(); + if (!apiKey) { + return undefined; + } + + let xhttp = new XMLHttpRequest(); + let serverAddress = "https://api.cognitive.microsoft.com"; + let endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize"; + + let results = await new Promise((resolve, reject) => { + let result: any; + xhttp.onreadystatechange = function () { + if (this.readyState === 4) { + try { + result = JSON.parse(xhttp.responseText); + } catch (e) { + return reject(e); + } + switch (this.status) { + case 200: + return resolve(result); + case 400: + default: + return reject(result); + } + } + }; + + xhttp.open("PUT", endpoint, true); + xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); + xhttp.setRequestHeader('Content-Type', 'application/json'); + xhttp.send(JSON.stringify(toHandwritingUnit(inkData))); + }); + + let recognizedText = results.recognitionUnits.map((unit: any) => unit.recognizedText); + let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); + target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results.recognitionUnits, "Ink Analysis"); + target.handwriting = individualWords.join(" "); + }); + }; + + const toHandwritingUnit = (inkData: InkData): HandwritingUnit => { + let entries = inkData.entries(), next = entries.next(); + let strokes: AzureStrokeData[] = []; + let id = 0; + while (!next.done) { + let entry = next.value; + let data = { + id: id++, + points: entry[1].pathData.map(point => `${point.x},${point.y}`).join(","), + language: "en-US" + }; + strokes.push(data); + next = entries.next(); + } + return { + version: 1, + language: "en-US", + unit: "mm", + strokes: strokes + }; + }; + + } + } \ No newline at end of file diff --git a/src/client/views/InkingCanvas.tsx b/src/client/views/InkingCanvas.tsx index 3e0d7b476..e48b45b56 100644 --- a/src/client/views/InkingCanvas.tsx +++ b/src/client/views/InkingCanvas.tsx @@ -10,6 +10,8 @@ import { undoBatch, UndoManager } from "../util/UndoManager"; import { StrokeData, InkField, InkTool } from "../../new_fields/InkField"; import { Doc } from "../../new_fields/Doc"; import { Cast, PromiseValue, NumCast } from "../../new_fields/Types"; +import { CognitiveServices } from "../cognitive_services/CognitiveServices"; +import { ContextMenu } from "./ContextMenu"; interface InkCanvasProps { getScreenTransform: () => Transform; @@ -178,7 +180,7 @@ export class InkingCanvas extends React.Component { render() { let svgCanvasStyle = InkingControl.Instance.selectedTool !== InkTool.None ? "canSelect" : "noSelect"; return ( -
+
{this.props.children()} {this.drawnPaths} diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 94a4835a1..53be0278e 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -394,6 +394,7 @@ export class MainView extends React.Component {
    • +
    • {btns.map(btn => @@ -444,7 +445,6 @@ export class MainView extends React.Component { this.isSearchVisible = !this.isSearchVisible; } - render() { return (
    diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 703873681..3fd86e950 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -31,6 +31,7 @@ import { ScriptField } from "../../../../new_fields/ScriptField"; import { OverlayView, OverlayElementOptions } from "../../OverlayView"; import { ScriptBox } from "../../ScriptBox"; import { CompileScript } from "../../../util/Scripting"; +import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; export const panZoomSchema = createSchema({ @@ -51,6 +52,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { private _lastY: number = 0; private get _pwidth() { return this.props.PanelWidth(); } private get _pheight() { return this.props.PanelHeight(); } + private inkKey = "ink"; @computed get contentBounds() { let bounds = this.props.fitToBox && !NumCast(this.nativeWidth) ? Doc.ComputeContentBounds(DocListCast(this.props.Document.data)) : undefined; @@ -477,6 +479,15 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { }, "arrange contents"); } }); + ContextMenu.Instance.addItem({ + description: "Analyze Strokes", event: async () => { + let data = Cast(this.fieldExtensionDoc[this.inkKey], InkField); + if (!data) { + return; + } + CognitiveServices.Inking.analyze(data.inkData, Doc.GetProto(this.props.Document)); + } + }); ContextMenu.Instance.addItem({ description: "Add freeform arrangement", event: () => { diff --git a/src/client/views/presentationview/PresentationView.tsx b/src/client/views/presentationview/PresentationView.tsx index f80840f96..966ade3de 100644 --- a/src/client/views/presentationview/PresentationView.tsx +++ b/src/client/views/presentationview/PresentationView.tsx @@ -31,6 +31,8 @@ export interface PresViewProps { Documents: List; } +const expandedWidth = 400; + @observer export class PresentationView extends React.Component { public static Instance: PresentationView; @@ -67,6 +69,14 @@ export class PresentationView extends React.Component { PresentationView.Instance = this; } + toggle = (forcedValue: boolean | undefined) => { + if (forcedValue !== undefined) { + this.curPresentation.width = forcedValue ? expandedWidth : 0; + } else { + this.curPresentation.width = this.curPresentation.width === expandedWidth ? 0 : expandedWidth; + } + } + //The first lifecycle function that gets called to set up the current presentation. async componentWillMount() { this.props.Documents.forEach(async (doc, index: number) => { @@ -543,7 +553,7 @@ export class PresentationView extends React.Component { this.curPresentation.data = new List([doc]); } - this.curPresentation.width = 400; + this.toggle(true); } //Function that sets the store of the children docs. diff --git a/src/new_fields/FieldSymbols.ts b/src/new_fields/FieldSymbols.ts index a436dcf2b..b5b3aa588 100644 --- a/src/new_fields/FieldSymbols.ts +++ b/src/new_fields/FieldSymbols.ts @@ -7,4 +7,4 @@ export const Id = Symbol("Id"); export const OnUpdate = Symbol("OnUpdate"); export const Parent = Symbol("Parent"); export const Copy = Symbol("Copy"); -export const ToScriptString = Symbol("Copy"); \ No newline at end of file +export const ToScriptString = Symbol("ToScriptString"); \ No newline at end of file diff --git a/src/new_fields/InkField.ts b/src/new_fields/InkField.ts index 39c6c8ce3..8f64c1c2e 100644 --- a/src/new_fields/InkField.ts +++ b/src/new_fields/InkField.ts @@ -19,6 +19,8 @@ export interface StrokeData { page: number; } +export type InkData = Map; + const pointSchema = createSimpleSchema({ x: true, y: true }); @@ -31,9 +33,9 @@ const strokeDataSchema = createSimpleSchema({ @Deserializable("ink") export class InkField extends ObjectField { @serializable(map(object(strokeDataSchema))) - readonly inkData: Map; + readonly inkData: InkData; - constructor(data?: Map) { + constructor(data?: InkData) { super(); this.inkData = data || new Map; } diff --git a/src/server/index.ts b/src/server/index.ts index 5b086a2cf..66c982adc 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -284,18 +284,15 @@ addSecureRoute( RouteStore.getCurrUser ); +const ServicesApiKeyMap = new Map([ + ["face", process.env.FACE], + ["vision", process.env.VISION], + ["handwriting", process.env.HANDWRITING] +]); + addSecureRoute(Method.GET, (user, res, req) => { - let requested = req.params.requestedservice; - switch (requested) { - case "face": - res.send(process.env.FACE); - break; - case "vision": - res.send(process.env.VISION); - break; - default: - res.send(undefined); - } + let service = req.params.requestedservice; + res.send(ServicesApiKeyMap.get(service)); }, undefined, `${RouteStore.cognitiveServices}/:requestedservice`); class NodeCanvasFactory { -- cgit v1.2.3-70-g09d2 From 80493ba3d5e578a37f703b1c0f9e04879c43c134 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 24 Jul 2019 00:22:33 -0400 Subject: cleanup --- src/client/cognitive_services/CognitiveServices.ts | 31 +++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 71e4276dc..a778bd47b 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -160,8 +160,10 @@ export namespace CognitiveServices { let serverAddress = "https://api.cognitive.microsoft.com"; let endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize"; - let results = await new Promise((resolve, reject) => { + let requestExecutor = (resolve: any, reject: any) => { let result: any; + let body = format(inkData); + xhttp.onreadystatechange = function () { if (this.readyState === 4) { try { @@ -182,36 +184,35 @@ export namespace CognitiveServices { xhttp.open("PUT", endpoint, true); xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); xhttp.setRequestHeader('Content-Type', 'application/json'); - xhttp.send(JSON.stringify(toHandwritingUnit(inkData))); - }); + xhttp.send(body); + }; + + let results = (await new Promise(requestExecutor)).recognitionUnits; - let recognizedText = results.recognitionUnits.map((unit: any) => unit.recognizedText); + target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); + let recognizedText = results.map((item: any) => item.recognizedText); let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); - target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results.recognitionUnits, "Ink Analysis"); target.handwriting = individualWords.join(" "); }); }; - const toHandwritingUnit = (inkData: InkData): HandwritingUnit => { + const format = (inkData: InkData): string => { let entries = inkData.entries(), next = entries.next(); - let strokes: AzureStrokeData[] = []; - let id = 0; + let strokes: AzureStrokeData[] = [], id = 0; while (!next.done) { - let entry = next.value; - let data = { + strokes.push({ id: id++, - points: entry[1].pathData.map(point => `${point.x},${point.y}`).join(","), + points: next.value[1].pathData.map(point => `${point.x},${point.y}`).join(","), language: "en-US" - }; - strokes.push(data); + }); next = entries.next(); } - return { + return JSON.stringify({ version: 1, language: "en-US", unit: "mm", strokes: strokes - }; + }); }; } -- cgit v1.2.3-70-g09d2 From 99ebc483059f5637e0731c9d0b43ddd3d531f2e3 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 24 Jul 2019 10:43:55 -0400 Subject: wrapped presentation view toggle in an action, cognitive services cleanup --- src/client/cognitive_services/CognitiveServices.ts | 15 +++++++++------ src/client/views/presentationview/PresentationView.tsx | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index a778bd47b..dcd27f858 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -187,12 +187,15 @@ export namespace CognitiveServices { xhttp.send(body); }; - let results = (await new Promise(requestExecutor)).recognitionUnits; - - target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); - let recognizedText = results.map((item: any) => item.recognizedText); - let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); - target.handwriting = individualWords.join(" "); + let results = await new Promise(requestExecutor); + + if (results) { + results.recognitionUnits && (results = results.recognitionUnits); + target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); + let recognizedText = results.map((item: any) => item.recognizedText); + let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); + target.handwriting = individualWords.join(" "); + } }); }; diff --git a/src/client/views/presentationview/PresentationView.tsx b/src/client/views/presentationview/PresentationView.tsx index 966ade3de..b318f0321 100644 --- a/src/client/views/presentationview/PresentationView.tsx +++ b/src/client/views/presentationview/PresentationView.tsx @@ -69,6 +69,7 @@ export class PresentationView extends React.Component { PresentationView.Instance = this; } + @action toggle = (forcedValue: boolean | undefined) => { if (forcedValue !== undefined) { this.curPresentation.width = forcedValue ? expandedWidth : 0; -- cgit v1.2.3-70-g09d2 From d6cef0815815c2587b9cc791e6a37c742aba1b45 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 25 Jul 2019 03:31:34 -0400 Subject: cognitive services refactor, buxton python script fixes, covered up imagebox context menu bug --- src/client/cognitive_services/CognitiveServices.ts | 151 +++++++++++---------- src/client/views/InkingCanvas.tsx | 4 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 2 +- src/client/views/nodes/ImageBox.tsx | 6 +- src/scraping/buxton/scraper.py | 17 +-- 5 files changed, 93 insertions(+), 87 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index dcd27f858..d689b424d 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -1,5 +1,5 @@ import * as request from "request-promise"; -import { Doc, Field } from "../../new_fields/Doc"; +import { Doc, Field, Opt } from "../../new_fields/Doc"; import { Cast } from "../../new_fields/Types"; import { ImageField } from "../../new_fields/URLField"; import { List } from "../../new_fields/List"; @@ -10,7 +10,15 @@ import { CompileScript } from "../util/Scripting"; import { ComputedField } from "../../new_fields/ScriptField"; import { InkData } from "../../new_fields/InkField"; -export enum Services { +type APIManager = { requester: RequestExecutor, applier: AnalysisApplier }; +type RequestExecutor = (apiKey: string, data: D, service: Service) => Promise; +type AnalysisApplier = (target: Doc, ...args: any) => any; +type Converter = (results: any) => Field; + +export type Tag = { name: string, confidence: number }; +export type Rectangle = { top: number, left: number, width: number, height: number }; + +export enum Service { ComputerVision = "vision", Face = "face", Handwriting = "handwriting" @@ -25,11 +33,6 @@ export enum Confidence { Excellent = 0.95 } -export type Tag = { name: string, confidence: number }; -export type Rectangle = { top: number, left: number, width: number, height: number }; -export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle }; -export type Converter = (results: any) => Field; - /** * A file that handles all interactions with Microsoft Azure's Cognitive * Services APIs. These machine learning endpoints allow basic data analytics for @@ -37,19 +40,33 @@ export type Converter = (results: any) => Field; */ export namespace CognitiveServices { + const executeQuery = async (service: Service, executor: RequestExecutor, data: D): Promise> => { + return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { + let apiKey = await response.text(); + if (!apiKey) { + return undefined; + } + + let results: Opt; + try { + results = await executor(apiKey, data, service).then(json => JSON.parse(json)); + } catch { + results = undefined; + } + return results; + }); + }; + export namespace Image { - export const analyze = async (imageUrl: string, service: Services) => { - return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { - let apiKey = await response.text(); - if (!apiKey) { - return undefined; - } + export const Manager: APIManager = { + + requester: (async (apiKey: string, imageUrl: string, service: Service) => { let uriBase; let parameters; switch (service) { - case Services.Face: + case Service.Face: uriBase = 'face/v1.0/detect'; parameters = { 'returnFaceId': 'true', @@ -58,7 +75,7 @@ export namespace CognitiveServices { 'emotion,hair,makeup,occlusion,accessories,blur,exposure,noise' }; break; - case Services.ComputerVision: + case Service.ComputerVision: uriBase = 'vision/v2.0/analyze'; parameters = { 'visualFeatures': 'Categories,Description,Color,Objects,Tags,Adult', @@ -78,35 +95,32 @@ export namespace CognitiveServices { } }; - let results: any; - try { - results = await request.post(options).then(response => JSON.parse(response)); - } catch (e) { - results = undefined; - } - return results; - }); - }; + return request.post(options); + }) as RequestExecutor, - const analyzeDocument = async (target: Doc, service: Services, converter: Converter, storageKey: string) => { - let imageData = Cast(target.data, ImageField); - if (!imageData || await Cast(target[storageKey], Doc)) { - return; - } - let toStore: any; - let results = await analyze(imageData.url.href, service); - if (!results) { - toStore = "Cognitive Services could not process the given image URL."; - } else { - if (!results.length) { - toStore = converter(results); + applier: (async (target: Doc, service: Service, converter: Converter, storageKey: string) => { + let imageData = Cast(target.data, ImageField); + if (!imageData || await Cast(target[storageKey], Doc)) { + return; + } + let toStore: any; + let results = await executeQuery(service, Manager.requester, imageData.url.href); + if (!results) { + toStore = "Cognitive Services could not process the given image URL."; } else { - toStore = results.length > 0 ? converter(results) : "Empty list returned."; + if (!results.length) { + toStore = converter(results); + } else { + toStore = results.length > 0 ? converter(results) : "Empty list returned."; + } } - } - target[storageKey] = toStore; + target[storageKey] = toStore; + }) as AnalysisApplier + }; + export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle }; + export const generateMetadata = async (target: Doc, threshold: Confidence = Confidence.Excellent) => { let converter = (results: any) => { let tagDoc = new Doc; @@ -120,7 +134,7 @@ export namespace CognitiveServices { tagDoc.confidence = threshold; return tagDoc; }; - analyzeDocument(target, Services.ComputerVision, converter, "generatedTags"); + Manager.applier(target, Service.ComputerVision, converter, "generatedTags"); }; export const extractFaces = async (target: Doc) => { @@ -129,48 +143,24 @@ export namespace CognitiveServices { results.map((face: Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); return faceDocs; }; - analyzeDocument(target, Services.Face, converter, "faces"); + Manager.applier(target, Service.Face, converter, "faces"); }; } export namespace Inking { - export interface AzureStrokeData { - id: number; - points: string; - language?: string; - } - - export interface HandwritingUnit { - version: number; - language: string; - unit: string; - strokes: AzureStrokeData[]; - } - - export const analyze = async (inkData: InkData, target: Doc) => { - return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${Services.Handwriting}`)).then(async response => { - let apiKey = await response.text(); - if (!apiKey) { - return undefined; - } + export const Manager: APIManager = { + requester: (async (apiKey: string, inkData: InkData) => { let xhttp = new XMLHttpRequest(); let serverAddress = "https://api.cognitive.microsoft.com"; let endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize"; - let requestExecutor = (resolve: any, reject: any) => { - let result: any; - let body = format(inkData); - + let promisified = (resolve: any, reject: any) => { xhttp.onreadystatechange = function () { if (this.readyState === 4) { - try { - result = JSON.parse(xhttp.responseText); - } catch (e) { - return reject(e); - } + let result = xhttp.responseText; switch (this.status) { case 200: return resolve(result); @@ -184,11 +174,14 @@ export namespace CognitiveServices { xhttp.open("PUT", endpoint, true); xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); xhttp.setRequestHeader('Content-Type', 'application/json'); - xhttp.send(body); + xhttp.send(format(inkData)); }; - let results = await new Promise(requestExecutor); + return new Promise(promisified); + }) as RequestExecutor, + applier: (async (target: Doc, inkData: InkData) => { + let results = await executeQuery(Service.Handwriting, Manager.requester, inkData); if (results) { results.recognitionUnits && (results = results.recognitionUnits); target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); @@ -196,9 +189,23 @@ export namespace CognitiveServices { let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); target.handwriting = individualWords.join(" "); } - }); + }) as AnalysisApplier + }; + export interface AzureStrokeData { + id: number; + points: string; + language?: string; + } + + export interface HandwritingUnit { + version: number; + language: string; + unit: string; + strokes: AzureStrokeData[]; + } + const format = (inkData: InkData): string => { let entries = inkData.entries(), next = entries.next(); let strokes: AzureStrokeData[] = [], id = 0; diff --git a/src/client/views/InkingCanvas.tsx b/src/client/views/InkingCanvas.tsx index e48b45b56..c4cd863d1 100644 --- a/src/client/views/InkingCanvas.tsx +++ b/src/client/views/InkingCanvas.tsx @@ -6,12 +6,10 @@ import "./InkingCanvas.scss"; import { InkingControl } from "./InkingControl"; import { InkingStroke } from "./InkingStroke"; import React = require("react"); -import { undoBatch, UndoManager } from "../util/UndoManager"; +import { UndoManager } from "../util/UndoManager"; import { StrokeData, InkField, InkTool } from "../../new_fields/InkField"; import { Doc } from "../../new_fields/Doc"; import { Cast, PromiseValue, NumCast } from "../../new_fields/Types"; -import { CognitiveServices } from "../cognitive_services/CognitiveServices"; -import { ContextMenu } from "./ContextMenu"; interface InkCanvasProps { getScreenTransform: () => Transform; diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index bd5c6f84b..4b7ddd49d 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -519,7 +519,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { if (!data) { return; } - CognitiveServices.Inking.analyze(data.inkData, Doc.GetProto(this.props.Document)); + CognitiveServices.Inking.Manager.applier(Doc.GetProto(this.props.Document), data.inkData); } }); } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 89ad18dd6..3d77696fe 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -216,9 +216,9 @@ export class ImageBox extends DocComponent(ImageD }); let modes: ContextMenuProps[] = []; - let dataDoc = Doc.GetProto(this.Document); - modes.push({ description: "Generate Tags", event: () => CognitiveServices.Image.generateMetadata(dataDoc), icon: "tag" }); - modes.push({ description: "Find Faces", event: () => CognitiveServices.Image.extractFaces(dataDoc), icon: "camera" }); + // let dataDoc = Doc.GetProto(this.Document); + modes.push({ description: "Generate Tags", event: () => CognitiveServices.Image.generateMetadata(this.Document), icon: "tag" }); + modes.push({ description: "Find Faces", event: () => CognitiveServices.Image.extractFaces(this.Document), icon: "camera" }); ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: funcs }); ContextMenu.Instance.addItem({ description: "Analyze...", subitems: modes }); diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 700269727..14490cfe4 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -15,6 +15,7 @@ source = "./source" dist = "../../server/public/files" db = MongoClient("localhost", 27017)["Dash"] +target_collection = db.newDocuments schema_guids = [] @@ -84,7 +85,7 @@ def write_schema(parse_results, display_fields, storage_key): "height": 600, "panX": 0, "panY": 0, - "zoomBasis": 0.5, + "zoomBasis": 1, "zIndex": 2, "libraryBrush": False, "viewType": 2 @@ -106,8 +107,8 @@ def write_schema(parse_results, display_fields, storage_key): fields["isPrototype"] = True fields["page"] = -1 - db.newDocuments.insert_one(data_doc) - db.newDocuments.insert_one(view_doc) + target_collection.insert_one(data_doc) + target_collection.insert_one(view_doc) data_doc_guid = data_doc["_id"] print(f"inserted view document ({view_doc_guid})") @@ -158,8 +159,8 @@ def write_text_doc(content): "__type": "Doc" } - db.newDocuments.insert_one(view_doc) - db.newDocuments.insert_one(data_doc) + target_collection.insert_one(view_doc) + target_collection.insert_one(data_doc) return view_doc_guid @@ -209,8 +210,8 @@ def write_image(folder, name): "__type": "Doc" } - db.newDocuments.insert_one(view_doc) - db.newDocuments.insert_one(data_doc) + target_collection.insert_one(view_doc) + target_collection.insert_one(data_doc) return view_doc_guid @@ -372,7 +373,7 @@ parent_guid = write_schema({ }, ["title", "short_description", "original_price"], "data") print("appending parent schema to main workspace...\n") -db.newDocuments.update_one( +target_collection.update_one( {"fields.title": "WS collection 1"}, {"$push": {"fields.data.fields": {"fieldId": parent_guid, "__type": "proxy"}}} ) -- cgit v1.2.3-70-g09d2 From 1846c217adb814a2b0297b8ab808c2f587741936 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 25 Jul 2019 09:26:33 -0400 Subject: final cognitive services refactor --- src/client/cognitive_services/CognitiveServices.ts | 84 +++++++++++----------- .../collectionFreeForm/CollectionFreeFormView.tsx | 4 +- 2 files changed, 47 insertions(+), 41 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d689b424d..0cda17325 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -10,9 +10,10 @@ import { CompileScript } from "../util/Scripting"; import { ComputedField } from "../../new_fields/ScriptField"; import { InkData } from "../../new_fields/InkField"; -type APIManager = { requester: RequestExecutor, applier: AnalysisApplier }; -type RequestExecutor = (apiKey: string, data: D, service: Service) => Promise; -type AnalysisApplier = (target: Doc, ...args: any) => any; +type APIManager = { converter: BodyConverter, requester: RequestExecutor, analyzer: AnalysisApplier }; +type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise; +type AnalysisApplier = (target: Doc, relevantKeys: string[], ...args: any) => any; +type BodyConverter = (data: D) => string; type Converter = (results: any) => Field; export type Tag = { name: string, confidence: number }; @@ -40,7 +41,7 @@ export enum Confidence { */ export namespace CognitiveServices { - const executeQuery = async (service: Service, executor: RequestExecutor, data: D): Promise> => { + const executeQuery = async (service: Service, manager: APIManager, data: D): Promise> => { return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { let apiKey = await response.text(); if (!apiKey) { @@ -49,7 +50,7 @@ export namespace CognitiveServices { let results: Opt; try { - results = await executor(apiKey, data, service).then(json => JSON.parse(json)); + results = await manager.requester(apiKey, manager.converter(data), service).then(json => JSON.parse(json)); } catch { results = undefined; } @@ -61,7 +62,9 @@ export namespace CognitiveServices { export const Manager: APIManager = { - requester: (async (apiKey: string, imageUrl: string, service: Service) => { + converter: (imageUrl: string) => JSON.stringify({ url: imageUrl }), + + requester: async (apiKey: string, body: string, service: Service) => { let uriBase; let parameters; @@ -88,7 +91,7 @@ export namespace CognitiveServices { const options = { uri: 'https://eastus.api.cognitive.microsoft.com/' + uriBase, qs: parameters, - body: `{"url": "${imageUrl}"}`, + body: body, headers: { 'Content-Type': 'application/json', 'Ocp-Apim-Subscription-Key': apiKey @@ -96,15 +99,16 @@ export namespace CognitiveServices { }; return request.post(options); - }) as RequestExecutor, + }, - applier: (async (target: Doc, service: Service, converter: Converter, storageKey: string) => { + analyzer: async (target: Doc, keys: string[], service: Service, converter: Converter) => { let imageData = Cast(target.data, ImageField); + let storageKey = keys[0]; if (!imageData || await Cast(target[storageKey], Doc)) { return; } let toStore: any; - let results = await executeQuery(service, Manager.requester, imageData.url.href); + let results = await executeQuery(service, Manager, imageData.url.href); if (!results) { toStore = "Cognitive Services could not process the given image URL."; } else { @@ -115,7 +119,7 @@ export namespace CognitiveServices { } } target[storageKey] = toStore; - }) as AnalysisApplier + } }; @@ -134,7 +138,7 @@ export namespace CognitiveServices { tagDoc.confidence = threshold; return tagDoc; }; - Manager.applier(target, Service.ComputerVision, converter, "generatedTags"); + Manager.analyzer(target, ["generatedTags"], Service.ComputerVision, converter); }; export const extractFaces = async (target: Doc) => { @@ -143,7 +147,7 @@ export namespace CognitiveServices { results.map((face: Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); return faceDocs; }; - Manager.applier(target, Service.Face, converter, "faces"); + Manager.analyzer(target, ["faces"], Service.Face, converter); }; } @@ -152,7 +156,26 @@ export namespace CognitiveServices { export const Manager: APIManager = { - requester: (async (apiKey: string, inkData: InkData) => { + converter: (inkData: InkData): string => { + let entries = inkData.entries(), next = entries.next(); + let strokes: AzureStrokeData[] = [], id = 0; + while (!next.done) { + strokes.push({ + id: id++, + points: next.value[1].pathData.map(point => `${point.x},${point.y}`).join(","), + language: "en-US" + }); + next = entries.next(); + } + return JSON.stringify({ + version: 1, + language: "en-US", + unit: "mm", + strokes: strokes + }); + }, + + requester: async (apiKey: string, body: string) => { let xhttp = new XMLHttpRequest(); let serverAddress = "https://api.cognitive.microsoft.com"; let endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize"; @@ -174,22 +197,22 @@ export namespace CognitiveServices { xhttp.open("PUT", endpoint, true); xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); xhttp.setRequestHeader('Content-Type', 'application/json'); - xhttp.send(format(inkData)); + xhttp.send(body); }; return new Promise(promisified); - }) as RequestExecutor, + }, - applier: (async (target: Doc, inkData: InkData) => { - let results = await executeQuery(Service.Handwriting, Manager.requester, inkData); + analyzer: async (target: Doc, keys: string[], inkData: InkData) => { + let results = await executeQuery(Service.Handwriting, Manager, inkData); if (results) { results.recognitionUnits && (results = results.recognitionUnits); - target.inkAnalysis = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); + target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); let recognizedText = results.map((item: any) => item.recognizedText); let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); - target.handwriting = individualWords.join(" "); + target[keys[1]] = individualWords.join(" "); } - }) as AnalysisApplier + } }; @@ -206,25 +229,6 @@ export namespace CognitiveServices { strokes: AzureStrokeData[]; } - const format = (inkData: InkData): string => { - let entries = inkData.entries(), next = entries.next(); - let strokes: AzureStrokeData[] = [], id = 0; - while (!next.done) { - strokes.push({ - id: id++, - points: next.value[1].pathData.map(point => `${point.x},${point.y}`).join(","), - language: "en-US" - }); - next = entries.next(); - } - return JSON.stringify({ - version: 1, - language: "en-US", - unit: "mm", - strokes: strokes - }); - }; - } } \ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 4b7ddd49d..d39d6f255 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -519,7 +519,9 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { if (!data) { return; } - CognitiveServices.Inking.Manager.applier(Doc.GetProto(this.props.Document), data.inkData); + let target = Doc.GetProto(this.props.Document); + let relevantKeys = ["inkAnalysis", "handwriting"]; + CognitiveServices.Inking.Manager.analyzer(target, relevantKeys, data.inkData); } }); } -- cgit v1.2.3-70-g09d2 From 216ea7dc77b1ac5529b02204e96f1d2c697afbec Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 25 Jul 2019 09:31:06 -0400 Subject: console log on no .env or key found --- src/client/cognitive_services/CognitiveServices.ts | 1 + 1 file changed, 1 insertion(+) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 0cda17325..d69378d0e 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -45,6 +45,7 @@ export namespace CognitiveServices { return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { let apiKey = await response.text(); if (!apiKey) { + console.log(`No API key found for ${service}: ensure index.ts has access to a .env file in your root directory`); return undefined; } -- cgit v1.2.3-70-g09d2 From 3152e69dfafe1c393bed38f3aad1e55881e62a33 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 26 Jul 2019 02:39:43 -0400 Subject: initial commit --- deploy/assets/Sunflower.mp3 | Bin 0 -> 7682122 bytes package.json | 1 + src/client/cognitive_services/CognitiveServices.ts | 27 +++++++++- src/client/views/MainView.tsx | 3 ++ src/server/RouteStore.ts | 1 + src/server/index.ts | 59 ++++++++++++++++++++- 6 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 deploy/assets/Sunflower.mp3 (limited to 'src/client/cognitive_services') diff --git a/deploy/assets/Sunflower.mp3 b/deploy/assets/Sunflower.mp3 new file mode 100644 index 000000000..ab04baac4 Binary files /dev/null and b/deploy/assets/Sunflower.mp3 differ diff --git a/package.json b/package.json index 4a15cbb2f..12f0cd302 100644 --- a/package.json +++ b/package.json @@ -139,6 +139,7 @@ "jsonwebtoken": "^8.5.0", "jsx-to-string": "^1.4.0", "lodash": "^4.17.11", + "microsoft-cognitiveservices-speech-sdk": "^1.6.0", "mobile-detect": "^1.4.3", "mobx": "^5.9.0", "mobx-react": "^5.3.5", diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d69378d0e..40bbe55a1 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -9,6 +9,10 @@ import { Utils } from "../../Utils"; import { CompileScript } from "../util/Scripting"; import { ComputedField } from "../../new_fields/ScriptField"; import { InkData } from "../../new_fields/InkField"; +import "microsoft-cognitiveservices-speech-sdk"; +import "fs"; +import { AudioInputStream } from "microsoft-cognitiveservices-speech-sdk"; +import { createReadStream, ReadStream } from "fs"; type APIManager = { converter: BodyConverter, requester: RequestExecutor, analyzer: AnalysisApplier }; type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise; @@ -22,7 +26,8 @@ export type Rectangle = { top: number, left: number, width: number, height: numb export enum Service { ComputerVision = "vision", Face = "face", - Handwriting = "handwriting" + Handwriting = "handwriting", + Transcription = "transcription" } export enum Confidence { @@ -232,4 +237,24 @@ export namespace CognitiveServices { } + export namespace Transcription { + + export const Manager: APIManager = { + + converter: (data: string) => data, + + requester: async (apiKey: string, body: string, service: Service) => { + let analysis = await fetch(`${RouteStore.audioData}/${body}`).then(async response => JSON.parse(await response.json())); + console.log(analysis); + return ""; + }, + + analyzer: async (doc: Doc, keys: string[], filename: string) => { + let results = await executeQuery(Service.Transcription, Manager, filename); + } + + }; + + } + } \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 61a013963..ca75ab2c4 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -39,6 +39,7 @@ import { PreviewCursor } from './PreviewCursor'; import { FilterBox } from './search/FilterBox'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ClientUtils } from '../util/ClientUtils'; +import { CognitiveServices } from '../cognitive_services/CognitiveServices'; @observer export class MainView extends React.Component { @@ -67,6 +68,8 @@ export class MainView extends React.Component { componentWillMount() { var tag = document.createElement('script'); + CognitiveServices.Transcription.Manager.analyzer(new Doc, ["hello", "world"], "Sunflower.mp3"); + tag.src = "https://www.youtube.com/iframe_api"; var firstScriptTag = document.getElementsByTagName('script')[0]; firstScriptTag.parentNode!.insertBefore(tag, firstScriptTag); diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index e30015e39..53f176c81 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -13,6 +13,7 @@ export enum RouteStore { upload = "/upload", dataUriToImage = "/uploadURI", images = "/images", + audioData = "/audioData", // USER AND WORKSPACES getCurrUser = "/getCurrentUser", diff --git a/src/server/index.ts b/src/server/index.ts index 40c0e7981..0a02b667e 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -40,6 +40,8 @@ import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); import { Response } from 'express-serve-static-core'; +import { AudioInputStream, AudioConfig, SpeechConfig, SpeechRecognizer, SpeechRecognitionResult } from 'microsoft-cognitiveservices-speech-sdk'; +import { Opt } from '../new_fields/Doc'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -297,7 +299,8 @@ addSecureRoute( const ServicesApiKeyMap = new Map([ ["face", process.env.FACE], ["vision", process.env.VISION], - ["handwriting", process.env.HANDWRITING] + ["handwriting", process.env.HANDWRITING], + ["transcription", process.env.TRANSCRIPTION] ]); addSecureRoute(Method.GET, (user, res, req) => { @@ -305,6 +308,60 @@ addSecureRoute(Method.GET, (user, res, req) => { res.send(ServicesApiKeyMap.get(service)); }, undefined, `${RouteStore.cognitiveServices}/:requestedservice`); +addSecureRoute( + Method.GET, + (user, res, req) => { + let asset = req.params.asset; + let pushStream = AudioInputStream.createPushStream(); + let readStream = fs.createReadStream(path.join(__dirname, '../../deploy/assets/' + asset)); + + let apiKey = process.env.TRANSCRIPTION; + if (!apiKey) { + res.send(undefined); + return; + } + + console.log("API KEY FOUND: ", apiKey); + + readStream.on('data', arrayBuffer => { + pushStream.write(arrayBuffer.buffer); + console.log(arrayBuffer.buffer); + }); + readStream.on('end', () => pushStream.close()); + readStream.on('error', (error) => { + console.log("ERROR! ", error); + res.end(error); + }); + + let audioConfig = AudioConfig.fromStreamInput(pushStream); + let speechConfig = SpeechConfig.fromSubscription(apiKey, "eastus"); + + console.log("Here are the configs!"); + console.log(audioConfig); + console.log(speechConfig); + + speechConfig.speechRecognitionLanguage = "en-US"; + + let recognizer: Opt = new SpeechRecognizer(speechConfig, audioConfig); + recognizer.recognizeOnceAsync( + (result: SpeechRecognitionResult) => { + console.log("RESULT! ", result); + res.send(result); + recognizer && recognizer.close(); + recognizer = undefined; + }, + (error: string) => { + console.log("RESULT ERROR: ", error); + res.send(error); + recognizer && recognizer.close(); + recognizer = undefined; + }, + ); + }, + undefined, + `${RouteStore.audioData}/:asset` +); + class NodeCanvasFactory { create = (width: number, height: number) => { var canvas = createCanvas(width, height); -- cgit v1.2.3-70-g09d2 From d009df0c7142c04947193fb5670974017919bd05 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 27 Jul 2019 16:38:55 -0400 Subject: undo for link dragging, image rotation and cognitive services --- src/client/cognitive_services/CognitiveServices.ts | 5 ++++ src/client/util/DragManager.ts | 2 ++ src/client/views/DocumentDecorations.tsx | 10 ++++++++ src/client/views/nodes/DocumentView.tsx | 1 + src/client/views/nodes/ImageBox.tsx | 29 +++++++++++----------- 5 files changed, 33 insertions(+), 14 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d69378d0e..bbc438a9b 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -9,6 +9,7 @@ import { Utils } from "../../Utils"; import { CompileScript } from "../util/Scripting"; import { ComputedField } from "../../new_fields/ScriptField"; import { InkData } from "../../new_fields/InkField"; +import { undoBatch, UndoManager } from "../util/UndoManager"; type APIManager = { converter: BodyConverter, requester: RequestExecutor, analyzer: AnalysisApplier }; type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise; @@ -103,6 +104,7 @@ export namespace CognitiveServices { }, analyzer: async (target: Doc, keys: string[], service: Service, converter: Converter) => { + let batch = UndoManager.StartBatch("Image Analysis"); let imageData = Cast(target.data, ImageField); let storageKey = keys[0]; if (!imageData || await Cast(target[storageKey], Doc)) { @@ -120,6 +122,7 @@ export namespace CognitiveServices { } } target[storageKey] = toStore; + batch.end(); } }; @@ -205,6 +208,7 @@ export namespace CognitiveServices { }, analyzer: async (target: Doc, keys: string[], inkData: InkData) => { + let batch = UndoManager.StartBatch("Ink Analysis"); let results = await executeQuery(Service.Handwriting, Manager, inkData); if (results) { results.recognitionUnits && (results = results.recognitionUnits); @@ -213,6 +217,7 @@ export namespace CognitiveServices { let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); target[keys[1]] = individualWords.join(" "); } + batch.end(); } }; diff --git a/src/client/util/DragManager.ts b/src/client/util/DragManager.ts index 5271f2f5d..95416cd53 100644 --- a/src/client/util/DragManager.ts +++ b/src/client/util/DragManager.ts @@ -8,6 +8,7 @@ import * as globalCssVariables from "../views/globalCssVariables.scss"; import { DocumentManager } from "./DocumentManager"; import { LinkManager } from "./LinkManager"; import { SelectionManager } from "./SelectionManager"; +import { DocumentDecorations } from "../views/DocumentDecorations"; export type dropActionType = "alias" | "copy" | undefined; export function SetupDrag( @@ -421,6 +422,7 @@ export namespace DragManager { if (options) { options.handlers.dragComplete({}); } + DocumentDecorations.Instance.endLinkDragBatch(); }; AbortDrag = () => { diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 255855b45..c4d958338 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -54,6 +54,7 @@ export class DocumentDecorations extends React.Component<{}, { value: string }> private _downY = 0; private _iconDoc?: Doc = undefined; private _resizeUndo?: UndoManager.Batch; + private _linkDrag?: UndoManager.Batch; @observable private _minimizedX = 0; @observable private _minimizedY = 0; @observable private _title: string = ""; @@ -376,7 +377,16 @@ export class DocumentDecorations extends React.Component<{}, { value: string }> } } + endLinkDragBatch = () => { + if (!this._linkDrag) { + return; + } + this._linkDrag.end(); + this._linkDrag = undefined; + } + onLinkerButtonDown = (e: React.PointerEvent): void => { + this._linkDrag = UndoManager.StartBatch("Drag Link"); e.stopPropagation(); document.removeEventListener("pointermove", this.onLinkerButtonMoved); document.addEventListener("pointermove", this.onLinkerButtonMoved); diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 392faf3e9..ebb2da506 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -40,6 +40,7 @@ import { ScriptingRepl } from '../ScriptingRepl'; import { ClientUtils } from '../../util/ClientUtils'; import { EditableView } from '../EditableView'; import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-icons'; +import { DocumentDecorations } from '../DocumentDecorations'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faTrash); diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index a36885616..bdb50bcf0 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -192,6 +192,20 @@ export class ImageBox extends DocComponent(ImageD }); } + @undoBatch + rotate = action(() => { + let proto = Doc.GetProto(this.props.Document); + let nw = this.props.Document.nativeWidth; + let nh = this.props.Document.nativeHeight; + let w = this.props.Document.width; + let h = this.props.Document.height; + proto.rotation = (NumCast(this.props.Document.rotation) + 90) % 360; + proto.nativeWidth = nh; + proto.nativeHeight = nw; + this.props.Document.width = h; + this.props.Document.height = w; + }); + specificContextMenu = (e: React.MouseEvent): void => { let field = Cast(this.Document[this.props.fieldKey], ImageField); if (field) { @@ -199,20 +213,7 @@ export class ImageBox extends DocComponent(ImageD let funcs: ContextMenuProps[] = []; funcs.push({ description: "Copy path", event: () => Utils.CopyText(url), icon: "expand-arrows-alt" }); funcs.push({ description: "Record 1sec audio", event: this.recordAudioAnnotation, icon: "expand-arrows-alt" }); - funcs.push({ - description: "Rotate", event: action(() => { - let proto = Doc.GetProto(this.props.Document); - let nw = this.props.Document.nativeWidth; - let nh = this.props.Document.nativeHeight; - let w = this.props.Document.width; - let h = this.props.Document.height; - proto.rotation = (NumCast(this.props.Document.rotation) + 90) % 360; - proto.nativeWidth = nh; - proto.nativeHeight = nw; - this.props.Document.width = h; - this.props.Document.height = w; - }), icon: "expand-arrows-alt" - }); + funcs.push({ description: "Rotate", event: this.rotate, icon: "expand-arrows-alt" }); let modes: ContextMenuProps[] = []; let dataDoc = Doc.GetProto(this.props.Document); -- cgit v1.2.3-70-g09d2 From 5248a770123e312e1684d7147ecb7118dd6ef1e7 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 30 Jul 2019 01:24:37 -0400 Subject: tag generation and faces on extensionDoc --- src/client/cognitive_services/CognitiveServices.ts | 40 ++++-------------- src/client/views/nodes/FaceRectangles.tsx | 2 +- src/client/views/nodes/ImageBox.tsx | 49 +++++++++++++++++++--- 3 files changed, 54 insertions(+), 37 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index bbc438a9b..076907f09 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -42,7 +42,7 @@ export enum Confidence { */ export namespace CognitiveServices { - const executeQuery = async (service: Service, manager: APIManager, data: D): Promise> => { + const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise> => { return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { let apiKey = await response.text(); if (!apiKey) { @@ -103,15 +103,15 @@ export namespace CognitiveServices { return request.post(options); }, - analyzer: async (target: Doc, keys: string[], service: Service, converter: Converter) => { + analyzer: async (target: Doc, keys: string[], url: string, service: Service, converter: Converter) => { let batch = UndoManager.StartBatch("Image Analysis"); - let imageData = Cast(target.data, ImageField); + let storageKey = keys[0]; - if (!imageData || await Cast(target[storageKey], Doc)) { + if (!url || await Cast(target[storageKey], Doc)) { return; } let toStore: any; - let results = await executeQuery(service, Manager, imageData.url.href); + let results = await ExecuteQuery(service, Manager, url); if (!results) { toStore = "Cognitive Services could not process the given image URL."; } else { @@ -122,6 +122,7 @@ export namespace CognitiveServices { } } target[storageKey] = toStore; + batch.end(); } @@ -129,31 +130,6 @@ export namespace CognitiveServices { export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle }; - export const generateMetadata = async (target: Doc, threshold: Confidence = Confidence.Excellent) => { - let converter = (results: any) => { - let tagDoc = new Doc; - results.tags.map((tag: Tag) => { - let sanitized = tag.name.replace(" ", "_"); - let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`; - let computed = CompileScript(script, { params: { this: "Doc" } }); - computed.compiled && (tagDoc[sanitized] = new ComputedField(computed)); - }); - tagDoc.title = "Generated Tags"; - tagDoc.confidence = threshold; - return tagDoc; - }; - Manager.analyzer(target, ["generatedTags"], Service.ComputerVision, converter); - }; - - export const extractFaces = async (target: Doc) => { - let converter = (results: any) => { - let faceDocs = new List(); - results.map((face: Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); - return faceDocs; - }; - Manager.analyzer(target, ["faces"], Service.Face, converter); - }; - } export namespace Inking { @@ -209,7 +185,8 @@ export namespace CognitiveServices { analyzer: async (target: Doc, keys: string[], inkData: InkData) => { let batch = UndoManager.StartBatch("Ink Analysis"); - let results = await executeQuery(Service.Handwriting, Manager, inkData); + + let results = await ExecuteQuery(Service.Handwriting, Manager, inkData); if (results) { results.recognitionUnits && (results = results.recognitionUnits); target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); @@ -217,6 +194,7 @@ export namespace CognitiveServices { let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); target[keys[1]] = individualWords.join(" "); } + batch.end(); } diff --git a/src/client/views/nodes/FaceRectangles.tsx b/src/client/views/nodes/FaceRectangles.tsx index 3570531b2..acf1aced3 100644 --- a/src/client/views/nodes/FaceRectangles.tsx +++ b/src/client/views/nodes/FaceRectangles.tsx @@ -20,7 +20,7 @@ export interface RectangleTemplate { export default class FaceRectangles extends React.Component { render() { - let faces = DocListCast(Doc.GetProto(this.props.document).faces); + let faces = DocListCast(this.props.document.faces); let templates: RectangleTemplate[] = faces.map(faceDoc => { let rectangle = Cast(faceDoc.faceRectangle, Doc) as Doc; let style = { diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 29a76b0c8..5554475ec 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -25,9 +25,12 @@ import { Docs, DocumentType } from '../../documents/Documents'; import { DocServer } from '../../DocServer'; import { Font } from '@react-pdf/renderer'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { CognitiveServices } from '../../cognitive_services/CognitiveServices'; +import { CognitiveServices, Service, Tag, Confidence } from '../../cognitive_services/CognitiveServices'; import FaceRectangles from './FaceRectangles'; import { faEye } from '@fortawesome/free-regular-svg-icons'; +import { ComputedField } from '../../../new_fields/ScriptField'; +import { CompileScript } from '../../util/Scripting'; +import { thisExpression } from 'babel-types'; var requestImageSize = require('../../util/request-image-size'); var path = require('path'); const { Howl } = require('howler'); @@ -226,20 +229,56 @@ export class ImageBox extends DocComponent(ImageD funcs.push({ description: "Rotate", event: this.rotate, icon: "expand-arrows-alt" }); let modes: ContextMenuProps[] = []; - let dataDoc = Doc.GetProto(this.props.Document); - modes.push({ description: "Generate Tags", event: () => CognitiveServices.Image.generateMetadata(dataDoc), icon: "tag" }); - modes.push({ description: "Find Faces", event: () => CognitiveServices.Image.extractFaces(dataDoc), icon: "camera" }); + modes.push({ description: "Generate Tags", event: this.generateMetadata, icon: "tag" }); + modes.push({ description: "Find Faces", event: this.extractFaces, icon: "camera" }); ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: funcs, icon: "asterisk" }); ContextMenu.Instance.addItem({ description: "Analyze...", subitems: modes, icon: "eye" }); } } + extractFaces = () => { + let converter = (results: any) => { + let faceDocs = new List(); + results.map((face: CognitiveServices.Image.Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); + return faceDocs; + }; + CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["faces"], this.url, Service.Face, converter); + } + + generateMetadata = (threshold: Confidence = Confidence.Excellent) => { + let converter = (results: any) => { + let tagDoc = new Doc; + let tagsList = new List(); + results.tags.map((tag: Tag) => { + tagsList.push(tag.name); + let sanitized = tag.name.replace(" ", "_"); + let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`; + let computed = CompileScript(script, { params: { this: "Doc" } }); + computed.compiled && (tagDoc[sanitized] = new ComputedField(computed)); + }); + this.extensionDoc.generatedTags = tagsList; + tagDoc.title = "Generated Tags Doc"; + tagDoc.confidence = threshold; + return tagDoc; + }; + CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter); + } + @action onDotDown(index: number) { this.Document.curPage = index; } + @computed get fieldExtensionDoc() { + return Doc.resolvedFieldDataDoc(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey, "true"); + } + + @computed private get url() { + let data = Cast(Doc.GetProto(this.props.Document).data, ImageField); + return data ? data.url.href : undefined; + } + dots(paths: string[]) { let nativeWidth = FieldValue(this.Document.nativeWidth, 1); let dist = Math.min(nativeWidth / paths.length, 40); @@ -394,7 +433,7 @@ export class ImageBox extends DocComponent(ImageD style={{ color: [DocListCast(this.extensionDoc.audioAnnotations).length ? "blue" : "gray", "green", "red"][this._audioState] }} icon={faFileAudio} size="sm" />
    {/* {this.lightbox(paths)} */} - +
); } } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From e0851a9bcde60b71dbf64a256c611288a1da8025 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 30 Jul 2019 11:00:57 -0400 Subject: clean up --- src/client/cognitive_services/CognitiveServices.ts | 8 +++----- src/client/views/MainView.tsx | 1 - src/server/index.ts | 4 +--- 3 files changed, 4 insertions(+), 9 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 720892b61..b9c718cfa 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -5,8 +5,6 @@ import { Docs } from "../documents/Documents"; import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { InkData } from "../../new_fields/InkField"; -import "microsoft-cognitiveservices-speech-sdk"; -import "fs"; import { UndoManager } from "../util/UndoManager"; type APIManager = { converter: BodyConverter, requester: RequestExecutor, analyzer: AnalysisApplier }; @@ -27,7 +25,7 @@ export type Rectangle = { top: number, left: number, width: number, height: numb export enum Service { ComputerVision = "vision", Face = "face", - Handwriting = "handwriting", + Handwriting = "handwriting" } export enum Confidence { @@ -221,7 +219,7 @@ export namespace CognitiveServices { export namespace Transcription { - export const analyzer = (doc: Doc, keys: string[]) => { + export const analyzer = (target: Doc, keys: string[]) => { let { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; let recognizer = new webkitSpeechRecognition(); recognizer.interimResults = true; @@ -229,7 +227,7 @@ export namespace CognitiveServices { recognizer.onresult = (e: any) => { let result = e.results[0][0]; - doc[keys[0]] = result.transcript; + target[keys[0]] = result.transcript; }; recognizer.start(); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index ccf8f571e..91c8fe57c 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -38,7 +38,6 @@ import { PreviewCursor } from './PreviewCursor'; import { FilterBox } from './search/FilterBox'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ClientUtils } from '../util/ClientUtils'; -import { CognitiveServices } from '../cognitive_services/CognitiveServices'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; @observer diff --git a/src/server/index.ts b/src/server/index.ts index f4bbd4423..adf218be6 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -40,8 +40,6 @@ import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); import { Response } from 'express-serve-static-core'; -import { AudioInputStream, AudioConfig, SpeechConfig, SpeechRecognizer, SpeechRecognitionResult } from 'microsoft-cognitiveservices-speech-sdk'; -import { Opt } from '../new_fields/Doc'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -299,7 +297,7 @@ addSecureRoute( const ServicesApiKeyMap = new Map([ ["face", process.env.FACE], ["vision", process.env.VISION], - ["handwriting", process.env.HANDWRITING], + ["handwriting", process.env.HANDWRITING] ]); addSecureRoute(Method.GET, (user, res, req) => { -- cgit v1.2.3-70-g09d2 From 1aac1e8820c62a5f06d7e7630394e0bd58b19a94 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 30 Jul 2019 12:56:25 -0400 Subject: refactored and implemented dictation manager and shift keyboard shortcut for voice commands --- package.json | 2 +- src/client/cognitive_services/CognitiveServices.ts | 24 ------------- src/client/util/DictationManager.ts | 39 ++++++++++++++++++++++ src/client/views/ContextMenu.tsx | 8 +++-- src/client/views/GlobalKeyHandler.ts | 29 ++++++++++++++-- src/client/views/nodes/DocumentView.tsx | 8 ++++- 6 files changed, 79 insertions(+), 31 deletions(-) create mode 100644 src/client/util/DictationManager.ts (limited to 'src/client/cognitive_services') diff --git a/package.json b/package.json index 3f100a3ef..37052fde3 100644 --- a/package.json +++ b/package.json @@ -203,4 +203,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} \ No newline at end of file +} diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index b9c718cfa..c118d91d3 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -13,12 +13,6 @@ type AnalysisApplier = (target: Doc, relevantKeys: string[], ...args: any) => an type BodyConverter = (data: D) => string; type Converter = (results: any) => Field; -namespace CORE { - export interface IWindow extends Window { - webkitSpeechRecognition: any; - } -} - export type Tag = { name: string, confidence: number }; export type Rectangle = { top: number, left: number, width: number, height: number }; @@ -217,22 +211,4 @@ export namespace CognitiveServices { } - export namespace Transcription { - - export const analyzer = (target: Doc, keys: string[]) => { - let { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; - let recognizer = new webkitSpeechRecognition(); - recognizer.interimResults = true; - recognizer.continuous = true; - - recognizer.onresult = (e: any) => { - let result = e.results[0][0]; - target[keys[0]] = result.transcript; - }; - - recognizer.start(); - }; - - } - } \ No newline at end of file diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts new file mode 100644 index 000000000..b58bdb6c7 --- /dev/null +++ b/src/client/util/DictationManager.ts @@ -0,0 +1,39 @@ +namespace CORE { + export interface IWindow extends Window { + webkitSpeechRecognition: any; + } +} + +const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; + +export default class DictationManager { + public static Instance = new DictationManager(); + private isListening = false; + private recognizer: any; + + constructor() { + this.recognizer = new webkitSpeechRecognition(); + this.recognizer.interimResults = false; + this.recognizer.continuous = true; + } + + finish = (handler: any, data: any) => { + handler(data); + this.isListening = false; + this.recognizer.stop(); + } + + listen = () => { + if (this.isListening) { + return undefined; + } + this.isListening = true; + this.recognizer.start(); + return new Promise((resolve, reject) => { + this.recognizer.onresult = (e: any) => this.finish(resolve, e.results[0][0].transcript); + this.recognizer.onerror = (e: any) => this.finish(reject, e); + }); + + } + +} \ No newline at end of file diff --git a/src/client/views/ContextMenu.tsx b/src/client/views/ContextMenu.tsx index a608e448a..98025ac31 100644 --- a/src/client/views/ContextMenu.tsx +++ b/src/client/views/ContextMenu.tsx @@ -38,8 +38,12 @@ export class ContextMenu extends React.Component { this._items = []; } - findByDescription = (target: string) => { - return this._items.find(menuItem => menuItem.description === target); + findByDescription = (target: string, toLowerCase = false) => { + return this._items.find(menuItem => { + let reference = menuItem.description; + toLowerCase && (reference = reference.toLowerCase()); + reference === target; + }); } @action diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index e31b44514..373584b4e 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -5,9 +5,13 @@ import { MainView } from "./MainView"; import { DragManager } from "../util/DragManager"; import { action } from "mobx"; import { Doc } from "../../new_fields/Doc"; +import { CognitiveServices } from "../cognitive_services/CognitiveServices"; +import DictationManager from "../util/DictationManager"; +import { ContextMenu } from "./ContextMenu"; +import { ContextMenuProps } from "./ContextMenuItem"; const modifiers = ["control", "meta", "shift", "alt"]; -type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo; +type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo | Promise; type KeyControlInfo = { preventDefault: boolean, stopPropagation: boolean @@ -25,9 +29,10 @@ export default class KeyManager { this.router.set(isMac ? "0001" : "0100", this.ctrl); this.router.set(isMac ? "0100" : "0010", this.alt); this.router.set(isMac ? "1001" : "1100", this.ctrl_shift); + this.router.set("1000", this.shift); } - public handle = (e: KeyboardEvent) => { + public handle = async (e: KeyboardEvent) => { let keyname = e.key.toLowerCase(); this.handleGreedy(keyname); @@ -43,7 +48,7 @@ export default class KeyManager { return; } - let control = handleConstrained(keyname, e); + let control = await handleConstrained(keyname, e); control.stopPropagation && e.stopPropagation(); control.preventDefault && e.preventDefault(); @@ -95,6 +100,24 @@ export default class KeyManager { }; }); + private shift = async (keyname: string) => { + let stopPropagation = true; + let preventDefault = true; + + switch (keyname) { + case " ": + let transcript = await DictationManager.Instance.listen(); + console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`); + let command: ContextMenuProps | undefined; + transcript && (command = ContextMenu.Instance.findByDescription(transcript, true)) && "event" in command && command.event(); + } + + return { + stopPropagation: stopPropagation, + preventDefault: preventDefault + }; + } + private alt = action((keyname: string) => { let stopPropagation = true; let preventDefault = true; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index fd53cd451..dc56c1c8f 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -42,6 +42,7 @@ import { EditableView } from '../EditableView'; import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-icons'; import { DocumentDecorations } from '../DocumentDecorations'; import { CognitiveServices } from '../../cognitive_services/CognitiveServices'; +import DictationManager from '../../util/DictationManager'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faTrash); @@ -536,6 +537,11 @@ export class DocumentView extends DocComponent(Docu this.props.Document.lockedPosition = BoolCast(this.props.Document.lockedPosition) ? undefined : true; } + listen = async () => { + let transcript = await DictationManager.Instance.listen(); + transcript && (Doc.GetProto(this.props.Document).transcript = transcript); + } + @action onContextMenu = async (e: React.MouseEvent): Promise => { e.persist(); @@ -559,7 +565,7 @@ export class DocumentView extends DocComponent(Docu cm.addItem({ description: BoolCast(this.props.Document.ignoreAspect, false) || !this.props.Document.nativeWidth || !this.props.Document.nativeHeight ? "Freeze" : "Unfreeze", event: this.freezeNativeDimensions, icon: "snowflake" }); cm.addItem({ description: "Pin to Presentation", event: () => PresentationView.Instance.PinDoc(this.props.Document), icon: "map-pin" }); cm.addItem({ description: BoolCast(this.props.Document.lockedPosition) ? "Unlock Position" : "Lock Position", event: this.toggleLockPosition, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" }); - cm.addItem({ description: "Transcribe Speech", event: () => CognitiveServices.Transcription.analyzer(Doc.GetProto(this.props.Document), ["transcript"]), icon: "microphone" }); + cm.addItem({ description: "Transcribe Speech", event: this.listen, icon: "microphone" }); let makes: ContextMenuProps[] = []; makes.push({ description: "Make Background", event: this.makeBackground, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" }); makes.push({ description: this.props.Document.isButton ? "Remove Button" : "Make Button", event: this.makeBtnClicked, icon: "concierge-bell" }); -- cgit v1.2.3-70-g09d2