From 3152e69dfafe1c393bed38f3aad1e55881e62a33 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 26 Jul 2019 02:39:43 -0400 Subject: initial commit --- src/client/cognitive_services/CognitiveServices.ts | 27 +++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d69378d0e..40bbe55a1 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -9,6 +9,10 @@ import { Utils } from "../../Utils"; import { CompileScript } from "../util/Scripting"; import { ComputedField } from "../../new_fields/ScriptField"; import { InkData } from "../../new_fields/InkField"; +import "microsoft-cognitiveservices-speech-sdk"; +import "fs"; +import { AudioInputStream } from "microsoft-cognitiveservices-speech-sdk"; +import { createReadStream, ReadStream } from "fs"; type APIManager = { converter: BodyConverter, requester: RequestExecutor, analyzer: AnalysisApplier }; type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise; @@ -22,7 +26,8 @@ export type Rectangle = { top: number, left: number, width: number, height: numb export enum Service { ComputerVision = "vision", Face = "face", - Handwriting = "handwriting" + Handwriting = "handwriting", + Transcription = "transcription" } export enum Confidence { @@ -232,4 +237,24 @@ export namespace CognitiveServices { } + export namespace Transcription { + + export const Manager: APIManager = { + + converter: (data: string) => data, + + requester: async (apiKey: string, body: string, service: Service) => { + let analysis = await fetch(`${RouteStore.audioData}/${body}`).then(async response => JSON.parse(await response.json())); + console.log(analysis); + return ""; + }, + + analyzer: async (doc: Doc, keys: string[], filename: string) => { + let results = await executeQuery(Service.Transcription, Manager, filename); + } + + }; + + } + } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 5248a770123e312e1684d7147ecb7118dd6ef1e7 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 30 Jul 2019 01:24:37 -0400 Subject: tag generation and faces on extensionDoc --- src/client/cognitive_services/CognitiveServices.ts | 40 ++++-------------- src/client/views/nodes/FaceRectangles.tsx | 2 +- src/client/views/nodes/ImageBox.tsx | 49 +++++++++++++++++++--- 3 files changed, 54 insertions(+), 37 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index bbc438a9b..076907f09 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -42,7 +42,7 @@ export enum Confidence { */ export namespace CognitiveServices { - const executeQuery = async (service: Service, manager: APIManager, data: D): Promise> => { + const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise> => { return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { let apiKey = await response.text(); if (!apiKey) { @@ -103,15 +103,15 @@ export namespace CognitiveServices { return request.post(options); }, - analyzer: async (target: Doc, keys: string[], service: Service, converter: Converter) => { + analyzer: async (target: Doc, keys: string[], url: string, service: Service, converter: Converter) => { let batch = UndoManager.StartBatch("Image Analysis"); - let imageData = Cast(target.data, ImageField); + let storageKey = keys[0]; - if (!imageData || await Cast(target[storageKey], Doc)) { + if (!url || await Cast(target[storageKey], Doc)) { return; } let toStore: any; - let results = await executeQuery(service, Manager, imageData.url.href); + let results = await ExecuteQuery(service, Manager, url); if (!results) { toStore = "Cognitive Services could not process the given image URL."; } else { @@ -122,6 +122,7 @@ export namespace CognitiveServices { } } target[storageKey] = toStore; + batch.end(); } @@ -129,31 +130,6 @@ export namespace CognitiveServices { export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle }; - export const generateMetadata = async (target: Doc, threshold: Confidence = Confidence.Excellent) => { - let converter = (results: any) => { - let tagDoc = new Doc; - results.tags.map((tag: Tag) => { - let sanitized = tag.name.replace(" ", "_"); - let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`; - let computed = CompileScript(script, { params: { this: "Doc" } }); - computed.compiled && (tagDoc[sanitized] = new ComputedField(computed)); - }); - tagDoc.title = "Generated Tags"; - tagDoc.confidence = threshold; - return tagDoc; - }; - Manager.analyzer(target, ["generatedTags"], Service.ComputerVision, converter); - }; - - export const extractFaces = async (target: Doc) => { - let converter = (results: any) => { - let faceDocs = new List(); - results.map((face: Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); - return faceDocs; - }; - Manager.analyzer(target, ["faces"], Service.Face, converter); - }; - } export namespace Inking { @@ -209,7 +185,8 @@ export namespace CognitiveServices { analyzer: async (target: Doc, keys: string[], inkData: InkData) => { let batch = UndoManager.StartBatch("Ink Analysis"); - let results = await executeQuery(Service.Handwriting, Manager, inkData); + + let results = await ExecuteQuery(Service.Handwriting, Manager, inkData); if (results) { results.recognitionUnits && (results = results.recognitionUnits); target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); @@ -217,6 +194,7 @@ export namespace CognitiveServices { let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); target[keys[1]] = individualWords.join(" "); } + batch.end(); } diff --git a/src/client/views/nodes/FaceRectangles.tsx b/src/client/views/nodes/FaceRectangles.tsx index 3570531b2..acf1aced3 100644 --- a/src/client/views/nodes/FaceRectangles.tsx +++ b/src/client/views/nodes/FaceRectangles.tsx @@ -20,7 +20,7 @@ export interface RectangleTemplate { export default class FaceRectangles extends React.Component { render() { - let faces = DocListCast(Doc.GetProto(this.props.document).faces); + let faces = DocListCast(this.props.document.faces); let templates: RectangleTemplate[] = faces.map(faceDoc => { let rectangle = Cast(faceDoc.faceRectangle, Doc) as Doc; let style = { diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 29a76b0c8..5554475ec 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -25,9 +25,12 @@ import { Docs, DocumentType } from '../../documents/Documents'; import { DocServer } from '../../DocServer'; import { Font } from '@react-pdf/renderer'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { CognitiveServices } from '../../cognitive_services/CognitiveServices'; +import { CognitiveServices, Service, Tag, Confidence } from '../../cognitive_services/CognitiveServices'; import FaceRectangles from './FaceRectangles'; import { faEye } from '@fortawesome/free-regular-svg-icons'; +import { ComputedField } from '../../../new_fields/ScriptField'; +import { CompileScript } from '../../util/Scripting'; +import { thisExpression } from 'babel-types'; var requestImageSize = require('../../util/request-image-size'); var path = require('path'); const { Howl } = require('howler'); @@ -226,20 +229,56 @@ export class ImageBox extends DocComponent(ImageD funcs.push({ description: "Rotate", event: this.rotate, icon: "expand-arrows-alt" }); let modes: ContextMenuProps[] = []; - let dataDoc = Doc.GetProto(this.props.Document); - modes.push({ description: "Generate Tags", event: () => CognitiveServices.Image.generateMetadata(dataDoc), icon: "tag" }); - modes.push({ description: "Find Faces", event: () => CognitiveServices.Image.extractFaces(dataDoc), icon: "camera" }); + modes.push({ description: "Generate Tags", event: this.generateMetadata, icon: "tag" }); + modes.push({ description: "Find Faces", event: this.extractFaces, icon: "camera" }); ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: funcs, icon: "asterisk" }); ContextMenu.Instance.addItem({ description: "Analyze...", subitems: modes, icon: "eye" }); } } + extractFaces = () => { + let converter = (results: any) => { + let faceDocs = new List(); + results.map((face: CognitiveServices.Image.Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); + return faceDocs; + }; + CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["faces"], this.url, Service.Face, converter); + } + + generateMetadata = (threshold: Confidence = Confidence.Excellent) => { + let converter = (results: any) => { + let tagDoc = new Doc; + let tagsList = new List(); + results.tags.map((tag: Tag) => { + tagsList.push(tag.name); + let sanitized = tag.name.replace(" ", "_"); + let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`; + let computed = CompileScript(script, { params: { this: "Doc" } }); + computed.compiled && (tagDoc[sanitized] = new ComputedField(computed)); + }); + this.extensionDoc.generatedTags = tagsList; + tagDoc.title = "Generated Tags Doc"; + tagDoc.confidence = threshold; + return tagDoc; + }; + CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter); + } + @action onDotDown(index: number) { this.Document.curPage = index; } + @computed get fieldExtensionDoc() { + return Doc.resolvedFieldDataDoc(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey, "true"); + } + + @computed private get url() { + let data = Cast(Doc.GetProto(this.props.Document).data, ImageField); + return data ? data.url.href : undefined; + } + dots(paths: string[]) { let nativeWidth = FieldValue(this.Document.nativeWidth, 1); let dist = Math.min(nativeWidth / paths.length, 40); @@ -394,7 +433,7 @@ export class ImageBox extends DocComponent(ImageD style={{ color: [DocListCast(this.extensionDoc.audioAnnotations).length ? "blue" : "gray", "green", "red"][this._audioState] }} icon={faFileAudio} size="sm" /> {/* {this.lightbox(paths)} */} - + ); } } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From e0851a9bcde60b71dbf64a256c611288a1da8025 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 30 Jul 2019 11:00:57 -0400 Subject: clean up --- src/client/cognitive_services/CognitiveServices.ts | 8 +++----- src/client/views/MainView.tsx | 1 - src/server/index.ts | 4 +--- 3 files changed, 4 insertions(+), 9 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 720892b61..b9c718cfa 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -5,8 +5,6 @@ import { Docs } from "../documents/Documents"; import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { InkData } from "../../new_fields/InkField"; -import "microsoft-cognitiveservices-speech-sdk"; -import "fs"; import { UndoManager } from "../util/UndoManager"; type APIManager = { converter: BodyConverter, requester: RequestExecutor, analyzer: AnalysisApplier }; @@ -27,7 +25,7 @@ export type Rectangle = { top: number, left: number, width: number, height: numb export enum Service { ComputerVision = "vision", Face = "face", - Handwriting = "handwriting", + Handwriting = "handwriting" } export enum Confidence { @@ -221,7 +219,7 @@ export namespace CognitiveServices { export namespace Transcription { - export const analyzer = (doc: Doc, keys: string[]) => { + export const analyzer = (target: Doc, keys: string[]) => { let { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; let recognizer = new webkitSpeechRecognition(); recognizer.interimResults = true; @@ -229,7 +227,7 @@ export namespace CognitiveServices { recognizer.onresult = (e: any) => { let result = e.results[0][0]; - doc[keys[0]] = result.transcript; + target[keys[0]] = result.transcript; }; recognizer.start(); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index ccf8f571e..91c8fe57c 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -38,7 +38,6 @@ import { PreviewCursor } from './PreviewCursor'; import { FilterBox } from './search/FilterBox'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ClientUtils } from '../util/ClientUtils'; -import { CognitiveServices } from '../cognitive_services/CognitiveServices'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; @observer diff --git a/src/server/index.ts b/src/server/index.ts index f4bbd4423..adf218be6 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -40,8 +40,6 @@ import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); import { Response } from 'express-serve-static-core'; -import { AudioInputStream, AudioConfig, SpeechConfig, SpeechRecognizer, SpeechRecognitionResult } from 'microsoft-cognitiveservices-speech-sdk'; -import { Opt } from '../new_fields/Doc'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -299,7 +297,7 @@ addSecureRoute( const ServicesApiKeyMap = new Map([ ["face", process.env.FACE], ["vision", process.env.VISION], - ["handwriting", process.env.HANDWRITING], + ["handwriting", process.env.HANDWRITING] ]); addSecureRoute(Method.GET, (user, res, req) => { -- cgit v1.2.3-70-g09d2 From 1aac1e8820c62a5f06d7e7630394e0bd58b19a94 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 30 Jul 2019 12:56:25 -0400 Subject: refactored and implemented dictation manager and shift keyboard shortcut for voice commands --- package.json | 2 +- src/client/cognitive_services/CognitiveServices.ts | 24 ------------- src/client/util/DictationManager.ts | 39 ++++++++++++++++++++++ src/client/views/ContextMenu.tsx | 8 +++-- src/client/views/GlobalKeyHandler.ts | 29 ++++++++++++++-- src/client/views/nodes/DocumentView.tsx | 8 ++++- 6 files changed, 79 insertions(+), 31 deletions(-) create mode 100644 src/client/util/DictationManager.ts (limited to 'src/client/cognitive_services') diff --git a/package.json b/package.json index 3f100a3ef..37052fde3 100644 --- a/package.json +++ b/package.json @@ -203,4 +203,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} \ No newline at end of file +} diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index b9c718cfa..c118d91d3 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -13,12 +13,6 @@ type AnalysisApplier = (target: Doc, relevantKeys: string[], ...args: any) => an type BodyConverter = (data: D) => string; type Converter = (results: any) => Field; -namespace CORE { - export interface IWindow extends Window { - webkitSpeechRecognition: any; - } -} - export type Tag = { name: string, confidence: number }; export type Rectangle = { top: number, left: number, width: number, height: number }; @@ -217,22 +211,4 @@ export namespace CognitiveServices { } - export namespace Transcription { - - export const analyzer = (target: Doc, keys: string[]) => { - let { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; - let recognizer = new webkitSpeechRecognition(); - recognizer.interimResults = true; - recognizer.continuous = true; - - recognizer.onresult = (e: any) => { - let result = e.results[0][0]; - target[keys[0]] = result.transcript; - }; - - recognizer.start(); - }; - - } - } \ No newline at end of file diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts new file mode 100644 index 000000000..b58bdb6c7 --- /dev/null +++ b/src/client/util/DictationManager.ts @@ -0,0 +1,39 @@ +namespace CORE { + export interface IWindow extends Window { + webkitSpeechRecognition: any; + } +} + +const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; + +export default class DictationManager { + public static Instance = new DictationManager(); + private isListening = false; + private recognizer: any; + + constructor() { + this.recognizer = new webkitSpeechRecognition(); + this.recognizer.interimResults = false; + this.recognizer.continuous = true; + } + + finish = (handler: any, data: any) => { + handler(data); + this.isListening = false; + this.recognizer.stop(); + } + + listen = () => { + if (this.isListening) { + return undefined; + } + this.isListening = true; + this.recognizer.start(); + return new Promise((resolve, reject) => { + this.recognizer.onresult = (e: any) => this.finish(resolve, e.results[0][0].transcript); + this.recognizer.onerror = (e: any) => this.finish(reject, e); + }); + + } + +} \ No newline at end of file diff --git a/src/client/views/ContextMenu.tsx b/src/client/views/ContextMenu.tsx index a608e448a..98025ac31 100644 --- a/src/client/views/ContextMenu.tsx +++ b/src/client/views/ContextMenu.tsx @@ -38,8 +38,12 @@ export class ContextMenu extends React.Component { this._items = []; } - findByDescription = (target: string) => { - return this._items.find(menuItem => menuItem.description === target); + findByDescription = (target: string, toLowerCase = false) => { + return this._items.find(menuItem => { + let reference = menuItem.description; + toLowerCase && (reference = reference.toLowerCase()); + reference === target; + }); } @action diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index e31b44514..373584b4e 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -5,9 +5,13 @@ import { MainView } from "./MainView"; import { DragManager } from "../util/DragManager"; import { action } from "mobx"; import { Doc } from "../../new_fields/Doc"; +import { CognitiveServices } from "../cognitive_services/CognitiveServices"; +import DictationManager from "../util/DictationManager"; +import { ContextMenu } from "./ContextMenu"; +import { ContextMenuProps } from "./ContextMenuItem"; const modifiers = ["control", "meta", "shift", "alt"]; -type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo; +type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo | Promise; type KeyControlInfo = { preventDefault: boolean, stopPropagation: boolean @@ -25,9 +29,10 @@ export default class KeyManager { this.router.set(isMac ? "0001" : "0100", this.ctrl); this.router.set(isMac ? "0100" : "0010", this.alt); this.router.set(isMac ? "1001" : "1100", this.ctrl_shift); + this.router.set("1000", this.shift); } - public handle = (e: KeyboardEvent) => { + public handle = async (e: KeyboardEvent) => { let keyname = e.key.toLowerCase(); this.handleGreedy(keyname); @@ -43,7 +48,7 @@ export default class KeyManager { return; } - let control = handleConstrained(keyname, e); + let control = await handleConstrained(keyname, e); control.stopPropagation && e.stopPropagation(); control.preventDefault && e.preventDefault(); @@ -95,6 +100,24 @@ export default class KeyManager { }; }); + private shift = async (keyname: string) => { + let stopPropagation = true; + let preventDefault = true; + + switch (keyname) { + case " ": + let transcript = await DictationManager.Instance.listen(); + console.log(`I heard${transcript ? `: ${transcript.toLowerCase()}` : " nothing: I thought I was still listening from an earlier session."}`); + let command: ContextMenuProps | undefined; + transcript && (command = ContextMenu.Instance.findByDescription(transcript, true)) && "event" in command && command.event(); + } + + return { + stopPropagation: stopPropagation, + preventDefault: preventDefault + }; + } + private alt = action((keyname: string) => { let stopPropagation = true; let preventDefault = true; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index fd53cd451..dc56c1c8f 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -42,6 +42,7 @@ import { EditableView } from '../EditableView'; import { faHandPointer, faHandPointRight } from '@fortawesome/free-regular-svg-icons'; import { DocumentDecorations } from '../DocumentDecorations'; import { CognitiveServices } from '../../cognitive_services/CognitiveServices'; +import DictationManager from '../../util/DictationManager'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faTrash); @@ -536,6 +537,11 @@ export class DocumentView extends DocComponent(Docu this.props.Document.lockedPosition = BoolCast(this.props.Document.lockedPosition) ? undefined : true; } + listen = async () => { + let transcript = await DictationManager.Instance.listen(); + transcript && (Doc.GetProto(this.props.Document).transcript = transcript); + } + @action onContextMenu = async (e: React.MouseEvent): Promise => { e.persist(); @@ -559,7 +565,7 @@ export class DocumentView extends DocComponent(Docu cm.addItem({ description: BoolCast(this.props.Document.ignoreAspect, false) || !this.props.Document.nativeWidth || !this.props.Document.nativeHeight ? "Freeze" : "Unfreeze", event: this.freezeNativeDimensions, icon: "snowflake" }); cm.addItem({ description: "Pin to Presentation", event: () => PresentationView.Instance.PinDoc(this.props.Document), icon: "map-pin" }); cm.addItem({ description: BoolCast(this.props.Document.lockedPosition) ? "Unlock Position" : "Lock Position", event: this.toggleLockPosition, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" }); - cm.addItem({ description: "Transcribe Speech", event: () => CognitiveServices.Transcription.analyzer(Doc.GetProto(this.props.Document), ["transcript"]), icon: "microphone" }); + cm.addItem({ description: "Transcribe Speech", event: this.listen, icon: "microphone" }); let makes: ContextMenuProps[] = []; makes.push({ description: "Make Background", event: this.makeBackground, icon: BoolCast(this.props.Document.lockedPosition) ? "unlock" : "lock" }); makes.push({ description: this.props.Document.isButton ? "Remove Button" : "Make Button", event: this.makeBtnClicked, icon: "concierge-bell" }); -- cgit v1.2.3-70-g09d2 From c48345b74fadc558062ced96591f041eb4f2729d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 1 Aug 2019 15:55:07 -0400 Subject: cognitive services --- src/client/cognitive_services/CognitiveServices.ts | 32 ++++++++++++++-------- .../collectionFreeForm/CollectionFreeFormView.tsx | 2 +- src/client/views/nodes/ImageBox.tsx | 8 ++++-- 3 files changed, 27 insertions(+), 15 deletions(-) (limited to 'src/client/cognitive_services') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index c118d91d3..08fcb4883 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -7,9 +7,9 @@ import { Utils } from "../../Utils"; import { InkData } from "../../new_fields/InkField"; import { UndoManager } from "../util/UndoManager"; -type APIManager = { converter: BodyConverter, requester: RequestExecutor, analyzer: AnalysisApplier }; +type APIManager = { converter: BodyConverter, requester: RequestExecutor }; type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise; -type AnalysisApplier = (target: Doc, relevantKeys: string[], ...args: any) => any; +type AnalysisApplier = (target: Doc, relevantKeys: string[], data: D, ...args: any) => any; type BodyConverter = (data: D) => string; type Converter = (results: any) => Field; @@ -38,7 +38,7 @@ export enum Confidence { */ export namespace CognitiveServices { - const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise> => { + const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise => { return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { let apiKey = await response.text(); if (!apiKey) { @@ -46,7 +46,7 @@ export namespace CognitiveServices { return undefined; } - let results: Opt; + let results: any; try { results = await manager.requester(apiKey, manager.converter(data), service).then(json => JSON.parse(json)); } catch { @@ -99,7 +99,11 @@ export namespace CognitiveServices { return request.post(options); }, - analyzer: async (target: Doc, keys: string[], url: string, service: Service, converter: Converter) => { + }; + + export namespace Appliers { + + export const ProcessImage: AnalysisApplier = async (target: Doc, keys: string[], url: string, service: Service, converter: Converter) => { let batch = UndoManager.StartBatch("Image Analysis"); let storageKey = keys[0]; @@ -107,7 +111,7 @@ export namespace CognitiveServices { return; } let toStore: any; - let results = await ExecuteQuery(service, Manager, url); + let results = await ExecuteQuery(service, Manager, url); if (!results) { toStore = "Cognitive Services could not process the given image URL."; } else { @@ -120,9 +124,9 @@ export namespace CognitiveServices { target[storageKey] = toStore; batch.end(); - } + }; - }; + } export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle }; @@ -179,10 +183,14 @@ export namespace CognitiveServices { return new Promise(promisified); }, - analyzer: async (target: Doc, keys: string[], inkData: InkData) => { + }; + + export namespace Appliers { + + export const ConcatenateHandwriting: AnalysisApplier = async (target: Doc, keys: string[], inkData: InkData) => { let batch = UndoManager.StartBatch("Ink Analysis"); - let results = await ExecuteQuery(Service.Handwriting, Manager, inkData); + let results = await ExecuteQuery(Service.Handwriting, Manager, inkData); if (results) { results.recognitionUnits && (results = results.recognitionUnits); target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); @@ -192,9 +200,9 @@ export namespace CognitiveServices { } batch.end(); - } + }; - }; + } export interface AzureStrokeData { id: number; diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index fa49e7175..9535a6f78 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -555,7 +555,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { return; } let relevantKeys = ["inkAnalysis", "handwriting"]; - CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData); + CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.fieldExtensionDoc, relevantKeys, data.inkData); } onContextMenu = (e: React.MouseEvent) => { diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 9a0615d68..879a91fa1 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -237,7 +237,9 @@ export class ImageBox extends DocComponent(ImageD results.map((face: CognitiveServices.Image.Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); return faceDocs; }; - CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["faces"], this.url, Service.Face, converter); + if (this.url) { + CognitiveServices.Image.Appliers.ProcessImage(this.extensionDoc, ["faces"], this.url, Service.Face, converter); + } } generateMetadata = (threshold: Confidence = Confidence.Excellent) => { @@ -256,7 +258,9 @@ export class ImageBox extends DocComponent(ImageD tagDoc.confidence = threshold; return tagDoc; }; - CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter); + if (this.url) { + CognitiveServices.Image.Appliers.ProcessImage(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter); + } } @action -- cgit v1.2.3-70-g09d2