diff options
Diffstat (limited to 'src/client/cognitive_services')
| -rw-r--r-- | src/client/cognitive_services/CognitiveServices.ts | 189 |
1 files changed, 147 insertions, 42 deletions
diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index d4085cf76..bbc438a9b 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -1,5 +1,5 @@ import * as request from "request-promise"; -import { Doc, Field } from "../../new_fields/Doc"; +import { Doc, Field, Opt } from "../../new_fields/Doc"; import { Cast } from "../../new_fields/Types"; import { ImageField } from "../../new_fields/URLField"; import { List } from "../../new_fields/List"; @@ -8,10 +8,22 @@ import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { CompileScript } from "../util/Scripting"; import { ComputedField } from "../../new_fields/ScriptField"; +import { InkData } from "../../new_fields/InkField"; +import { undoBatch, UndoManager } from "../util/UndoManager"; -export enum Services { +type APIManager<D> = { converter: BodyConverter<D>, requester: RequestExecutor, analyzer: AnalysisApplier }; +type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise<string>; +type AnalysisApplier = (target: Doc, relevantKeys: string[], ...args: any) => any; +type BodyConverter<D> = (data: D) => string; +type Converter = (results: any) => Field; + +export type Tag = { name: string, confidence: number }; +export type Rectangle = { top: number, left: number, width: number, height: number }; + +export enum Service { ComputerVision = "vision", - Face = "face" + Face = "face", + Handwriting = "handwriting" } export enum Confidence { @@ -23,11 +35,6 @@ export enum Confidence { Excellent = 0.95 } -export type Tag = { name: string, confidence: number }; -export type Rectangle = { top: number, left: number, width: number, height: number }; -export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle }; -export type Converter = (results: any) => Field; - /** * A file that handles all interactions with Microsoft Azure's Cognitive * Services APIs. These machine learning endpoints allow basic data analytics for @@ -35,19 +42,36 @@ export type Converter = (results: any) => Field; */ export namespace CognitiveServices { + const executeQuery = async <D, R>(service: Service, manager: APIManager<D>, data: D): Promise<Opt<R>> => { + return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { + let apiKey = await response.text(); + if (!apiKey) { + console.log(`No API key found for ${service}: ensure index.ts has access to a .env file in your root directory`); + return undefined; + } + + let results: Opt<R>; + try { + results = await manager.requester(apiKey, manager.converter(data), service).then(json => JSON.parse(json)); + } catch { + results = undefined; + } + return results; + }); + }; + export namespace Image { - export const analyze = async (imageUrl: string, service: Services) => { - return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { - let apiKey = await response.text(); - if (!apiKey) { - return undefined; - } + export const Manager: APIManager<string> = { + + converter: (imageUrl: string) => JSON.stringify({ url: imageUrl }), + + requester: async (apiKey: string, body: string, service: Service) => { let uriBase; let parameters; switch (service) { - case Services.Face: + case Service.Face: uriBase = 'face/v1.0/detect'; parameters = { 'returnFaceId': 'true', @@ -56,7 +80,7 @@ export namespace CognitiveServices { 'emotion,hair,makeup,occlusion,accessories,blur,exposure,noise' }; break; - case Services.ComputerVision: + case Service.ComputerVision: uriBase = 'vision/v2.0/analyze'; parameters = { 'visualFeatures': 'Categories,Description,Color,Objects,Tags,Adult', @@ -69,42 +93,42 @@ export namespace CognitiveServices { const options = { uri: 'https://eastus.api.cognitive.microsoft.com/' + uriBase, qs: parameters, - body: `{"url": "${imageUrl}"}`, + body: body, headers: { 'Content-Type': 'application/json', 'Ocp-Apim-Subscription-Key': apiKey } }; - let results: any; - try { - results = await request.post(options).then(response => JSON.parse(response)); - } catch (e) { - results = undefined; - } - return results; - }); - }; + return request.post(options); + }, - const analyzeDocument = async (target: Doc, service: Services, converter: Converter, storageKey: string) => { - let imageData = Cast(target.data, ImageField); - if (!imageData || await Cast(target[storageKey], Doc)) { - return; - } - let toStore: any; - let results = await analyze(imageData.url.href, service); - if (!results) { - toStore = "Cognitive Services could not process the given image URL."; - } else { - if (!results.length) { - toStore = converter(results); + analyzer: async (target: Doc, keys: string[], service: Service, converter: Converter) => { + let batch = UndoManager.StartBatch("Image Analysis"); + let imageData = Cast(target.data, ImageField); + let storageKey = keys[0]; + if (!imageData || await Cast(target[storageKey], Doc)) { + return; + } + let toStore: any; + let results = await executeQuery<string, any>(service, Manager, imageData.url.href); + if (!results) { + toStore = "Cognitive Services could not process the given image URL."; } else { - toStore = results.length > 0 ? converter(results) : "Empty list returned."; + if (!results.length) { + toStore = converter(results); + } else { + toStore = results.length > 0 ? converter(results) : "Empty list returned."; + } } + target[storageKey] = toStore; + batch.end(); } - target[storageKey] = toStore; + }; + export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle }; + export const generateMetadata = async (target: Doc, threshold: Confidence = Confidence.Excellent) => { let converter = (results: any) => { let tagDoc = new Doc; @@ -118,7 +142,7 @@ export namespace CognitiveServices { tagDoc.confidence = threshold; return tagDoc; }; - analyzeDocument(target, Services.ComputerVision, converter, "generatedTags"); + Manager.analyzer(target, ["generatedTags"], Service.ComputerVision, converter); }; export const extractFaces = async (target: Doc) => { @@ -127,9 +151,90 @@ export namespace CognitiveServices { results.map((face: Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); return faceDocs; }; - analyzeDocument(target, Services.Face, converter, "faces"); + Manager.analyzer(target, ["faces"], Service.Face, converter); + }; + + } + + export namespace Inking { + + export const Manager: APIManager<InkData> = { + + converter: (inkData: InkData): string => { + let entries = inkData.entries(), next = entries.next(); + let strokes: AzureStrokeData[] = [], id = 0; + while (!next.done) { + strokes.push({ + id: id++, + points: next.value[1].pathData.map(point => `${point.x},${point.y}`).join(","), + language: "en-US" + }); + next = entries.next(); + } + return JSON.stringify({ + version: 1, + language: "en-US", + unit: "mm", + strokes: strokes + }); + }, + + requester: async (apiKey: string, body: string) => { + let xhttp = new XMLHttpRequest(); + let serverAddress = "https://api.cognitive.microsoft.com"; + let endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize"; + + let promisified = (resolve: any, reject: any) => { + xhttp.onreadystatechange = function () { + if (this.readyState === 4) { + let result = xhttp.responseText; + switch (this.status) { + case 200: + return resolve(result); + case 400: + default: + return reject(result); + } + } + }; + + xhttp.open("PUT", endpoint, true); + xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); + xhttp.setRequestHeader('Content-Type', 'application/json'); + xhttp.send(body); + }; + + return new Promise<any>(promisified); + }, + + analyzer: async (target: Doc, keys: string[], inkData: InkData) => { + let batch = UndoManager.StartBatch("Ink Analysis"); + let results = await executeQuery<InkData, any>(Service.Handwriting, Manager, inkData); + if (results) { + results.recognitionUnits && (results = results.recognitionUnits); + target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis"); + let recognizedText = results.map((item: any) => item.recognizedText); + let individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1); + target[keys[1]] = individualWords.join(" "); + } + batch.end(); + } + }; + export interface AzureStrokeData { + id: number; + points: string; + language?: string; + } + + export interface HandwritingUnit { + version: number; + language: string; + unit: string; + strokes: AzureStrokeData[]; + } + } }
\ No newline at end of file |