/* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable camelcase */ /* eslint-disable no-useless-catch */ /* eslint-disable no-use-before-define */ import * as rp from 'request-promise'; import { Doc, FieldType } from '../../fields/Doc'; import { InkData } from '../../fields/InkField'; import { List } from '../../fields/List'; import { Cast } from '../../fields/Types'; import { UndoManager } from '../util/UndoManager'; import { ClientUtils } from '../../ClientUtils'; type APIManager = { converter: BodyConverter; requester: RequestExecutor }; type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise; type AnalysisApplier = (target: Doc, relevantKeys: string[], data: D, ...args: any) => any; type BodyConverter = (data: D) => string; type Converter = (results: any) => FieldType; type TextConverter = (results: any, data: string) => Promise<{ keyterms: FieldType; external_recommendations: any; kp_string: string[] }>; type BingConverter = (results: any) => Promise<{ title_vals: string[]; url_vals: string[] }>; export type Tag = { name: string; confidence: number }; export type Rectangle = { top: number; left: number; width: number; height: number }; export enum Service { ComputerVision = 'vision', Face = 'face', Handwriting = 'handwriting', Text = 'text', Bing = 'bing', } export enum Confidence { Yikes = 0.0, Unlikely = 0.2, Poor = 0.4, Fair = 0.6, Good = 0.8, Excellent = 0.95, } /** * A file that handles all interactions with Microsoft Azure's Cognitive * Services APIs. These machine learning endpoints allow basic data analytics for * various media types. */ export namespace CognitiveServices { const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise => { const apiKey = process.env[service.toUpperCase()]; if (!apiKey) { console.log(`No API key found for ${service}: ensure youe root directory has .env file with _CLIENT_${service.toUpperCase()}.`); return undefined; } let results: any; try { results = await manager.requester(apiKey, manager.converter(data), service).then(json => JSON.parse(json)); } catch (e) { throw e; } return results; }; export namespace Image { export const Manager: APIManager = { converter: (imageUrl: string) => JSON.stringify({ url: imageUrl }), requester: async (apiKey: string, body: string, service: Service) => { let uriBase; let parameters; switch (service) { case Service.Face: uriBase = 'face/v1.0/detect'; parameters = { returnFaceId: 'true', returnFaceLandmarks: 'false', returnFaceAttributes: 'age,gender,headPose,smile,facialHair,glasses,emotion,hair,makeup,occlusion,accessories,blur,exposure,noise', }; break; case Service.ComputerVision: uriBase = 'vision/v2.0/analyze'; parameters = { visualFeatures: 'Categories,Description,Color,Objects,Tags,Adult', details: 'Celebrities,Landmarks', language: 'en', }; break; default: } const options = { uri: 'https://eastus.api.cognitive.microsoft.com/' + uriBase, qs: parameters, body: body, headers: { 'Content-Type': 'application/json', 'Ocp-Apim-Subscription-Key': apiKey, }, }; return rp.post(options); }, }; export namespace Appliers { export const ProcessImage: AnalysisApplier = async (target: Doc, keys: string[], url: string, service: Service, converter: Converter) => { const batch = UndoManager.StartBatch('Image Analysis'); const storageKey = keys[0]; if (!url || (await Cast(target[storageKey], Doc))) { return; } let toStore: any; const results = await ExecuteQuery(service, Manager, url); if (!results) { toStore = 'Cognitive Services could not process the given image URL.'; } else if (!results.length) { toStore = converter(results); } else { toStore = results.length > 0 ? converter(results) : 'Empty list returned.'; } target[storageKey] = toStore; batch.end(); }; } export type Face = { faceAttributes: any; faceId: string; faceRectangle: Rectangle }; } export namespace Inking { export const Manager: APIManager = { converter: (inkData: InkData[]): string => { let id = 0; const strokes: AzureStrokeData[] = inkData.map(points => ({ id: id++, points: points.map(({ X: x, Y: y }) => `${x},${y}`).join(','), language: 'en-US', })); return JSON.stringify({ version: 1, language: 'en-US', unit: 'mm', strokes, }); }, requester: async (apiKey: string, body: string) => { const xhttp = new XMLHttpRequest(); const serverAddress = 'https://api.cognitive.microsoft.com'; const endpoint = serverAddress + '/inkrecognizer/v1.0-preview/recognize'; return new Promise((resolve, reject) => { xhttp.onreadystatechange = function (this: XMLHttpRequest) { if (this.readyState === 4) { const result = xhttp.responseText; switch (this.status) { case 200: return resolve(result); case 400: default: return reject(result); } } return undefined; }; xhttp.open('PUT', endpoint, true); xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); xhttp.setRequestHeader('Content-Type', 'application/json'); xhttp.send(body); }); }, }; export namespace Appliers { export const ConcatenateHandwriting: AnalysisApplier = async (target: Doc, keys: string[], inkData: InkData[]) => { const batch = UndoManager.StartBatch('Ink Analysis'); let results = await ExecuteQuery(Service.Handwriting, Manager, inkData); if (results) { results.recognitionUnits && (results = results.recognitionUnits); target[keys[0]] = Doc.Get.FromJson({ data: results, title: 'Ink Analysis' }); const recognizedText = results.map((item: any) => item.recognizedText); const recognizedObjects = results.map((item: any) => item.recognizedObject); const individualWords = recognizedText.filter((text: string) => text && text.split(' ').length === 1); target[keys[1]] = individualWords.length ? individualWords.join(' ') : recognizedObjects.join(', '); } batch.end(); }; export const InterpretStrokes = async (strokes: InkData[]) => { let results = await ExecuteQuery(Service.Handwriting, Manager, strokes); if (results) { results.recognitionUnits && (results = results.recognitionUnits); } return results; }; } export interface AzureStrokeData { id: number; points: string; language?: string; } export interface HandwritingUnit { version: number; language: string; unit: string; strokes: AzureStrokeData[]; } } export namespace BingSearch { export const Manager: APIManager = { converter: (data: string) => data, requester: async (apiKey: string, query: string) => { const xhttp = new XMLHttpRequest(); const serverAddress = 'https://api.cognitive.microsoft.com'; const endpoint = serverAddress + '/bing/v5.0/search?q=' + encodeURIComponent(query); const promisified = (resolve: any, reject: any) => { xhttp.onreadystatechange = function (this: XMLHttpRequest) { if (this.readyState === 4) { const result = xhttp.responseText; switch (this.status) { case 200: return resolve(result); case 400: default: return reject(result); } } return undefined; }; if (apiKey) { xhttp.open('GET', endpoint, true); xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); xhttp.setRequestHeader('Content-Type', 'application/json'); xhttp.send(); } else { console.log('API key for BING unavailable'); } }; return new Promise(promisified); }, }; export namespace Appliers { export const analyzer = async (query: string, converter: BingConverter) => { const results = await ExecuteQuery(Service.Bing, Manager, query); console.log('Bing results: ', results); const { title_vals, url_vals } = await converter(results); return { title_vals, url_vals }; }; } } export namespace HathiTrust { export const Manager: APIManager = { converter: (data: string) => data, requester: async (apiKey: string, query: string) => { const xhttp = new XMLHttpRequest(); const serverAddress = 'https://babel.hathitrust.org/cgi/htd/​'; const endpoint = serverAddress + '/bing/v5.0/search?q=' + encodeURIComponent(query); const promisified = (resolve: any, reject: any) => { xhttp.onreadystatechange = function (this: XMLHttpRequest) { if (this.readyState === 4) { const result = xhttp.responseText; switch (this.status) { case 200: return resolve(result); case 400: default: return reject(result); } } return undefined; }; if (apiKey) { xhttp.open('GET', endpoint, true); xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); xhttp.setRequestHeader('Content-Type', 'application/json'); xhttp.send(); } else { console.log('API key for BING unavailable'); } }; return new Promise(promisified); }, }; export namespace Appliers { export const analyzer = async (query: string, converter: BingConverter) => { const results = await ExecuteQuery(Service.Bing, Manager, query); console.log('Bing results: ', results); const { title_vals, url_vals } = await converter(results); return { title_vals, url_vals }; }; } } export namespace Text { export const Manager: APIManager = { converter: (data: string) => JSON.stringify({ documents: [ { id: 1, language: 'en', text: data, }, ], }), requester: async (apiKey: string, body: string, service: Service) => { const serverAddress = 'https://eastus.api.cognitive.microsoft.com'; const endpoint = serverAddress + '/text/analytics/v2.1/keyPhrases'; const sampleBody = { documents: [ { language: 'en', id: 1, text: 'Hello world. This is some input text that I love.', }, ], }; const actualBody = body; const options = { uri: endpoint, body: actualBody, headers: { 'Content-Type': 'application/json', 'Ocp-Apim-Subscription-Key': apiKey, }, }; return rp.post(options); }, }; export namespace Appliers { export async function vectorize(keyterms: any, dataDoc: Doc, mainDoc: boolean = false) { console.log('vectorizing...'); // keyterms = ["father", "king"]; const args = { method: 'POST', uri: ClientUtils.prepend('/recommender'), body: { keyphrases: keyterms }, json: true }; await rp.post(args).then(async wordvecs => { if (wordvecs) { const indices = Object.keys(wordvecs); console.log('successful vectorization!'); const vectorValues = new List(); indices.forEach((ind: any) => { vectorValues.push(wordvecs[ind]); }); // ClientRecommender.Instance.processVector(vectorValues, dataDoc, mainDoc); } // adds document to internal doc set else { console.log('unsuccessful :( word(s) not in vocabulary'); } }); } export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: TextConverter, isMainDoc: boolean = false, isInternal: boolean = true) => { const results = await ExecuteQuery(Service.Text, Manager, data); console.log('Cognitive Services keyphrases: ', results); const { keyterms, external_recommendations, kp_string } = await converter(results, data); target[keys[0]] = keyterms; if (isInternal) { // await vectorize([data], dataDoc, isMainDoc); await vectorize(kp_string, dataDoc, isMainDoc); } else { return { recs: external_recommendations, keyterms: keyterms }; } return undefined; }; // export async function countFrequencies() } } }