1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
|
import * as request from "request-promise";
import { Doc, Field } from "../../new_fields/Doc";
import { Cast } from "../../new_fields/Types";
import { ImageField } from "../../new_fields/URLField";
import { List } from "../../new_fields/List";
import { Docs } from "../documents/Documents";
import { RouteStore } from "../../server/RouteStore";
import { Utils } from "../../Utils";
import { CompileScript } from "../util/Scripting";
import { ComputedField } from "../../new_fields/ScriptField";
export enum Services {
ComputerVision = "vision",
Face = "face"
}
export enum Confidence {
Yikes = 0.0,
Unlikely = 0.2,
Poor = 0.4,
Fair = 0.6,
Good = 0.8,
Excellent = 0.95
}
export type Tag = { name: string, confidence: number };
export type Rectangle = { top: number, left: number, width: number, height: number };
export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle };
export type Converter = (results: any) => Field;
/**
* A file that handles all interactions with Microsoft Azure's Cognitive
* Services APIs. These machine learning endpoints allow basic data analytics for
* various media types.
*/
export namespace CognitiveServices {
export namespace Image {
export const analyze = async (imageUrl: string, service: Services) => {
return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => {
let apiKey = await response.text();
if (!apiKey) {
return undefined;
}
let uriBase;
let parameters;
switch (service) {
case Services.Face:
uriBase = 'face/v1.0/detect';
parameters = {
'returnFaceId': 'true',
'returnFaceLandmarks': 'false',
'returnFaceAttributes': 'age,gender,headPose,smile,facialHair,glasses,' +
'emotion,hair,makeup,occlusion,accessories,blur,exposure,noise'
};
break;
case Services.ComputerVision:
uriBase = 'vision/v2.0/analyze';
parameters = {
'visualFeatures': 'Categories,Description,Color,Objects,Tags,Adult',
'details': 'Celebrities,Landmarks',
'language': 'en',
};
break;
}
const options = {
uri: 'https://eastus.api.cognitive.microsoft.com/' + uriBase,
qs: parameters,
body: `{"url": "${imageUrl}"}`,
headers: {
'Content-Type': 'application/json',
'Ocp-Apim-Subscription-Key': apiKey
}
};
let results: any;
try {
results = await request.post(options).then(response => JSON.parse(response));
} catch (e) {
results = undefined;
}
return results;
});
};
const analyzeDocument = async (target: Doc, service: Services, converter: Converter, storageKey: string) => {
let imageData = Cast(target.data, ImageField);
if (!imageData || await Cast(target[storageKey], Doc)) {
return;
}
let toStore: any;
let results = await analyze(imageData.url.href, service);
if (!results) {
toStore = "Cognitive Services could not process the given image URL.";
} else {
if (!results.length) {
toStore = converter(results);
} else {
toStore = results.length > 0 ? converter(results) : "Empty list returned.";
}
}
target[storageKey] = toStore;
};
export const generateMetadata = async (target: Doc, threshold: Confidence = Confidence.Excellent) => {
let converter = (results: any) => {
let tagDoc = new Doc;
results.tags.map((tag: Tag) => {
let sanitized = tag.name.replace(" ", "_");
let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`;
let computed = CompileScript(script, { params: { this: "Doc" } });
computed.compiled && (tagDoc[sanitized] = new ComputedField(computed));
});
tagDoc.title = "Generated Tags";
tagDoc.confidence = threshold;
return tagDoc;
};
analyzeDocument(target, Services.ComputerVision, converter, "generatedTags");
};
export const extractFaces = async (target: Doc) => {
let converter = (results: any) => {
let faceDocs = new List<Doc>();
results.map((face: Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!));
return faceDocs;
};
analyzeDocument(target, Services.Face, converter, "faces");
};
}
}
|
