From 131f3fa7d1f1a42be58de63ee54fecd20e6d9fbf Mon Sep 17 00:00:00 2001 From: ab Date: Thu, 1 Aug 2019 16:29:56 -0400 Subject: key words successfully extracted --- src/client/views/nodes/ImageBox.tsx | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src/client/views') diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 9a0615d68..c12fae8e9 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -225,6 +225,7 @@ export class ImageBox extends DocComponent(ImageD let modes: ContextMenuProps[] = []; modes.push({ description: "Generate Tags", event: this.generateMetadata, icon: "tag" }); modes.push({ description: "Find Faces", event: this.extractFaces, icon: "camera" }); + modes.push({ description: "Recommend", event: this.extractText, icon: "brain" }); ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: funcs, icon: "asterisk" }); ContextMenu.Instance.addItem({ description: "Analyze...", subitems: modes, icon: "eye" }); @@ -240,6 +241,18 @@ export class ImageBox extends DocComponent(ImageD CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["faces"], this.url, Service.Face, converter); } + extractText = () => { + let data = StrCast(this.dataDoc.title); + console.log(data); + let converter = (results: any) => { + results.documents.forEach((doc: any) => { + console.log(doc.keyPhrases); + }); + return new Doc(); + }; + CognitiveServices.Text.Manager.analyzer(this.extensionDoc, ["key words", "key word strings"], data, converter); + } + generateMetadata = (threshold: Confidence = Confidence.Excellent) => { let converter = (results: any) => { let tagDoc = new Doc; -- cgit v1.2.3-70-g09d2 From 38317d4a3687e02a727693b569dfc5980db81854 Mon Sep 17 00:00:00 2001 From: ab Date: Mon, 5 Aug 2019 16:15:59 -0400 Subject: recommender --- package.json | 1 + src/client/cognitive_services/CognitiveServices.ts | 3 ++- src/client/util/Recommender.tsx | 22 ++++++++++++++++++++++ src/client/views/nodes/ImageBox.tsx | 8 +++++--- 4 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 src/client/util/Recommender.tsx (limited to 'src/client/views') diff --git a/package.json b/package.json index d469f0c49..9012ff1f7 100644 --- a/package.json +++ b/package.json @@ -214,6 +214,7 @@ "typescript-collections": "^1.3.2", "url-loader": "^1.1.2", "uuid": "^3.3.2", + "word2vec": "^1.1.4", "xoauth2": "^1.2.0", "youtube": "^0.1.0" } diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 8aac20084..6afd2571a 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -251,8 +251,9 @@ export namespace CognitiveServices { analyzer: async (target: Doc, keys: string[], data: string, converter: Converter) => { let results = await ExecuteQuery(Service.Text, Manager, data); console.log(results); - converter(results); + let keyterms = converter(results); //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis"); + target[keys[0]] = keyterms; console.log("analyzed!"); return null; } diff --git a/src/client/util/Recommender.tsx b/src/client/util/Recommender.tsx new file mode 100644 index 000000000..88d0c85f5 --- /dev/null +++ b/src/client/util/Recommender.tsx @@ -0,0 +1,22 @@ +import React = require("react"); +import { observer } from "mobx-react"; +import { observable, runInAction } from "mobx"; +var w2v = require('word2vec'); + +@observer +export default class Recommender extends React.Component { + + /*** + * Converts text to n-dimensional vector using pretrained word2vec model + */ + text_to_vec(text: string) { + + } + + render() { + return ( +
Recommender System!!!
+ ) + } + +} \ No newline at end of file diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index c12fae8e9..7388b532c 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -245,12 +245,14 @@ export class ImageBox extends DocComponent(ImageD let data = StrCast(this.dataDoc.title); console.log(data); let converter = (results: any) => { + let keyterms = new List(); results.documents.forEach((doc: any) => { - console.log(doc.keyPhrases); + let keyPhrases = doc.keyPhrases; + keyPhrases.map((kp: string) => keyterms.push(kp)); }); - return new Doc(); + return keyterms; }; - CognitiveServices.Text.Manager.analyzer(this.extensionDoc, ["key words", "key word strings"], data, converter); + CognitiveServices.Text.Manager.analyzer(this.extensionDoc, ["key words"], data, converter); } generateMetadata = (threshold: Confidence = Confidence.Excellent) => { -- cgit v1.2.3-70-g09d2 From 06b59a4ec2f1871846696da22928fc7d54ae02d6 Mon Sep 17 00:00:00 2001 From: ab Date: Tue, 6 Aug 2019 16:01:57 -0400 Subject: word2vec is functional --- package.json | 8 +++++- src/client/cognitive_services/CognitiveServices.ts | 18 ++++++++++++- src/client/views/nodes/ImageBox.tsx | 8 ++++-- src/server/Recommender.ts | 30 ++++++++++++++++++---- src/server/index.ts | 19 ++++++++++---- webpack.config.js | 5 +++- 6 files changed, 73 insertions(+), 15 deletions(-) (limited to 'src/client/views') diff --git a/package.json b/package.json index 9012ff1f7..44d5287bd 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,12 @@ "version": "1.0.0", "description": "", "main": "index.js", + "browser": { + "child_process": false + }, + "node": { + "child_process": "empty" + }, "scripts": { "start": "cross-env NODE_OPTIONS=--max_old_space_size=4096 ts-node-dev -- src/server/index.ts", "debug": "cross-env NODE_OPTIONS=--max_old_space_size=8192 ts-node-dev --inspect -- src/server/index.ts", @@ -218,4 +224,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} +} \ No newline at end of file diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 6afd2571a..863236b60 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -6,6 +6,8 @@ import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { InkData } from "../../new_fields/InkField"; import { UndoManager } from "../util/UndoManager"; +import requestPromise = require("request-promise"); +import { List } from "../../new_fields/List"; type APIManager = { converter: BodyConverter, requester: RequestExecutor, analyzer: AnalysisApplier }; type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise; @@ -255,9 +257,23 @@ export namespace CognitiveServices { //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis"); target[keys[0]] = keyterms; console.log("analyzed!"); + vectorize(keyterms); return null; - } + }, + + }; + function vectorize(keyterms: any) { + console.log("vectorizing..."); + keyterms = ["father", "king"]; + let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true }; + requestPromise.post(args).then((value) => { + value.forEach((wordvec: any) => { + console.log(wordvec.word); + }); + }); + } + } } \ No newline at end of file diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 7388b532c..161226c0d 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -1,5 +1,5 @@ import { library } from '@fortawesome/fontawesome-svg-core'; -import { faImage, faFileAudio, faPaintBrush, faAsterisk } from '@fortawesome/free-solid-svg-icons'; +import { faImage, faFileAudio, faPaintBrush, faAsterisk, faBrain } from '@fortawesome/free-solid-svg-icons'; import { action, observable, computed, runInAction } from 'mobx'; import { observer } from "mobx-react"; import Lightbox from 'react-image-lightbox'; @@ -31,12 +31,14 @@ import { faEye } from '@fortawesome/free-regular-svg-icons'; import { ComputedField } from '../../../new_fields/ScriptField'; import { CompileScript } from '../../util/Scripting'; import { thisExpression } from 'babel-types'; +import { Recommender } from '../../../server/Recommender'; +import requestPromise = require('request-promise'); var requestImageSize = require('../../util/request-image-size'); var path = require('path'); const { Howl } = require('howler'); -library.add(faImage, faEye, faPaintBrush); +library.add(faImage, faEye, faPaintBrush, faBrain); library.add(faFileAudio, faAsterisk); @@ -253,6 +255,8 @@ export class ImageBox extends DocComponent(ImageD return keyterms; }; CognitiveServices.Text.Manager.analyzer(this.extensionDoc, ["key words"], data, converter); + // request recommender + //fetch(Utils.prepend("/recommender"), { body: body, method: "POST", headers: { "content-type": "application/json" } }).then((value) => console.log(value)); } generateMetadata = (threshold: Confidence = Confidence.Excellent) => { diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts index 6d9ca6486..3c71f3aa1 100644 --- a/src/server/Recommender.ts +++ b/src/server/Recommender.ts @@ -3,25 +3,27 @@ var w2v = require('word2vec'); export class Recommender { private _model: any; + static Instance: Recommender; constructor() { console.log("creating recommender..."); + Recommender.Instance = this; } - public loadModel(): Promise { + private loadModel(): Promise { let self = this; return new Promise(res => { w2v.loadModel("./node_modules/word2vec/vectors.txt", function (err: any, model: any) { - console.log(err); - console.log(model); self._model = model; - console.log(model.similarity('father', 'mother')); res(model); }); }); } - public testModel() { + public async testModel() { + if (!this._model) { + await this.loadModel(); + } if (this._model) { let similarity = this._model.similarity('father', 'mother'); console.log(similarity); @@ -30,4 +32,22 @@ export class Recommender { console.log("model not found :("); } } + + public async testInstance(text: string) { + if (!this._model) { + await this.loadModel(); + } + console.log(text); + } + + public async vectorize(text: string[]) { + if (!this._model) { + await this.loadModel(); + } + if (this._model) { + let word_vecs = this._model.getVectors(text); + console.log(word_vecs[0]); + return word_vecs; + } + } } diff --git a/src/server/index.ts b/src/server/index.ts index dda5a870a..67087fc1f 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -52,11 +52,6 @@ const probe = require("probe-image-size"); var SolrNode = require('solr-node'); var shell = require('shelljs'); -let recommender = new Recommender(); -recommender.loadModel().then(() => { - recommender.testModel(); -}); - const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); @@ -651,6 +646,20 @@ app.use(RouteStore.corsProxy, (req, res) => { }).pipe(res); }); +//// + +let recommender = new Recommender(); +recommender.testModel(); + +app.post("/recommender", async (req, res) => { + let keyphrases = req.body.keyphrases; + let wordvecs = await recommender.vectorize(keyphrases); + res.send(wordvecs); +}); + + +///// + app.get(RouteStore.delete, (req, res) => { if (release) { res.send("no"); diff --git a/webpack.config.js b/webpack.config.js index 5e0a6a883..6a14dfcda 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -93,5 +93,8 @@ module.exports = { warnings: true, errors: true } - } + }, + externals: [ + 'child_process' + ] }; \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 17b27d3575d3f91f461262e5ad72a457238d198a Mon Sep 17 00:00:00 2001 From: ab Date: Wed, 7 Aug 2019 16:28:51 -0400 Subject: correlation matrix completed --- package.json | 2 +- src/client/ClientRecommender.ts | 101 +++++++++++++++++++++ src/client/cognitive_services/CognitiveServices.ts | 22 +++-- src/client/views/MainView.tsx | 7 ++ .../collectionFreeForm/CollectionFreeFormView.tsx | 19 +++- src/client/views/nodes/ImageBox.tsx | 14 +-- src/server/Recommender.ts | 27 +++++- 7 files changed, 166 insertions(+), 26 deletions(-) create mode 100644 src/client/ClientRecommender.ts (limited to 'src/client/views') diff --git a/package.json b/package.json index 44d5287bd..ebb0c35a8 100644 --- a/package.json +++ b/package.json @@ -224,4 +224,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} \ No newline at end of file +} diff --git a/src/client/ClientRecommender.ts b/src/client/ClientRecommender.ts new file mode 100644 index 000000000..7ff79ab50 --- /dev/null +++ b/src/client/ClientRecommender.ts @@ -0,0 +1,101 @@ +import { Doc } from "../new_fields/Doc"; +import { StrCast } from "../new_fields/Types"; +import { List } from "../new_fields/List"; +import { CognitiveServices } from "./cognitive_services/CognitiveServices"; + + +var assert = require('assert'); + +export class ClientRecommender { + + static Instance: ClientRecommender; + private docVectors: Set; + + constructor() { + //console.log("creating client recommender..."); + ClientRecommender.Instance = this; + this.docVectors = new Set(); + } + + + /*** + * Computes the cosine similarity between two vectors in Euclidean space. + */ + + private distance(vector1: number[], vector2: number[]) { + assert(vector1.length === vector2.length, "Vectors are not the same length"); + var dotproduct = 0; + var mA = 0; + var mB = 0; + for (let i = 0; i < vector1.length; i++) { // here you missed the i++ + dotproduct += (vector1[i] * vector2[i]); + mA += (vector1[i] * vector1[i]); + mB += (vector2[i] * vector2[i]); + } + mA = Math.sqrt(mA); + mB = Math.sqrt(mB); + var similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets + return similarity; + } + + /*** + * Computes the mean of a set of vectors + */ + + public mean(paragraph: Set) { + const n = 200; + const num_words = paragraph.size; + let meanVector = new Array(n).fill(0); // mean vector + paragraph.forEach((wordvec: number[]) => { + for (let i = 0; i < n; i++) { + meanVector[i] += wordvec[i]; + } + }); + meanVector = meanVector.map(x => x / num_words); + this.addToDocSet(meanVector); + return meanVector; + } + + private addToDocSet(vector: number[]) { + if (this.docVectors) { + this.docVectors.add(vector); + } + } + + /*** + * Uses Cognitive Services to extract keywords from a document + */ + + public async extractText(dataDoc: Doc, extDoc: Doc) { + let data = StrCast(dataDoc.title); + //console.log(data); + let converter = (results: any) => { + let keyterms = new List(); + results.documents.forEach((doc: any) => { + let keyPhrases = doc.keyPhrases; + keyPhrases.map((kp: string) => keyterms.push(kp)); + }); + return keyterms; + }; + await CognitiveServices.Text.Manager.analyzer(extDoc, ["key words"], data, converter); + } + + /*** + * Creates distance matrix for all Documents analyzed + */ + + public createDistanceMatrix(documents: Set = this.docVectors) { + const documents_list = Array.from(documents); + const n = documents_list.length; + var matrix = new Array(n).fill(0).map(() => new Array(n).fill(0)); + for (let i = 0; i < n; i++) { + var doc1 = documents_list[i]; + for (let j = 0; j < n; j++) { + var doc2 = documents_list[j]; + matrix[i][j] = this.distance(doc1, doc2); + } + } + return matrix; + } + +} \ No newline at end of file diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 863236b60..cc366abc2 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -8,6 +8,7 @@ import { InkData } from "../../new_fields/InkField"; import { UndoManager } from "../util/UndoManager"; import requestPromise = require("request-promise"); import { List } from "../../new_fields/List"; +import { ClientRecommender } from "../ClientRecommender"; type APIManager = { converter: BodyConverter, requester: RequestExecutor, analyzer: AnalysisApplier }; type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise; @@ -257,20 +258,21 @@ export namespace CognitiveServices { //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis"); target[keys[0]] = keyterms; console.log("analyzed!"); - vectorize(keyterms); - return null; - }, - - + await vectorize(keyterms); + } }; - function vectorize(keyterms: any) { + async function vectorize(keyterms: any) { console.log("vectorizing..."); - keyterms = ["father", "king"]; + //keyterms = ["father", "king"]; let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true }; - requestPromise.post(args).then((value) => { - value.forEach((wordvec: any) => { - console.log(wordvec.word); + await requestPromise.post(args).then(async (wordvecs) => { + var vectorValues = new Set(); + wordvecs.forEach((wordvec: any) => { + //console.log(wordvec.word); + vectorValues.add(wordvec.values as number[]); }); + ClientRecommender.Instance.mean(vectorValues); + //console.log(vectorValues.size); }); } diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 2ecf5fd85..97964166a 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -39,6 +39,7 @@ import { FilterBox } from './search/FilterBox'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ClientUtils } from '../util/ClientUtils'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; +//import { DocumentManager } from '../util/DocumentManager'; @observer export class MainView extends React.Component { @@ -435,6 +436,12 @@ export class MainView extends React.Component { ; } + // clusterDocuments = () => { + // DocumentManager.Instance.DocumentViews(); + // } + + + @action diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 29f9b1429..9344b43d2 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1,6 +1,6 @@ import { library } from "@fortawesome/fontawesome-svg-core"; import { faEye } from "@fortawesome/free-regular-svg-icons"; -import { faCompass, faCompressArrowsAlt, faExpandArrowsAlt, faPaintBrush, faTable, faUpload } from "@fortawesome/free-solid-svg-icons"; +import { faCompass, faCompressArrowsAlt, faExpandArrowsAlt, faPaintBrush, faTable, faUpload, faBrain } from "@fortawesome/free-solid-svg-icons"; import { action, computed } from "mobx"; import { observer } from "mobx-react"; import { Doc, DocListCastAsync, HeightSym, WidthSym } from "../../../../new_fields/Doc"; @@ -37,8 +37,9 @@ import "./CollectionFreeFormView.scss"; import { MarqueeView } from "./MarqueeView"; import React = require("react"); import v5 = require("uuid/v5"); +import { ClientRecommender } from "../../../ClientRecommender"; -library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload); +library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBrain); export const panZoomSchema = createSchema({ panX: "number", @@ -596,6 +597,20 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { input.click(); } }); + ContextMenu.Instance.addItem({ + description: "Recommender System", + event: async () => { + new ClientRecommender(); + let activedocs = this.getActiveDocuments(); + await Promise.all(activedocs.map((doc: Doc) => { + console.log(StrCast(doc.title)); + const extdoc = doc.data_ext as Doc; + return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + })); + console.log(ClientRecommender.Instance.createDistanceMatrix()); + }, + icon: "brain" + }); } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 161226c0d..660772c0e 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -31,7 +31,7 @@ import { faEye } from '@fortawesome/free-regular-svg-icons'; import { ComputedField } from '../../../new_fields/ScriptField'; import { CompileScript } from '../../util/Scripting'; import { thisExpression } from 'babel-types'; -import { Recommender } from '../../../server/Recommender'; +//import { Recommender } from '../../../server/Recommender'; import requestPromise = require('request-promise'); var requestImageSize = require('../../util/request-image-size'); var path = require('path'); @@ -244,17 +244,7 @@ export class ImageBox extends DocComponent(ImageD } extractText = () => { - let data = StrCast(this.dataDoc.title); - console.log(data); - let converter = (results: any) => { - let keyterms = new List(); - results.documents.forEach((doc: any) => { - let keyPhrases = doc.keyPhrases; - keyPhrases.map((kp: string) => keyterms.push(kp)); - }); - return keyterms; - }; - CognitiveServices.Text.Manager.analyzer(this.extensionDoc, ["key words"], data, converter); + //Recommender.Instance.extractText(this.dataDoc, this.extensionDoc); // request recommender //fetch(Utils.prepend("/recommender"), { body: body, method: "POST", headers: { "content-type": "application/json" } }).then((value) => console.log(value)); } diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts index 3c71f3aa1..ea59703c3 100644 --- a/src/server/Recommender.ts +++ b/src/server/Recommender.ts @@ -1,4 +1,10 @@ +//import { Doc } from "../new_fields/Doc"; +//import { StrCast } from "../new_fields/Types"; +//import { List } from "../new_fields/List"; +//import { CognitiveServices } from "../client/cognitive_services/CognitiveServices"; + var w2v = require('word2vec'); +var assert = require('assert'); export class Recommender { @@ -10,6 +16,10 @@ export class Recommender { Recommender.Instance = this; } + /*** + * Loads pre-trained model from word2vec + */ + private loadModel(): Promise { let self = this; return new Promise(res => { @@ -20,6 +30,10 @@ export class Recommender { }); } + /*** + * Testing + */ + public async testModel() { if (!this._model) { await this.loadModel(); @@ -33,6 +47,10 @@ export class Recommender { } } + /*** + * Tests if instance exists + */ + public async testInstance(text: string) { if (!this._model) { await this.loadModel(); @@ -40,14 +58,21 @@ export class Recommender { console.log(text); } + /*** + * Uses model to convert words to vectors + */ + public async vectorize(text: string[]) { if (!this._model) { await this.loadModel(); } if (this._model) { let word_vecs = this._model.getVectors(text); - console.log(word_vecs[0]); return word_vecs; } } + + + + } -- cgit v1.2.3-70-g09d2 From 17f28e393e0c24fcace33a3ecd5564bc766fe685 Mon Sep 17 00:00:00 2001 From: ab Date: Thu, 8 Aug 2019 13:00:51 -0400 Subject: fuck you react --- src/client/ClientRecommender.ts | 101 -------------- src/client/ClientRecommender.tsx | 148 +++++++++++++++++++++ .../collectionFreeForm/CollectionFreeFormView.tsx | 4 +- src/server/Recommender.ts | 2 + 4 files changed, 153 insertions(+), 102 deletions(-) delete mode 100644 src/client/ClientRecommender.ts create mode 100644 src/client/ClientRecommender.tsx (limited to 'src/client/views') diff --git a/src/client/ClientRecommender.ts b/src/client/ClientRecommender.ts deleted file mode 100644 index 7ff79ab50..000000000 --- a/src/client/ClientRecommender.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { Doc } from "../new_fields/Doc"; -import { StrCast } from "../new_fields/Types"; -import { List } from "../new_fields/List"; -import { CognitiveServices } from "./cognitive_services/CognitiveServices"; - - -var assert = require('assert'); - -export class ClientRecommender { - - static Instance: ClientRecommender; - private docVectors: Set; - - constructor() { - //console.log("creating client recommender..."); - ClientRecommender.Instance = this; - this.docVectors = new Set(); - } - - - /*** - * Computes the cosine similarity between two vectors in Euclidean space. - */ - - private distance(vector1: number[], vector2: number[]) { - assert(vector1.length === vector2.length, "Vectors are not the same length"); - var dotproduct = 0; - var mA = 0; - var mB = 0; - for (let i = 0; i < vector1.length; i++) { // here you missed the i++ - dotproduct += (vector1[i] * vector2[i]); - mA += (vector1[i] * vector1[i]); - mB += (vector2[i] * vector2[i]); - } - mA = Math.sqrt(mA); - mB = Math.sqrt(mB); - var similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets - return similarity; - } - - /*** - * Computes the mean of a set of vectors - */ - - public mean(paragraph: Set) { - const n = 200; - const num_words = paragraph.size; - let meanVector = new Array(n).fill(0); // mean vector - paragraph.forEach((wordvec: number[]) => { - for (let i = 0; i < n; i++) { - meanVector[i] += wordvec[i]; - } - }); - meanVector = meanVector.map(x => x / num_words); - this.addToDocSet(meanVector); - return meanVector; - } - - private addToDocSet(vector: number[]) { - if (this.docVectors) { - this.docVectors.add(vector); - } - } - - /*** - * Uses Cognitive Services to extract keywords from a document - */ - - public async extractText(dataDoc: Doc, extDoc: Doc) { - let data = StrCast(dataDoc.title); - //console.log(data); - let converter = (results: any) => { - let keyterms = new List(); - results.documents.forEach((doc: any) => { - let keyPhrases = doc.keyPhrases; - keyPhrases.map((kp: string) => keyterms.push(kp)); - }); - return keyterms; - }; - await CognitiveServices.Text.Manager.analyzer(extDoc, ["key words"], data, converter); - } - - /*** - * Creates distance matrix for all Documents analyzed - */ - - public createDistanceMatrix(documents: Set = this.docVectors) { - const documents_list = Array.from(documents); - const n = documents_list.length; - var matrix = new Array(n).fill(0).map(() => new Array(n).fill(0)); - for (let i = 0; i < n; i++) { - var doc1 = documents_list[i]; - for (let j = 0; j < n; j++) { - var doc2 = documents_list[j]; - matrix[i][j] = this.distance(doc1, doc2); - } - } - return matrix; - } - -} \ No newline at end of file diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx new file mode 100644 index 000000000..2344ee490 --- /dev/null +++ b/src/client/ClientRecommender.tsx @@ -0,0 +1,148 @@ +import { Doc } from "../new_fields/Doc"; +import { StrCast } from "../new_fields/Types"; +import { List } from "../new_fields/List"; +import { CognitiveServices } from "./cognitive_services/CognitiveServices"; +import React = require("react"); +import { observer } from "mobx-react"; +import { observable, action, computed, reaction } from "mobx"; +var assert = require('assert'); +import Table from 'react-bootstrap/Table'; + +export interface RecommenderProps { + title: string; +} + +@observer +export class ClientRecommender extends React.Component { + + static Instance: ClientRecommender; + private docVectors: Set; + private corr_matrix = observable([[0, 0], [0, 0]]); + @observable private firstnum = 0; + //@observable private corr_matrix: number[][] = [[0, 0], [0, 0]]; + + constructor(props: RecommenderProps) { + //console.log("creating client recommender..."); + super(props); + ClientRecommender.Instance = this; + this.docVectors = new Set(); + //this.corr_matrix = [[0, 0], [0, 0]]; + } + + + /*** + * Computes the cosine similarity between two vectors in Euclidean space. + */ + + private distance(vector1: number[], vector2: number[]) { + assert(vector1.length === vector2.length, "Vectors are not the same length"); + var dotproduct = 0; + var mA = 0; + var mB = 0; + for (let i = 0; i < vector1.length; i++) { // here you missed the i++ + dotproduct += (vector1[i] * vector2[i]); + mA += (vector1[i] * vector1[i]); + mB += (vector2[i] * vector2[i]); + } + mA = Math.sqrt(mA); + mB = Math.sqrt(mB); + var similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets + return similarity; + } + + /*** + * Computes the mean of a set of vectors + */ + + public mean(paragraph: Set) { + const n = 200; + const num_words = paragraph.size; + let meanVector = new Array(n).fill(0); // mean vector + paragraph.forEach((wordvec: number[]) => { + for (let i = 0; i < n; i++) { + meanVector[i] += wordvec[i]; + } + }); + meanVector = meanVector.map(x => x / num_words); + this.addToDocSet(meanVector); + return meanVector; + } + + private addToDocSet(vector: number[]) { + if (this.docVectors) { + this.docVectors.add(vector); + } + } + + /*** + * Uses Cognitive Services to extract keywords from a document + */ + + public async extractText(dataDoc: Doc, extDoc: Doc) { + let data = StrCast(dataDoc.title); + //console.log(data); + let converter = (results: any) => { + let keyterms = new List(); + results.documents.forEach((doc: any) => { + let keyPhrases = doc.keyPhrases; + keyPhrases.map((kp: string) => keyterms.push(kp)); + }); + return keyterms; + }; + await CognitiveServices.Text.Manager.analyzer(extDoc, ["key words"], data, converter); + } + + /*** + * Creates distance matrix for all Documents analyzed + */ + + @action + public createDistanceMatrix(documents: Set = this.docVectors) { + //this.corr_matrix[0][0] = 500; + this.firstnum = 500; + const documents_list = Array.from(documents); + const n = documents_list.length; + var matrix = new Array(n).fill(0).map(() => new Array(n).fill(0)); + for (let i = 0; i < n; i++) { + var doc1 = documents_list[i]; + for (let j = 0; j < n; j++) { + var doc2 = documents_list[j]; + matrix[i][j] = this.distance(doc1, doc2); + } + } + //this.corr_matrix = matrix; + + return matrix; + } + + @computed get first_num() { + return this.firstnum; + } + + dumb_reaction = reaction( + () => this.first_num, + number => { + console.log("number has changed", number); + this.forceUpdate(); + } + ); + + render() { + return (
+

{this.props.title ? this.props.title : "hello"}

+ + + + + + + + + + + +
{this.first_num}{this.corr_matrix[0][1]}
{this.corr_matrix[1][0]}{this.corr_matrix[1][1]}
+
); + } + +} \ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 9344b43d2..5259b9b49 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -523,6 +523,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY)); } + @action onContextMenu = (e: React.MouseEvent) => { let layoutItems: ContextMenuProps[] = []; layoutItems.push({ @@ -600,7 +601,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { ContextMenu.Instance.addItem({ description: "Recommender System", event: async () => { - new ClientRecommender(); + // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); let activedocs = this.getActiveDocuments(); await Promise.all(activedocs.map((doc: Doc) => { console.log(StrCast(doc.title)); @@ -715,6 +716,7 @@ class CollectionFreeFormViewPannableContents extends React.Component otherwise, reactions won't fire return
{this.props.children} +
; } } \ No newline at end of file diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts index ea59703c3..d175b67c7 100644 --- a/src/server/Recommender.ts +++ b/src/server/Recommender.ts @@ -10,6 +10,7 @@ export class Recommender { private _model: any; static Instance: Recommender; + private dimension: number = 0; constructor() { console.log("creating recommender..."); @@ -25,6 +26,7 @@ export class Recommender { return new Promise(res => { w2v.loadModel("./node_modules/word2vec/vectors.txt", function (err: any, model: any) { self._model = model; + self.dimension = model.size; res(model); }); }); -- cgit v1.2.3-70-g09d2 From 116f218c2eeec377fb465027bcfaa7521d9af492 Mon Sep 17 00:00:00 2001 From: ab Date: Thu, 8 Aug 2019 17:48:58 -0400 Subject: euclidian --- src/client/ClientRecommender.scss | 8 ++ src/client/ClientRecommender.tsx | 100 +++++++++++++-------- .../collectionFreeForm/CollectionFreeFormView.tsx | 3 +- 3 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 src/client/ClientRecommender.scss (limited to 'src/client/views') diff --git a/src/client/ClientRecommender.scss b/src/client/ClientRecommender.scss new file mode 100644 index 000000000..710d80f34 --- /dev/null +++ b/src/client/ClientRecommender.scss @@ -0,0 +1,8 @@ +@import "/views/globalCssVariables.scss"; + +.space{ + border: 1px dashed blue; + border-spacing: 25px; + border-collapse: separate; + align-content: center; +} \ No newline at end of file diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 2344ee490..0793f0887 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -6,7 +6,7 @@ import React = require("react"); import { observer } from "mobx-react"; import { observable, action, computed, reaction } from "mobx"; var assert = require('assert'); -import Table from 'react-bootstrap/Table'; +import "./ClientRecommender.scss"; export interface RecommenderProps { title: string; @@ -17,37 +17,53 @@ export class ClientRecommender extends React.Component { static Instance: ClientRecommender; private docVectors: Set; - private corr_matrix = observable([[0, 0], [0, 0]]); - @observable private firstnum = 0; - //@observable private corr_matrix: number[][] = [[0, 0], [0, 0]]; + @observable private corr_matrix = [[0, 0], [0, 0]]; constructor(props: RecommenderProps) { //console.log("creating client recommender..."); super(props); - ClientRecommender.Instance = this; + if (!ClientRecommender.Instance) ClientRecommender.Instance = this; this.docVectors = new Set(); //this.corr_matrix = [[0, 0], [0, 0]]; } + @action + public reset_docs() { + this.docVectors = new Set(); + this.corr_matrix = [[0, 0], [0, 0]]; + } /*** * Computes the cosine similarity between two vectors in Euclidean space. */ - private distance(vector1: number[], vector2: number[]) { + private distance(vector1: number[], vector2: number[], metric: string = "cosine") { assert(vector1.length === vector2.length, "Vectors are not the same length"); - var dotproduct = 0; - var mA = 0; - var mB = 0; - for (let i = 0; i < vector1.length; i++) { // here you missed the i++ - dotproduct += (vector1[i] * vector2[i]); - mA += (vector1[i] * vector1[i]); - mB += (vector2[i] * vector2[i]); + let similarity: number; + switch (metric) { + case "cosine": + var dotproduct = 0; + var mA = 0; + var mB = 0; + for (let i = 0; i < vector1.length; i++) { // here you missed the i++ + dotproduct += (vector1[i] * vector2[i]); + mA += (vector1[i] * vector1[i]); + mB += (vector2[i] * vector2[i]); + } + mA = Math.sqrt(mA); + mB = Math.sqrt(mB); + similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets + return similarity; + case "euclidian": + var sum = 0; + for (let i = 0; i < vector1.length; i++) { + sum += Math.pow(vector1[i] - vector2[i], 2); + } + similarity = Math.sqrt(sum); + return similarity; + default: + return 0; } - mA = Math.sqrt(mA); - mB = Math.sqrt(mB); - var similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets - return similarity; } /*** @@ -98,8 +114,6 @@ export class ClientRecommender extends React.Component { @action public createDistanceMatrix(documents: Set = this.docVectors) { - //this.corr_matrix[0][0] = 500; - this.firstnum = 500; const documents_list = Array.from(documents); const n = documents_list.length; var matrix = new Array(n).fill(0).map(() => new Array(n).fill(0)); @@ -107,40 +121,48 @@ export class ClientRecommender extends React.Component { var doc1 = documents_list[i]; for (let j = 0; j < n; j++) { var doc2 = documents_list[j]; - matrix[i][j] = this.distance(doc1, doc2); + matrix[i][j] = this.distance(doc1, doc2, "euclidian"); } } - //this.corr_matrix = matrix; - + this.corr_matrix = matrix; return matrix; } - @computed get first_num() { - return this.firstnum; - } - - dumb_reaction = reaction( - () => this.first_num, - number => { - console.log("number has changed", number); - this.forceUpdate(); + @computed + private get generateRows() { + const n = this.corr_matrix.length; + let rows: React.ReactElement[] = []; + for (let i = 0; i < n; i++) { + let children: React.ReactElement[] = []; + for (let j = 0; j < n; j++) { + let cell = React.createElement("td", this.corr_matrix[i][j]); + children.push(cell); + } + let row = React.createElement("tr", { children: children }); + rows.push(row); } - ); + return rows; + } render() { return (

{this.props.title ? this.props.title : "hello"}

- + {/*
- - - + + + - - - + + + +
{this.first_num}{this.corr_matrix[0][1]}
{this.corr_matrix[0][0].toFixed(4)}{this.corr_matrix[0][1].toFixed(4)}
{this.corr_matrix[1][0]}{this.corr_matrix[1][1]}
{this.corr_matrix[1][0].toFixed(4)}{this.corr_matrix[1][1].toFixed(4)}
*/} + + + {this.generateRows} +
); } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 5259b9b49..2b9f32136 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -603,6 +603,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { event: async () => { // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); let activedocs = this.getActiveDocuments(); + ClientRecommender.Instance.reset_docs(); await Promise.all(activedocs.map((doc: Doc) => { console.log(StrCast(doc.title)); const extdoc = doc.data_ext as Doc; @@ -716,7 +717,7 @@ class CollectionFreeFormViewPannableContents extends React.Component otherwise, reactions won't fire return
{this.props.children} - +
; } } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From b8ab5a823ae22b021c09dfd713b77211a51b3eae Mon Sep 17 00:00:00 2001 From: ab Date: Fri, 9 Aug 2019 17:04:25 -0400 Subject: idk man --- solr-8.1.1/server/solr/dash/conf/schema.xml | 2 +- src/client/ClientRecommender.scss | 4 ++++ src/client/ClientRecommender.tsx | 17 ++++++++++------- src/client/util/SearchUtil.ts | 21 ++++++++++++++++++++- src/client/views/SearchBox.tsx | 2 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 6 +++++- src/server/Search.ts | 2 +- 7 files changed, 42 insertions(+), 12 deletions(-) (limited to 'src/client/views') diff --git a/solr-8.1.1/server/solr/dash/conf/schema.xml b/solr-8.1.1/server/solr/dash/conf/schema.xml index 8610786af..c0a4bab07 100644 --- a/solr-8.1.1/server/solr/dash/conf/schema.xml +++ b/solr-8.1.1/server/solr/dash/conf/schema.xml @@ -52,7 +52,7 @@ - + diff --git a/src/client/ClientRecommender.scss b/src/client/ClientRecommender.scss index 710d80f34..49163cdc8 100644 --- a/src/client/ClientRecommender.scss +++ b/src/client/ClientRecommender.scss @@ -5,4 +5,8 @@ border-spacing: 25px; border-collapse: separate; align-content: center; +} + +.wrapper{ + text-align: -webkit-center; } \ No newline at end of file diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 0793f0887..ddaa8a7fc 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -7,6 +7,7 @@ import { observer } from "mobx-react"; import { observable, action, computed, reaction } from "mobx"; var assert = require('assert'); import "./ClientRecommender.scss"; +import { JSXElement } from "babel-types"; export interface RecommenderProps { title: string; @@ -131,22 +132,24 @@ export class ClientRecommender extends React.Component { @computed private get generateRows() { const n = this.corr_matrix.length; - let rows: React.ReactElement[] = []; + let rows: JSX.Element[] = []; for (let i = 0; i < n; i++) { - let children: React.ReactElement[] = []; + let children: JSX.Element[] = []; for (let j = 0; j < n; j++) { - let cell = React.createElement("td", this.corr_matrix[i][j]); + //let cell = React.createElement("td", this.corr_matrix[i][j]); + let cell = {this.corr_matrix[i][j].toFixed(4)}; children.push(cell); } - let row = React.createElement("tr", { children: children }); + //let row = React.createElement("tr", { children: children, key: i }); + let row = {children}; rows.push(row); } return rows; } render() { - return (
-

{this.props.title ? this.props.title : "hello"}

+ return (
+

{this.props.title ? this.props.title : "hello"}

{/* @@ -159,7 +162,7 @@ export class ClientRecommender extends React.Component {
*/} - +
{this.generateRows} diff --git a/src/client/util/SearchUtil.ts b/src/client/util/SearchUtil.ts index ee5a83710..3a3ba1803 100644 --- a/src/client/util/SearchUtil.ts +++ b/src/client/util/SearchUtil.ts @@ -77,4 +77,23 @@ export namespace SearchUtil { aliasContexts.forEach(result => contexts.aliasContexts.push(...result.ids)); return contexts; } -} \ No newline at end of file + + export async function GetAllDocs() { + const query = "*"; + let response = await rp.get(Utils.prepend('/search'), { + qs: { + query + } + }); + let res: string[] = JSON.parse(response); + const fields = await DocServer.GetRefFields(res); + const docs: Doc[] = []; + for (const id of res) { + const field = fields[id]; + if (field instanceof Doc) { + docs.push(field); + } + } + return docs; + } +} diff --git a/src/client/views/SearchBox.tsx b/src/client/views/SearchBox.tsx index 33cb63df5..17f99fa05 100644 --- a/src/client/views/SearchBox.tsx +++ b/src/client/views/SearchBox.tsx @@ -47,7 +47,7 @@ export class SearchBox extends React.Component { } @action - getResults = async (query: string) => { + public getResults = async (query: string) => { let response = await rp.get(Utils.prepend('/search'), { qs: { query diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 2b9f32136..0beb0086b 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -38,6 +38,8 @@ import { MarqueeView } from "./MarqueeView"; import React = require("react"); import v5 = require("uuid/v5"); import { ClientRecommender } from "../../../ClientRecommender"; +import { SearchUtil } from "../../../util/SearchUtil"; +import { SearchBox } from "../../SearchBox"; library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBrain); @@ -603,9 +605,11 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { event: async () => { // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); let activedocs = this.getActiveDocuments(); + let allDocs = await SearchUtil.GetAllDocs(); + allDocs.forEach(doc => console.log(doc.title)); ClientRecommender.Instance.reset_docs(); await Promise.all(activedocs.map((doc: Doc) => { - console.log(StrCast(doc.title)); + //console.log(StrCast(doc.title)); const extdoc = doc.data_ext as Doc; return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); })); diff --git a/src/server/Search.ts b/src/server/Search.ts index 723dc101b..44c1b1298 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -14,7 +14,7 @@ export class Search { }); return res; } catch (e) { - // console.warn("Search error: " + e + document); + console.warn("Search error: " + e + document); } } -- cgit v1.2.3-70-g09d2 From 9dd2a31b72e5e527e2dae3b68f856ab8da879e93 Mon Sep 17 00:00:00 2001 From: ab Date: Mon, 12 Aug 2019 16:41:23 -0400 Subject: documentation --- package.json | 2 +- src/client/ClientRecommender.tsx | 18 +++++----- src/client/cognitive_services/CognitiveServices.ts | 42 ++++++++++++---------- src/client/util/SearchUtil.ts | 13 ++++--- .../collectionFreeForm/CollectionFreeFormView.tsx | 1 + src/server/Recommender.ts | 1 + 6 files changed, 45 insertions(+), 32 deletions(-) (limited to 'src/client/views') diff --git a/package.json b/package.json index 1e2c74411..1c7a10ac8 100644 --- a/package.json +++ b/package.json @@ -225,4 +225,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} \ No newline at end of file +} diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index ddaa8a7fc..63f85c737 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -75,13 +75,15 @@ export class ClientRecommender extends React.Component { const n = 200; const num_words = paragraph.size; let meanVector = new Array(n).fill(0); // mean vector - paragraph.forEach((wordvec: number[]) => { - for (let i = 0; i < n; i++) { - meanVector[i] += wordvec[i]; - } - }); - meanVector = meanVector.map(x => x / num_words); - this.addToDocSet(meanVector); + if (num_words > 0) { // check to see if paragraph actually was vectorized + paragraph.forEach((wordvec: number[]) => { + for (let i = 0; i < n; i++) { + meanVector[i] += wordvec[i]; + } + }); + meanVector = meanVector.map(x => x / num_words); + this.addToDocSet(meanVector); + } return meanVector; } @@ -106,7 +108,7 @@ export class ClientRecommender extends React.Component { }); return keyterms; }; - await CognitiveServices.Text.Manager.analyzer(extDoc, ["key words"], data, converter); + await CognitiveServices.Text.Appliers.analyzer(extDoc, ["key words"], data, converter); } /*** diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 954a05585..75d0760ed 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -258,32 +258,38 @@ export namespace CognitiveServices { }; console.log("requested!"); return request.post(options); - }, - analyzer: async (target: Doc, keys: string[], data: string, converter: Converter) => { - let results = await ExecuteQuery(Service.Text, Manager, data); + } + }; + + export namespace Appliers { + + export async function vectorize(keyterms: any) { + console.log("vectorizing..."); + //keyterms = ["father", "king"]; + let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true }; + await requestPromise.post(args).then(async (wordvecs) => { + var vectorValues = new Set(); + wordvecs.forEach((wordvec: any) => { + //console.log(wordvec.word); + vectorValues.add(wordvec.values as number[]); + }); + ClientRecommender.Instance.mean(vectorValues); + //console.log(vectorValues.size); + }); + } + + export const analyzer = async (target: Doc, keys: string[], data: string, converter: Converter) => { + let results = await ExecuteQuery(Service.Text, Manager, data); console.log(results); let keyterms = converter(results); //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis"); target[keys[0]] = keyterms; console.log("analyzed!"); await vectorize(keyterms); - } - }; - async function vectorize(keyterms: any) { - console.log("vectorizing..."); - //keyterms = ["father", "king"]; - let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true }; - await requestPromise.post(args).then(async (wordvecs) => { - var vectorValues = new Set(); - wordvecs.forEach((wordvec: any) => { - //console.log(wordvec.word); - vectorValues.add(wordvec.values as number[]); - }); - ClientRecommender.Instance.mean(vectorValues); - //console.log(vectorValues.size); - }); + }; } } + } \ No newline at end of file diff --git a/src/client/util/SearchUtil.ts b/src/client/util/SearchUtil.ts index 3a3ba1803..1fce995d7 100644 --- a/src/client/util/SearchUtil.ts +++ b/src/client/util/SearchUtil.ts @@ -82,18 +82,21 @@ export namespace SearchUtil { const query = "*"; let response = await rp.get(Utils.prepend('/search'), { qs: { - query + q: query } }); - let res: string[] = JSON.parse(response); - const fields = await DocServer.GetRefFields(res); + let result: IdSearchResult = JSON.parse(response); + const { ids, numFound, highlighting } = result; + const docMap = await DocServer.GetRefFields(ids); const docs: Doc[] = []; - for (const id of res) { - const field = fields[id]; + for (const id of ids) { + const field = docMap[id]; if (field instanceof Doc) { docs.push(field); } } return docs; + // const docs = ids.map((id: string) => docMap[id]).filter((doc: any) => doc instanceof Doc); + // return docs as Doc[]; } } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index e9791df4e..d1e8031fd 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -894,6 +894,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { let activedocs = this.getActiveDocuments(); let allDocs = await SearchUtil.GetAllDocs(); allDocs.forEach(doc => console.log(doc.title)); + // clears internal representation of documents as vectors ClientRecommender.Instance.reset_docs(); await Promise.all(activedocs.map((doc: Doc) => { //console.log(StrCast(doc.title)); diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts index d175b67c7..1c95d7ea4 100644 --- a/src/server/Recommender.ts +++ b/src/server/Recommender.ts @@ -70,6 +70,7 @@ export class Recommender { } if (this._model) { let word_vecs = this._model.getVectors(text); + return word_vecs; } } -- cgit v1.2.3-70-g09d2 From 843ac47f04f8c9bd687600f9f07fe4c15744e4ec Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed Date: Tue, 13 Aug 2019 11:12:41 -0400 Subject: recommendations --- src/Recommendations.scss | 21 ++++++ src/Recommendations.tsx | 28 ++++++++ src/client/views/nodes/DocumentView.tsx | 113 +++++++++++++++++--------------- 3 files changed, 108 insertions(+), 54 deletions(-) create mode 100644 src/Recommendations.scss create mode 100644 src/Recommendations.tsx (limited to 'src/client/views') diff --git a/src/Recommendations.scss b/src/Recommendations.scss new file mode 100644 index 000000000..5129a59d9 --- /dev/null +++ b/src/Recommendations.scss @@ -0,0 +1,21 @@ +.recommendation-content *{ + display: inline-block; + margin: auto; + border: 1px dashed grey; + padding: 2px 2px; +} + +.recommendation-content { + float: left; + border: 1px solid green; + width: 200px; + align-content: center; +} + +.rec-scroll { + overflow-y: scroll; + height: 300px; + width: auto; + position: absolute; + background: #cdcdcd; +} \ No newline at end of file diff --git a/src/Recommendations.tsx b/src/Recommendations.tsx new file mode 100644 index 000000000..ca1123ef9 --- /dev/null +++ b/src/Recommendations.tsx @@ -0,0 +1,28 @@ +import { observer } from "mobx-react"; +import React = require("react"); +import { Doc } from "./new_fields/Doc"; +import { NumCast } from "./new_fields/Types"; + +export interface RecProps { + documents: { preview: string, similarity: number }[], + node: Doc; +} + +@observer +export class Recommendations extends React.Component { + render() { + const transform = "translate(" + (NumCast(this.props.node.x) + 350) + "px, " + NumCast(this.props.node.y) + "px" + return ( +
+ {this.props.documents.map(doc => { + return ( +
+ +
{doc.similarity}
+
+ ) + })} +
+ ) + } +} \ No newline at end of file diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index c8eab85c2..24bcc0217 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -40,6 +40,7 @@ import React = require("react"); import { DictationManager } from '../../util/DictationManager'; import { MainView } from '../MainView'; import requestPromise = require('request-promise'); +import { Recommendations } from '../../../Recommendations'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faTrash); @@ -680,6 +681,7 @@ export class DocumentView extends DocComponent(Docu } render() { + let documents = [{ preview: "hi", similarity: 0 }]; let backgroundColor = this.layoutDoc.isBackground || (this.props.ContainingCollectionView && this.props.ContainingCollectionView.props.Document.clusterOverridesDefaultBackground && this.layoutDoc.backgroundColor === this.layoutDoc.defaultBackgroundColor) ? this.props.backgroundColor(this.layoutDoc) || StrCast(this.layoutDoc.backgroundColor) : StrCast(this.layoutDoc.backgroundColor) || this.props.backgroundColor(this.layoutDoc); @@ -699,61 +701,64 @@ export class DocumentView extends DocComponent(Docu let showTextTitle = showTitle && StrCast(this.layoutDoc.layout).startsWith(" - {!showTitle && !showCaption ? this.contents : -
- -
- {this.contents} -
- {!showTitle ? (null) : -
- StrCast((this.layoutDoc.isTemplate || !this.dataDoc ? this.layoutDoc : this.dataDoc)[showTitle!])} - SetValue={(value: string) => (Doc.GetProto(this.layoutDoc)[showTitle!] = value) ? true : true} - /> -
- } - {!showCaption ? (null) : -
- +
+
+ {!showTitle && !showCaption ? this.contents : +
+ +
+ {this.contents}
- } -
- } + {!showTitle ? (null) : +
+ StrCast((this.layoutDoc.isTemplate || !this.dataDoc ? this.layoutDoc : this.dataDoc)[showTitle!])} + SetValue={(value: string) => (Doc.GetProto(this.layoutDoc)[showTitle!] = value) ? true : true} + /> +
+ } + {!showCaption ? (null) : +
+ +
+ } +
+ } +
+
); } -- cgit v1.2.3-70-g09d2 From e03a1b2cc90e0fdb7789f4826e482e9040aa7075 Mon Sep 17 00:00:00 2001 From: ab Date: Tue, 13 Aug 2019 17:26:49 -0400 Subject: 80% done, garbage collection is much needed --- src/Recommendations.scss | 21 --- src/Recommendations.tsx | 28 ---- src/client/util/SearchUtil.ts | 7 +- src/client/views/MainView.tsx | 2 + src/client/views/Recommendations.scss | 65 ++++++++ src/client/views/Recommendations.tsx | 169 +++++++++++++++++++++ .../collectionFreeForm/CollectionFreeFormView.tsx | 13 +- src/client/views/nodes/DocumentView.tsx | 34 ++++- src/client/views/nodes/ImageBox.tsx | 21 ++- 9 files changed, 297 insertions(+), 63 deletions(-) delete mode 100644 src/Recommendations.scss delete mode 100644 src/Recommendations.tsx create mode 100644 src/client/views/Recommendations.scss create mode 100644 src/client/views/Recommendations.tsx (limited to 'src/client/views') diff --git a/src/Recommendations.scss b/src/Recommendations.scss deleted file mode 100644 index 5129a59d9..000000000 --- a/src/Recommendations.scss +++ /dev/null @@ -1,21 +0,0 @@ -.recommendation-content *{ - display: inline-block; - margin: auto; - border: 1px dashed grey; - padding: 2px 2px; -} - -.recommendation-content { - float: left; - border: 1px solid green; - width: 200px; - align-content: center; -} - -.rec-scroll { - overflow-y: scroll; - height: 300px; - width: auto; - position: absolute; - background: #cdcdcd; -} \ No newline at end of file diff --git a/src/Recommendations.tsx b/src/Recommendations.tsx deleted file mode 100644 index ca1123ef9..000000000 --- a/src/Recommendations.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import { observer } from "mobx-react"; -import React = require("react"); -import { Doc } from "./new_fields/Doc"; -import { NumCast } from "./new_fields/Types"; - -export interface RecProps { - documents: { preview: string, similarity: number }[], - node: Doc; -} - -@observer -export class Recommendations extends React.Component { - render() { - const transform = "translate(" + (NumCast(this.props.node.x) + 350) + "px, " + NumCast(this.props.node.y) + "px" - return ( -
- {this.props.documents.map(doc => { - return ( -
- -
{doc.similarity}
-
- ) - })} -
- ) - } -} \ No newline at end of file diff --git a/src/client/util/SearchUtil.ts b/src/client/util/SearchUtil.ts index 1fce995d7..85e593529 100644 --- a/src/client/util/SearchUtil.ts +++ b/src/client/util/SearchUtil.ts @@ -81,12 +81,13 @@ export namespace SearchUtil { export async function GetAllDocs() { const query = "*"; let response = await rp.get(Utils.prepend('/search'), { - qs: { - q: query - } + qs: + { start: 0, rows: 10000, q: query }, + }); let result: IdSearchResult = JSON.parse(response); const { ids, numFound, highlighting } = result; + console.log(ids.length); const docMap = await DocServer.GetRefFields(ids); const docs: Doc[] = []; for (const id of ids) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 419b15697..0b6fe3876 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -41,6 +41,7 @@ import { ClientUtils } from '../util/ClientUtils'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; //import { DocumentManager } from '../util/DocumentManager'; import { DictationManager } from '../util/DictationManager'; +import { Recommendations } from './Recommendations'; @observer export class MainView extends React.Component { @@ -581,6 +582,7 @@ export class MainView extends React.Component { {this.mainContent} + {this.nodesMenu()} {this.miscButtons} diff --git a/src/client/views/Recommendations.scss b/src/client/views/Recommendations.scss new file mode 100644 index 000000000..5d8f17e37 --- /dev/null +++ b/src/client/views/Recommendations.scss @@ -0,0 +1,65 @@ +@import "globalCssVariables"; + +.rec-content *{ + display: inline-block; + margin: auto; + width: 50; + height: 30px; + border: 1px dashed grey; + padding: 10px 10px; +} + +.rec-content { + float: left; + width: inherit; + align-content: center; +} + +.rec-scroll { + overflow-y: scroll; + overflow-x: hidden; + position: absolute; + // display: flex; + z-index: 10000; + box-shadow: gray 0.2vw 0.2vw 0.4vw; + // flex-direction: column; + background: whitesmoke; + padding-bottom: 10px; + border-radius: 15px; + border: solid #BBBBBBBB 1px; + width: 200px; + text-align: center; + max-height: 250px; + text-transform: uppercase; + color: grey; + letter-spacing: 2px; +} + +.content { + padding: 10px; + display: flex; + flex-direction: row; + align-items: center; + justify-content: center; +} + +.image-background { + pointer-events: none; + background-color: transparent; + width: 50%; + text-align: center; + height: 35px; + margin-left: 5px; +} + +img{ + width: 100%; + height: 100%; +} + +.score { + // margin-left: 15px; + width: 50%; + height: 100%; + text-align: center; +} diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx new file mode 100644 index 000000000..8569996b3 --- /dev/null +++ b/src/client/views/Recommendations.tsx @@ -0,0 +1,169 @@ +import { observer } from "mobx-react"; +import React = require("react"); +import { observable, action } from "mobx"; +import Measure from "react-measure"; +import "./Recommendations.scss"; +import { Doc, DocListCast, WidthSym, HeightSym } from "../../new_fields/Doc"; +import { DocumentIcon } from "./nodes/DocumentIcon"; +import { StrCast, NumCast } from "../../new_fields/Types"; +import { returnFalse, emptyFunction, returnEmptyString, returnOne } from "../../Utils"; +import { Transform } from "../util/Transform"; +import { ObjectField } from "../../new_fields/ObjectField"; +import { DocumentView } from "./nodes/DocumentView"; +import { DocumentType } from "../documents/Documents"; + + +export interface RecProps { + documents: { preview: Doc, similarity: number }[]; + node: Doc; +} + +@observer +export class Recommendations extends React.Component<{}> { + + static Instance: Recommendations; + @observable private _display: boolean = false; + @observable private _pageX: number = 0; + @observable private _pageY: number = 0; + @observable private _width: number = 0; + @observable private _height: number = 0; + @observable private _documents: { preview: Doc, score: number }[] = []; + + constructor(props: {}) { + super(props); + Recommendations.Instance = this; + } + + private DocumentIcon(doc: Doc) { + let layoutresult = StrCast(doc.type); + let renderDoc = doc; + //let box: number[] = []; + if (layoutresult.indexOf(DocumentType.COL) !== -1) { + renderDoc = Doc.MakeDelegate(renderDoc); + let bounds = DocListCast(renderDoc.data).reduce((bounds, doc) => { + var [sptX, sptY] = [NumCast(doc.x), NumCast(doc.y)]; + let [bptX, bptY] = [sptX + doc[WidthSym](), sptY + doc[HeightSym]()]; + return { + x: Math.min(sptX, bounds.x), y: Math.min(sptY, bounds.y), + r: Math.max(bptX, bounds.r), b: Math.max(bptY, bounds.b) + }; + }, { x: Number.MAX_VALUE, y: Number.MAX_VALUE, r: Number.MIN_VALUE, b: Number.MIN_VALUE }); + } + let returnXDimension = () => 50; + let returnYDimension = () => 50; + let scale = () => returnXDimension() / NumCast(renderDoc.nativeWidth, returnXDimension()); + let newRenderDoc = Doc.MakeDelegate(renderDoc); /// newRenderDoc -> renderDoc -> render"data"Doc -> TextProt + const docview =
+ {/* onPointerDown={action(() => { + this._useIcons = !this._useIcons; + this._displayDim = this._useIcons ? 50 : Number(SEARCH_THUMBNAIL_SIZE); + })} + onPointerEnter={action(() => this._displayDim = this._useIcons ? 50 : Number(SEARCH_THUMBNAIL_SIZE))} + onPointerLeave={action(() => this._displayDim = 50)} > */} + +
; + const data = renderDoc.data; + if (data instanceof ObjectField) newRenderDoc.data = ObjectField.MakeCopy(data); + newRenderDoc.preview = true; + return docview; + + } + + @action + closeMenu = () => { + this._display = false; + } + + @action + resetDocuments = () => { + this._documents = []; + } + + @action + addDocuments = (documents: { preview: Doc, score: number }[]) => { + this._documents = documents; + } + + @action + displayRecommendations(x: number, y: number) { + this._pageX = x; + this._pageY = y; + this._display = true; + } + + static readonly buffer = 20; + + get pageX() { + const x = this._pageX; + if (x < 0) { + return 0; + } + const width = this._width; + if (x + width > window.innerWidth - Recommendations.buffer) { + return window.innerWidth - Recommendations.buffer - width; + } + return x; + } + + get pageY() { + const y = this._pageY; + if (y < 0) { + return 0; + } + const height = this._height; + if (y + height > window.innerHeight - Recommendations.buffer) { + return window.innerHeight - Recommendations.buffer - height; + } + return y; + } + + render() { + if (!this._display) { + return null; + } + let style = { left: this.pageX, top: this.pageY }; + //const transform = "translate(" + (NumCast(this.props.node.x) + 350) + "px, " + NumCast(this.props.node.y) + "px" + return ( + { this._width = r.offset.width; this._height = r.offset.height; })}> + {({ measureRef }) => ( +
+

Recommendations

+ {this._documents.map(doc => { + return ( +
+ + {this.DocumentIcon(doc.preview)} + + {doc.score} +
+ ); + })} + +
+ ) + } + + + ); + } +} \ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index d1e8031fd..50f7e2dc8 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -896,10 +896,13 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { allDocs.forEach(doc => console.log(doc.title)); // clears internal representation of documents as vectors ClientRecommender.Instance.reset_docs(); - await Promise.all(activedocs.map((doc: Doc) => { - //console.log(StrCast(doc.title)); - const extdoc = doc.data_ext as Doc; - return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + await Promise.all(allDocs.map((doc: Doc) => { + console.log(StrCast(doc.title)); + if (doc.type === DocumentType.IMG) { + console.log(doc.title); + const extdoc = doc.data_ext as Doc; + return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + } })); console.log(ClientRecommender.Instance.createDistanceMatrix()); }, @@ -1013,7 +1016,7 @@ class CollectionFreeFormViewPannableContents extends React.Component otherwise, reactions won't fire return
{this.props.children} - + {/* */}
; } } \ No newline at end of file diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 24bcc0217..3ce4dbd4f 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -40,9 +40,13 @@ import React = require("react"); import { DictationManager } from '../../util/DictationManager'; import { MainView } from '../MainView'; import requestPromise = require('request-promise'); -import { Recommendations } from '../../../Recommendations'; +import { Recommendations } from '../Recommendations'; +import { SearchUtil } from '../../util/SearchUtil'; +import { ClientRecommender } from '../../ClientRecommender'; +import { DocumentType } from '../../documents/Documents'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? +library.add(fa.faBrain); library.add(fa.faTrash); library.add(fa.faShare); library.add(fa.faDownload); @@ -610,6 +614,33 @@ export class DocumentView extends DocComponent(Docu a.click(); } }); + cm.addItem({ + description: "Recommender System", + event: async () => { + if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); + let documents: Doc[] = []; + let allDocs = await SearchUtil.GetAllDocs(); + allDocs.forEach(doc => console.log(doc.title)); + // clears internal representation of documents as vectors + ClientRecommender.Instance.reset_docs(); + await Promise.all(allDocs.map((doc: Doc) => { + if (doc.type === DocumentType.IMG) { + console.log(StrCast(doc.title)); + documents.push(doc); + const extdoc = doc.data_ext as Doc; + return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + } + })); + console.log(ClientRecommender.Instance.createDistanceMatrix()); + let recDocs: { preview: Doc, score: number }[] = []; + for (let i = 0; i < documents.length; i++) { + recDocs.push({ preview: documents[i], score: i }); + } + Recommendations.Instance.addDocuments(recDocs); + Recommendations.Instance.displayRecommendations(e.pageX + 100, e.pageY); + }, + icon: "brain" + }); cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" }); type User = { email: string, userDocumentId: string }; let usersMenu: ContextMenuProps[] = []; @@ -758,7 +789,6 @@ export class DocumentView extends DocComponent(Docu
} - ); } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 73b892e26..45d389ba6 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -28,6 +28,9 @@ import FaceRectangles from './FaceRectangles'; import { FieldView, FieldViewProps } from './FieldView'; import "./ImageBox.scss"; import React = require("react"); +import { SearchUtil } from '../../util/SearchUtil'; +import { ClientRecommender } from '../../ClientRecommender'; +import { DocumentType } from '../../documents/Documents'; var requestImageSize = require('../../util/request-image-size'); var path = require('path'); const { Howl } = require('howler'); @@ -240,10 +243,20 @@ export class ImageBox extends DocComponent(ImageD } } - extractText = () => { - //Recommender.Instance.extractText(this.dataDoc, this.extensionDoc); - // request recommender - //fetch(Utils.prepend("/recommender"), { body: body, method: "POST", headers: { "content-type": "application/json" } }).then((value) => console.log(value)); + extractText = async () => { + //let activedocs = this.getActiveDocuments(); + let allDocs = await SearchUtil.GetAllDocs(); + allDocs.forEach(doc => console.log(doc.title)); + // clears internal representation of documents as vectors + ClientRecommender.Instance.reset_docs(); + await Promise.all(allDocs.map((doc: Doc) => { + //console.log(StrCast(doc.title)); + if (doc.type === DocumentType.IMG) { + const extdoc = doc.data_ext as Doc; + return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + } + })); + console.log(ClientRecommender.Instance.createDistanceMatrix()); } generateMetadata = (threshold: Confidence = Confidence.Excellent) => { -- cgit v1.2.3-70-g09d2 From e0bfe978e029268b3901b5d098f946b1a6fc7d0d Mon Sep 17 00:00:00 2001 From: ab Date: Thu, 29 Aug 2019 18:43:32 -0400 Subject: ui fixes, datadoc resolved --- src/client/ClientRecommender.tsx | 83 +++++++++++++++------- src/client/cognitive_services/CognitiveServices.ts | 24 ++++--- src/client/views/MainView.tsx | 1 + src/client/views/Recommendations.tsx | 21 ++++-- .../collectionFreeForm/CollectionFreeFormView.tsx | 21 ------ src/client/views/nodes/DocumentView.tsx | 26 ++++--- src/client/views/nodes/ImageBox.tsx | 16 ----- 7 files changed, 104 insertions(+), 88 deletions(-) (limited to 'src/client/views') diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 63f85c737..a6d1a32b3 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -1,5 +1,5 @@ import { Doc } from "../new_fields/Doc"; -import { StrCast } from "../new_fields/Types"; +import { StrCast, Cast } from "../new_fields/Types"; import { List } from "../new_fields/List"; import { CognitiveServices } from "./cognitive_services/CognitiveServices"; import React = require("react"); @@ -8,30 +8,42 @@ import { observable, action, computed, reaction } from "mobx"; var assert = require('assert'); import "./ClientRecommender.scss"; import { JSXElement } from "babel-types"; +import { ToPlainText, RichTextField } from "../new_fields/RichTextField"; export interface RecommenderProps { title: string; } +export interface RecommenderDocument { + actualDoc: Doc; + vectorDoc: number[]; + score: number; +} + @observer export class ClientRecommender extends React.Component { static Instance: ClientRecommender; - private docVectors: Set; + private mainDoc?: RecommenderDocument; + private docVectors: Set = new Set(); @observable private corr_matrix = [[0, 0], [0, 0]]; constructor(props: RecommenderProps) { //console.log("creating client recommender..."); super(props); if (!ClientRecommender.Instance) ClientRecommender.Instance = this; - this.docVectors = new Set(); - //this.corr_matrix = [[0, 0], [0, 0]]; + ClientRecommender.Instance.docVectors = new Set(); + //ClientRecommender.Instance.corr_matrix = [[0, 0], [0, 0]]; } @action public reset_docs() { - this.docVectors = new Set(); - this.corr_matrix = [[0, 0], [0, 0]]; + ClientRecommender.Instance.docVectors = new Set(); + ClientRecommender.Instance.corr_matrix = [[0, 0], [0, 0]]; + } + + public deleteDocs() { + console.log("deleting previews..."); } /*** @@ -67,11 +79,24 @@ export class ClientRecommender extends React.Component { } } + public computeSimilarities() { + ClientRecommender.Instance.docVectors.forEach((doc: RecommenderDocument) => { + if (ClientRecommender.Instance.mainDoc) { + const distance = ClientRecommender.Instance.distance(ClientRecommender.Instance.mainDoc.vectorDoc, doc.vectorDoc, "euclidian"); + doc.score = distance; + } + } + ); + let doclist = Array.from(ClientRecommender.Instance.docVectors); + doclist.sort((a: RecommenderDocument, b: RecommenderDocument) => a.score - b.score); + return doclist; + } + /*** * Computes the mean of a set of vectors */ - public mean(paragraph: Set) { + public mean(paragraph: Set, dataDoc: Doc, mainDoc: boolean) { const n = 200; const num_words = paragraph.size; let meanVector = new Array(n).fill(0); // mean vector @@ -82,14 +107,16 @@ export class ClientRecommender extends React.Component { } }); meanVector = meanVector.map(x => x / num_words); - this.addToDocSet(meanVector); + const internalDoc: RecommenderDocument = { actualDoc: dataDoc, vectorDoc: meanVector, score: 0 }; + if (mainDoc) ClientRecommender.Instance.mainDoc = internalDoc; + ClientRecommender.Instance.addToDocSet(internalDoc); } return meanVector; } - private addToDocSet(vector: number[]) { - if (this.docVectors) { - this.docVectors.add(vector); + private addToDocSet(internalDoc: RecommenderDocument) { + if (ClientRecommender.Instance.docVectors) { + ClientRecommender.Instance.docVectors.add(internalDoc); } } @@ -97,9 +124,11 @@ export class ClientRecommender extends React.Component { * Uses Cognitive Services to extract keywords from a document */ - public async extractText(dataDoc: Doc, extDoc: Doc) { - let data = StrCast(dataDoc.title); - //console.log(data); + public async extractText(dataDoc: Doc, extDoc: Doc, mainDoc: boolean = false) { + let fielddata = Cast(dataDoc.data, RichTextField); + let data: string; + fielddata ? data = fielddata[ToPlainText]() : data = ""; + console.log(data); let converter = (results: any) => { let keyterms = new List(); results.documents.forEach((doc: any) => { @@ -108,7 +137,7 @@ export class ClientRecommender extends React.Component { }); return keyterms; }; - await CognitiveServices.Text.Appliers.analyzer(extDoc, ["key words"], data, converter); + await CognitiveServices.Text.Appliers.analyzer(dataDoc, extDoc, ["key words"], data, converter, mainDoc); } /*** @@ -116,7 +145,7 @@ export class ClientRecommender extends React.Component { */ @action - public createDistanceMatrix(documents: Set = this.docVectors) { + public createDistanceMatrix(documents: Set = ClientRecommender.Instance.docVectors) { const documents_list = Array.from(documents); const n = documents_list.length; var matrix = new Array(n).fill(0).map(() => new Array(n).fill(0)); @@ -124,22 +153,22 @@ export class ClientRecommender extends React.Component { var doc1 = documents_list[i]; for (let j = 0; j < n; j++) { var doc2 = documents_list[j]; - matrix[i][j] = this.distance(doc1, doc2, "euclidian"); + matrix[i][j] = ClientRecommender.Instance.distance(doc1.vectorDoc, doc2.vectorDoc, "euclidian"); } } - this.corr_matrix = matrix; + ClientRecommender.Instance.corr_matrix = matrix; return matrix; } @computed private get generateRows() { - const n = this.corr_matrix.length; + const n = ClientRecommender.Instance.corr_matrix.length; let rows: JSX.Element[] = []; for (let i = 0; i < n; i++) { let children: JSX.Element[] = []; for (let j = 0; j < n; j++) { - //let cell = React.createElement("td", this.corr_matrix[i][j]); - let cell = ; + //let cell = React.createElement("td", ClientRecommender.Instance.corr_matrix[i][j]); + let cell = ; children.push(cell); } //let row = React.createElement("tr", { children: children, key: i }); @@ -151,22 +180,22 @@ export class ClientRecommender extends React.Component { render() { return (
-

{this.props.title ? this.props.title : "hello"}

+

{ClientRecommender.Instance.props.title ? ClientRecommender.Instance.props.title : "hello"}

{/*
{this.corr_matrix[i][j].toFixed(4)}{ClientRecommender.Instance.corr_matrix[i][j].toFixed(4)}
- - + + - - + +
{this.corr_matrix[0][0].toFixed(4)}{this.corr_matrix[0][1].toFixed(4)}{ClientRecommender.Instance.corr_matrix[0][0].toFixed(4)}{ClientRecommender.Instance.corr_matrix[0][1].toFixed(4)}
{this.corr_matrix[1][0].toFixed(4)}{this.corr_matrix[1][1].toFixed(4)}{ClientRecommender.Instance.corr_matrix[1][0].toFixed(4)}{ClientRecommender.Instance.corr_matrix[1][1].toFixed(4)}
*/} - {this.generateRows} + {ClientRecommender.Instance.generateRows}
); diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 75d0760ed..874ee433d 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -263,29 +263,35 @@ export namespace CognitiveServices { export namespace Appliers { - export async function vectorize(keyterms: any) { + export async function vectorize(keyterms: any, dataDoc: Doc, mainDoc: boolean = false) { console.log("vectorizing..."); //keyterms = ["father", "king"]; let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true }; await requestPromise.post(args).then(async (wordvecs) => { - var vectorValues = new Set(); - wordvecs.forEach((wordvec: any) => { - //console.log(wordvec.word); - vectorValues.add(wordvec.values as number[]); - }); - ClientRecommender.Instance.mean(vectorValues); + if (wordvecs.length > 0) { + console.log("successful vectorization!"); + var vectorValues = new Set(); + wordvecs.forEach((wordvec: any) => { + //console.log(wordvec.word); + vectorValues.add(wordvec.values as number[]); + }); + ClientRecommender.Instance.mean(vectorValues, dataDoc, mainDoc); + } // adds document to internal doc set + else { + console.log("unsuccessful :( word(s) not in vocabulary"); + } //console.log(vectorValues.size); }); } - export const analyzer = async (target: Doc, keys: string[], data: string, converter: Converter) => { + export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: Converter, mainDoc: boolean = false) => { let results = await ExecuteQuery(Service.Text, Manager, data); console.log(results); let keyterms = converter(results); //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis"); target[keys[0]] = keyterms; console.log("analyzed!"); - await vectorize(keyterms); + await vectorize(keyterms, dataDoc, mainDoc); }; } diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 57eb30439..3a5795077 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -204,6 +204,7 @@ export class MainView extends React.Component { const targets = document.elementsFromPoint(e.x, e.y); if (targets && targets.length && targets[0].className.toString().indexOf("contextMenu") === -1) { ContextMenu.Instance.closeMenu(); + Recommendations.Instance.closeMenu(); } }); diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx index 8569996b3..cf1974c69 100644 --- a/src/client/views/Recommendations.tsx +++ b/src/client/views/Recommendations.tsx @@ -10,8 +10,10 @@ import { returnFalse, emptyFunction, returnEmptyString, returnOne } from "../../ import { Transform } from "../util/Transform"; import { ObjectField } from "../../new_fields/ObjectField"; import { DocumentView } from "./nodes/DocumentView"; -import { DocumentType } from "../documents/Documents"; - +import { DocumentType } from '../documents/DocumentTypes'; +import { ClientRecommender } from "../ClientRecommender"; +import { DocServer } from "../DocServer"; +import { Id } from "../../new_fields/FieldSymbols"; export interface RecProps { documents: { preview: Doc, similarity: number }[]; @@ -28,6 +30,7 @@ export class Recommendations extends React.Component<{}> { @observable private _width: number = 0; @observable private _height: number = 0; @observable private _documents: { preview: Doc, score: number }[] = []; + private previewDocs: Doc[] = []; constructor(props: {}) { super(props); @@ -52,7 +55,8 @@ export class Recommendations extends React.Component<{}> { let returnXDimension = () => 50; let returnYDimension = () => 50; let scale = () => returnXDimension() / NumCast(renderDoc.nativeWidth, returnXDimension()); - let newRenderDoc = Doc.MakeDelegate(renderDoc); /// newRenderDoc -> renderDoc -> render"data"Doc -> TextProt + //let scale = () => 1; + //let newRenderDoc = Doc.MakeDelegate(renderDoc); /// newRenderDoc -> renderDoc -> render"data"Doc -> TextProt const docview =
{/* onPointerDown={action(() => { this._useIcons = !this._useIcons; @@ -62,7 +66,7 @@ export class Recommendations extends React.Component<{}> { onPointerLeave={action(() => this._displayDim = 50)} > */} { ContentScaling={scale} />
; - const data = renderDoc.data; - if (data instanceof ObjectField) newRenderDoc.data = ObjectField.MakeCopy(data); - newRenderDoc.preview = true; + // const data = renderDoc.data; + // if (data instanceof ObjectField) newRenderDoc.data = ObjectField.MakeCopy(data); + // newRenderDoc.preview = true; + // this.previewDocs.push(newRenderDoc); return docview; } @@ -92,6 +97,8 @@ export class Recommendations extends React.Component<{}> { @action closeMenu = () => { this._display = false; + this.previewDocs.forEach(doc => DocServer.DeleteDocument(doc[Id])); + this.previewDocs = []; } @action diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 2d4775070..3cef93383 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -857,27 +857,6 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { input.click(); } }); - ContextMenu.Instance.addItem({ - description: "Recommender System", - event: async () => { - // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); - let activedocs = this.getActiveDocuments(); - let allDocs = await SearchUtil.GetAllDocs(); - allDocs.forEach(doc => console.log(doc.title)); - // clears internal representation of documents as vectors - ClientRecommender.Instance.reset_docs(); - await Promise.all(allDocs.map((doc: Doc) => { - console.log(StrCast(doc.title)); - if (doc.type === DocumentType.IMG) { - console.log(doc.title); - const extdoc = doc.data_ext as Doc; - return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); - } - })); - console.log(ClientRecommender.Instance.createDistanceMatrix()); - }, - icon: "brain" - }); layoutItems.push({ description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`, event: this.fitToContainer, icon: !this.fitToBox ? "expand-arrows-alt" : "compress-arrows-alt" }); layoutItems.push({ description: "reset view", event: () => { this.props.Document.panX = this.props.Document.panY = 0; this.props.Document.scale = 1; }, icon: "compress-arrows-alt" }); layoutItems.push({ diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 2a6e91272..f708a7a3a 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -648,21 +648,31 @@ export class DocumentView extends DocComponent(Docu if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); let documents: Doc[] = []; let allDocs = await SearchUtil.GetAllDocs(); - allDocs.forEach(doc => console.log(doc.title)); + //allDocs.forEach(doc => console.log(doc.title)); // clears internal representation of documents as vectors ClientRecommender.Instance.reset_docs(); await Promise.all(allDocs.map((doc: Doc) => { - if (doc.type === DocumentType.IMG) { - console.log(StrCast(doc.title)); - documents.push(doc); - const extdoc = doc.data_ext as Doc; - return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + let mainDoc: boolean = false; + const dataDoc = Doc.GetDataDoc(doc); + if (doc.type === DocumentType.TEXT) { + if (dataDoc === Doc.GetDataDoc(this.props.Document)) { + mainDoc = true; + console.log(StrCast(doc.title)); + } + if (!documents.includes(dataDoc)) { + documents.push(dataDoc); + const extdoc = doc.data_ext as Doc; + return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc, mainDoc); + } } })); console.log(ClientRecommender.Instance.createDistanceMatrix()); + const doclist = ClientRecommender.Instance.computeSimilarities(); let recDocs: { preview: Doc, score: number }[] = []; - for (let i = 0; i < documents.length; i++) { - recDocs.push({ preview: documents[i], score: i }); + // tslint:disable-next-line: prefer-for-of + for (let i = 0; i < doclist.length; i++) { + console.log(doclist[i].score); + recDocs.push({ preview: doclist[i].actualDoc, score: doclist[i].score }); } Recommendations.Instance.addDocuments(recDocs); Recommendations.Instance.displayRecommendations(e.pageX + 100, e.pageY); diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index ec35465eb..d94e92847 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -240,22 +240,6 @@ export class ImageBox extends DocComponent(ImageD } } - extractText = async () => { - //let activedocs = this.getActiveDocuments(); - let allDocs = await SearchUtil.GetAllDocs(); - allDocs.forEach(doc => console.log(doc.title)); - // clears internal representation of documents as vectors - ClientRecommender.Instance.reset_docs(); - await Promise.all(allDocs.map((doc: Doc) => { - //console.log(StrCast(doc.title)); - if (doc.type === DocumentType.IMG) { - const extdoc = doc.data_ext as Doc; - return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); - } - })); - console.log(ClientRecommender.Instance.createDistanceMatrix()); - } - generateMetadata = (threshold: Confidence = Confidence.Excellent) => { let converter = (results: any) => { let tagDoc = new Doc; -- cgit v1.2.3-70-g09d2 From 6f6e5c763d9a398e897df94b89fcc7d3845c318e Mon Sep 17 00:00:00 2001 From: ab Date: Fri, 30 Aug 2019 17:57:39 -0400 Subject: minor fixes --- src/client/ClientRecommender.tsx | 6 +++++- src/client/views/Recommendations.scss | 3 ++- src/client/views/Recommendations.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) (limited to 'src/client/views') diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index a6d1a32b3..9953700cc 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -39,6 +39,7 @@ export class ClientRecommender extends React.Component { @action public reset_docs() { ClientRecommender.Instance.docVectors = new Set(); + ClientRecommender.Instance.mainDoc = undefined; ClientRecommender.Instance.corr_matrix = [[0, 0], [0, 0]]; } @@ -133,7 +134,10 @@ export class ClientRecommender extends React.Component { let keyterms = new List(); results.documents.forEach((doc: any) => { let keyPhrases = doc.keyPhrases; - keyPhrases.map((kp: string) => keyterms.push(kp)); + keyPhrases.map((kp: string) => { + const words = kp.split(" "); + words.forEach((word) => keyterms.push(word)); + }); }); return keyterms; }; diff --git a/src/client/views/Recommendations.scss b/src/client/views/Recommendations.scss index 5d8f17e37..6619d8df3 100644 --- a/src/client/views/Recommendations.scss +++ b/src/client/views/Recommendations.scss @@ -27,7 +27,7 @@ padding-bottom: 10px; border-radius: 15px; border: solid #BBBBBBBB 1px; - width: 200px; + width: 250px; text-align: center; max-height: 250px; text-transform: uppercase; @@ -62,4 +62,5 @@ img{ width: 50%; height: 100%; text-align: center; + margin-left: 10px; } diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx index cf1974c69..63aa12c29 100644 --- a/src/client/views/Recommendations.tsx +++ b/src/client/views/Recommendations.tsx @@ -161,7 +161,7 @@ export class Recommendations extends React.Component<{}> { {this.DocumentIcon(doc.preview)} - {doc.score} + {doc.score.toFixed(4)}
); })} diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index f708a7a3a..d51c90b61 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -648,7 +648,7 @@ export class DocumentView extends DocComponent(Docu if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); let documents: Doc[] = []; let allDocs = await SearchUtil.GetAllDocs(); - //allDocs.forEach(doc => console.log(doc.title)); + // allDocs.forEach(doc => console.log(doc.title)); // clears internal representation of documents as vectors ClientRecommender.Instance.reset_docs(); await Promise.all(allDocs.map((doc: Doc) => { -- cgit v1.2.3-70-g09d2 From a1c2afe27c75354d4365a79ea202eca94516069e Mon Sep 17 00:00:00 2001 From: ab Date: Wed, 4 Sep 2019 09:52:39 -0400 Subject: stopwords, frequency, proto arxiv --- package.json | 2 + src/client/ClientRecommender.tsx | 75 ++++++++++++++++++++-- src/client/cognitive_services/CognitiveServices.ts | 13 ++-- src/client/util/TooltipTextMenu.scss | 2 +- src/client/views/nodes/DocumentView.tsx | 1 + src/server/Recommender.ts | 29 +++++++++ src/server/index.ts | 1 + 7 files changed, 113 insertions(+), 10 deletions(-) (limited to 'src/client/views') diff --git a/package.json b/package.json index ec5af93b1..d4b5bdab6 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", + "arxiv-api-node": "0.0.2", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", @@ -218,6 +219,7 @@ "socket.io-client": "^2.2.0", "solr-node": "^1.2.1", "standard-http-error": "^2.0.1", + "stopword": "^0.3.3", "typescript-collections": "^1.3.2", "url-loader": "^1.1.2", "uuid": "^3.3.2", diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 9953700cc..66f0ae745 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -6,6 +6,7 @@ import React = require("react"); import { observer } from "mobx-react"; import { observable, action, computed, reaction } from "mobx"; var assert = require('assert'); +var sw = require('stopword'); import "./ClientRecommender.scss"; import { JSXElement } from "babel-types"; import { ToPlainText, RichTextField } from "../new_fields/RichTextField"; @@ -130,20 +131,86 @@ export class ClientRecommender extends React.Component { let data: string; fielddata ? data = fielddata[ToPlainText]() : data = ""; console.log(data); - let converter = (results: any) => { + let converter = (results: any, data: string) => { let keyterms = new List(); + let keyterms_counted = new List(); results.documents.forEach((doc: any) => { let keyPhrases = doc.keyPhrases; keyPhrases.map((kp: string) => { - const words = kp.split(" "); - words.forEach((word) => keyterms.push(word)); + const frequency = this.countFrequencies(kp, data); + let words = kp.split(" "); // separates phrase into words + words = this.removeStopWords(words); + words.forEach((word) => { + keyterms.push(word); + for (let i = 0; i < frequency; i++) { + keyterms_counted.push(word); + } + }); + }); + }); + return { keyterms: keyterms, keyterms_counted: keyterms_counted }; + }; + let test = (results: any, data: string) => { + results.documents.forEach((doc: any) => { + let kps = doc.keyPhrases; + kps.map((kp: string) => { + this.countFrequencies(kp, data); }); }); - return keyterms; }; await CognitiveServices.Text.Appliers.analyzer(dataDoc, extDoc, ["key words"], data, converter, mainDoc); } + private countFrequencies(keyphrase: string, paragraph: string) { + let data = paragraph.split(" "); + let kp_array = keyphrase.split(" "); + let num_keywords = kp_array.length; + let par_length = data.length; + let frequency = 0; + // console.log("Paragraph: ", data); + // console.log("Keyphrases:", kp_array); + for (let i = 0; i <= par_length - num_keywords; i++) { + const window = data.slice(i, i + num_keywords); + if (JSON.stringify(window) === JSON.stringify(kp_array)) { + frequency++; + } + } + return frequency; + } + + private removeStopWords(word_array: string[]) { + //console.log(sw.removeStopwords(word_array)); + return sw.removeStopwords(word_array); + } + + /** + * Request to the arXiv server for ML articles. + */ + + arxivrequest = async (query: string) => { + let xhttp = new XMLHttpRequest(); + let serveraddress = "http://export.arxiv.org/api" + let endpoint = serveraddress + "/query?search_query=all:" + query + "&start=0&max_results=5"; + let promisified = (resolve: any, reject: any) => { + xhttp.onreadystatechange = function () { + if (this.readyState === 4) { + let result = xhttp.response; + switch (this.status) { + case 200: + console.log(result); + return resolve(result); + case 400: + default: + return reject(result); + } + } + }; + xhttp.open("GET", endpoint, true); + xhttp.send(); + }; + return new Promise(promisified); + } + /*** * Creates distance matrix for all Documents analyzed */ diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 874ee433d..eb1dd5197 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -15,6 +15,7 @@ type RequestExecutor = (apiKey: string, body: string, service: Service) => Promi type AnalysisApplier = (target: Doc, relevantKeys: string[], data: D, ...args: any) => any; type BodyConverter = (data: D) => string; type Converter = (results: any) => Field; +type TextConverter = (results: any, data: string) => { keyterms: Field, keyterms_counted: Field }; export type Tag = { name: string, confidence: number }; export type Rectangle = { top: number, left: number, width: number, height: number }; @@ -263,7 +264,7 @@ export namespace CognitiveServices { export namespace Appliers { - export async function vectorize(keyterms: any, dataDoc: Doc, mainDoc: boolean = false) { + export async function vectorize(keyterms: any, dataDoc: Doc, mainDoc: boolean = false, data: string) { console.log("vectorizing..."); //keyterms = ["father", "king"]; let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true }; @@ -284,15 +285,17 @@ export namespace CognitiveServices { }); } - export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: Converter, mainDoc: boolean = false) => { + export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: TextConverter, mainDoc: boolean = false) => { let results = await ExecuteQuery(Service.Text, Manager, data); console.log(results); - let keyterms = converter(results); + let keyterms = converter(results, data); //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis"); - target[keys[0]] = keyterms; + target[keys[0]] = keyterms.keyterms; console.log("analyzed!"); - await vectorize(keyterms, dataDoc, mainDoc); + await vectorize(keyterms.keyterms_counted, dataDoc, mainDoc, data); }; + + // export async function countFrequencies() } } diff --git a/src/client/util/TooltipTextMenu.scss b/src/client/util/TooltipTextMenu.scss index ebf833dbe..ab6cee763 100644 --- a/src/client/util/TooltipTextMenu.scss +++ b/src/client/util/TooltipTextMenu.scss @@ -351,5 +351,5 @@ .dragger{ color: #eee; - margin-left: 5px; + margin: 5px; } \ No newline at end of file diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index d51c90b61..a1e64f1c5 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -651,6 +651,7 @@ export class DocumentView extends DocComponent(Docu // allDocs.forEach(doc => console.log(doc.title)); // clears internal representation of documents as vectors ClientRecommender.Instance.reset_docs(); + ClientRecommender.Instance.arxivrequest("electrons"); await Promise.all(allDocs.map((doc: Doc) => { let mainDoc: boolean = false; const dataDoc = Doc.GetDataDoc(doc); diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts index 781974208..efb5fbbbf 100644 --- a/src/server/Recommender.ts +++ b/src/server/Recommender.ts @@ -5,6 +5,9 @@ var w2v = require('word2vec'); var assert = require('assert'); +var arxivapi = require('arxiv-api-node'); +import requestPromise = require("request-promise"); + export class Recommender { @@ -75,6 +78,32 @@ export class Recommender { } } + public async arxivRequest(query: string) { + // let xhttp = new XMLHttpRequest(); + // let serveraddress = "http://export.arxiv.org/api/query?search_query=all:electron&start=0&max_results=1"; + // let promisified = (resolve: any, reject: any) => { + // xhttp.onreadystatechange = function () { + // if (this.readyState === 4) { + // let result = xhttp.response; + // switch (this.status) { + // case 200: + // console.log(result); + // return resolve(result); + // case 400: + // default: + // return reject(result); + // } + // } + // }; + // xhttp.open("GET", serveraddress, true); + // xhttp.send(); + // }; + // return new Promise(promisified); + + let res = await arxivapi.query("all:electrons"); + console.log(res); + } + diff --git a/src/server/index.ts b/src/server/index.ts index e1ecc4ac0..24ffc466f 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -688,6 +688,7 @@ app.use(RouteStore.corsProxy, (req, res) => { let recommender = new Recommender(); recommender.testModel(); +recommender.arxivRequest("Triangle-GAN"); app.post("/recommender", async (req, res) => { let keyphrases = req.body.keyphrases; -- cgit v1.2.3-70-g09d2 From 435145b366e877c9c10a91408f6fdfc5b612e7f4 Mon Sep 17 00:00:00 2001 From: ab Date: Sat, 7 Sep 2019 17:38:52 -0400 Subject: parser for arxiv api response --- package.json | 1 + src/client/ClientRecommender.tsx | 57 ++++++++++++++++++++++------ src/client/views/Recommendations.tsx | 6 ++- src/client/views/nodes/DocumentView.tsx | 67 ++++++++++++++++----------------- 4 files changed, 84 insertions(+), 47 deletions(-) (limited to 'src/client/views') diff --git a/package.json b/package.json index d4b5bdab6..3486f752b 100644 --- a/package.json +++ b/package.json @@ -140,6 +140,7 @@ "express-session": "^1.15.6", "express-validator": "^5.3.1", "expressjs": "^1.0.1", + "feedparser": "^2.2.9", "flexlayout-react": "^0.3.3", "font-awesome": "^4.7.0", "formidable": "^1.2.1", diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 66f0ae745..0e344dae9 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -7,6 +7,7 @@ import { observer } from "mobx-react"; import { observable, action, computed, reaction } from "mobx"; var assert = require('assert'); var sw = require('stopword'); +var FeedParser = require('feedparser'); import "./ClientRecommender.scss"; import { JSXElement } from "babel-types"; import { ToPlainText, RichTextField } from "../new_fields/RichTextField"; @@ -130,14 +131,23 @@ export class ClientRecommender extends React.Component { let fielddata = Cast(dataDoc.data, RichTextField); let data: string; fielddata ? data = fielddata[ToPlainText]() : data = ""; - console.log(data); let converter = (results: any, data: string) => { let keyterms = new List(); let keyterms_counted = new List(); + let highKP: string[] = [""]; + let high = 0; results.documents.forEach((doc: any) => { let keyPhrases = doc.keyPhrases; keyPhrases.map((kp: string) => { const frequency = this.countFrequencies(kp, data); + if (frequency > high) { + high = frequency; + highKP = [kp]; + + } + else if (frequency === high) { + highKP.push(kp); + } let words = kp.split(" "); // separates phrase into words words = this.removeStopWords(words); words.forEach((word) => { @@ -148,19 +158,18 @@ export class ClientRecommender extends React.Component { }); }); }); + console.log(highKP); + this.sendRequest(highKP); return { keyterms: keyterms, keyterms_counted: keyterms_counted }; }; - let test = (results: any, data: string) => { - results.documents.forEach((doc: any) => { - let kps = doc.keyPhrases; - kps.map((kp: string) => { - this.countFrequencies(kp, data); - }); - }); - }; await CognitiveServices.Text.Appliers.analyzer(dataDoc, extDoc, ["key words"], data, converter, mainDoc); } + private findImportantKPs(keyterms_counted: string[], paragraph: string) { + let imporantSet = new Set(); + + } + private countFrequencies(keyphrase: string, paragraph: string) { let data = paragraph.split(" "); let kp_array = keyphrase.split(" "); @@ -183,6 +192,12 @@ export class ClientRecommender extends React.Component { return sw.removeStopwords(word_array); } + private async sendRequest(keywords: string[]) { + let query = ""; + keywords.forEach((kp: string) => query += kp); + await this.arxivrequest(query); + } + /** * Request to the arXiv server for ML articles. */ @@ -190,14 +205,28 @@ export class ClientRecommender extends React.Component { arxivrequest = async (query: string) => { let xhttp = new XMLHttpRequest(); let serveraddress = "http://export.arxiv.org/api" - let endpoint = serveraddress + "/query?search_query=all:" + query + "&start=0&max_results=5"; + let endpoint = serveraddress + "/query?search_query=all:" + query + "&start=0&max_results=1"; let promisified = (resolve: any, reject: any) => { xhttp.onreadystatechange = function () { if (this.readyState === 4) { let result = xhttp.response; + let xml = xhttp.responseXML; + console.log(xml); switch (this.status) { case 200: - console.log(result); + //console.log(result); + if (xml) { + let titles = xml.getElementsByTagName("title"); + if (titles && titles.length > 1) { + let text = titles[1].childNodes[0].nodeValue; + console.log(text); + } + let ids = xml.getElementsByTagName("id"); + if (ids && ids.length > 1) { + let text = ids[1].childNodes[0].nodeValue; + console.log(text); + } + } return resolve(result); case 400: default: @@ -211,6 +240,12 @@ export class ClientRecommender extends React.Component { return new Promise(promisified); } + processArxivResult = (result: any) => { + var xmlDoc = result as XMLDocument; + let text = xmlDoc.getElementsByTagName("title")[0].childNodes[0].nodeValue; + console.log(text); + } + /*** * Creates distance matrix for all Documents analyzed */ diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx index 63aa12c29..d0105ee18 100644 --- a/src/client/views/Recommendations.tsx +++ b/src/client/views/Recommendations.tsx @@ -56,7 +56,9 @@ export class Recommendations extends React.Component<{}> { let returnYDimension = () => 50; let scale = () => returnXDimension() / NumCast(renderDoc.nativeWidth, returnXDimension()); //let scale = () => 1; - //let newRenderDoc = Doc.MakeDelegate(renderDoc); /// newRenderDoc -> renderDoc -> render"data"Doc -> TextProt + let newRenderDoc = Doc.MakeAlias(renderDoc); /// newRenderDoc -> renderDoc -> render"data"Doc -> TextProt + newRenderDoc.height = 50; + newRenderDoc.autoHeight = false; const docview =
{/* onPointerDown={action(() => { this._useIcons = !this._useIcons; @@ -66,7 +68,7 @@ export class Recommendations extends React.Component<{}> { onPointerLeave={action(() => this._displayDim = 50)} > */} (Docu }); cm.addItem({ description: "Recommender System", - event: async () => { - if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); - let documents: Doc[] = []; - let allDocs = await SearchUtil.GetAllDocs(); - // allDocs.forEach(doc => console.log(doc.title)); - // clears internal representation of documents as vectors - ClientRecommender.Instance.reset_docs(); - ClientRecommender.Instance.arxivrequest("electrons"); - await Promise.all(allDocs.map((doc: Doc) => { - let mainDoc: boolean = false; - const dataDoc = Doc.GetDataDoc(doc); - if (doc.type === DocumentType.TEXT) { - if (dataDoc === Doc.GetDataDoc(this.props.Document)) { - mainDoc = true; - console.log(StrCast(doc.title)); - } - if (!documents.includes(dataDoc)) { - documents.push(dataDoc); - const extdoc = doc.data_ext as Doc; - return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc, mainDoc); - } - } - })); - console.log(ClientRecommender.Instance.createDistanceMatrix()); - const doclist = ClientRecommender.Instance.computeSimilarities(); - let recDocs: { preview: Doc, score: number }[] = []; - // tslint:disable-next-line: prefer-for-of - for (let i = 0; i < doclist.length; i++) { - console.log(doclist[i].score); - recDocs.push({ preview: doclist[i].actualDoc, score: doclist[i].score }); - } - Recommendations.Instance.addDocuments(recDocs); - Recommendations.Instance.displayRecommendations(e.pageX + 100, e.pageY); - }, + event: () => this.recommender(e), icon: "brain" }); cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" }); @@ -744,6 +711,38 @@ export class DocumentView extends DocComponent(Docu }); } + recommender = async (e: React.MouseEvent) => { + if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); + let documents: Doc[] = []; + let allDocs = await SearchUtil.GetAllDocs(); + // allDocs.forEach(doc => console.log(doc.title)); + // clears internal representation of documents as vectors + ClientRecommender.Instance.reset_docs(); + //ClientRecommender.Instance.arxivrequest("electrons"); + await Promise.all(allDocs.map((doc: Doc) => { + let mainDoc: boolean = false; + const dataDoc = Doc.GetDataDoc(doc); + if (doc.type === DocumentType.TEXT) { + if (dataDoc === Doc.GetDataDoc(this.props.Document)) { + mainDoc = true; + } + if (!documents.includes(dataDoc)) { + documents.push(dataDoc); + const extdoc = doc.data_ext as Doc; + return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc, mainDoc); + } + } + })); + const doclist = ClientRecommender.Instance.computeSimilarities(); + let recDocs: { preview: Doc, score: number }[] = []; + // tslint:disable-next-line: prefer-for-of + for (let i = 0; i < doclist.length; i++) { + recDocs.push({ preview: doclist[i].actualDoc, score: doclist[i].score }); + } + Recommendations.Instance.addDocuments(recDocs); + Recommendations.Instance.displayRecommendations(e.pageX + 100, e.pageY); + } + onPointerEnter = (e: React.PointerEvent): void => { Doc.BrushDoc(this.props.Document); }; onPointerLeave = (e: React.PointerEvent): void => { Doc.UnBrushDoc(this.props.Document); }; -- cgit v1.2.3-70-g09d2 From ee03fa6e04dd9dba3099f75154de6ffab566ff5c Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Fri, 13 Sep 2019 16:43:09 -0400 Subject: Trial 5(Most succesful yet) --- package.json | 2 + src/client/documents/DocumentTypes.ts | 1 + src/client/documents/Documents.ts | 10 +- src/client/views/MainView.tsx | 5 +- src/client/views/nodes/DocumentContentsView.tsx | 4 +- src/client/views/webcam/DashWebCam.tsx | 302 ++++++++++++++++++++++++ src/new_fields/URLField.ts | 2 + 7 files changed, 323 insertions(+), 3 deletions(-) create mode 100644 src/client/views/webcam/DashWebCam.tsx (limited to 'src/client/views') diff --git a/package.json b/package.json index c2085fb45..d0718345f 100644 --- a/package.json +++ b/package.json @@ -100,6 +100,7 @@ "@types/react-color": "^2.14.1", "@types/react-measure": "^2.0.4", "@types/react-table": "^6.7.22", + "@types/react-webcam": "^3.0.0", "@types/request": "^2.48.1", "@types/request-promise": "^4.1.42", "@types/sharp": "^0.22.2", @@ -201,6 +202,7 @@ "react-simple-dropdown": "^3.2.3", "react-split-pane": "^0.1.85", "react-table": "^6.9.2", + "react-webcam": "^3.0.1", "readline": "^1.3.0", "request": "^2.88.0", "request-promise": "^4.2.4", diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index 381981e1b..4430ffd62 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -17,6 +17,7 @@ export enum DocumentType { TEMPLATE = "template", EXTENSION = "extension", YOUTUBE = "youtube", + WEBCAM = "webcam", DRAGBOX = "dragbox", PRES = "presentation", LINKFOLLOW = "linkfollow", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 602a7f9ad..036cc75a0 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -22,7 +22,7 @@ import { MINIMIZED_ICON_SIZE } from "../views/globalCssVariables.scss"; import { IconBox } from "../views/nodes/IconBox"; import { Field, Doc, Opt } from "../../new_fields/Doc"; import { OmitKeys, JSONUtils } from "../../Utils"; -import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField } from "../../new_fields/URLField"; +import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField, WebCamField } from "../../new_fields/URLField"; import { HtmlField } from "../../new_fields/HtmlField"; import { List } from "../../new_fields/List"; import { Cast, NumCast } from "../../new_fields/Types"; @@ -46,6 +46,7 @@ import { ComputedField } from "../../new_fields/ScriptField"; import { ProxyField } from "../../new_fields/Proxy"; import { DocumentType } from "./DocumentTypes"; import { LinkFollowBox } from "../views/linking/LinkFollowBox"; +import { DashWebCam } from "../views/webcam/DashWebCam"; //import { PresBox } from "../views/nodes/PresBox"; //import { PresField } from "../../new_fields/PresField"; var requestImageSize = require('../util/request-image-size'); @@ -176,6 +177,9 @@ export namespace Docs { }], [DocumentType.LINKFOLLOW, { layout: { view: LinkFollowBox } + }], + [DocumentType.WEBCAM, { + layout: { view: DashWebCam } }] ]); @@ -357,6 +361,10 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.YOUTUBE), new YoutubeField(new URL(url)), options); } + export function WebCamDocument(url: string, options: DocumentOptions = {}) { + return InstanceFromProto(Prototypes.get(DocumentType.WEBCAM), "", options); + } + export function AudioDocument(url: string, options: DocumentOptions = {}) { return InstanceFromProto(Prototypes.get(DocumentType.AUDIO), new AudioField(new URL(url)), options); } diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index b64986084..62d4fb1b8 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -1,5 +1,5 @@ import { IconName, library } from '@fortawesome/fontawesome-svg-core'; -import { faLink, faArrowDown, faArrowUp, faBolt, faCaretUp, faCat, faCheck, faClone, faCloudUploadAlt, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faLongArrowAltRight, faMusic, faObjectGroup, faPause, faPenNib, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faUndoAlt, faTv } from '@fortawesome/free-solid-svg-icons'; +import { faLink, faArrowDown, faArrowUp, faBolt, faCaretUp, faCat, faCheck, faClone, faCloudUploadAlt, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faLongArrowAltRight, faMusic, faObjectGroup, faPause, faPenNib, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faUndoAlt, faTv, faVideo } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, computed, configure, observable, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; @@ -195,6 +195,7 @@ export class MainView extends React.Component { library.add(faArrowUp); library.add(faCloudUploadAlt); library.add(faBolt); + library.add(faVideo); this.initEventListeners(); this.initAuthenticationRouters(); } @@ -456,6 +457,7 @@ export class MainView extends React.Component { let addImportCollectionNode = action(() => Docs.Create.DirectoryImportDocument({ title: "Directory Import", width: 400, height: 400 })); // let youtubeurl = "https://www.youtube.com/embed/TqcApsGRzWw"; // let addYoutubeSearcher = action(() => Docs.Create.YoutubeDocument(youtubeurl, { width: 600, height: 600, title: "youtube search" })); + let addWebCam = action(() => Docs.Create.WebCamDocument("", {})); let btns: [React.RefObject, IconName, string, () => Doc][] = [ [React.createRef(), "object-group", "Add Collection", addColNode], @@ -464,6 +466,7 @@ export class MainView extends React.Component { [React.createRef(), "bolt", "Add Button", addButtonDocument], [React.createRef(), "file", "Add Document Dragger", addDragboxNode], [React.createRef(), "cloud-upload-alt", "Import Directory", addImportCollectionNode], //remove at some point in favor of addImportCollectionNode + [React.createRef(), "video", "Add WebCam", addWebCam] //[React.createRef(), "play", "Add Youtube Searcher", addYoutubeSearcher], ]; if (!ClientUtils.RELEASE) btns.unshift([React.createRef(), "cat", "Add Cat Image", addImageNode]); diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index d0e117fe4..df5ff04dd 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -32,6 +32,8 @@ import { Doc } from "../../../new_fields/Doc"; import DirectoryImportBox from "../../util/Import & Export/DirectoryImportBox"; import { ScriptField } from "../../../new_fields/ScriptField"; import { fromPromise } from "mobx-utils"; +import { DashWebCam } from "../../views/webcam/DashWebCam"; + const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? type BindingProps = Without; @@ -110,7 +112,7 @@ export class DocumentContentsView extends React.Component void; + onUserMediaError: (error: string) => void; + screenshotFormat: "image/webp" | "image/png" | "image/jpeg"; + screenshotQuality: number; + videoConstraints?: MediaStreamConstraints["video"]; +} + +@observer +export class DashWebCam extends React.Component, WebcamState> { + static defaultProps = { + audio: true, + imageSmoothing: true, + onUserMedia: () => { }, + onUserMediaError: () => { }, + screenshotFormat: "image/webp", + screenshotQuality: 0.92 + }; + + private static mountedInstances: DashWebCam[] = []; + private static userMediaRequested = false; + private canvas: HTMLCanvasElement | undefined; + private ctx: CanvasRenderingContext2D | null = null; + private stream: MediaStream | undefined; + private video: HTMLVideoElement | null | undefined; + + @observable private hasUserMedia: boolean | undefined; + @observable private src: string | undefined; + + // constructor(props: any) { + // super(props); + // this.state = { + // hasUserMedia: false + // }; + // } + + componentDidMount() { + if (!hasGetUserMedia()) return; + + const { state } = this; + + DashWebCam.mountedInstances.push(this); + + if (!state.hasUserMedia && !DashWebCam.userMediaRequested) { + this.requestUserMedia(); + } + } + + componentDidUpdate(nextProps: WebcamProps) { + const { props } = this; + if ( + JSON.stringify(nextProps.audioConstraints) !== + JSON.stringify(props.audioConstraints) || + JSON.stringify(nextProps.videoConstraints) !== + JSON.stringify(props.videoConstraints) + ) { + this.requestUserMedia(); + } + } + + componentWillUnmount() { + //const { state } = this; + const index = DashWebCam.mountedInstances.indexOf(this); + DashWebCam.mountedInstances.splice(index, 1); + + DashWebCam.userMediaRequested = false; + if (DashWebCam.mountedInstances.length === 0 && this.hasUserMedia) { + if (this.stream!.getVideoTracks && this.stream!.getAudioTracks) { + this.stream!.getVideoTracks().map(track => track.stop()); + this.stream!.getAudioTracks().map(track => track.stop()); + } else { + ((this.stream as unknown) as MediaStreamTrack).stop(); + } + + if (this.src) { + window.URL.revokeObjectURL(this.src); + } + } + } + + getScreenshot() { + const { props } = this; + + if (!this.hasUserMedia) return null; + + const canvas = this.getCanvas(); + return ( + canvas && + canvas.toDataURL(props.screenshotFormat, props.screenshotQuality) + ); + } + + getCanvas() { + const { props } = this; + + if (!this.video) { + return null; + } + + if (!this.hasUserMedia || !this.video.videoHeight) return null; + + if (!this.ctx) { + const canvas = document.createElement("canvas"); + const aspectRatio = this.video.videoWidth / this.video.videoHeight; + + let canvasWidth = props.minScreenshotWidth || this.video.clientWidth; + let canvasHeight = canvasWidth / aspectRatio; + + if ( + props.minScreenshotHeight && + canvasHeight < props.minScreenshotHeight + ) { + canvasHeight = props.minScreenshotHeight; + canvasWidth = canvasHeight * aspectRatio; + } + + canvas.width = canvasWidth; + canvas.height = canvasHeight; + + this.canvas = canvas; + this.ctx = canvas.getContext("2d"); + } + + const { ctx, canvas } = this; + + if (ctx) { + ctx.imageSmoothingEnabled = props.imageSmoothing; + ctx.drawImage(this.video, 0, 0, canvas!.width, canvas!.height); + } + + return canvas; + } + + requestUserMedia() { + const { props } = this; + + navigator.getUserMedia = + navigator.mediaDevices.getUserMedia; + + const sourceSelected = (audioConstraints, videoConstraints) => { + const constraints: MediaStreamConstraints = { + video: typeof videoConstraints !== "undefined" ? videoConstraints : true + }; + + if (props.audio) { + constraints.audio = + typeof audioConstraints !== "undefined" ? audioConstraints : true; + } + + navigator.mediaDevices + .getUserMedia(constraints) + .then(stream => { + DashWebCam.mountedInstances.forEach(instance => + instance.handleUserMedia(null, stream) + ); + }) + .catch(e => { + DashWebCam.mountedInstances.forEach(instance => + instance.handleUserMedia(e) + ); + }); + }; + + if ("mediaDevices" in navigator) { + sourceSelected(props.audioConstraints, props.videoConstraints); + } else { + const optionalSource = id => ({ optional: [{ sourceId: id }] }); + + const constraintToSourceId = constraint => { + const { deviceId } = constraint; + + if (typeof deviceId === "string") { + return deviceId; + } + + if (Array.isArray(deviceId) && deviceId.length > 0) { + return deviceId[0]; + } + + if (typeof deviceId === "object" && deviceId.ideal) { + return deviceId.ideal; + } + + return null; + }; + + // @ts-ignore: deprecated api + MediaStreamTrack.getSources(sources => { + let audioSource = null; + let videoSource = null; + + sources.forEach(source => { + if (source.kind === "audio") { + audioSource = source.id; + } else if (source.kind === "video") { + videoSource = source.id; + } + }); + + const audioSourceId = constraintToSourceId(props.audioConstraints); + if (audioSourceId) { + audioSource = audioSourceId; + } + + const videoSourceId = constraintToSourceId(props.videoConstraints); + if (videoSourceId) { + videoSource = videoSourceId; + } + + sourceSelected( + optionalSource(audioSource), + optionalSource(videoSource) + ); + }); + } + + DashWebCam.userMediaRequested = true; + } + + handleUserMedia(err, stream?: MediaStream) { + const { props } = this; + + if (err || !stream) { + this.setState({ hasUserMedia: false }); + props.onUserMediaError(err); + + return; + } + + this.stream = stream; + + try { + if (this.video) { + this.video.srcObject = stream; + } + this.setState({ hasUserMedia: true }); + } catch (error) { + this.setState({ + hasUserMedia: true, + src: window.URL.createObjectURL(stream) + }); + } + + props.onUserMedia(); + } + + + + + + public static LayoutString() { return FieldView.LayoutString(DashWebCam); } + + + render() { + const { props } = this; + + const { + audio, + onUserMedia, + onUserMediaError, + screenshotFormat, + screenshotQuality, + minScreenshotWidth, + minScreenshotHeight, + audioConstraints, + videoConstraints, + imageSmoothing, + ...rest + } = props; + + + return ( +
; + + + let frozen = !local.isSelected() || DocumentDecorations.Instance.Interacting; + let classname = "webBox-cont" + (local.isSelected() && !InkingControl.Instance.selectedTool && !DocumentDecorations.Instance.Interacting ? "-interactive" : ""); return ( -