From 77be33c927b52f93e862b868321f79c59bfc050d Mon Sep 17 00:00:00 2001 From: ab Date: Sat, 28 Sep 2019 17:18:08 -0400 Subject: tensorflow model, server posting fix needed --- src/client/cognitive_services/CognitiveServices.ts | 5 +- src/server/Recommender.ts | 105 +++++++++++---------- src/server/index.ts | 8 +- 3 files changed, 65 insertions(+), 53 deletions(-) (limited to 'src') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index baafb63a1..b23441552 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -269,7 +269,7 @@ export namespace CognitiveServices { //keyterms = ["father", "king"]; let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true }; await requestPromise.post(args).then(async (wordvecs) => { - if (wordvecs.length > 0) { + if (wordvecs.shape[0] > 0) { console.log("successful vectorization!"); var vectorValues = new Set(); wordvecs.forEach((wordvec: any) => { @@ -282,7 +282,8 @@ export namespace CognitiveServices { console.log("unsuccessful :( word(s) not in vocabulary"); } //console.log(vectorValues.size); - }); + } + ); } export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: TextConverter, mainDoc: boolean = false, internal: boolean = true) => { diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts index d47257550..d014ba344 100644 --- a/src/server/Recommender.ts +++ b/src/server/Recommender.ts @@ -7,6 +7,8 @@ var w2v = require('word2vec'); var assert = require('assert'); var arxivapi = require('arxiv-api-node'); import requestPromise = require("request-promise"); +import * as use from '@tensorflow-models/universal-sentence-encoder'; +import { Tensor } from "@tensorflow/tfjs-core/dist/tensor"; //http://gnuwin32.sourceforge.net/packages/make.htm @@ -15,12 +17,31 @@ export class Recommender { private _model: any; static Instance: Recommender; private dimension: number = 0; + private choice: string = ""; constructor() { console.log("creating recommender..."); Recommender.Instance = this; } + /*** + * Loads pre-trained model from TF + */ + + public async loadTFModel() { + let self = this; + return new Promise(res => { + use.load().then(model => { + self.choice = "TF"; + self._model = model; + self.dimension = 512; + res(model); + }); + } + + ); + } + /*** * Loads pre-trained model from word2vec */ @@ -29,6 +50,7 @@ export class Recommender { let self = this; return new Promise(res => { w2v.loadModel("./node_modules/word2vec/examples/fixtures/vectors.txt", function (err: any, model: any) { + self.choice = "WV"; self._model = model; self.dimension = model.size; res(model); @@ -42,40 +64,56 @@ export class Recommender { public async testModel() { if (!this._model) { - await this.loadModel(); + await this.loadTFModel(); } if (this._model) { - let similarity = this._model.similarity('father', 'mother'); - console.log(similarity); + if (this.choice === "WV") { + let similarity = this._model.similarity('father', 'mother'); + console.log(similarity); + } + else if (this.choice === "TF") { + const model = this._model as use.UniversalSentenceEncoder; + // Embed an array of sentences. + const sentences = [ + 'Hello.', + 'How are you?' + ]; + const embeddings = await this.vectorize(sentences); + if (embeddings) embeddings.print(true /*verbose*/); + // model.embed(sentences).then(embeddings => { + // // `embeddings` is a 2D tensor consisting of the 512-dimensional embeddings for each sentence. + // // So in this example `embeddings` has the shape [2, 512]. + // embeddings.print(true /* verbose */); + // }); + } } else { console.log("model not found :("); } } - /*** - * Tests if instance exists - */ - - public async testInstance(text: string) { - if (!this._model) { - await this.loadModel(); - } - console.log(text); - } - /*** * Uses model to convert words to vectors */ - public async vectorize(text: string[]) { + public async vectorize(text: string[]): Promise { if (!this._model) { - await this.loadModel(); + await this.loadTFModel(); } if (this._model) { - let word_vecs = this._model.getVectors(text); - - return word_vecs; + if (this.choice === "WV") { + let word_vecs = this._model.getVectors(text); + return word_vecs; + } + else if (this.choice === "TF") { + const model = this._model as use.UniversalSentenceEncoder; + return new Promise(res => { + model.embed(text).then(embeddings => { + res(embeddings); + }); + }); + + } } } @@ -95,33 +133,4 @@ export class Recommender { console.log("phrased!!!"); } - public async arxivRequest(query: string) { - // let xhttp = new XMLHttpRequest(); - // let serveraddress = "http://export.arxiv.org/api/query?search_query=all:electron&start=0&max_results=1"; - // let promisified = (resolve: any, reject: any) => { - // xhttp.onreadystatechange = function () { - // if (this.readyState === 4) { - // let result = xhttp.response; - // switch (this.status) { - // case 200: - // console.log(result); - // return resolve(result); - // case 400: - // default: - // return reject(result); - // } - // } - // }; - // xhttp.open("GET", serveraddress, true); - // xhttp.send(); - // }; - // return new Promise(promisified); - - let res = await arxivapi.query("all:electrons"); - console.log(res); - } - - - - } diff --git a/src/server/index.ts b/src/server/index.ts index e7d49579d..ac803a253 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -688,13 +688,15 @@ app.use(RouteStore.corsProxy, (req, res) => { let recommender = new Recommender(); recommender.testModel(); -recommender.trainModel(); -recommender.arxivRequest("Triangle-GAN"); app.post("/recommender", async (req, res) => { let keyphrases = req.body.keyphrases; let wordvecs = await recommender.vectorize(keyphrases); - res.send(wordvecs); + let embedding: number[][] = []; + if (wordvecs && wordvecs.array()) { + wordvecs.array().then(array => embedding = array as number[][]); + } + res.send(embedding); }); -- cgit v1.2.3-70-g09d2