diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/client/ClientRecommender.tsx | 8 | ||||
| -rw-r--r-- | src/client/apis/IBM_Recommender.ts | 66 | ||||
| -rw-r--r-- | src/server/ApiManagers/UtilManager.ts | 44 | ||||
| -rw-r--r-- | src/server/Recommender.ts | 274 |
4 files changed, 196 insertions, 196 deletions
diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index cb1674943..0e67a6e57 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -5,10 +5,10 @@ import { CognitiveServices, Confidence, Tag, Service } from "./cognitive_service import React = require("react"); import { observer } from "mobx-react"; import { observable, action, computed, reaction } from "mobx"; -var assert = require('assert'); -var sw = require('stopword'); -var FeedParser = require('feedparser'); -var https = require('https'); +// var assert = require('assert'); +// var sw = require('stopword'); +// var FeedParser = require('feedparser'); +// var https = require('https'); import "./ClientRecommender.scss"; import { JSXElement } from "babel-types"; import { RichTextField } from "../new_fields/RichTextField"; diff --git a/src/client/apis/IBM_Recommender.ts b/src/client/apis/IBM_Recommender.ts index da6257f28..4e1c541c8 100644 --- a/src/client/apis/IBM_Recommender.ts +++ b/src/client/apis/IBM_Recommender.ts @@ -1,40 +1,40 @@ -import { Opt } from "../../new_fields/Doc"; +// import { Opt } from "../../new_fields/Doc"; -const NaturalLanguageUnderstandingV1 = require('ibm-watson/natural-language-understanding/v1'); -const { IamAuthenticator } = require('ibm-watson/auth'); +// const NaturalLanguageUnderstandingV1 = require('ibm-watson/natural-language-understanding/v1'); +// const { IamAuthenticator } = require('ibm-watson/auth'); -export namespace IBM_Recommender { +// export namespace IBM_Recommender { - // pass to IBM account is Browngfx1 +// // pass to IBM account is Browngfx1 - const naturalLanguageUnderstanding = new NaturalLanguageUnderstandingV1({ - version: '2019-07-12', - authenticator: new IamAuthenticator({ - apikey: 'tLiYwbRim3CnBcCO4phubpf-zEiGcub1uh0V-sD9OKhw', - }), - url: 'https://gateway-wdc.watsonplatform.net/natural-language-understanding/api' - }); +// const naturalLanguageUnderstanding = new NaturalLanguageUnderstandingV1({ +// version: '2019-07-12', +// authenticator: new IamAuthenticator({ +// apikey: 'tLiYwbRim3CnBcCO4phubpf-zEiGcub1uh0V-sD9OKhw', +// }), +// url: 'https://gateway-wdc.watsonplatform.net/natural-language-understanding/api' +// }); - const analyzeParams = { - 'text': 'this is a test of the keyword extraction feature I am integrating into the program', - 'features': { - 'keywords': { - 'sentiment': true, - 'emotion': true, - 'limit': 3 - }, - } - }; +// const analyzeParams = { +// 'text': 'this is a test of the keyword extraction feature I am integrating into the program', +// 'features': { +// 'keywords': { +// 'sentiment': true, +// 'emotion': true, +// 'limit': 3 +// }, +// } +// }; - export const analyze = async (_parameters: any): Promise<Opt<string>> => { - try { - const response = await naturalLanguageUnderstanding.analyze(_parameters); - console.log(response); - return (JSON.stringify(response, null, 2)); - } catch (err) { - console.log('error: ', err); - return undefined; - } - }; +// export const analyze = async (_parameters: any): Promise<Opt<string>> => { +// try { +// const response = await naturalLanguageUnderstanding.analyze(_parameters); +// console.log(response); +// return (JSON.stringify(response, null, 2)); +// } catch (err) { +// console.log('error: ', err); +// return undefined; +// } +// }; -}
\ No newline at end of file +// }
\ No newline at end of file diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts index d18529cf2..ad8119bf4 100644 --- a/src/server/ApiManagers/UtilManager.ts +++ b/src/server/ApiManagers/UtilManager.ts @@ -3,11 +3,11 @@ import { Method } from "../RouteManager"; import { exec } from 'child_process'; import RouteSubscriber from "../RouteSubscriber"; import { red } from "colors"; -import { IBM_Recommender } from "../../client/apis/IBM_Recommender"; -import { Recommender } from "../Recommender"; +// import { IBM_Recommender } from "../../client/apis/IBM_Recommender"; +// import { Recommender } from "../Recommender"; -const recommender = new Recommender(); -recommender.testModel(); +// const recommender = new Recommender(); +// recommender.testModel(); import executeImport from "../../scraping/buxton/final/BuxtonImporter"; export default class UtilManager extends ApiManager { @@ -27,25 +27,25 @@ export default class UtilManager extends ApiManager { } }); - register({ - method: Method.POST, - subscription: "/IBMAnalysis", - secureHandler: async ({ req, res }) => res.send(await IBM_Recommender.analyze(req.body)) - }); + // register({ + // method: Method.POST, + // subscription: "/IBMAnalysis", + // secureHandler: async ({ req, res }) => res.send(await IBM_Recommender.analyze(req.body)) + // }); - register({ - method: Method.POST, - subscription: "/recommender", - secureHandler: async ({ req, res }) => { - const keyphrases = req.body.keyphrases; - const wordvecs = await recommender.vectorize(keyphrases); - let embedding: Float32Array = new Float32Array(); - if (wordvecs && wordvecs.dataSync()) { - embedding = wordvecs.dataSync() as Float32Array; - } - res.send(embedding); - } - }); + // register({ + // method: Method.POST, + // subscription: "/recommender", + // secureHandler: async ({ req, res }) => { + // const keyphrases = req.body.keyphrases; + // const wordvecs = await recommender.vectorize(keyphrases); + // let embedding: Float32Array = new Float32Array(); + // if (wordvecs && wordvecs.dataSync()) { + // embedding = wordvecs.dataSync() as Float32Array; + // } + // res.send(embedding); + // } + // }); register({ diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts index 1d2cb3858..aacdb4053 100644 --- a/src/server/Recommender.ts +++ b/src/server/Recommender.ts @@ -1,137 +1,137 @@ -//import { Doc } from "../new_fields/Doc"; -//import { StrCast } from "../new_fields/Types"; -//import { List } from "../new_fields/List"; -//import { CognitiveServices } from "../client/cognitive_services/CognitiveServices"; - -// var w2v = require('word2vec'); -var assert = require('assert'); -var arxivapi = require('arxiv-api-node'); -import requestPromise = require("request-promise"); -import * as use from '@tensorflow-models/universal-sentence-encoder'; -import { Tensor } from "@tensorflow/tfjs-core/dist/tensor"; -require('@tensorflow/tfjs-node'); - -//http://gnuwin32.sourceforge.net/packages/make.htm - -export class Recommender { - - private _model: any; - static Instance: Recommender; - private dimension: number = 0; - private choice: string = ""; // Tensorflow or Word2Vec - - constructor() { - console.log("creating recommender..."); - Recommender.Instance = this; - } - - /*** - * Loads pre-trained model from TF - */ - - public async loadTFModel() { - let self = this; - return new Promise(res => { - use.load().then(model => { - self.choice = "TF"; - self._model = model; - self.dimension = 512; - res(model); - }); - } - - ); - } - - /*** - * Loads pre-trained model from word2vec - */ - - // private loadModel(): Promise<any> { - // let self = this; - // return new Promise(res => { - // w2v.loadModel("./node_modules/word2vec/examples/fixtures/vectors.txt", function (err: any, model: any) { - // self.choice = "WV"; - // self._model = model; - // self.dimension = model.size; - // res(model); - // }); - // }); - // } - - /*** - * Testing - */ - - public async testModel() { - if (!this._model) { - await this.loadTFModel(); - } - if (this._model) { - if (this.choice === "WV") { - let similarity = this._model.similarity('father', 'mother'); - console.log(similarity); - } - else if (this.choice === "TF") { - const model = this._model as use.UniversalSentenceEncoder; - // Embed an array of sentences. - const sentences = [ - 'Hello.', - 'How are you?' - ]; - const embeddings = await this.vectorize(sentences); - if (embeddings) embeddings.print(true /*verbose*/); - // model.embed(sentences).then(embeddings => { - // // `embeddings` is a 2D tensor consisting of the 512-dimensional embeddings for each sentence. - // // So in this example `embeddings` has the shape [2, 512]. - // embeddings.print(true /* verbose */); - // }); - } - } - else { - console.log("model not found :("); - } - } - - /*** - * Uses model to convert words to vectors - */ - - public async vectorize(text: string[]): Promise<Tensor | undefined> { - if (!this._model) { - await this.loadTFModel(); - } - if (this._model) { - if (this.choice === "WV") { - let word_vecs = this._model.getVectors(text); - return word_vecs; - } - else if (this.choice === "TF") { - const model = this._model as use.UniversalSentenceEncoder; - return new Promise<Tensor>(res => { - model.embed(text).then(embeddings => { - res(embeddings); - }); - }); - - } - } - } - - // public async trainModel() { - // console.log("phrasing..."); - // w2v.word2vec("./node_modules/word2vec/examples/eng_news-typical_2016_1M-sentences.txt", './node_modules/word2vec/examples/my_phrases.txt', { - // cbow: 1, - // size: 200, - // window: 8, - // negative: 25, - // hs: 0, - // sample: 1e-4, - // threads: 20, - // iter: 200, - // minCount: 2 - // }); - // console.log("phrased!!!"); - // } - -} +// //import { Doc } from "../new_fields/Doc"; +// //import { StrCast } from "../new_fields/Types"; +// //import { List } from "../new_fields/List"; +// //import { CognitiveServices } from "../client/cognitive_services/CognitiveServices"; + +// // var w2v = require('word2vec'); +// var assert = require('assert'); +// var arxivapi = require('arxiv-api-node'); +// import requestPromise = require("request-promise"); +// import * as use from '@tensorflow-models/universal-sentence-encoder'; +// import { Tensor } from "@tensorflow/tfjs-core/dist/tensor"; +// require('@tensorflow/tfjs-node'); + +// //http://gnuwin32.sourceforge.net/packages/make.htm + +// export class Recommender { + +// private _model: any; +// static Instance: Recommender; +// private dimension: number = 0; +// private choice: string = ""; // Tensorflow or Word2Vec + +// constructor() { +// console.log("creating recommender..."); +// Recommender.Instance = this; +// } + +// /*** +// * Loads pre-trained model from TF +// */ + +// public async loadTFModel() { +// let self = this; +// return new Promise(res => { +// use.load().then(model => { +// self.choice = "TF"; +// self._model = model; +// self.dimension = 512; +// res(model); +// }); +// } + +// ); +// } + +// /*** +// * Loads pre-trained model from word2vec +// */ + +// // private loadModel(): Promise<any> { +// // let self = this; +// // return new Promise(res => { +// // w2v.loadModel("./node_modules/word2vec/examples/fixtures/vectors.txt", function (err: any, model: any) { +// // self.choice = "WV"; +// // self._model = model; +// // self.dimension = model.size; +// // res(model); +// // }); +// // }); +// // } + +// /*** +// * Testing +// */ + +// public async testModel() { +// if (!this._model) { +// await this.loadTFModel(); +// } +// if (this._model) { +// if (this.choice === "WV") { +// let similarity = this._model.similarity('father', 'mother'); +// console.log(similarity); +// } +// else if (this.choice === "TF") { +// const model = this._model as use.UniversalSentenceEncoder; +// // Embed an array of sentences. +// const sentences = [ +// 'Hello.', +// 'How are you?' +// ]; +// const embeddings = await this.vectorize(sentences); +// if (embeddings) embeddings.print(true /*verbose*/); +// // model.embed(sentences).then(embeddings => { +// // // `embeddings` is a 2D tensor consisting of the 512-dimensional embeddings for each sentence. +// // // So in this example `embeddings` has the shape [2, 512]. +// // embeddings.print(true /* verbose */); +// // }); +// } +// } +// else { +// console.log("model not found :("); +// } +// } + +// /*** +// * Uses model to convert words to vectors +// */ + +// public async vectorize(text: string[]): Promise<Tensor | undefined> { +// if (!this._model) { +// await this.loadTFModel(); +// } +// if (this._model) { +// if (this.choice === "WV") { +// let word_vecs = this._model.getVectors(text); +// return word_vecs; +// } +// else if (this.choice === "TF") { +// const model = this._model as use.UniversalSentenceEncoder; +// return new Promise<Tensor>(res => { +// model.embed(text).then(embeddings => { +// res(embeddings); +// }); +// }); + +// } +// } +// } + +// // public async trainModel() { +// // console.log("phrasing..."); +// // w2v.word2vec("./node_modules/word2vec/examples/eng_news-typical_2016_1M-sentences.txt", './node_modules/word2vec/examples/my_phrases.txt', { +// // cbow: 1, +// // size: 200, +// // window: 8, +// // negative: 25, +// // hs: 0, +// // sample: 1e-4, +// // threads: 20, +// // iter: 200, +// // minCount: 2 +// // }); +// // console.log("phrased!!!"); +// // } + +// } |