From 1f8bf407ef49aab33294c3e7393718606dfa65dd Mon Sep 17 00:00:00 2001 From: Abdullah Ahmed Date: Fri, 11 Oct 2019 17:40:09 -0400 Subject: fixes + refactoring --- src/client/ClientRecommender.tsx | 95 ++++++---------------- src/client/cognitive_services/CognitiveServices.ts | 6 +- src/client/views/Recommendations.tsx | 22 ++--- src/server/Recommender.ts | 2 +- 4 files changed, 37 insertions(+), 88 deletions(-) (limited to 'src') diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 9ce7df366..bc1cd139c 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -35,7 +35,6 @@ export class ClientRecommender extends React.Component { static Instance: ClientRecommender; private mainDoc?: RecommenderDocument; private docVectors: Set = new Set(); - private highKP: string[] = []; @observable private corr_matrix = [[0, 0], [0, 0]]; @@ -94,7 +93,7 @@ export class ClientRecommender extends React.Component { public computeSimilarities() { ClientRecommender.Instance.docVectors.forEach((doc: RecommenderDocument) => { if (ClientRecommender.Instance.mainDoc) { - const distance = ClientRecommender.Instance.distance(ClientRecommender.Instance.mainDoc.vectorDoc, doc.vectorDoc, "euclidian"); + const distance = ClientRecommender.Instance.distance(ClientRecommender.Instance.mainDoc.vectorDoc, doc.vectorDoc, "cosine"); doc.score = distance; } } @@ -151,17 +150,21 @@ export class ClientRecommender extends React.Component { let converter = async (results: any, data: string) => { let keyterms = new List(); // raw keywords let keyterms_counted = new List(); // keywords, where each keyword is repeated as - let highKP: string[] = [""]; // most frequent + let kp_string: string = ""; + let highKP: string[] = [""]; // most frequent keyphrase let high = 0; results.documents.forEach((doc: any) => { let keyPhrases = doc.keyPhrases; keyPhrases.map((kp: string) => { const frequency = this.countFrequencies(kp, data); keyterms.push(kp); + kp_string += kp + ", "; + // replaces highKP with new one if (frequency > high) { high = frequency; highKP = [kp]; } + // appends to current highKP phrase else if (frequency === high) { highKP.push(kp); } @@ -175,13 +178,11 @@ export class ClientRecommender extends React.Component { }); }); }); - this.highKP = highKP; - //console.log(highKP); const kts_counted = new List(); keyterms_counted.forEach(kt => kts_counted.push(kt.toLowerCase())); let values = ""; if (!internal) values = await this.sendRequest(highKP); - return { keyterms: keyterms, keyterms_counted: kts_counted, values }; + return { keyterms: keyterms, keyterms_counted: kts_counted, values, kp_string: [kp_string] }; }; if (data != "") { return CognitiveServices.Text.Appliers.analyzer(dataDoc, extDoc, ["key words"], data, converter, mainDoc, internal); @@ -189,6 +190,10 @@ export class ClientRecommender extends React.Component { return; } + /** + * + * Counts frequencies of keyphrase in paragraph. + */ private countFrequencies(keyphrase: string, paragraph: string) { let data = paragraph.split(" "); @@ -196,8 +201,7 @@ export class ClientRecommender extends React.Component { let num_keywords = kp_array.length; let par_length = data.length; let frequency = 0; - // console.log("Paragraph: ", data); - // console.log("Keyphrases:", kp_array); + // slides keyphrase windows across paragraph and checks if it matches with corresponding paragraph slice for (let i = 0; i <= par_length - num_keywords; i++) { const window = data.slice(i, i + num_keywords); if (JSON.stringify(window).toLowerCase() === JSON.stringify(kp_array).toLowerCase() || kp_array.every(val => window.includes(val))) { @@ -207,11 +211,21 @@ export class ClientRecommender extends React.Component { return frequency; } + /** + * + * Removes stopwords from list of strings representing a sentence + */ + private removeStopWords(word_array: string[]) { //console.log(sw.removeStopwords(word_array)); return sw.removeStopwords(word_array); } + /** + * + * API for sending arXiv request. + */ + private async sendRequest(keywords: string[]) { let query = ""; keywords.forEach((kp: string) => query += " " + kp); @@ -221,13 +235,14 @@ export class ClientRecommender extends React.Component { } /** - * Request to the arXiv server for ML articles. + * Actual request to the arXiv server for ML articles. */ arxivrequest = async (query: string) => { let xhttp = new XMLHttpRequest(); let serveraddress = "http://export.arxiv.org/api"; - let endpoint = serveraddress + "/query?search_query=all:" + query + "&start=0&max_results=5"; + const maxresults = 5; + let endpoint = serveraddress + "/query?search_query=all:" + query + "&start=0&max_results=" + maxresults.toString(); let promisified = (resolve: any, reject: any) => { xhttp.onreadystatechange = function () { if (this.readyState === 4) { @@ -243,7 +258,7 @@ export class ClientRecommender extends React.Component { let titles = xml.getElementsByTagName("title"); let counter = 1; if (titles && titles.length > 1) { - while (counter <= 5) { + while (counter <= maxresults) { const title = titles[counter].childNodes[0].nodeValue!; console.log(title) title_vals.push(title); @@ -253,7 +268,7 @@ export class ClientRecommender extends React.Component { let ids = xml.getElementsByTagName("id"); counter = 1; if (ids && ids.length > 1) { - while (counter <= 5) { + while (counter <= maxresults) { const url = ids[counter].childNodes[0].nodeValue!; console.log(url); url_vals.push(url); @@ -280,64 +295,8 @@ export class ClientRecommender extends React.Component { console.log(text); } - /*** - * Creates distance matrix for all Documents analyzed - */ - - @action - public createDistanceMatrix(documents: Set = ClientRecommender.Instance.docVectors) { - const documents_list = Array.from(documents); - const n = documents_list.length; - var matrix = new Array(n).fill(0).map(() => new Array(n).fill(0)); - for (let i = 0; i < n; i++) { - var doc1 = documents_list[i]; - for (let j = 0; j < n; j++) { - var doc2 = documents_list[j]; - matrix[i][j] = ClientRecommender.Instance.distance(doc1.vectorDoc, doc2.vectorDoc, "euclidian"); - } - } - ClientRecommender.Instance.corr_matrix = matrix; - return matrix; - } - - @computed - private get generateRows() { - const n = ClientRecommender.Instance.corr_matrix.length; - let rows: JSX.Element[] = []; - for (let i = 0; i < n; i++) { - let children: JSX.Element[] = []; - for (let j = 0; j < n; j++) { - //let cell = React.createElement("td", ClientRecommender.Instance.corr_matrix[i][j]); - let cell = {ClientRecommender.Instance.corr_matrix[i][j].toFixed(4)}; - children.push(cell); - } - //let row = React.createElement("tr", { children: children, key: i }); - let row = {children}; - rows.push(row); - } - return rows; - } - render() { return (
-

{ClientRecommender.Instance.props.title ? ClientRecommender.Instance.props.title : "hello"}

- {/* - - - - - - - - - - -
{ClientRecommender.Instance.corr_matrix[0][0].toFixed(4)}{ClientRecommender.Instance.corr_matrix[0][1].toFixed(4)}
{ClientRecommender.Instance.corr_matrix[1][0].toFixed(4)}{ClientRecommender.Instance.corr_matrix[1][1].toFixed(4)}
*/} - - - {ClientRecommender.Instance.generateRows} - -
); } diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index c138c68b7..eb088763d 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -15,7 +15,7 @@ type RequestExecutor = (apiKey: string, body: string, service: Service) => Promi type AnalysisApplier = (target: Doc, relevantKeys: string[], data: D, ...args: any) => any; type BodyConverter = (data: D) => string; type Converter = (results: any) => Field; -type TextConverter = (results: any, data: string) => Promise<{ keyterms: Field, keyterms_counted: Field, values: any }>; +type TextConverter = (results: any, data: string) => Promise<{ keyterms: Field, keyterms_counted: Field, values: any, kp_string: string[] }>; export type Tag = { name: string, confidence: number }; export type Rectangle = { top: number, left: number, width: number, height: number }; @@ -290,12 +290,12 @@ export namespace CognitiveServices { export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: TextConverter, mainDoc: boolean = false, internal: boolean = true) => { let results = await ExecuteQuery(Service.Text, Manager, data); console.log(results); - let { keyterms, values, keyterms_counted } = await converter(results, data); + let { keyterms, values, keyterms_counted, kp_string } = await converter(results, data); //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis"); target[keys[0]] = keyterms; console.log("analyzed!"); if (internal) { - await vectorize(keyterms, dataDoc, mainDoc, data); + await vectorize(kp_string, dataDoc, mainDoc, data); } else { return values; } diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx index 5dc62105d..f965d655b 100644 --- a/src/client/views/Recommendations.tsx +++ b/src/client/views/Recommendations.tsx @@ -70,15 +70,9 @@ export class RecommendationsBox extends React.Component { newRenderDoc.height = NumCast(this.props.Document.documentIconHeight); newRenderDoc.autoHeight = false; const docview =
- {/* onPointerDown={action(() => { - this._useIcons = !this._useIcons; - this._displayDim = this._useIcons ? 50 : Number(SEARCH_THUMBNAIL_SIZE); - })} - onPointerEnter={action(() => this._displayDim = this._useIcons ? 50 : Number(SEARCH_THUMBNAIL_SIZE))} - onPointerLeave={action(() => this._displayDim = 50)} > */} { ContentScaling={scale} />
; - // const data = renderDoc.data; - // if (data instanceof ObjectField) newRenderDoc.data = ObjectField.MakeCopy(data); - // newRenderDoc.preview = true; - // this.previewDocs.push(newRenderDoc); return docview; } @@ -171,16 +161,16 @@ export class RecommendationsBox extends React.Component { {DocListCast(this.props.Document.data).map(doc => { return (
- {/* + {this.DocumentIcon(doc)} - */} + {NumCast(doc.score).toFixed(4)}
DocumentManager.Instance.jumpToDocument(doc, false)}>
- {/*
DocUtils.MakeLink({ doc: this.props.Document.sourceDoc as Doc }, { doc: doc }, "User Selected Link", "Generated from Recommender", undefined)}> +
DocUtils.MakeLink({ doc: this.props.Document.sourceDoc as Doc }, { doc: doc }, "User Selected Link", "Generated from Recommender", undefined)}> -
*/} +
); })} diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts index d014ba344..d974d7ef6 100644 --- a/src/server/Recommender.ts +++ b/src/server/Recommender.ts @@ -17,7 +17,7 @@ export class Recommender { private _model: any; static Instance: Recommender; private dimension: number = 0; - private choice: string = ""; + private choice: string = ""; // Tensorflow or Word2Vec constructor() { console.log("creating recommender..."); -- cgit v1.2.3-70-g09d2