Merge branch 'recommender_system' of https://github.com/browngraphicslab/Dash-Web into recommender_ui

3 files changed, 46 insertions, 23 deletions

diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx
index 0e344dae9..217c89297 100644
--- a/src/client/ClientRecommender.tsx
+++ b/src/client/ClientRecommender.tsx
@@ -16,6 +16,12 @@ export interface RecommenderProps {
     title: string;
 }
 
+/**
+ * actualDoc: datadoc
+ * vectorDoc: mean vector of text
+ * score: similarity score to main doc
+ */
+
 export interface RecommenderDocument {
     actualDoc: Doc;
     vectorDoc: number[];
@@ -28,6 +34,8 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
     static Instance: ClientRecommender;
     private mainDoc?: RecommenderDocument;
     private docVectors: Set<RecommenderDocument> = new Set();
+    private highKP: string[] = [];
+
     @observable private corr_matrix = [[0, 0], [0, 0]];
 
     constructor(props: RecommenderProps) {
@@ -45,10 +53,6 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
         ClientRecommender.Instance.corr_matrix = [[0, 0], [0, 0]];
     }
 
-    public deleteDocs() {
-        console.log("deleting previews...");
-    }
-
     /***
      * Computes the cosine similarity between two vectors in Euclidean space. 
      */
@@ -82,6 +86,10 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
         }
     }
 
+    /**
+     * Returns list of {doc, similarity (to main doc)} in increasing score
+     */
+
     public computeSimilarities() {
         ClientRecommender.Instance.docVectors.forEach((doc: RecommenderDocument) => {
             if (ClientRecommender.Instance.mainDoc) {
@@ -127,14 +135,14 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
      * Uses Cognitive Services to extract keywords from a document
      */
 
-    public async extractText(dataDoc: Doc, extDoc: Doc, mainDoc: boolean = false) {
+    public async extractText(dataDoc: Doc, extDoc: Doc, internal: boolean = true, mainDoc: boolean = false) {
         let fielddata = Cast(dataDoc.data, RichTextField);
         let data: string;
         fielddata ? data = fielddata[ToPlainText]() : data = "";
         let converter = (results: any, data: string) => {
-            let keyterms = new List<string>();
-            let keyterms_counted = new List<string>();
-            let highKP: string[] = [""];
+            let keyterms = new List<string>(); // raw keywords
+            let keyterms_counted = new List<string>(); // keywords, where each keyword is repeated as 
+            let highKP: string[] = [""]; // most frequent 
             let high = 0;
             results.documents.forEach((doc: any) => {
                 let keyPhrases = doc.keyPhrases;
@@ -143,13 +151,12 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
                     if (frequency > high) {
                         high = frequency;
                         highKP = [kp];
-
                     }
                     else if (frequency === high) {
                         highKP.push(kp);
                     }
                     let words = kp.split(" "); // separates phrase into words
-                    words = this.removeStopWords(words);
+                    words = this.removeStopWords(words); // removes stop words if they appear in phrases
                     words.forEach((word) => {
                         keyterms.push(word);
                         for (let i = 0; i < frequency; i++) {
@@ -158,16 +165,12 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
                     });
                 });
             });
+            this.highKP = highKP;
             console.log(highKP);
             this.sendRequest(highKP);
             return { keyterms: keyterms, keyterms_counted: keyterms_counted };
         };
-        await CognitiveServices.Text.Appliers.analyzer(dataDoc, extDoc, ["key words"], data, converter, mainDoc);
-    }
-
-    private findImportantKPs(keyterms_counted: string[], paragraph: string) {
-        let imporantSet = new Set<string>();
-
+        await CognitiveServices.Text.Appliers.analyzer(dataDoc, extDoc, ["key words"], data, converter, mainDoc, internal);
     }
 
     private countFrequencies(keyphrase: string, paragraph: string) {
@@ -180,7 +183,7 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
         // console.log("Keyphrases:", kp_array);
         for (let i = 0; i <= par_length - num_keywords; i++) {
             const window = data.slice(i, i + num_keywords);
-            if (JSON.stringify(window) === JSON.stringify(kp_array)) {
+            if (JSON.stringify(window).toLowerCase() === JSON.stringify(kp_array).toLowerCase() || kp_array.every(val => window.includes(val))) {
                 frequency++;
             }
         }
@@ -194,7 +197,7 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
 
     private async sendRequest(keywords: string[]) {
         let query = "";
-        keywords.forEach((kp: string) => query += kp);
+        keywords.forEach((kp: string) => query += " " + kp);
         await this.arxivrequest(query);
     }
 
diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts
index eb1dd5197..8a58355a8 100644
--- a/src/client/cognitive_services/CognitiveServices.ts
+++ b/src/client/cognitive_services/CognitiveServices.ts
@@ -285,14 +285,14 @@ export namespace CognitiveServices {
                 });
             }
 
-            export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: TextConverter, mainDoc: boolean = false) => {
+            export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: TextConverter, mainDoc: boolean = false, internal: boolean = true) => {
                 let results = await ExecuteQuery(Service.Text, Manager, data);
                 console.log(results);
                 let keyterms = converter(results, data);
                 //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis");
                 target[keys[0]] = keyterms.keyterms;
                 console.log("analyzed!");
-                await vectorize(keyterms.keyterms_counted, dataDoc, mainDoc, data);
+                if (internal) await vectorize(keyterms.keyterms_counted, dataDoc, mainDoc, data);
             };
 
             // export async function countFrequencies() 
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index d1e87b663..2ae71f1da 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -642,11 +642,23 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                 a.click();
             }
         });
-        cm.addItem({
-            description: "Recommender System",
+        let recommender_subitems: ContextMenuProps[] = [];
+
+        recommender_subitems.push({
+            description: "Internal recommendations",
             event: () => this.recommender(e),
             icon: "brain"
         });
+
+        recommender_subitems.push({
+            description: "External recommendations",
+            event: () => this.externalRecommendation(e),
+            icon: "brain"
+        });
+
+        cm.addItem({ description: "Recommender System", subitems: recommender_subitems, icon: "brain" });
+
+
         cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" });
         type User = { email: string, userDocumentId: string };
         let usersMenu: ContextMenuProps[] = [];
@@ -729,7 +741,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                 if (!documents.includes(dataDoc)) {
                     documents.push(dataDoc);
                     const extdoc = doc.data_ext as Doc;
-                    return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc, mainDoc);
+                    return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc, true, mainDoc);
                 }
             }
         }));
@@ -757,6 +769,14 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
         // RecommendationsBox.Instance.displayRecommendations(e.pageX + 100, e.pageY);
     }
 
+    externalRecommendation = async (e: React.MouseEvent) => {
+        if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" });
+        ClientRecommender.Instance.reset_docs();
+        const doc = Doc.GetDataDoc(this.props.Document);
+        const extdoc = doc.data_ext as Doc;
+        return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc, false);
+    }
+
     onPointerEnter = (e: React.PointerEvent): void => { Doc.BrushDoc(this.props.Document); };
     onPointerLeave = (e: React.PointerEvent): void => { Doc.UnBrushDoc(this.props.Document); };