From 17b27d3575d3f91f461262e5ad72a457238d198a Mon Sep 17 00:00:00 2001
From: ab <abdullah_ahmed@brown.edu>
Date: Wed, 7 Aug 2019 16:28:51 -0400
Subject: correlation matrix completed

---
 .../collectionFreeForm/CollectionFreeFormView.tsx     | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

(limited to 'src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx')

diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 29f9b1429..9344b43d2 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -1,6 +1,6 @@
 import { library } from "@fortawesome/fontawesome-svg-core";
 import { faEye } from "@fortawesome/free-regular-svg-icons";
-import { faCompass, faCompressArrowsAlt, faExpandArrowsAlt, faPaintBrush, faTable, faUpload } from "@fortawesome/free-solid-svg-icons";
+import { faCompass, faCompressArrowsAlt, faExpandArrowsAlt, faPaintBrush, faTable, faUpload, faBrain } from "@fortawesome/free-solid-svg-icons";
 import { action, computed } from "mobx";
 import { observer } from "mobx-react";
 import { Doc, DocListCastAsync, HeightSym, WidthSym } from "../../../../new_fields/Doc";
@@ -37,8 +37,9 @@ import "./CollectionFreeFormView.scss";
 import { MarqueeView } from "./MarqueeView";
 import React = require("react");
 import v5 = require("uuid/v5");
+import { ClientRecommender } from "../../../ClientRecommender";
 
-library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload);
+library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBrain);
 
 export const panZoomSchema = createSchema({
     panX: "number",
@@ -596,6 +597,20 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                 input.click();
             }
         });
+        ContextMenu.Instance.addItem({
+            description: "Recommender System",
+            event: async () => {
+                new ClientRecommender();
+                let activedocs = this.getActiveDocuments();
+                await Promise.all(activedocs.map((doc: Doc) => {
+                    console.log(StrCast(doc.title));
+                    const extdoc = doc.data_ext as Doc;
+                    return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc);
+                }));
+                console.log(ClientRecommender.Instance.createDistanceMatrix());
+            },
+            icon: "brain"
+        });
     }
 
 
-- 
cgit v1.2.3-70-g09d2


From 17f28e393e0c24fcace33a3ecd5564bc766fe685 Mon Sep 17 00:00:00 2001
From: ab <abdullah_ahmed@brown.edu>
Date: Thu, 8 Aug 2019 13:00:51 -0400
Subject: fuck you react

---
 src/client/ClientRecommender.ts                    | 101 --------------
 src/client/ClientRecommender.tsx                   | 148 +++++++++++++++++++++
 .../collectionFreeForm/CollectionFreeFormView.tsx  |   4 +-
 src/server/Recommender.ts                          |   2 +
 4 files changed, 153 insertions(+), 102 deletions(-)
 delete mode 100644 src/client/ClientRecommender.ts
 create mode 100644 src/client/ClientRecommender.tsx

(limited to 'src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx')

diff --git a/src/client/ClientRecommender.ts b/src/client/ClientRecommender.ts
deleted file mode 100644
index 7ff79ab50..000000000
--- a/src/client/ClientRecommender.ts
+++ /dev/null
@@ -1,101 +0,0 @@
-import { Doc } from "../new_fields/Doc";
-import { StrCast } from "../new_fields/Types";
-import { List } from "../new_fields/List";
-import { CognitiveServices } from "./cognitive_services/CognitiveServices";
-
-
-var assert = require('assert');
-
-export class ClientRecommender {
-
-    static Instance: ClientRecommender;
-    private docVectors: Set<number[]>;
-
-    constructor() {
-        //console.log("creating client recommender...");
-        ClientRecommender.Instance = this;
-        this.docVectors = new Set<number[]>();
-    }
-
-
-    /***
-     * Computes the cosine similarity between two vectors in Euclidean space. 
-     */
-
-    private distance(vector1: number[], vector2: number[]) {
-        assert(vector1.length === vector2.length, "Vectors are not the same length");
-        var dotproduct = 0;
-        var mA = 0;
-        var mB = 0;
-        for (let i = 0; i < vector1.length; i++) { // here you missed the i++
-            dotproduct += (vector1[i] * vector2[i]);
-            mA += (vector1[i] * vector1[i]);
-            mB += (vector2[i] * vector2[i]);
-        }
-        mA = Math.sqrt(mA);
-        mB = Math.sqrt(mB);
-        var similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets
-        return similarity;
-    }
-
-    /***
-     * Computes the mean of a set of vectors
-     */
-
-    public mean(paragraph: Set<number[]>) {
-        const n = 200;
-        const num_words = paragraph.size;
-        let meanVector = new Array<number>(n).fill(0); // mean vector
-        paragraph.forEach((wordvec: number[]) => {
-            for (let i = 0; i < n; i++) {
-                meanVector[i] += wordvec[i];
-            }
-        });
-        meanVector = meanVector.map(x => x / num_words);
-        this.addToDocSet(meanVector);
-        return meanVector;
-    }
-
-    private addToDocSet(vector: number[]) {
-        if (this.docVectors) {
-            this.docVectors.add(vector);
-        }
-    }
-
-    /***
-     * Uses Cognitive Services to extract keywords from a document
-     */
-
-    public async extractText(dataDoc: Doc, extDoc: Doc) {
-        let data = StrCast(dataDoc.title);
-        //console.log(data);
-        let converter = (results: any) => {
-            let keyterms = new List<string>();
-            results.documents.forEach((doc: any) => {
-                let keyPhrases = doc.keyPhrases;
-                keyPhrases.map((kp: string) => keyterms.push(kp));
-            });
-            return keyterms;
-        };
-        await CognitiveServices.Text.Manager.analyzer(extDoc, ["key words"], data, converter);
-    }
-
-    /***
-     * Creates distance matrix for all Documents analyzed
-     */
-
-    public createDistanceMatrix(documents: Set<number[]> = this.docVectors) {
-        const documents_list = Array.from(documents);
-        const n = documents_list.length;
-        var matrix = new Array(n).fill(0).map(() => new Array(n).fill(0));
-        for (let i = 0; i < n; i++) {
-            var doc1 = documents_list[i];
-            for (let j = 0; j < n; j++) {
-                var doc2 = documents_list[j];
-                matrix[i][j] = this.distance(doc1, doc2);
-            }
-        }
-        return matrix;
-    }
-
-}
\ No newline at end of file
diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx
new file mode 100644
index 000000000..2344ee490
--- /dev/null
+++ b/src/client/ClientRecommender.tsx
@@ -0,0 +1,148 @@
+import { Doc } from "../new_fields/Doc";
+import { StrCast } from "../new_fields/Types";
+import { List } from "../new_fields/List";
+import { CognitiveServices } from "./cognitive_services/CognitiveServices";
+import React = require("react");
+import { observer } from "mobx-react";
+import { observable, action, computed, reaction } from "mobx";
+var assert = require('assert');
+import Table from 'react-bootstrap/Table';
+
+export interface RecommenderProps {
+    title: string;
+}
+
+@observer
+export class ClientRecommender extends React.Component<RecommenderProps> {
+
+    static Instance: ClientRecommender;
+    private docVectors: Set<number[]>;
+    private corr_matrix = observable([[0, 0], [0, 0]]);
+    @observable private firstnum = 0;
+    //@observable private corr_matrix: number[][] = [[0, 0], [0, 0]];
+
+    constructor(props: RecommenderProps) {
+        //console.log("creating client recommender...");
+        super(props);
+        ClientRecommender.Instance = this;
+        this.docVectors = new Set<number[]>();
+        //this.corr_matrix = [[0, 0], [0, 0]];
+    }
+
+
+    /***
+     * Computes the cosine similarity between two vectors in Euclidean space. 
+     */
+
+    private distance(vector1: number[], vector2: number[]) {
+        assert(vector1.length === vector2.length, "Vectors are not the same length");
+        var dotproduct = 0;
+        var mA = 0;
+        var mB = 0;
+        for (let i = 0; i < vector1.length; i++) { // here you missed the i++
+            dotproduct += (vector1[i] * vector2[i]);
+            mA += (vector1[i] * vector1[i]);
+            mB += (vector2[i] * vector2[i]);
+        }
+        mA = Math.sqrt(mA);
+        mB = Math.sqrt(mB);
+        var similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets
+        return similarity;
+    }
+
+    /***
+     * Computes the mean of a set of vectors
+     */
+
+    public mean(paragraph: Set<number[]>) {
+        const n = 200;
+        const num_words = paragraph.size;
+        let meanVector = new Array<number>(n).fill(0); // mean vector
+        paragraph.forEach((wordvec: number[]) => {
+            for (let i = 0; i < n; i++) {
+                meanVector[i] += wordvec[i];
+            }
+        });
+        meanVector = meanVector.map(x => x / num_words);
+        this.addToDocSet(meanVector);
+        return meanVector;
+    }
+
+    private addToDocSet(vector: number[]) {
+        if (this.docVectors) {
+            this.docVectors.add(vector);
+        }
+    }
+
+    /***
+     * Uses Cognitive Services to extract keywords from a document
+     */
+
+    public async extractText(dataDoc: Doc, extDoc: Doc) {
+        let data = StrCast(dataDoc.title);
+        //console.log(data);
+        let converter = (results: any) => {
+            let keyterms = new List<string>();
+            results.documents.forEach((doc: any) => {
+                let keyPhrases = doc.keyPhrases;
+                keyPhrases.map((kp: string) => keyterms.push(kp));
+            });
+            return keyterms;
+        };
+        await CognitiveServices.Text.Manager.analyzer(extDoc, ["key words"], data, converter);
+    }
+
+    /***
+     * Creates distance matrix for all Documents analyzed
+     */
+
+    @action
+    public createDistanceMatrix(documents: Set<number[]> = this.docVectors) {
+        //this.corr_matrix[0][0] = 500;
+        this.firstnum = 500;
+        const documents_list = Array.from(documents);
+        const n = documents_list.length;
+        var matrix = new Array<number>(n).fill(0).map(() => new Array<number>(n).fill(0));
+        for (let i = 0; i < n; i++) {
+            var doc1 = documents_list[i];
+            for (let j = 0; j < n; j++) {
+                var doc2 = documents_list[j];
+                matrix[i][j] = this.distance(doc1, doc2);
+            }
+        }
+        //this.corr_matrix = matrix;
+
+        return matrix;
+    }
+
+    @computed get first_num() {
+        return this.firstnum;
+    }
+
+    dumb_reaction = reaction(
+        () => this.first_num,
+        number => {
+            console.log("number has changed", number);
+            this.forceUpdate();
+        }
+    );
+
+    render() {
+        return (<div>
+            <h3>{this.props.title ? this.props.title : "hello"}</h3>
+            <table>
+                <tbody>
+                    <tr>
+                        <td>{this.first_num}</td>
+                        <td>{this.corr_matrix[0][1]}</td>
+                    </tr>
+                    <tr>
+                        <td>{this.corr_matrix[1][0]}</td>
+                        <td>{this.corr_matrix[1][1]}</td>
+                    </tr>
+                </tbody>
+            </table>
+        </div>);
+    }
+
+}
\ No newline at end of file
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 9344b43d2..5259b9b49 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -523,6 +523,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY));
     }
 
+    @action
     onContextMenu = (e: React.MouseEvent) => {
         let layoutItems: ContextMenuProps[] = [];
         layoutItems.push({
@@ -600,7 +601,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
         ContextMenu.Instance.addItem({
             description: "Recommender System",
             event: async () => {
-                new ClientRecommender();
+                // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" });
                 let activedocs = this.getActiveDocuments();
                 await Promise.all(activedocs.map((doc: Doc) => {
                     console.log(StrCast(doc.title));
@@ -715,6 +716,7 @@ class CollectionFreeFormViewPannableContents extends React.Component<CollectionF
         const zoom = this.props.zoomScaling();// needs to be a variable outside of the <Measure> otherwise, reactions won't fire
         return <div className={freeformclass} style={{ borderRadius: "inherit", transform: `translate(${cenx}px, ${ceny}px) scale(${zoom}, ${zoom}) translate(${panx}px, ${pany}px)` }}>
             {this.props.children}
+            <ClientRecommender title="Correlation Matrix" />
         </div>;
     }
 }
\ No newline at end of file
diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts
index ea59703c3..d175b67c7 100644
--- a/src/server/Recommender.ts
+++ b/src/server/Recommender.ts
@@ -10,6 +10,7 @@ export class Recommender {
 
     private _model: any;
     static Instance: Recommender;
+    private dimension: number = 0;
 
     constructor() {
         console.log("creating recommender...");
@@ -25,6 +26,7 @@ export class Recommender {
         return new Promise(res => {
             w2v.loadModel("./node_modules/word2vec/vectors.txt", function (err: any, model: any) {
                 self._model = model;
+                self.dimension = model.size;
                 res(model);
             });
         });
-- 
cgit v1.2.3-70-g09d2


From 116f218c2eeec377fb465027bcfaa7521d9af492 Mon Sep 17 00:00:00 2001
From: ab <abdullah_ahmed@brown.edu>
Date: Thu, 8 Aug 2019 17:48:58 -0400
Subject: euclidian

---
 src/client/ClientRecommender.scss                  |   8 ++
 src/client/ClientRecommender.tsx                   | 100 +++++++++++++--------
 .../collectionFreeForm/CollectionFreeFormView.tsx  |   3 +-
 3 files changed, 71 insertions(+), 40 deletions(-)
 create mode 100644 src/client/ClientRecommender.scss

(limited to 'src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx')

diff --git a/src/client/ClientRecommender.scss b/src/client/ClientRecommender.scss
new file mode 100644
index 000000000..710d80f34
--- /dev/null
+++ b/src/client/ClientRecommender.scss
@@ -0,0 +1,8 @@
+@import "/views/globalCssVariables.scss";
+
+.space{
+    border: 1px dashed blue;
+    border-spacing: 25px;
+    border-collapse: separate;
+    align-content: center;
+}
\ No newline at end of file
diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx
index 2344ee490..0793f0887 100644
--- a/src/client/ClientRecommender.tsx
+++ b/src/client/ClientRecommender.tsx
@@ -6,7 +6,7 @@ import React = require("react");
 import { observer } from "mobx-react";
 import { observable, action, computed, reaction } from "mobx";
 var assert = require('assert');
-import Table from 'react-bootstrap/Table';
+import "./ClientRecommender.scss";
 
 export interface RecommenderProps {
     title: string;
@@ -17,37 +17,53 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
 
     static Instance: ClientRecommender;
     private docVectors: Set<number[]>;
-    private corr_matrix = observable([[0, 0], [0, 0]]);
-    @observable private firstnum = 0;
-    //@observable private corr_matrix: number[][] = [[0, 0], [0, 0]];
+    @observable private corr_matrix = [[0, 0], [0, 0]];
 
     constructor(props: RecommenderProps) {
         //console.log("creating client recommender...");
         super(props);
-        ClientRecommender.Instance = this;
+        if (!ClientRecommender.Instance) ClientRecommender.Instance = this;
         this.docVectors = new Set<number[]>();
         //this.corr_matrix = [[0, 0], [0, 0]];
     }
 
+    @action
+    public reset_docs() {
+        this.docVectors = new Set();
+        this.corr_matrix = [[0, 0], [0, 0]];
+    }
 
     /***
      * Computes the cosine similarity between two vectors in Euclidean space. 
      */
 
-    private distance(vector1: number[], vector2: number[]) {
+    private distance(vector1: number[], vector2: number[], metric: string = "cosine") {
         assert(vector1.length === vector2.length, "Vectors are not the same length");
-        var dotproduct = 0;
-        var mA = 0;
-        var mB = 0;
-        for (let i = 0; i < vector1.length; i++) { // here you missed the i++
-            dotproduct += (vector1[i] * vector2[i]);
-            mA += (vector1[i] * vector1[i]);
-            mB += (vector2[i] * vector2[i]);
+        let similarity: number;
+        switch (metric) {
+            case "cosine":
+                var dotproduct = 0;
+                var mA = 0;
+                var mB = 0;
+                for (let i = 0; i < vector1.length; i++) { // here you missed the i++
+                    dotproduct += (vector1[i] * vector2[i]);
+                    mA += (vector1[i] * vector1[i]);
+                    mB += (vector2[i] * vector2[i]);
+                }
+                mA = Math.sqrt(mA);
+                mB = Math.sqrt(mB);
+                similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets
+                return similarity;
+            case "euclidian":
+                var sum = 0;
+                for (let i = 0; i < vector1.length; i++) {
+                    sum += Math.pow(vector1[i] - vector2[i], 2);
+                }
+                similarity = Math.sqrt(sum);
+                return similarity;
+            default:
+                return 0;
         }
-        mA = Math.sqrt(mA);
-        mB = Math.sqrt(mB);
-        var similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets
-        return similarity;
     }
 
     /***
@@ -98,8 +114,6 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
 
     @action
     public createDistanceMatrix(documents: Set<number[]> = this.docVectors) {
-        //this.corr_matrix[0][0] = 500;
-        this.firstnum = 500;
         const documents_list = Array.from(documents);
         const n = documents_list.length;
         var matrix = new Array<number>(n).fill(0).map(() => new Array<number>(n).fill(0));
@@ -107,40 +121,48 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
             var doc1 = documents_list[i];
             for (let j = 0; j < n; j++) {
                 var doc2 = documents_list[j];
-                matrix[i][j] = this.distance(doc1, doc2);
+                matrix[i][j] = this.distance(doc1, doc2, "euclidian");
             }
         }
-        //this.corr_matrix = matrix;
-
+        this.corr_matrix = matrix;
         return matrix;
     }
 
-    @computed get first_num() {
-        return this.firstnum;
-    }
-
-    dumb_reaction = reaction(
-        () => this.first_num,
-        number => {
-            console.log("number has changed", number);
-            this.forceUpdate();
+    @computed
+    private get generateRows() {
+        const n = this.corr_matrix.length;
+        let rows: React.ReactElement[] = [];
+        for (let i = 0; i < n; i++) {
+            let children: React.ReactElement[] = [];
+            for (let j = 0; j < n; j++) {
+                let cell = React.createElement("td", this.corr_matrix[i][j]);
+                children.push(cell);
+            }
+            let row = React.createElement("tr", { children: children });
+            rows.push(row);
         }
-    );
+        return rows;
+    }
 
     render() {
         return (<div>
             <h3>{this.props.title ? this.props.title : "hello"}</h3>
-            <table>
+            {/* <table className="space" >
                 <tbody>
-                    <tr>
-                        <td>{this.first_num}</td>
-                        <td>{this.corr_matrix[0][1]}</td>
+                    <tr key="1">
+                        <td key="1">{this.corr_matrix[0][0].toFixed(4)}</td>
+                        <td key="2">{this.corr_matrix[0][1].toFixed(4)}</td>
                     </tr>
-                    <tr>
-                        <td>{this.corr_matrix[1][0]}</td>
-                        <td>{this.corr_matrix[1][1]}</td>
+                    <tr key="2">
+                        <td key="1">{this.corr_matrix[1][0].toFixed(4)}</td>
+                        <td key="2">{this.corr_matrix[1][1].toFixed(4)}</td>
                     </tr>
                 </tbody>
+            </table> */}
+            <table>
+                <tbody>
+                    {this.generateRows}
+                </tbody>
             </table>
         </div>);
     }
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 5259b9b49..2b9f32136 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -603,6 +603,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
             event: async () => {
                 // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" });
                 let activedocs = this.getActiveDocuments();
+                ClientRecommender.Instance.reset_docs();
                 await Promise.all(activedocs.map((doc: Doc) => {
                     console.log(StrCast(doc.title));
                     const extdoc = doc.data_ext as Doc;
@@ -716,7 +717,7 @@ class CollectionFreeFormViewPannableContents extends React.Component<CollectionF
         const zoom = this.props.zoomScaling();// needs to be a variable outside of the <Measure> otherwise, reactions won't fire
         return <div className={freeformclass} style={{ borderRadius: "inherit", transform: `translate(${cenx}px, ${ceny}px) scale(${zoom}, ${zoom}) translate(${panx}px, ${pany}px)` }}>
             {this.props.children}
-            <ClientRecommender title="Correlation Matrix" />
+            <ClientRecommender title="Distance Matrix" />
         </div>;
     }
 }
\ No newline at end of file
-- 
cgit v1.2.3-70-g09d2


From b8ab5a823ae22b021c09dfd713b77211a51b3eae Mon Sep 17 00:00:00 2001
From: ab <abdullah_ahmed@brown.edu>
Date: Fri, 9 Aug 2019 17:04:25 -0400
Subject: idk man

---
 solr-8.1.1/server/solr/dash/conf/schema.xml         |  2 +-
 src/client/ClientRecommender.scss                   |  4 ++++
 src/client/ClientRecommender.tsx                    | 17 ++++++++++-------
 src/client/util/SearchUtil.ts                       | 21 ++++++++++++++++++++-
 src/client/views/SearchBox.tsx                      |  2 +-
 .../collectionFreeForm/CollectionFreeFormView.tsx   |  6 +++++-
 src/server/Search.ts                                |  2 +-
 7 files changed, 42 insertions(+), 12 deletions(-)

(limited to 'src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx')

diff --git a/solr-8.1.1/server/solr/dash/conf/schema.xml b/solr-8.1.1/server/solr/dash/conf/schema.xml
index 8610786af..c0a4bab07 100644
--- a/solr-8.1.1/server/solr/dash/conf/schema.xml
+++ b/solr-8.1.1/server/solr/dash/conf/schema.xml
@@ -52,7 +52,7 @@
   <dynamicField name="*_i" type="string" indexed="true" stored="true" uninvertible="false" docValues="false"/>
   <dynamicField name="*_b" type="boolean" indexed="true" stored="true" uninvertible="false" docValues="false"/>
 
-  <dynamicField name="*_a" type="text" indexed="true" stored="false" uninvertible="false" docValues="false"/>
+  <dynamicField name="*_a" type="text" indexed="true" stored="false" uninvertible="false" docValues="false" multiValued="true"/>
 
   <copyField source="*_t" dest="text"/>
 
diff --git a/src/client/ClientRecommender.scss b/src/client/ClientRecommender.scss
index 710d80f34..49163cdc8 100644
--- a/src/client/ClientRecommender.scss
+++ b/src/client/ClientRecommender.scss
@@ -5,4 +5,8 @@
     border-spacing: 25px;
     border-collapse: separate;
     align-content: center;
+}
+
+.wrapper{
+    text-align: -webkit-center;
 }
\ No newline at end of file
diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx
index 0793f0887..ddaa8a7fc 100644
--- a/src/client/ClientRecommender.tsx
+++ b/src/client/ClientRecommender.tsx
@@ -7,6 +7,7 @@ import { observer } from "mobx-react";
 import { observable, action, computed, reaction } from "mobx";
 var assert = require('assert');
 import "./ClientRecommender.scss";
+import { JSXElement } from "babel-types";
 
 export interface RecommenderProps {
     title: string;
@@ -131,22 +132,24 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
     @computed
     private get generateRows() {
         const n = this.corr_matrix.length;
-        let rows: React.ReactElement[] = [];
+        let rows: JSX.Element[] = [];
         for (let i = 0; i < n; i++) {
-            let children: React.ReactElement[] = [];
+            let children: JSX.Element[] = [];
             for (let j = 0; j < n; j++) {
-                let cell = React.createElement("td", this.corr_matrix[i][j]);
+                //let cell = React.createElement("td", this.corr_matrix[i][j]);
+                let cell = <td>{this.corr_matrix[i][j].toFixed(4)}</td>;
                 children.push(cell);
             }
-            let row = React.createElement("tr", { children: children });
+            //let row = React.createElement("tr", { children: children, key: i });
+            let row = <tr>{children}</tr>;
             rows.push(row);
         }
         return rows;
     }
 
     render() {
-        return (<div>
-            <h3>{this.props.title ? this.props.title : "hello"}</h3>
+        return (<div className="wrapper">
+            <h3 >{this.props.title ? this.props.title : "hello"}</h3>
             {/* <table className="space" >
                 <tbody>
                     <tr key="1">
@@ -159,7 +162,7 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
                     </tr>
                 </tbody>
             </table> */}
-            <table>
+            <table className="space">
                 <tbody>
                     {this.generateRows}
                 </tbody>
diff --git a/src/client/util/SearchUtil.ts b/src/client/util/SearchUtil.ts
index ee5a83710..3a3ba1803 100644
--- a/src/client/util/SearchUtil.ts
+++ b/src/client/util/SearchUtil.ts
@@ -77,4 +77,23 @@ export namespace SearchUtil {
         aliasContexts.forEach(result => contexts.aliasContexts.push(...result.ids));
         return contexts;
     }
-}
\ No newline at end of file
+
+    export async function GetAllDocs() {
+        const query = "*";
+        let response = await rp.get(Utils.prepend('/search'), {
+            qs: {
+                query
+            }
+        });
+        let res: string[] = JSON.parse(response);
+        const fields = await DocServer.GetRefFields(res);
+        const docs: Doc[] = [];
+        for (const id of res) {
+            const field = fields[id];
+            if (field instanceof Doc) {
+                docs.push(field);
+            }
+        }
+        return docs;
+    }
+}
diff --git a/src/client/views/SearchBox.tsx b/src/client/views/SearchBox.tsx
index 33cb63df5..17f99fa05 100644
--- a/src/client/views/SearchBox.tsx
+++ b/src/client/views/SearchBox.tsx
@@ -47,7 +47,7 @@ export class SearchBox extends React.Component {
     }
 
     @action
-    getResults = async (query: string) => {
+    public getResults = async (query: string) => {
         let response = await rp.get(Utils.prepend('/search'), {
             qs: {
                 query
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 2b9f32136..0beb0086b 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -38,6 +38,8 @@ import { MarqueeView } from "./MarqueeView";
 import React = require("react");
 import v5 = require("uuid/v5");
 import { ClientRecommender } from "../../../ClientRecommender";
+import { SearchUtil } from "../../../util/SearchUtil";
+import { SearchBox } from "../../SearchBox";
 
 library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBrain);
 
@@ -603,9 +605,11 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
             event: async () => {
                 // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" });
                 let activedocs = this.getActiveDocuments();
+                let allDocs = await SearchUtil.GetAllDocs();
+                allDocs.forEach(doc => console.log(doc.title));
                 ClientRecommender.Instance.reset_docs();
                 await Promise.all(activedocs.map((doc: Doc) => {
-                    console.log(StrCast(doc.title));
+                    //console.log(StrCast(doc.title));
                     const extdoc = doc.data_ext as Doc;
                     return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc);
                 }));
diff --git a/src/server/Search.ts b/src/server/Search.ts
index 723dc101b..44c1b1298 100644
--- a/src/server/Search.ts
+++ b/src/server/Search.ts
@@ -14,7 +14,7 @@ export class Search {
             });
             return res;
         } catch (e) {
-            // console.warn("Search error: " + e + document);
+            console.warn("Search error: " + e + document);
         }
     }
 
-- 
cgit v1.2.3-70-g09d2


From 9dd2a31b72e5e527e2dae3b68f856ab8da879e93 Mon Sep 17 00:00:00 2001
From: ab <abdullah_ahmed@brown.edu>
Date: Mon, 12 Aug 2019 16:41:23 -0400
Subject: documentation

---
 package.json                                       |  2 +-
 src/client/ClientRecommender.tsx                   | 18 +++++-----
 src/client/cognitive_services/CognitiveServices.ts | 42 ++++++++++++----------
 src/client/util/SearchUtil.ts                      | 13 ++++---
 .../collectionFreeForm/CollectionFreeFormView.tsx  |  1 +
 src/server/Recommender.ts                          |  1 +
 6 files changed, 45 insertions(+), 32 deletions(-)

(limited to 'src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx')

diff --git a/package.json b/package.json
index 1e2c74411..1c7a10ac8 100644
--- a/package.json
+++ b/package.json
@@ -225,4 +225,4 @@
         "xoauth2": "^1.2.0",
         "youtube": "^0.1.0"
     }
-}
\ No newline at end of file
+}
diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx
index ddaa8a7fc..63f85c737 100644
--- a/src/client/ClientRecommender.tsx
+++ b/src/client/ClientRecommender.tsx
@@ -75,13 +75,15 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
         const n = 200;
         const num_words = paragraph.size;
         let meanVector = new Array<number>(n).fill(0); // mean vector
-        paragraph.forEach((wordvec: number[]) => {
-            for (let i = 0; i < n; i++) {
-                meanVector[i] += wordvec[i];
-            }
-        });
-        meanVector = meanVector.map(x => x / num_words);
-        this.addToDocSet(meanVector);
+        if (num_words > 0) { // check to see if paragraph actually was vectorized
+            paragraph.forEach((wordvec: number[]) => {
+                for (let i = 0; i < n; i++) {
+                    meanVector[i] += wordvec[i];
+                }
+            });
+            meanVector = meanVector.map(x => x / num_words);
+            this.addToDocSet(meanVector);
+        }
         return meanVector;
     }
 
@@ -106,7 +108,7 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
             });
             return keyterms;
         };
-        await CognitiveServices.Text.Manager.analyzer(extDoc, ["key words"], data, converter);
+        await CognitiveServices.Text.Appliers.analyzer(extDoc, ["key words"], data, converter);
     }
 
     /***
diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts
index 954a05585..75d0760ed 100644
--- a/src/client/cognitive_services/CognitiveServices.ts
+++ b/src/client/cognitive_services/CognitiveServices.ts
@@ -258,32 +258,38 @@ export namespace CognitiveServices {
                 };
                 console.log("requested!");
                 return request.post(options);
-            },
-            analyzer: async (target: Doc, keys: string[], data: string, converter: Converter) => {
-                let results = await ExecuteQuery<string, any>(Service.Text, Manager, data);
+            }
+        };
+
+        export namespace Appliers {
+
+            export async function vectorize(keyterms: any) {
+                console.log("vectorizing...");
+                //keyterms = ["father", "king"];
+                let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true };
+                await requestPromise.post(args).then(async (wordvecs) => {
+                    var vectorValues = new Set<number[]>();
+                    wordvecs.forEach((wordvec: any) => {
+                        //console.log(wordvec.word);
+                        vectorValues.add(wordvec.values as number[]);
+                    });
+                    ClientRecommender.Instance.mean(vectorValues);
+                    //console.log(vectorValues.size);
+                });
+            }
+
+            export const analyzer = async (target: Doc, keys: string[], data: string, converter: Converter) => {
+                let results = await ExecuteQuery(Service.Text, Manager, data);
                 console.log(results);
                 let keyterms = converter(results);
                 //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis");
                 target[keys[0]] = keyterms;
                 console.log("analyzed!");
                 await vectorize(keyterms);
-            }
-        };
-        async function vectorize(keyterms: any) {
-            console.log("vectorizing...");
-            //keyterms = ["father", "king"];
-            let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true };
-            await requestPromise.post(args).then(async (wordvecs) => {
-                var vectorValues = new Set<number[]>();
-                wordvecs.forEach((wordvec: any) => {
-                    //console.log(wordvec.word);
-                    vectorValues.add(wordvec.values as number[]);
-                });
-                ClientRecommender.Instance.mean(vectorValues);
-                //console.log(vectorValues.size);
-            });
+            };
         }
 
     }
 
+
 }
\ No newline at end of file
diff --git a/src/client/util/SearchUtil.ts b/src/client/util/SearchUtil.ts
index 3a3ba1803..1fce995d7 100644
--- a/src/client/util/SearchUtil.ts
+++ b/src/client/util/SearchUtil.ts
@@ -82,18 +82,21 @@ export namespace SearchUtil {
         const query = "*";
         let response = await rp.get(Utils.prepend('/search'), {
             qs: {
-                query
+                q: query
             }
         });
-        let res: string[] = JSON.parse(response);
-        const fields = await DocServer.GetRefFields(res);
+        let result: IdSearchResult = JSON.parse(response);
+        const { ids, numFound, highlighting } = result;
+        const docMap = await DocServer.GetRefFields(ids);
         const docs: Doc[] = [];
-        for (const id of res) {
-            const field = fields[id];
+        for (const id of ids) {
+            const field = docMap[id];
             if (field instanceof Doc) {
                 docs.push(field);
             }
         }
         return docs;
+        // const docs = ids.map((id: string) => docMap[id]).filter((doc: any) => doc instanceof Doc);
+        // return docs as Doc[];
     }
 }
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index e9791df4e..d1e8031fd 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -894,6 +894,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                 let activedocs = this.getActiveDocuments();
                 let allDocs = await SearchUtil.GetAllDocs();
                 allDocs.forEach(doc => console.log(doc.title));
+                // clears internal representation of documents as vectors
                 ClientRecommender.Instance.reset_docs();
                 await Promise.all(activedocs.map((doc: Doc) => {
                     //console.log(StrCast(doc.title));
diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts
index d175b67c7..1c95d7ea4 100644
--- a/src/server/Recommender.ts
+++ b/src/server/Recommender.ts
@@ -70,6 +70,7 @@ export class Recommender {
         }
         if (this._model) {
             let word_vecs = this._model.getVectors(text);
+
             return word_vecs;
         }
     }
-- 
cgit v1.2.3-70-g09d2


From e03a1b2cc90e0fdb7789f4826e482e9040aa7075 Mon Sep 17 00:00:00 2001
From: ab <abdullah_ahmed@brown.edu>
Date: Tue, 13 Aug 2019 17:26:49 -0400
Subject: 80% done, garbage collection is much needed

---
 src/Recommendations.scss                           |  21 ---
 src/Recommendations.tsx                            |  28 ----
 src/client/util/SearchUtil.ts                      |   7 +-
 src/client/views/MainView.tsx                      |   2 +
 src/client/views/Recommendations.scss              |  65 ++++++++
 src/client/views/Recommendations.tsx               | 169 +++++++++++++++++++++
 .../collectionFreeForm/CollectionFreeFormView.tsx  |  13 +-
 src/client/views/nodes/DocumentView.tsx            |  34 ++++-
 src/client/views/nodes/ImageBox.tsx                |  21 ++-
 9 files changed, 297 insertions(+), 63 deletions(-)
 delete mode 100644 src/Recommendations.scss
 delete mode 100644 src/Recommendations.tsx
 create mode 100644 src/client/views/Recommendations.scss
 create mode 100644 src/client/views/Recommendations.tsx

(limited to 'src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx')

diff --git a/src/Recommendations.scss b/src/Recommendations.scss
deleted file mode 100644
index 5129a59d9..000000000
--- a/src/Recommendations.scss
+++ /dev/null
@@ -1,21 +0,0 @@
-.recommendation-content *{
-    display: inline-block;
-    margin: auto;
-    border: 1px dashed grey;
-    padding: 2px 2px;
-}
-
-.recommendation-content {
-    float: left;
-    border: 1px solid green;
-    width: 200px;
-    align-content: center;
-}
-
-.rec-scroll {
-    overflow-y: scroll;
-    height: 300px; 
-    width: auto;
-    position: absolute;
-    background: #cdcdcd;
-}
\ No newline at end of file
diff --git a/src/Recommendations.tsx b/src/Recommendations.tsx
deleted file mode 100644
index ca1123ef9..000000000
--- a/src/Recommendations.tsx
+++ /dev/null
@@ -1,28 +0,0 @@
-import { observer } from "mobx-react";
-import React = require("react");
-import { Doc } from "./new_fields/Doc";
-import { NumCast } from "./new_fields/Types";
-
-export interface RecProps {
-    documents: { preview: string, similarity: number }[],
-    node: Doc;
-}
-
-@observer
-export class Recommendations extends React.Component<RecProps> {
-    render() {
-        const transform = "translate(" + (NumCast(this.props.node.x) + 350) + "px, " + NumCast(this.props.node.y) + "px"
-        return (
-            <div className="rec-scroll" style={{ transform: transform }}>
-                {this.props.documents.map(doc => {
-                    return (
-                        <div className="recommendation-content">
-                            <img src={doc.preview} />
-                            <div>{doc.similarity}</div>
-                        </div>
-                    )
-                })}
-            </div>
-        )
-    }
-}
\ No newline at end of file
diff --git a/src/client/util/SearchUtil.ts b/src/client/util/SearchUtil.ts
index 1fce995d7..85e593529 100644
--- a/src/client/util/SearchUtil.ts
+++ b/src/client/util/SearchUtil.ts
@@ -81,12 +81,13 @@ export namespace SearchUtil {
     export async function GetAllDocs() {
         const query = "*";
         let response = await rp.get(Utils.prepend('/search'), {
-            qs: {
-                q: query
-            }
+            qs:
+                { start: 0, rows: 10000, q: query },
+
         });
         let result: IdSearchResult = JSON.parse(response);
         const { ids, numFound, highlighting } = result;
+        console.log(ids.length);
         const docMap = await DocServer.GetRefFields(ids);
         const docs: Doc[] = [];
         for (const id of ids) {
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 419b15697..0b6fe3876 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -41,6 +41,7 @@ import { ClientUtils } from '../util/ClientUtils';
 import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField';
 //import { DocumentManager } from '../util/DocumentManager';
 import { DictationManager } from '../util/DictationManager';
+import { Recommendations } from './Recommendations';
 
 @observer
 export class MainView extends React.Component {
@@ -581,6 +582,7 @@ export class MainView extends React.Component {
                 {this.mainContent}
                 <PreviewCursor />
                 <ContextMenu />
+                <Recommendations />
                 {this.nodesMenu()}
                 {this.miscButtons}
                 <PDFMenu />
diff --git a/src/client/views/Recommendations.scss b/src/client/views/Recommendations.scss
new file mode 100644
index 000000000..5d8f17e37
--- /dev/null
+++ b/src/client/views/Recommendations.scss
@@ -0,0 +1,65 @@
+@import "globalCssVariables";
+
+.rec-content *{
+    display: inline-block;
+    margin: auto;
+    width: 50;
+    height: 30px;
+    border: 1px dashed grey;
+    padding: 10px 10px;
+}
+
+.rec-content {
+    float: left;
+    width: inherit;
+    align-content: center;
+}
+
+.rec-scroll {
+    overflow-y: scroll;
+    overflow-x: hidden;
+    position: absolute;
+    // display: flex;
+    z-index: 10000;
+    box-shadow: gray 0.2vw 0.2vw 0.4vw;
+    // flex-direction: column;
+    background: whitesmoke;
+    padding-bottom: 10px;
+    border-radius: 15px;
+    border: solid #BBBBBBBB 1px;
+    width: 200px;
+    text-align: center;
+    max-height: 250px;
+    text-transform: uppercase;
+    color: grey;
+    letter-spacing: 2px;
+}
+
+.content {
+    padding: 10px;
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    justify-content: center;
+}
+
+.image-background {
+    pointer-events: none;
+    background-color: transparent;
+    width: 50%;
+    text-align: center;
+    height: 35px;
+    margin-left: 5px;
+}
+
+img{
+    width: 100%;
+    height: 100%;
+}
+
+.score {
+    // margin-left: 15px;
+    width: 50%;
+    height: 100%;
+    text-align: center;
+}
diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx
new file mode 100644
index 000000000..8569996b3
--- /dev/null
+++ b/src/client/views/Recommendations.tsx
@@ -0,0 +1,169 @@
+import { observer } from "mobx-react";
+import React = require("react");
+import { observable, action } from "mobx";
+import Measure from "react-measure";
+import "./Recommendations.scss";
+import { Doc, DocListCast, WidthSym, HeightSym } from "../../new_fields/Doc";
+import { DocumentIcon } from "./nodes/DocumentIcon";
+import { StrCast, NumCast } from "../../new_fields/Types";
+import { returnFalse, emptyFunction, returnEmptyString, returnOne } from "../../Utils";
+import { Transform } from "../util/Transform";
+import { ObjectField } from "../../new_fields/ObjectField";
+import { DocumentView } from "./nodes/DocumentView";
+import { DocumentType } from "../documents/Documents";
+
+
+export interface RecProps {
+    documents: { preview: Doc, similarity: number }[];
+    node: Doc;
+}
+
+@observer
+export class Recommendations extends React.Component<{}> {
+
+    static Instance: Recommendations;
+    @observable private _display: boolean = false;
+    @observable private _pageX: number = 0;
+    @observable private _pageY: number = 0;
+    @observable private _width: number = 0;
+    @observable private _height: number = 0;
+    @observable private _documents: { preview: Doc, score: number }[] = [];
+
+    constructor(props: {}) {
+        super(props);
+        Recommendations.Instance = this;
+    }
+
+    private DocumentIcon(doc: Doc) {
+        let layoutresult = StrCast(doc.type);
+        let renderDoc = doc;
+        //let box: number[] = [];
+        if (layoutresult.indexOf(DocumentType.COL) !== -1) {
+            renderDoc = Doc.MakeDelegate(renderDoc);
+            let bounds = DocListCast(renderDoc.data).reduce((bounds, doc) => {
+                var [sptX, sptY] = [NumCast(doc.x), NumCast(doc.y)];
+                let [bptX, bptY] = [sptX + doc[WidthSym](), sptY + doc[HeightSym]()];
+                return {
+                    x: Math.min(sptX, bounds.x), y: Math.min(sptY, bounds.y),
+                    r: Math.max(bptX, bounds.r), b: Math.max(bptY, bounds.b)
+                };
+            }, { x: Number.MAX_VALUE, y: Number.MAX_VALUE, r: Number.MIN_VALUE, b: Number.MIN_VALUE });
+        }
+        let returnXDimension = () => 50;
+        let returnYDimension = () => 50;
+        let scale = () => returnXDimension() / NumCast(renderDoc.nativeWidth, returnXDimension());
+        let newRenderDoc = Doc.MakeDelegate(renderDoc); ///   newRenderDoc -> renderDoc -> render"data"Doc -> TextProt
+        const docview = <div>
+            {/* onPointerDown={action(() => {
+                this._useIcons = !this._useIcons;
+                this._displayDim = this._useIcons ? 50 : Number(SEARCH_THUMBNAIL_SIZE);
+            })}
+            onPointerEnter={action(() => this._displayDim = this._useIcons ? 50 : Number(SEARCH_THUMBNAIL_SIZE))}
+            onPointerLeave={action(() => this._displayDim = 50)} > */}
+            <DocumentView
+                fitToBox={StrCast(doc.type).indexOf(DocumentType.COL) !== -1}
+                Document={doc}
+                addDocument={returnFalse}
+                removeDocument={returnFalse}
+                ScreenToLocalTransform={Transform.Identity}
+                addDocTab={returnFalse}
+                renderDepth={1}
+                PanelWidth={returnXDimension}
+                PanelHeight={returnYDimension}
+                focus={emptyFunction}
+                backgroundColor={returnEmptyString}
+                selectOnLoad={false}
+                parentActive={returnFalse}
+                whenActiveChanged={returnFalse}
+                bringToFront={emptyFunction}
+                zoomToScale={emptyFunction}
+                getScale={returnOne}
+                ContainingCollectionView={undefined}
+                ContentScaling={scale}
+            />
+        </div>;
+        const data = renderDoc.data;
+        if (data instanceof ObjectField) newRenderDoc.data = ObjectField.MakeCopy(data);
+        newRenderDoc.preview = true;
+        return docview;
+
+    }
+
+    @action
+    closeMenu = () => {
+        this._display = false;
+    }
+
+    @action
+    resetDocuments = () => {
+        this._documents = [];
+    }
+
+    @action
+    addDocuments = (documents: { preview: Doc, score: number }[]) => {
+        this._documents = documents;
+    }
+
+    @action
+    displayRecommendations(x: number, y: number) {
+        this._pageX = x;
+        this._pageY = y;
+        this._display = true;
+    }
+
+    static readonly buffer = 20;
+
+    get pageX() {
+        const x = this._pageX;
+        if (x < 0) {
+            return 0;
+        }
+        const width = this._width;
+        if (x + width > window.innerWidth - Recommendations.buffer) {
+            return window.innerWidth - Recommendations.buffer - width;
+        }
+        return x;
+    }
+
+    get pageY() {
+        const y = this._pageY;
+        if (y < 0) {
+            return 0;
+        }
+        const height = this._height;
+        if (y + height > window.innerHeight - Recommendations.buffer) {
+            return window.innerHeight - Recommendations.buffer - height;
+        }
+        return y;
+    }
+
+    render() {
+        if (!this._display) {
+            return null;
+        }
+        let style = { left: this.pageX, top: this.pageY };
+        //const transform = "translate(" + (NumCast(this.props.node.x) + 350) + "px, " + NumCast(this.props.node.y) + "px"
+        return (
+            <Measure offset onResize={action((r: any) => { this._width = r.offset.width; this._height = r.offset.height; })}>
+                {({ measureRef }) => (
+                    <div className="rec-scroll" style={style} ref={measureRef}>
+                        <p>Recommendations</p>
+                        {this._documents.map(doc => {
+                            return (
+                                <div className="content">
+                                    <span className="image-background">
+                                        {this.DocumentIcon(doc.preview)}
+                                    </span>
+                                    <span className="score">{doc.score}</span>
+                                </div>
+                            );
+                        })}
+
+                    </div>
+                )
+                }
+
+            </Measure>
+        );
+    }
+}
\ No newline at end of file
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index d1e8031fd..50f7e2dc8 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -896,10 +896,13 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                 allDocs.forEach(doc => console.log(doc.title));
                 // clears internal representation of documents as vectors
                 ClientRecommender.Instance.reset_docs();
-                await Promise.all(activedocs.map((doc: Doc) => {
-                    //console.log(StrCast(doc.title));
-                    const extdoc = doc.data_ext as Doc;
-                    return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc);
+                await Promise.all(allDocs.map((doc: Doc) => {
+                    console.log(StrCast(doc.title));
+                    if (doc.type === DocumentType.IMG) {
+                        console.log(doc.title);
+                        const extdoc = doc.data_ext as Doc;
+                        return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc);
+                    }
                 }));
                 console.log(ClientRecommender.Instance.createDistanceMatrix());
             },
@@ -1013,7 +1016,7 @@ class CollectionFreeFormViewPannableContents extends React.Component<CollectionF
         const zoom = this.props.zoomScaling();// needs to be a variable outside of the <Measure> otherwise, reactions won't fire
         return <div className={freeformclass} style={{ borderRadius: "inherit", transform: `translate(${cenx}px, ${ceny}px) scale(${zoom}, ${zoom}) translate(${panx}px, ${pany}px)` }}>
             {this.props.children}
-            <ClientRecommender title="Distance Matrix" />
+            {/* <ClientRecommender title="Distance Matrix" /> */}
         </div>;
     }
 }
\ No newline at end of file
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 24bcc0217..3ce4dbd4f 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -40,9 +40,13 @@ import React = require("react");
 import { DictationManager } from '../../util/DictationManager';
 import { MainView } from '../MainView';
 import requestPromise = require('request-promise');
-import { Recommendations } from '../../../Recommendations';
+import { Recommendations } from '../Recommendations';
+import { SearchUtil } from '../../util/SearchUtil';
+import { ClientRecommender } from '../../ClientRecommender';
+import { DocumentType } from '../../documents/Documents';
 const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this?
 
+library.add(fa.faBrain);
 library.add(fa.faTrash);
 library.add(fa.faShare);
 library.add(fa.faDownload);
@@ -610,6 +614,33 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                 a.click();
             }
         });
+        cm.addItem({
+            description: "Recommender System",
+            event: async () => {
+                if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" });
+                let documents: Doc[] = [];
+                let allDocs = await SearchUtil.GetAllDocs();
+                allDocs.forEach(doc => console.log(doc.title));
+                // clears internal representation of documents as vectors
+                ClientRecommender.Instance.reset_docs();
+                await Promise.all(allDocs.map((doc: Doc) => {
+                    if (doc.type === DocumentType.IMG) {
+                        console.log(StrCast(doc.title));
+                        documents.push(doc);
+                        const extdoc = doc.data_ext as Doc;
+                        return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc);
+                    }
+                }));
+                console.log(ClientRecommender.Instance.createDistanceMatrix());
+                let recDocs: { preview: Doc, score: number }[] = [];
+                for (let i = 0; i < documents.length; i++) {
+                    recDocs.push({ preview: documents[i], score: i });
+                }
+                Recommendations.Instance.addDocuments(recDocs);
+                Recommendations.Instance.displayRecommendations(e.pageX + 100, e.pageY);
+            },
+            icon: "brain"
+        });
         cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" });
         type User = { email: string, userDocumentId: string };
         let usersMenu: ContextMenuProps[] = [];
@@ -758,7 +789,6 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                         </div>
                     }
                 </div>
-                <Recommendations documents={documents} node={this.props.Document}></Recommendations>
             </div>
         );
     }
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index 73b892e26..45d389ba6 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -28,6 +28,9 @@ import FaceRectangles from './FaceRectangles';
 import { FieldView, FieldViewProps } from './FieldView';
 import "./ImageBox.scss";
 import React = require("react");
+import { SearchUtil } from '../../util/SearchUtil';
+import { ClientRecommender } from '../../ClientRecommender';
+import { DocumentType } from '../../documents/Documents';
 var requestImageSize = require('../../util/request-image-size');
 var path = require('path');
 const { Howl } = require('howler');
@@ -240,10 +243,20 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
         }
     }
 
-    extractText = () => {
-        //Recommender.Instance.extractText(this.dataDoc, this.extensionDoc);
-        // request recommender 
-        //fetch(Utils.prepend("/recommender"), { body: body, method: "POST", headers: { "content-type": "application/json" } }).then((value) => console.log(value));
+    extractText = async () => {
+        //let activedocs = this.getActiveDocuments();
+        let allDocs = await SearchUtil.GetAllDocs();
+        allDocs.forEach(doc => console.log(doc.title));
+        // clears internal representation of documents as vectors
+        ClientRecommender.Instance.reset_docs();
+        await Promise.all(allDocs.map((doc: Doc) => {
+            //console.log(StrCast(doc.title));
+            if (doc.type === DocumentType.IMG) {
+                const extdoc = doc.data_ext as Doc;
+                return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc);
+            }
+        }));
+        console.log(ClientRecommender.Instance.createDistanceMatrix());
     }
 
     generateMetadata = (threshold: Confidence = Confidence.Excellent) => {
-- 
cgit v1.2.3-70-g09d2


From e0bfe978e029268b3901b5d098f946b1a6fc7d0d Mon Sep 17 00:00:00 2001
From: ab <abdullah_ahmed@brown.edu>
Date: Thu, 29 Aug 2019 18:43:32 -0400
Subject: ui fixes, datadoc resolved

---
 src/client/ClientRecommender.tsx                   | 83 +++++++++++++++-------
 src/client/cognitive_services/CognitiveServices.ts | 24 ++++---
 src/client/views/MainView.tsx                      |  1 +
 src/client/views/Recommendations.tsx               | 21 ++++--
 .../collectionFreeForm/CollectionFreeFormView.tsx  | 21 ------
 src/client/views/nodes/DocumentView.tsx            | 26 ++++---
 src/client/views/nodes/ImageBox.tsx                | 16 -----
 7 files changed, 104 insertions(+), 88 deletions(-)

(limited to 'src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx')

diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx
index 63f85c737..a6d1a32b3 100644
--- a/src/client/ClientRecommender.tsx
+++ b/src/client/ClientRecommender.tsx
@@ -1,5 +1,5 @@
 import { Doc } from "../new_fields/Doc";
-import { StrCast } from "../new_fields/Types";
+import { StrCast, Cast } from "../new_fields/Types";
 import { List } from "../new_fields/List";
 import { CognitiveServices } from "./cognitive_services/CognitiveServices";
 import React = require("react");
@@ -8,30 +8,42 @@ import { observable, action, computed, reaction } from "mobx";
 var assert = require('assert');
 import "./ClientRecommender.scss";
 import { JSXElement } from "babel-types";
+import { ToPlainText, RichTextField } from "../new_fields/RichTextField";
 
 export interface RecommenderProps {
     title: string;
 }
 
+export interface RecommenderDocument {
+    actualDoc: Doc;
+    vectorDoc: number[];
+    score: number;
+}
+
 @observer
 export class ClientRecommender extends React.Component<RecommenderProps> {
 
     static Instance: ClientRecommender;
-    private docVectors: Set<number[]>;
+    private mainDoc?: RecommenderDocument;
+    private docVectors: Set<RecommenderDocument> = new Set();
     @observable private corr_matrix = [[0, 0], [0, 0]];
 
     constructor(props: RecommenderProps) {
         //console.log("creating client recommender...");
         super(props);
         if (!ClientRecommender.Instance) ClientRecommender.Instance = this;
-        this.docVectors = new Set<number[]>();
-        //this.corr_matrix = [[0, 0], [0, 0]];
+        ClientRecommender.Instance.docVectors = new Set();
+        //ClientRecommender.Instance.corr_matrix = [[0, 0], [0, 0]];
     }
 
     @action
     public reset_docs() {
-        this.docVectors = new Set();
-        this.corr_matrix = [[0, 0], [0, 0]];
+        ClientRecommender.Instance.docVectors = new Set();
+        ClientRecommender.Instance.corr_matrix = [[0, 0], [0, 0]];
+    }
+
+    public deleteDocs() {
+        console.log("deleting previews...");
     }
 
     /***
@@ -67,11 +79,24 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
         }
     }
 
+    public computeSimilarities() {
+        ClientRecommender.Instance.docVectors.forEach((doc: RecommenderDocument) => {
+            if (ClientRecommender.Instance.mainDoc) {
+                const distance = ClientRecommender.Instance.distance(ClientRecommender.Instance.mainDoc.vectorDoc, doc.vectorDoc, "euclidian");
+                doc.score = distance;
+            }
+        }
+        );
+        let doclist = Array.from(ClientRecommender.Instance.docVectors);
+        doclist.sort((a: RecommenderDocument, b: RecommenderDocument) => a.score - b.score);
+        return doclist;
+    }
+
     /***
      * Computes the mean of a set of vectors
      */
 
-    public mean(paragraph: Set<number[]>) {
+    public mean(paragraph: Set<number[]>, dataDoc: Doc, mainDoc: boolean) {
         const n = 200;
         const num_words = paragraph.size;
         let meanVector = new Array<number>(n).fill(0); // mean vector
@@ -82,14 +107,16 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
                 }
             });
             meanVector = meanVector.map(x => x / num_words);
-            this.addToDocSet(meanVector);
+            const internalDoc: RecommenderDocument = { actualDoc: dataDoc, vectorDoc: meanVector, score: 0 };
+            if (mainDoc) ClientRecommender.Instance.mainDoc = internalDoc;
+            ClientRecommender.Instance.addToDocSet(internalDoc);
         }
         return meanVector;
     }
 
-    private addToDocSet(vector: number[]) {
-        if (this.docVectors) {
-            this.docVectors.add(vector);
+    private addToDocSet(internalDoc: RecommenderDocument) {
+        if (ClientRecommender.Instance.docVectors) {
+            ClientRecommender.Instance.docVectors.add(internalDoc);
         }
     }
 
@@ -97,9 +124,11 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
      * Uses Cognitive Services to extract keywords from a document
      */
 
-    public async extractText(dataDoc: Doc, extDoc: Doc) {
-        let data = StrCast(dataDoc.title);
-        //console.log(data);
+    public async extractText(dataDoc: Doc, extDoc: Doc, mainDoc: boolean = false) {
+        let fielddata = Cast(dataDoc.data, RichTextField);
+        let data: string;
+        fielddata ? data = fielddata[ToPlainText]() : data = "";
+        console.log(data);
         let converter = (results: any) => {
             let keyterms = new List<string>();
             results.documents.forEach((doc: any) => {
@@ -108,7 +137,7 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
             });
             return keyterms;
         };
-        await CognitiveServices.Text.Appliers.analyzer(extDoc, ["key words"], data, converter);
+        await CognitiveServices.Text.Appliers.analyzer(dataDoc, extDoc, ["key words"], data, converter, mainDoc);
     }
 
     /***
@@ -116,7 +145,7 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
      */
 
     @action
-    public createDistanceMatrix(documents: Set<number[]> = this.docVectors) {
+    public createDistanceMatrix(documents: Set<RecommenderDocument> = ClientRecommender.Instance.docVectors) {
         const documents_list = Array.from(documents);
         const n = documents_list.length;
         var matrix = new Array<number>(n).fill(0).map(() => new Array<number>(n).fill(0));
@@ -124,22 +153,22 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
             var doc1 = documents_list[i];
             for (let j = 0; j < n; j++) {
                 var doc2 = documents_list[j];
-                matrix[i][j] = this.distance(doc1, doc2, "euclidian");
+                matrix[i][j] = ClientRecommender.Instance.distance(doc1.vectorDoc, doc2.vectorDoc, "euclidian");
             }
         }
-        this.corr_matrix = matrix;
+        ClientRecommender.Instance.corr_matrix = matrix;
         return matrix;
     }
 
     @computed
     private get generateRows() {
-        const n = this.corr_matrix.length;
+        const n = ClientRecommender.Instance.corr_matrix.length;
         let rows: JSX.Element[] = [];
         for (let i = 0; i < n; i++) {
             let children: JSX.Element[] = [];
             for (let j = 0; j < n; j++) {
-                //let cell = React.createElement("td", this.corr_matrix[i][j]);
-                let cell = <td>{this.corr_matrix[i][j].toFixed(4)}</td>;
+                //let cell = React.createElement("td", ClientRecommender.Instance.corr_matrix[i][j]);
+                let cell = <td>{ClientRecommender.Instance.corr_matrix[i][j].toFixed(4)}</td>;
                 children.push(cell);
             }
             //let row = React.createElement("tr", { children: children, key: i });
@@ -151,22 +180,22 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
 
     render() {
         return (<div className="wrapper">
-            <h3 >{this.props.title ? this.props.title : "hello"}</h3>
+            <h3 >{ClientRecommender.Instance.props.title ? ClientRecommender.Instance.props.title : "hello"}</h3>
             {/* <table className="space" >
                 <tbody>
                     <tr key="1">
-                        <td key="1">{this.corr_matrix[0][0].toFixed(4)}</td>
-                        <td key="2">{this.corr_matrix[0][1].toFixed(4)}</td>
+                        <td key="1">{ClientRecommender.Instance.corr_matrix[0][0].toFixed(4)}</td>
+                        <td key="2">{ClientRecommender.Instance.corr_matrix[0][1].toFixed(4)}</td>
                     </tr>
                     <tr key="2">
-                        <td key="1">{this.corr_matrix[1][0].toFixed(4)}</td>
-                        <td key="2">{this.corr_matrix[1][1].toFixed(4)}</td>
+                        <td key="1">{ClientRecommender.Instance.corr_matrix[1][0].toFixed(4)}</td>
+                        <td key="2">{ClientRecommender.Instance.corr_matrix[1][1].toFixed(4)}</td>
                     </tr>
                 </tbody>
             </table> */}
             <table className="space">
                 <tbody>
-                    {this.generateRows}
+                    {ClientRecommender.Instance.generateRows}
                 </tbody>
             </table>
         </div>);
diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts
index 75d0760ed..874ee433d 100644
--- a/src/client/cognitive_services/CognitiveServices.ts
+++ b/src/client/cognitive_services/CognitiveServices.ts
@@ -263,29 +263,35 @@ export namespace CognitiveServices {
 
         export namespace Appliers {
 
-            export async function vectorize(keyterms: any) {
+            export async function vectorize(keyterms: any, dataDoc: Doc, mainDoc: boolean = false) {
                 console.log("vectorizing...");
                 //keyterms = ["father", "king"];
                 let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true };
                 await requestPromise.post(args).then(async (wordvecs) => {
-                    var vectorValues = new Set<number[]>();
-                    wordvecs.forEach((wordvec: any) => {
-                        //console.log(wordvec.word);
-                        vectorValues.add(wordvec.values as number[]);
-                    });
-                    ClientRecommender.Instance.mean(vectorValues);
+                    if (wordvecs.length > 0) {
+                        console.log("successful vectorization!");
+                        var vectorValues = new Set<number[]>();
+                        wordvecs.forEach((wordvec: any) => {
+                            //console.log(wordvec.word);
+                            vectorValues.add(wordvec.values as number[]);
+                        });
+                        ClientRecommender.Instance.mean(vectorValues, dataDoc, mainDoc);
+                    } // adds document to internal doc set
+                    else {
+                        console.log("unsuccessful :( word(s) not in vocabulary");
+                    }
                     //console.log(vectorValues.size);
                 });
             }
 
-            export const analyzer = async (target: Doc, keys: string[], data: string, converter: Converter) => {
+            export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: Converter, mainDoc: boolean = false) => {
                 let results = await ExecuteQuery(Service.Text, Manager, data);
                 console.log(results);
                 let keyterms = converter(results);
                 //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis");
                 target[keys[0]] = keyterms;
                 console.log("analyzed!");
-                await vectorize(keyterms);
+                await vectorize(keyterms, dataDoc, mainDoc);
             };
         }
 
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 57eb30439..3a5795077 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -204,6 +204,7 @@ export class MainView extends React.Component {
         const targets = document.elementsFromPoint(e.x, e.y);
         if (targets && targets.length && targets[0].className.toString().indexOf("contextMenu") === -1) {
             ContextMenu.Instance.closeMenu();
+            Recommendations.Instance.closeMenu();
         }
     });
 
diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx
index 8569996b3..cf1974c69 100644
--- a/src/client/views/Recommendations.tsx
+++ b/src/client/views/Recommendations.tsx
@@ -10,8 +10,10 @@ import { returnFalse, emptyFunction, returnEmptyString, returnOne } from "../../
 import { Transform } from "../util/Transform";
 import { ObjectField } from "../../new_fields/ObjectField";
 import { DocumentView } from "./nodes/DocumentView";
-import { DocumentType } from "../documents/Documents";
-
+import { DocumentType } from '../documents/DocumentTypes';
+import { ClientRecommender } from "../ClientRecommender";
+import { DocServer } from "../DocServer";
+import { Id } from "../../new_fields/FieldSymbols";
 
 export interface RecProps {
     documents: { preview: Doc, similarity: number }[];
@@ -28,6 +30,7 @@ export class Recommendations extends React.Component<{}> {
     @observable private _width: number = 0;
     @observable private _height: number = 0;
     @observable private _documents: { preview: Doc, score: number }[] = [];
+    private previewDocs: Doc[] = [];
 
     constructor(props: {}) {
         super(props);
@@ -52,7 +55,8 @@ export class Recommendations extends React.Component<{}> {
         let returnXDimension = () => 50;
         let returnYDimension = () => 50;
         let scale = () => returnXDimension() / NumCast(renderDoc.nativeWidth, returnXDimension());
-        let newRenderDoc = Doc.MakeDelegate(renderDoc); ///   newRenderDoc -> renderDoc -> render"data"Doc -> TextProt
+        //let scale = () => 1;
+        //let newRenderDoc = Doc.MakeDelegate(renderDoc); ///   newRenderDoc -> renderDoc -> render"data"Doc -> TextProt
         const docview = <div>
             {/* onPointerDown={action(() => {
                 this._useIcons = !this._useIcons;
@@ -62,7 +66,7 @@ export class Recommendations extends React.Component<{}> {
             onPointerLeave={action(() => this._displayDim = 50)} > */}
             <DocumentView
                 fitToBox={StrCast(doc.type).indexOf(DocumentType.COL) !== -1}
-                Document={doc}
+                Document={renderDoc}
                 addDocument={returnFalse}
                 removeDocument={returnFalse}
                 ScreenToLocalTransform={Transform.Identity}
@@ -82,9 +86,10 @@ export class Recommendations extends React.Component<{}> {
                 ContentScaling={scale}
             />
         </div>;
-        const data = renderDoc.data;
-        if (data instanceof ObjectField) newRenderDoc.data = ObjectField.MakeCopy(data);
-        newRenderDoc.preview = true;
+        // const data = renderDoc.data;
+        // if (data instanceof ObjectField) newRenderDoc.data = ObjectField.MakeCopy(data);
+        // newRenderDoc.preview = true;
+        // this.previewDocs.push(newRenderDoc);
         return docview;
 
     }
@@ -92,6 +97,8 @@ export class Recommendations extends React.Component<{}> {
     @action
     closeMenu = () => {
         this._display = false;
+        this.previewDocs.forEach(doc => DocServer.DeleteDocument(doc[Id]));
+        this.previewDocs = [];
     }
 
     @action
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 2d4775070..3cef93383 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -857,27 +857,6 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
                 input.click();
             }
         });
-        ContextMenu.Instance.addItem({
-            description: "Recommender System",
-            event: async () => {
-                // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" });
-                let activedocs = this.getActiveDocuments();
-                let allDocs = await SearchUtil.GetAllDocs();
-                allDocs.forEach(doc => console.log(doc.title));
-                // clears internal representation of documents as vectors
-                ClientRecommender.Instance.reset_docs();
-                await Promise.all(allDocs.map((doc: Doc) => {
-                    console.log(StrCast(doc.title));
-                    if (doc.type === DocumentType.IMG) {
-                        console.log(doc.title);
-                        const extdoc = doc.data_ext as Doc;
-                        return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc);
-                    }
-                }));
-                console.log(ClientRecommender.Instance.createDistanceMatrix());
-            },
-            icon: "brain"
-        });
         layoutItems.push({ description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`, event: this.fitToContainer, icon: !this.fitToBox ? "expand-arrows-alt" : "compress-arrows-alt" });
         layoutItems.push({ description: "reset view", event: () => { this.props.Document.panX = this.props.Document.panY = 0; this.props.Document.scale = 1; }, icon: "compress-arrows-alt" });
         layoutItems.push({
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 2a6e91272..f708a7a3a 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -648,21 +648,31 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                 if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" });
                 let documents: Doc[] = [];
                 let allDocs = await SearchUtil.GetAllDocs();
-                allDocs.forEach(doc => console.log(doc.title));
+                //allDocs.forEach(doc => console.log(doc.title));
                 // clears internal representation of documents as vectors
                 ClientRecommender.Instance.reset_docs();
                 await Promise.all(allDocs.map((doc: Doc) => {
-                    if (doc.type === DocumentType.IMG) {
-                        console.log(StrCast(doc.title));
-                        documents.push(doc);
-                        const extdoc = doc.data_ext as Doc;
-                        return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc);
+                    let mainDoc: boolean = false;
+                    const dataDoc = Doc.GetDataDoc(doc);
+                    if (doc.type === DocumentType.TEXT) {
+                        if (dataDoc === Doc.GetDataDoc(this.props.Document)) {
+                            mainDoc = true;
+                            console.log(StrCast(doc.title));
+                        }
+                        if (!documents.includes(dataDoc)) {
+                            documents.push(dataDoc);
+                            const extdoc = doc.data_ext as Doc;
+                            return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc, mainDoc);
+                        }
                     }
                 }));
                 console.log(ClientRecommender.Instance.createDistanceMatrix());
+                const doclist = ClientRecommender.Instance.computeSimilarities();
                 let recDocs: { preview: Doc, score: number }[] = [];
-                for (let i = 0; i < documents.length; i++) {
-                    recDocs.push({ preview: documents[i], score: i });
+                // tslint:disable-next-line: prefer-for-of
+                for (let i = 0; i < doclist.length; i++) {
+                    console.log(doclist[i].score);
+                    recDocs.push({ preview: doclist[i].actualDoc, score: doclist[i].score });
                 }
                 Recommendations.Instance.addDocuments(recDocs);
                 Recommendations.Instance.displayRecommendations(e.pageX + 100, e.pageY);
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index ec35465eb..d94e92847 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -240,22 +240,6 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
         }
     }
 
-    extractText = async () => {
-        //let activedocs = this.getActiveDocuments();
-        let allDocs = await SearchUtil.GetAllDocs();
-        allDocs.forEach(doc => console.log(doc.title));
-        // clears internal representation of documents as vectors
-        ClientRecommender.Instance.reset_docs();
-        await Promise.all(allDocs.map((doc: Doc) => {
-            //console.log(StrCast(doc.title));
-            if (doc.type === DocumentType.IMG) {
-                const extdoc = doc.data_ext as Doc;
-                return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc);
-            }
-        }));
-        console.log(ClientRecommender.Instance.createDistanceMatrix());
-    }
-
     generateMetadata = (threshold: Confidence = Confidence.Excellent) => {
         let converter = (results: any) => {
             let tagDoc = new Doc;
-- 
cgit v1.2.3-70-g09d2


From c6e2eee8e5b035ed4cab1e7bb1315a36d43255c5 Mon Sep 17 00:00:00 2001
From: ab <abdullah_ahmed@brown.edu>
Date: Tue, 8 Oct 2019 16:02:25 -0400
Subject: switching out

---
 src/client/views/MainView.tsx                           |  1 -
 src/client/views/Recommendations.tsx                    |  2 +-
 .../collectionFreeForm/CollectionFreeFormView.tsx       |  3 ---
 src/client/views/nodes/DocumentContentsView.tsx         |  6 ------
 src/client/views/nodes/DocumentView.tsx                 |  4 ++--
 src/client/views/nodes/ImageBox.tsx                     |  1 -
 src/new_fields/RichTextField.ts                         | 17 ++++++++++++++++-
 7 files changed, 19 insertions(+), 15 deletions(-)

(limited to 'src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx')

diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 17de708a2..e6f500e75 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -41,7 +41,6 @@ import { FilterBox } from './search/FilterBox';
 import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField';
 //import { DocumentManager } from '../util/DocumentManager';
 import { RecommendationsBox } from './Recommendations';
-import PresModeMenu from './presentationview/PresentationModeMenu';
 import { PresBox } from './nodes/PresBox';
 import { OverlayView } from './OverlayView';
 
diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx
index b7b1d84d0..c44dfc032 100644
--- a/src/client/views/Recommendations.tsx
+++ b/src/client/views/Recommendations.tsx
@@ -174,7 +174,7 @@ export class RecommendationsBox extends React.Component<FieldViewProps> {
                                 {this.DocumentIcon(doc)}
                             </span>
                             <span className="score">{NumCast(doc.score).toFixed(4)}</span>
-                            <div style={{ marginRight: 50 }} onClick={() => DocumentManager.Instance.jumpToDocument(doc, true, undefined, undefined, undefined, this.props.Document.sourceDocContext as Doc)}>
+                            <div style={{ marginRight: 50 }} onClick={() => DocumentManager.Instance.jumpToDocument(doc, false)}>
                                 <FontAwesomeIcon className="documentdecorations-icon" icon={"bullseye"} size="sm" />
                             </div>
                             <div style={{ marginRight: 50 }} onClick={() => DocUtils.MakeLink(this.props.Document.sourceDoc as Doc, doc, undefined, "User Selected Link", "Generated from Recommender", undefined)}>
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 3ee069e4c..d2b8afa02 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -12,7 +12,6 @@ import { BoolCast, Cast, DateCast, NumCast, StrCast } from "../../../../new_fiel
 import { CurrentUserUtils } from "../../../../server/authentication/models/current_user_utils";
 import { aggregateBounds, emptyFunction, intersectRect, returnEmptyString, returnOne, Utils } from "../../../../Utils";
 import { CognitiveServices } from "../../../cognitive_services/CognitiveServices";
-import { DocServer } from "../../../DocServer";
 import { Docs } from "../../../documents/Documents";
 import { DocumentType } from "../../../documents/DocumentTypes";
 import { DocumentManager } from "../../../util/DocumentManager";
@@ -28,7 +27,6 @@ import { InkingCanvas } from "../../InkingCanvas";
 import { CollectionFreeFormDocumentView, positionSchema } from "../../nodes/CollectionFreeFormDocumentView";
 import { DocumentContentsView } from "../../nodes/DocumentContentsView";
 import { documentSchema, DocumentViewProps } from "../../nodes/DocumentView";
-import { FormattedTextBox } from "../../nodes/FormattedTextBox";
 import { pageSchema } from "../../nodes/ImageBox";
 import PDFMenu from "../../pdf/PDFMenu";
 import { CollectionSubView } from "../CollectionSubView";
@@ -41,7 +39,6 @@ import React = require("react");
 import v5 = require("uuid/v5");
 import { ClientRecommender } from "../../../ClientRecommender";
 import { SearchUtil } from "../../../util/SearchUtil";
-import { SearchBox } from "../../SearchBox";
 import { RouteStore } from "../../../../server/RouteStore";
 import { string, number, elementType } from "prop-types";
 import { DocServer } from "../../../DocServer";
diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx
index e5eb8dbce..919c13055 100644
--- a/src/client/views/nodes/DocumentContentsView.tsx
+++ b/src/client/views/nodes/DocumentContentsView.tsx
@@ -2,8 +2,6 @@ import { computed } from "mobx";
 import { observer } from "mobx-react";
 import { Doc } from "../../../new_fields/Doc";
 import { ScriptField } from "../../../new_fields/ScriptField";
-import { Cast } from "../../../new_fields/Types";
-import { OmitKeys, Without } from "../../../Utils";
 import { HistogramBox } from "../../northstar/dash-nodes/HistogramBox";
 import DirectoryImportBox from "../../util/Import & Export/DirectoryImportBox";
 import { CollectionDockingView } from "../collections/CollectionDockingView";
@@ -30,14 +28,10 @@ import { PresElementBox } from "../presentationview/PresElementBox";
 import { VideoBox } from "./VideoBox";
 import { WebBox } from "./WebBox";
 import React = require("react");
-import { FieldViewProps } from "./FieldView";
 import { Without, OmitKeys } from "../../../Utils";
 import { Cast, StrCast, NumCast } from "../../../new_fields/Types";
 import { List } from "../../../new_fields/List";
-import { Doc } from "../../../new_fields/Doc";
-import DirectoryImportBox from "../../util/Import & Export/DirectoryImportBox";
 import { RecommendationsBox } from "../../views/Recommendations";
-import { ScriptField } from "../../../new_fields/ScriptField";
 const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this?
 
 type BindingProps = Without<FieldViewProps, 'fieldKey'>;
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 0064b98c3..070b1f426 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -621,7 +621,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
         recommendations.documentIconHeight = 150;
         recommendations.sourceDoc = this.props.Document;
         recommendations.sourceDocContext = this.props.ContainingCollectionView!.props.Document;
-        CollectionDockingView.Instance.AddRightSplit(recommendations, undefined);
+        CollectionDockingView.AddRightSplit(recommendations, undefined);
 
         // RecommendationsBox.Instance.displayRecommendations(e.pageX + 100, e.pageY);
     }
@@ -641,7 +641,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
             body.href = urls[i];
             bodies.push(body);
         }
-        CollectionDockingView.Instance.AddRightSplit(Docs.Create.SchemaDocument(headers, bodies, { title: `Showing External Recommendations for "${StrCast(doc.title)}"` }), undefined);
+        CollectionDockingView.AddRightSplit(Docs.Create.SchemaDocument(headers, bodies, { title: `Showing External Recommendations for "${StrCast(doc.title)}"` }), undefined);
     }
 
     onPointerEnter = (e: React.PointerEvent): void => { Doc.BrushDoc(this.props.Document); };
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index f36b9895f..7ffe64b9b 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -29,7 +29,6 @@ import "./ImageBox.scss";
 import React = require("react");
 import { SearchUtil } from '../../util/SearchUtil';
 import { ClientRecommender } from '../../ClientRecommender';
-import { DocumentType } from '../../documents/Documents';
 var requestImageSize = require('../../util/request-image-size');
 var path = require('path');
 const { Howl } = require('howler');
diff --git a/src/new_fields/RichTextField.ts b/src/new_fields/RichTextField.ts
index 390045ee1..f41ea0350 100644
--- a/src/new_fields/RichTextField.ts
+++ b/src/new_fields/RichTextField.ts
@@ -4,6 +4,9 @@ import { Deserializable } from "../client/util/SerializationHelper";
 import { Copy, ToScriptString, ToPlainText } from "./FieldSymbols";
 import { scriptingGlobal } from "../client/util/Scripting";
 
+const delimiter = "\n";
+const joiner = "";
+
 @scriptingGlobal
 @Deserializable("RichTextField")
 export class RichTextField extends ObjectField {
@@ -24,7 +27,19 @@ export class RichTextField extends ObjectField {
     }
 
     [ToPlainText]() {
-        return this.Data;
+        // Because we're working with plain text, just concatenate all paragraphs
+        let content = JSON.parse(this.Data).doc.content;
+        let paragraphs = content.filter((item: any) => item.type === "paragraph");
+
+        // Functions to flatten ProseMirror paragraph objects (and their components) to plain text
+        // While this function already exists in state.doc.textBeteen(), it doesn't account for newlines 
+        let blockText = (block: any) => block.text;
+        let concatenateParagraph = (p: any) => (p.content ? p.content.map(blockText).join(joiner) : "") + delimiter;
+
+        // Concatentate paragraphs and string the result together
+        let textParagraphs: string[] = paragraphs.map(concatenateParagraph);
+        let plainText = textParagraphs.join(joiner);
+        return plainText.substring(0, plainText.length - 1);
     }
 
 }
\ No newline at end of file
-- 
cgit v1.2.3-70-g09d2


From e103be2d58da2a6809dd4ad5f0b5f445d8d6c96b Mon Sep 17 00:00:00 2001
From: ab <abdullah_ahmed@brown.edu>
Date: Sat, 16 Nov 2019 16:45:41 -0500
Subject: ibm integrated, bing search -> server next time

---
 package.json                                       |  7 ++-
 src/client/ClientRecommender.tsx                   | 66 ++++++++++++++++++++--
 src/client/apis/IBM_Recommender.ts                 |  6 +-
 .../collectionFreeForm/CollectionFreeFormView.tsx  |  5 --
 src/client/views/nodes/DocumentView.tsx            |  2 +-
 src/server/index.ts                                |  2 +-
 6 files changed, 70 insertions(+), 18 deletions(-)

(limited to 'src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx')

diff --git a/package.json b/package.json
index 80b65cc68..d41119a47 100644
--- a/package.json
+++ b/package.json
@@ -57,9 +57,9 @@
         "@hig/theme-context": "^2.1.3",
         "@hig/theme-data": "^2.3.3",
         "@tensorflow-models/universal-sentence-encoder": "^1.2.0",
-        "@tensorflow/tfjs": "^1.3.1",
+        "@tensorflow/tfjs-converter": "^1.3.2",
         "@tensorflow/tfjs-core": "^1.3.1",
-        "@tensorflow/tfjs-node": "^1.3.1",
+        "@tensorflow/tfjs-node": "1.3.1",
         "@trendmicro/react-dropdown": "^1.3.0",
         "@types/adm-zip": "^0.4.32",
         "@types/animejs": "^2.0.2",
@@ -238,8 +238,9 @@
         "url-loader": "^1.1.2",
         "uuid": "^3.3.2",
         "wikijs": "^6.0.1",
+        "word2vec": "^1.1.4",
         "words-to-numbers": "^1.5.1",
         "xoauth2": "^1.2.0",
         "youtube": "^0.1.0"
     }
-}
\ No newline at end of file
+}
diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx
index 364ec0fe0..a37434c0d 100644
--- a/src/client/ClientRecommender.tsx
+++ b/src/client/ClientRecommender.tsx
@@ -8,6 +8,7 @@ import { observable, action, computed, reaction } from "mobx";
 var assert = require('assert');
 var sw = require('stopword');
 var FeedParser = require('feedparser');
+var https = require('https');
 import "./ClientRecommender.scss";
 import { JSXElement } from "babel-types";
 import { RichTextField } from "../new_fields/RichTextField";
@@ -111,7 +112,7 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
         }
         );
         let doclist = Array.from(ClientRecommender.Instance.docVectors);
-        if (distance_metric == "euclidian") {
+        if (distance_metric === "euclidian") {
             doclist.sort((a: RecommenderDocument, b: RecommenderDocument) => a.score - b.score);
         }
         else {
@@ -245,7 +246,27 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
             if (kp_string.length > 2) kp_string = kp_string.substring(0, kp_string.length - 2);
             console.log("kp string: ", kp_string);
             let values = "";
-            if (!internal) values = await this.sendRequest(highKP);
+            if (!internal) {
+                const parameters: any = {
+                    'language': 'en',
+                    'text': data,
+                    'features': {
+                        'keywords': {
+                            'sentiment': true,
+                            'emotion': true,
+                            'limit': 3
+                        }
+                    }
+                };
+                await Identified.PostToServer("/IBMAnalysis", parameters).then(response => {
+                    const sorted_keywords = response.result.keywords;
+                    if (sorted_keywords.length > 0) {
+                        console.log("IBM keyphrase", sorted_keywords[0]);
+                        highKP = [sorted_keywords[0].text];
+                    }
+                });
+                values = await this.sendRequest(highKP, "bing");
+            }
             return { keyterms: keyterms, external_recommendations: values, kp_string: [kp_string] };
         };
         if (data !== "") {
@@ -290,11 +311,46 @@ export class ClientRecommender extends React.Component<RecommenderProps> {
      * API for sending arXiv request.
      */
 
-    private async sendRequest(keywords: string[]) {
+    private async sendRequest(keywords: string[], api: string) {
         let query = "";
         keywords.forEach((kp: string) => query += " " + kp);
-        return new Promise<any>(resolve => {
-            this.arxivrequest(query).then(resolve);
+        if (api === "arxiv") {
+            return new Promise<any>(resolve => {
+                this.arxivrequest(query).then(resolve);
+            });
+        }
+        else if (api === "bing") {
+            await this.bingWebSearch(query);
+        }
+        else {
+            return new Promise<any>(resolve => {
+                this.arxivrequest(query).then(resolve);
+            });
+        }
+
+    }
+
+    bingWebSearch = async (query: string) => {
+        https.get({
+            hostname: 'api.cognitive.microsoft.com',
+            path: '/bing/v5.0/search?q=' + encodeURIComponent(query),
+            headers: { 'Ocp-Apim-Subscription-Key': process.env.BING },
+        }, (res: any) => {
+            let body = '';
+            res.on('data', (part: any) => body += part);
+            res.on('end', () => {
+                for (var header in res.headers) {
+                    if (header.startsWith("bingapis-") || header.startsWith("x-msedge-")) {
+                        console.log(header + ": " + res.headers[header])
+                    }
+                }
+                console.log('\nJSON Response:\n');
+                console.dir(JSON.parse(body), { colors: false, depth: null });
+            })
+            res.on('error', (e: any) => {
+                console.log('Error: ' + e.message);
+                throw e;
+            });
         });
     }
 
diff --git a/src/client/apis/IBM_Recommender.ts b/src/client/apis/IBM_Recommender.ts
index 5f80f5126..da6257f28 100644
--- a/src/client/apis/IBM_Recommender.ts
+++ b/src/client/apis/IBM_Recommender.ts
@@ -16,19 +16,19 @@ export namespace IBM_Recommender {
     });
 
     const analyzeParams = {
-        'url': 'www.ibm.com',
+        'text': 'this is a test of the keyword extraction feature I am integrating into the program',
         'features': {
             'keywords': {
                 'sentiment': true,
                 'emotion': true,
                 'limit': 3
-            }
+            },
         }
     };
 
     export const analyze = async (_parameters: any): Promise<Opt<string>> => {
         try {
-            const response = await naturalLanguageUnderstanding.analyze(analyzeParams);
+            const response = await naturalLanguageUnderstanding.analyze(_parameters);
             console.log(response);
             return (JSON.stringify(response, null, 2));
         } catch (err) {
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index bb5d99a28..bfec545c6 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -34,11 +34,6 @@ import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCurso
 import "./CollectionFreeFormView.scss";
 import { MarqueeView } from "./MarqueeView";
 import React = require("react");
-import v5 = require("uuid/v5");
-import { ClientRecommender } from "../../../ClientRecommender";
-import { SearchUtil } from "../../../util/SearchUtil";
-import { RouteStore } from "../../../../server/RouteStore";
-import { string, number, elementType } from "prop-types";
 import { DocServer } from "../../../DocServer";
 import { documentSchema, positionSchema } from "../../../../new_fields/documentSchemas";
 import { DocumentViewProps } from "../../nodes/DocumentView";
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 55063a52c..c6ad2f9d7 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -772,7 +772,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
     }
     render() {
         if (!this.props.Document) return (null);
-        trace();
+        //trace();
         const animDims = this.Document.animateToDimensions ? Array.from(this.Document.animateToDimensions) : undefined;
         const ruleColor = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleColor_" + this.Document.heading]) : undefined;
         const ruleRounding = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleRounding_" + this.Document.heading]) : undefined;
diff --git a/src/server/index.ts b/src/server/index.ts
index 2a669f147..45fc7fc07 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -1086,7 +1086,7 @@ addSecureRoute({
         const batched = BatchedArray.from<GooglePhotosUploadUtils.MediaInput>(media, { batchSize: 25 });
         const newMediaItems = await batched.batchedMapPatientInterval<NewMediaItem>(
             { magnitude: 100, unit: TimeUnit.Milliseconds },
-            async (batch, collector) => {
+            async (batch: any, collector: any): Promise<any> => {
                 for (let index = 0; index < batch.length; index++) {
                     const { url, description } = batch[index];
                     const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(url);
-- 
cgit v1.2.3-70-g09d2

