From 7f6666500f157c5884377d714137426cf05e7569 Mon Sep 17 00:00:00 2001
From: Tyler Schicke <tyler_schicke@brown.edu>
Date: Tue, 16 Jul 2019 13:40:20 -0400
Subject: Changed search to display extension documents as the extended
 documents

---
 src/client/views/search/SearchBox.tsx | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

(limited to 'src/client/views/search/SearchBox.tsx')

diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx
index ec778b346..dc76d1ff4 100644
--- a/src/client/views/search/SearchBox.tsx
+++ b/src/client/views/search/SearchBox.tsx
@@ -6,7 +6,7 @@ import "./FilterBox.scss";
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
 import { SetupDrag } from '../../util/DragManager';
 import { Docs } from '../../documents/Documents';
-import { NumCast } from '../../../new_fields/Types';
+import { NumCast, Cast } from '../../../new_fields/Types';
 import { Doc } from '../../../new_fields/Doc';
 import { SearchItem } from './SearchItem';
 import { DocServer } from '../../DocServer';
@@ -118,7 +118,7 @@ export class SearchBox extends React.Component {
 
     private get filterQuery() {
         const types = FilterBox.Instance.filterTypes;
-        return "proto_i:*" + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : "");
+        return "NOT baseProto_b:true" + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : "");
     }
 
 
@@ -129,15 +129,18 @@ export class SearchBox extends React.Component {
         }
         this.lockPromise = new Promise(async res => {
             while (this._results.length <= this._endIndex && (this._numTotalResults === -1 || this._maxSearchIndex < this._numTotalResults)) {
-                this._curRequest = SearchUtil.Search(query, this.filterQuery, true, this._maxSearchIndex, 10).then(action((res: SearchUtil.DocSearchResult) => {
+                this._curRequest = SearchUtil.Search(query, this.filterQuery, true, this._maxSearchIndex, 10).then(action(async (res: SearchUtil.DocSearchResult) => {
 
                     // happens at the beginning
                     if (res.numFound !== this._numTotalResults && this._numTotalResults === -1) {
                         this._numTotalResults = res.numFound;
                     }
 
-                    let filteredDocs = FilterBox.Instance.filterDocsByType(res.docs);
-                    this._results.push(...filteredDocs);
+                    const docs = await Promise.all(res.docs.map(doc => Cast(doc.extendsDoc, Doc, doc as any)));
+                    let filteredDocs = FilterBox.Instance.filterDocsByType(docs);
+                    runInAction(() => {
+                        this._results.push(...filteredDocs);
+                    });
 
                     this._curRequest = undefined;
                 }));
-- 
cgit v1.2.3-70-g09d2


From 967638b84c15f35bbfb1c464ac8e22916a1a2bc7 Mon Sep 17 00:00:00 2001
From: monikahedman <monika_hedman@brown.edu>
Date: Tue, 16 Jul 2019 14:31:20 -0400
Subject: changes

---
 src/client/views/search/FilterBox.tsx  |  3 +++
 src/client/views/search/SearchBox.scss |  4 ++--
 src/client/views/search/SearchBox.tsx  | 15 +++++++--------
 3 files changed, 12 insertions(+), 10 deletions(-)

(limited to 'src/client/views/search/SearchBox.tsx')

diff --git a/src/client/views/search/FilterBox.tsx b/src/client/views/search/FilterBox.tsx
index f11fb008c..757665bcf 100644
--- a/src/client/views/search/FilterBox.tsx
+++ b/src/client/views/search/FilterBox.tsx
@@ -87,6 +87,9 @@ export class FilterBox extends React.Component {
 
                     }
                 });
+
+                let el = acc[i] as HTMLElement;
+                el.click();
             }
         });
     }
diff --git a/src/client/views/search/SearchBox.scss b/src/client/views/search/SearchBox.scss
index 324ba3063..109b88ac9 100644
--- a/src/client/views/search/SearchBox.scss
+++ b/src/client/views/search/SearchBox.scss
@@ -41,11 +41,10 @@
 }
 
 .searchBox-results {
-    margin-right: 142px;
+    margin-right: 136px;
     top: 300px;
     display: flex;
     flex-direction: column;
-    margin-right: 72px;
     max-height: 560px;
     overflow: hidden;
     overflow-y: auto;
@@ -60,5 +59,6 @@
         text-transform: uppercase;
         text-align: left;
         font-weight: bold;
+        margin-left: 28px;
     }
 }
\ No newline at end of file
diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx
index 3830a6d16..661c9c10b 100644
--- a/src/client/views/search/SearchBox.tsx
+++ b/src/client/views/search/SearchBox.tsx
@@ -22,6 +22,7 @@ export class SearchBox extends React.Component {
 
     @observable private _searchString: string = "";
     @observable private _resultsOpen: boolean = false;
+    @observable private _searchbarOpen: boolean = false;
     @observable private _results: Doc[] = [];
     @observable private _openNoResults: boolean = false;
     @observable private _visibleElements: JSX.Element[] = [];
@@ -107,6 +108,7 @@ export class SearchBox extends React.Component {
 
         runInAction(() => {
             this._resultsOpen = true;
+            this._searchbarOpen = true;
             this._openNoResults = true;
             this.resultsScrolled();
         });
@@ -198,6 +200,7 @@ export class SearchBox extends React.Component {
         this._openNoResults = false;
         FilterBox.Instance.closeFilter();
         this._resultsOpen = true;
+        this._searchbarOpen = true;
         FilterBox.Instance._pointerTime = e.timeStamp;
     }
 
@@ -205,6 +208,7 @@ export class SearchBox extends React.Component {
     closeSearch = () => {
         FilterBox.Instance.closeFilter();
         this.closeResults();
+        this._searchbarOpen = false;
     }
 
     @action.bound
@@ -281,15 +285,10 @@ export class SearchBox extends React.Component {
     }
 
     @computed
-    get resFull() {
-        console.log(this._numTotalResults)
-        return this._numTotalResults <= 8;
-    }
+    get resFull() { return this._numTotalResults <= 8; }
 
     @computed
-    get resultHeight() {
-        return this._numTotalResults * 70;
-    }
+    get resultHeight() { return this._numTotalResults * 70; }
 
     render() {
         return (
@@ -300,7 +299,7 @@ export class SearchBox extends React.Component {
                     </span>
                     <input value={this._searchString} onChange={this.onChange} type="text" placeholder="Search..."
                         className="searchBox-barChild searchBox-input" onPointerDown={this.openSearch} onKeyPress={this.enter}
-                        style={{ width: this._resultsOpen ? "500px" : "100px" }} />
+                        style={{ width: this._searchbarOpen ? "500px" : "100px" }} />
                     <button className="searchBox-barChild searchBox-submit" onClick={this.submitSearch} onPointerDown={FilterBox.Instance.stopProp}>Submit</button>
                     <button className="searchBox-barChild searchBox-filter" onClick={FilterBox.Instance.openFilter} onPointerDown={FilterBox.Instance.stopProp}>Filter</button>
                 </div>
-- 
cgit v1.2.3-70-g09d2


From 1b5bf295d951858a9d54e95e2ada4882f1a39f40 Mon Sep 17 00:00:00 2001
From: Tyler Schicke <tyler_schicke@brown.edu>
Date: Tue, 16 Jul 2019 14:41:18 -0400
Subject: Added start of deleted documents to search

---
 solr/conf/schema.xml                  |  1 +
 src/client/views/search/SearchBox.tsx |  3 +-
 src/server/GarbageCollector.ts        | 87 +++++++++++++++++++++--------------
 3 files changed, 55 insertions(+), 36 deletions(-)

(limited to 'src/client/views/search/SearchBox.tsx')

diff --git a/solr/conf/schema.xml b/solr/conf/schema.xml
index 8610786af..7a21794cb 100644
--- a/solr/conf/schema.xml
+++ b/solr/conf/schema.xml
@@ -44,6 +44,7 @@
   <field name="text" type="text" indexed="true" stored="false" uninvertible="false" multiValued="true"/>
   <field name="id" type="string" indexed="true" stored="true" uninvertible="false" required="true"/>
   <field name="_version_" type="plong" indexed="true" stored="true"/>
+  <field name="deleted" type="boolean" indexed="true" stored="true" uninvertible="false" multiValued="false"/>
 
   <dynamicField name="*_t" type="text" indexed="true" stored="true" uninvertible="false" docValues="false"/>
   <dynamicField name="*_n" type="pdouble" indexed="true" stored="true" uninvertible="false" docValues="false"/>
diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx
index dc76d1ff4..733e93a62 100644
--- a/src/client/views/search/SearchBox.tsx
+++ b/src/client/views/search/SearchBox.tsx
@@ -118,7 +118,8 @@ export class SearchBox extends React.Component {
 
     private get filterQuery() {
         const types = FilterBox.Instance.filterTypes;
-        return "NOT baseProto_b:true" + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : "");
+        const includeDeleted = false;
+        return "NOT baseProto_b:true" + (includeDeleted ? "" : " AND NOT deleted:true") + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : "");
     }
 
 
diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts
index 59682e51e..ea5388004 100644
--- a/src/server/GarbageCollector.ts
+++ b/src/server/GarbageCollector.ts
@@ -59,7 +59,9 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) {
     }
 }
 
-async function GarbageCollect() {
+async function GarbageCollect(full: boolean = true) {
+    console.log("start GC");
+    const start = Date.now();
     // await new Promise(res => setTimeout(res, 3000));
     const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users');
     const users = await cursor.toArray();
@@ -68,7 +70,7 @@ async function GarbageCollect() {
     const files: { [name: string]: string[] } = {};
 
     while (ids.length) {
-        const count = Math.min(ids.length, 100);
+        const count = Math.min(ids.length, 1000);
         const index = ids.length - count;
         const fetchIds = ids.splice(index, count).filter(id => !visited.has(id));
         if (!fetchIds.length) {
@@ -91,43 +93,58 @@ async function GarbageCollect() {
 
     cursor.close();
 
-    const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }, { _id: 1 });
+    const notToDelete = Array.from(visited);
+    const toDeleteCursor = await Database.Instance.query({ _id: { $nin: notToDelete } }, { _id: 1 });
     const toDelete: string[] = (await toDeleteCursor.toArray()).map(doc => doc._id);
     toDeleteCursor.close();
-    let i = 0;
-    let deleted = 0;
-    while (i < toDelete.length) {
-        const count = Math.min(toDelete.length, 5000);
-        const toDeleteDocs = toDelete.slice(i, i + count);
-        i += count;
-        const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments");
-        deleted += result.deletedCount || 0;
-    }
-    // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments");
-    console.log(`${deleted} documents deleted`);
+    if (!full) {
+        await Database.Instance.updateMany({ _id: { $nin: notToDelete } }, { $set: { "deleted": true } });
+        await Database.Instance.updateMany({ _id: { $in: notToDelete } }, { $unset: { "deleted": true } });
+        console.log(await Search.Instance.updateDocuments(
+            notToDelete.map<any>(id => ({
+                id, deleted: { set: null }
+            }))
+                .concat(toDelete.map(id => ({
+                    id, deleted: { set: true }
+                })))));
+        console.log("Done with partial GC");
+        console.log(`Took ${(Date.now() - start) / 1000} seconds`);
+    } else {
+        let i = 0;
+        let deleted = 0;
+        while (i < toDelete.length) {
+            const count = Math.min(toDelete.length, 5000);
+            const toDeleteDocs = toDelete.slice(i, i + count);
+            i += count;
+            const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments");
+            deleted += result.deletedCount || 0;
+        }
+        // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments");
+        console.log(`${deleted} documents deleted`);
 
-    await Search.Instance.deleteDocuments(toDelete);
-    console.log("Cleared search documents");
+        await Search.Instance.deleteDocuments(toDelete);
+        console.log("Cleared search documents");
 
-    const folder = "./src/server/public/files/";
-    fs.readdir(folder, (_, fileList) => {
-        const filesToDelete = fileList.filter(file => {
-            const ext = path.extname(file);
-            let base = path.basename(file, ext);
-            const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext);
-            return file !== ".gitignore" && !existsInDb;
-        });
-        console.log(`Deleting ${filesToDelete.length} files`);
-        filesToDelete.forEach(file => {
-            console.log(`Deleting file ${file}`);
-            try {
-                fs.unlinkSync(folder + file);
-            } catch {
-                console.warn(`Couldn't delete file ${file}`);
-            }
+        const folder = "./src/server/public/files/";
+        fs.readdir(folder, (_, fileList) => {
+            const filesToDelete = fileList.filter(file => {
+                const ext = path.extname(file);
+                let base = path.basename(file, ext);
+                const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext);
+                return file !== ".gitignore" && !existsInDb;
+            });
+            console.log(`Deleting ${filesToDelete.length} files`);
+            filesToDelete.forEach(file => {
+                console.log(`Deleting file ${file}`);
+                try {
+                    fs.unlinkSync(folder + file);
+                } catch {
+                    console.warn(`Couldn't delete file ${file}`);
+                }
+            });
+            console.log(`Deleted ${filesToDelete.length} files`);
         });
-        console.log(`Deleted ${filesToDelete.length} files`);
-    });
+    }
 }
 
-GarbageCollect();
+GarbageCollect(false);
-- 
cgit v1.2.3-70-g09d2


From 189bb0ffd14fb0b8db0edbe813efe19f143a30fd Mon Sep 17 00:00:00 2001
From: monikahedman <monika_hedman@brown.edu>
Date: Tue, 16 Jul 2019 14:46:44 -0400
Subject: shes fixed

---
 src/client/views/search/SearchBox.tsx | 1 +
 1 file changed, 1 insertion(+)

(limited to 'src/client/views/search/SearchBox.tsx')

diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx
index 661c9c10b..0b2d792e0 100644
--- a/src/client/views/search/SearchBox.tsx
+++ b/src/client/views/search/SearchBox.tsx
@@ -94,6 +94,7 @@ export class SearchBox extends React.Component {
         this._results = [];
         this._isSearch = [];
         this._visibleElements = [];
+        FilterBox.Instance.closeFilter();
 
         //if there is no query there should be no result
         if (query === "") {
-- 
cgit v1.2.3-70-g09d2


From 6353522fbd8dc961b9c172b15d33af842fdab323 Mon Sep 17 00:00:00 2001
From: Tyler Schicke <tyler_schicke@brown.edu>
Date: Tue, 16 Jul 2019 17:03:41 -0400
Subject: Added deduplication of results

---
 src/client/views/search/SearchBox.tsx | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

(limited to 'src/client/views/search/SearchBox.tsx')

diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx
index 04db26aca..cb4aef961 100644
--- a/src/client/views/search/SearchBox.tsx
+++ b/src/client/views/search/SearchBox.tsx
@@ -24,6 +24,7 @@ export class SearchBox extends React.Component {
     @observable private _resultsOpen: boolean = false;
     @observable private _searchbarOpen: boolean = false;
     @observable private _results: Doc[] = [];
+    private _resultsSet = new Set<Doc>();
     @observable private _openNoResults: boolean = false;
     @observable private _visibleElements: JSX.Element[] = [];
 
@@ -61,6 +62,7 @@ export class SearchBox extends React.Component {
 
         this._openNoResults = false;
         this._results = [];
+        this._resultsSet.clear();
         this._visibleElements = [];
         this._numTotalResults = -1;
         this._endIndex = -1;
@@ -92,6 +94,7 @@ export class SearchBox extends React.Component {
         let query = this._searchString;
         query = FilterBox.Instance.getFinalQuery(query);
         this._results = [];
+        this._resultsSet.clear();
         this._isSearch = [];
         this._visibleElements = [];
 
@@ -120,7 +123,7 @@ export class SearchBox extends React.Component {
 
     private get filterQuery() {
         const types = FilterBox.Instance.filterTypes;
-        const includeDeleted = false;
+        const includeDeleted = FilterBox.Instance.getDataStatus();
         return "NOT baseProto_b:true" + (includeDeleted ? "" : " AND NOT deleted:true") + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : "");
     }
 
@@ -142,7 +145,13 @@ export class SearchBox extends React.Component {
                     const docs = await Promise.all(res.docs.map(doc => Cast(doc.extendsDoc, Doc, doc as any)));
                     let filteredDocs = FilterBox.Instance.filterDocsByType(docs);
                     runInAction(() => {
-                        this._results.push(...filteredDocs);
+                        // this._results.push(...filteredDocs);
+                        filteredDocs.forEach(doc => {
+                            if (!this._resultsSet.has(doc)) {
+                                this._results.push(doc);
+                                this._resultsSet.add(doc);
+                            }
+                        });
                     });
 
                     this._curRequest = undefined;
@@ -219,6 +228,7 @@ export class SearchBox extends React.Component {
     closeResults() {
         this._resultsOpen = false;
         this._results = [];
+        this._resultsSet.clear();
         this._visibleElements = [];
         this._numTotalResults = -1;
         this._endIndex = -1;
-- 
cgit v1.2.3-70-g09d2

