From 7f6666500f157c5884377d714137426cf05e7569 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 16 Jul 2019 13:40:20 -0400 Subject: Changed search to display extension documents as the extended documents --- src/client/views/search/SearchBox.tsx | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'src/client/views/search') diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx index ec778b346..dc76d1ff4 100644 --- a/src/client/views/search/SearchBox.tsx +++ b/src/client/views/search/SearchBox.tsx @@ -6,7 +6,7 @@ import "./FilterBox.scss"; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { SetupDrag } from '../../util/DragManager'; import { Docs } from '../../documents/Documents'; -import { NumCast } from '../../../new_fields/Types'; +import { NumCast, Cast } from '../../../new_fields/Types'; import { Doc } from '../../../new_fields/Doc'; import { SearchItem } from './SearchItem'; import { DocServer } from '../../DocServer'; @@ -118,7 +118,7 @@ export class SearchBox extends React.Component { private get filterQuery() { const types = FilterBox.Instance.filterTypes; - return "proto_i:*" + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : ""); + return "NOT baseProto_b:true" + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : ""); } @@ -129,15 +129,18 @@ export class SearchBox extends React.Component { } this.lockPromise = new Promise(async res => { while (this._results.length <= this._endIndex && (this._numTotalResults === -1 || this._maxSearchIndex < this._numTotalResults)) { - this._curRequest = SearchUtil.Search(query, this.filterQuery, true, this._maxSearchIndex, 10).then(action((res: SearchUtil.DocSearchResult) => { + this._curRequest = SearchUtil.Search(query, this.filterQuery, true, this._maxSearchIndex, 10).then(action(async (res: SearchUtil.DocSearchResult) => { // happens at the beginning if (res.numFound !== this._numTotalResults && this._numTotalResults === -1) { this._numTotalResults = res.numFound; } - let filteredDocs = FilterBox.Instance.filterDocsByType(res.docs); - this._results.push(...filteredDocs); + const docs = await Promise.all(res.docs.map(doc => Cast(doc.extendsDoc, Doc, doc as any))); + let filteredDocs = FilterBox.Instance.filterDocsByType(docs); + runInAction(() => { + this._results.push(...filteredDocs); + }); this._curRequest = undefined; })); -- cgit v1.2.3-70-g09d2 From 967638b84c15f35bbfb1c464ac8e22916a1a2bc7 Mon Sep 17 00:00:00 2001 From: monikahedman Date: Tue, 16 Jul 2019 14:31:20 -0400 Subject: changes --- src/client/views/search/FilterBox.tsx | 3 +++ src/client/views/search/SearchBox.scss | 4 ++-- src/client/views/search/SearchBox.tsx | 15 +++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) (limited to 'src/client/views/search') diff --git a/src/client/views/search/FilterBox.tsx b/src/client/views/search/FilterBox.tsx index f11fb008c..757665bcf 100644 --- a/src/client/views/search/FilterBox.tsx +++ b/src/client/views/search/FilterBox.tsx @@ -87,6 +87,9 @@ export class FilterBox extends React.Component { } }); + + let el = acc[i] as HTMLElement; + el.click(); } }); } diff --git a/src/client/views/search/SearchBox.scss b/src/client/views/search/SearchBox.scss index 324ba3063..109b88ac9 100644 --- a/src/client/views/search/SearchBox.scss +++ b/src/client/views/search/SearchBox.scss @@ -41,11 +41,10 @@ } .searchBox-results { - margin-right: 142px; + margin-right: 136px; top: 300px; display: flex; flex-direction: column; - margin-right: 72px; max-height: 560px; overflow: hidden; overflow-y: auto; @@ -60,5 +59,6 @@ text-transform: uppercase; text-align: left; font-weight: bold; + margin-left: 28px; } } \ No newline at end of file diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx index 3830a6d16..661c9c10b 100644 --- a/src/client/views/search/SearchBox.tsx +++ b/src/client/views/search/SearchBox.tsx @@ -22,6 +22,7 @@ export class SearchBox extends React.Component { @observable private _searchString: string = ""; @observable private _resultsOpen: boolean = false; + @observable private _searchbarOpen: boolean = false; @observable private _results: Doc[] = []; @observable private _openNoResults: boolean = false; @observable private _visibleElements: JSX.Element[] = []; @@ -107,6 +108,7 @@ export class SearchBox extends React.Component { runInAction(() => { this._resultsOpen = true; + this._searchbarOpen = true; this._openNoResults = true; this.resultsScrolled(); }); @@ -198,6 +200,7 @@ export class SearchBox extends React.Component { this._openNoResults = false; FilterBox.Instance.closeFilter(); this._resultsOpen = true; + this._searchbarOpen = true; FilterBox.Instance._pointerTime = e.timeStamp; } @@ -205,6 +208,7 @@ export class SearchBox extends React.Component { closeSearch = () => { FilterBox.Instance.closeFilter(); this.closeResults(); + this._searchbarOpen = false; } @action.bound @@ -281,15 +285,10 @@ export class SearchBox extends React.Component { } @computed - get resFull() { - console.log(this._numTotalResults) - return this._numTotalResults <= 8; - } + get resFull() { return this._numTotalResults <= 8; } @computed - get resultHeight() { - return this._numTotalResults * 70; - } + get resultHeight() { return this._numTotalResults * 70; } render() { return ( @@ -300,7 +299,7 @@ export class SearchBox extends React.Component { + style={{ width: this._searchbarOpen ? "500px" : "100px" }} /> -- cgit v1.2.3-70-g09d2 From c6b933c6e89d5f438c434e8676cce9a6a577edbc Mon Sep 17 00:00:00 2001 From: monikahedman Date: Tue, 16 Jul 2019 14:35:35 -0400 Subject: checkbox --- src/client/views/search/FieldFilters.tsx | 2 +- src/client/views/search/FilterBox.tsx | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'src/client/views/search') diff --git a/src/client/views/search/FieldFilters.tsx b/src/client/views/search/FieldFilters.tsx index 648aac20a..7a33282d2 100644 --- a/src/client/views/search/FieldFilters.tsx +++ b/src/client/views/search/FieldFilters.tsx @@ -34,7 +34,7 @@ export class FieldFilters extends React.Component {
- +
); } diff --git a/src/client/views/search/FilterBox.tsx b/src/client/views/search/FilterBox.tsx index 757665bcf..f28b074a7 100644 --- a/src/client/views/search/FilterBox.tsx +++ b/src/client/views/search/FilterBox.tsx @@ -40,7 +40,7 @@ export class FilterBox extends React.Component { @observable private _icons: string[] = this._allIcons; @observable private _titleFieldStatus: boolean = true; @observable private _authorFieldStatus: boolean = true; - @observable private _dataFieldStatus: boolean = true; + @observable public _deletedDocsStatus: boolean = false; @observable private _collectionStatus = false; @observable private _collectionSelfStatus = true; @observable private _collectionParentStatus = true; @@ -164,13 +164,13 @@ export class FilterBox extends React.Component { if (this._authorFieldStatus) { finalQuery = finalQuery + this.basicFieldFilters(query, Keys.AUTHOR); } - if (this._dataFieldStatus) { + if (this._deletedDocsStatus) { finalQuery = finalQuery + this.basicFieldFilters(query, Keys.DATA); } return finalQuery; } - get fieldFiltersApplied() { return !(this._dataFieldStatus && this._authorFieldStatus && this._titleFieldStatus); } + get fieldFiltersApplied() { return !(this._deletedDocsStatus && this._authorFieldStatus && this._titleFieldStatus); } //TODO: basically all of this //gets all of the collections of all the docviews that are selected @@ -308,7 +308,7 @@ export class FilterBox extends React.Component { updateAuthorStatus(newStat: boolean) { this._authorFieldStatus = newStat; } @action.bound - updateDataStatus(newStat: boolean) { this._dataFieldStatus = newStat; } + updateDataStatus(newStat: boolean) { this._deletedDocsStatus = newStat; } @action.bound updateCollectionStatus(newStat: boolean) { this._collectionStatus = newStat; } @@ -324,7 +324,7 @@ export class FilterBox extends React.Component { getParentCollectionStatus() { return this._collectionParentStatus; } getTitleStatus() { return this._titleFieldStatus; } getAuthorStatus() { return this._authorFieldStatus; } - getDataStatus() { return this._dataFieldStatus; } + getDataStatus() { return this._deletedDocsStatus; } // Useful queries: // Delegates of a document: {!join from=id to=proto_i}id:{protoId} @@ -376,7 +376,7 @@ export class FilterBox extends React.Component {
-- cgit v1.2.3-70-g09d2 From 1b5bf295d951858a9d54e95e2ada4882f1a39f40 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 16 Jul 2019 14:41:18 -0400 Subject: Added start of deleted documents to search --- solr/conf/schema.xml | 1 + src/client/views/search/SearchBox.tsx | 3 +- src/server/GarbageCollector.ts | 87 +++++++++++++++++++++-------------- 3 files changed, 55 insertions(+), 36 deletions(-) (limited to 'src/client/views/search') diff --git a/solr/conf/schema.xml b/solr/conf/schema.xml index 8610786af..7a21794cb 100644 --- a/solr/conf/schema.xml +++ b/solr/conf/schema.xml @@ -44,6 +44,7 @@ + diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx index dc76d1ff4..733e93a62 100644 --- a/src/client/views/search/SearchBox.tsx +++ b/src/client/views/search/SearchBox.tsx @@ -118,7 +118,8 @@ export class SearchBox extends React.Component { private get filterQuery() { const types = FilterBox.Instance.filterTypes; - return "NOT baseProto_b:true" + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : ""); + const includeDeleted = false; + return "NOT baseProto_b:true" + (includeDeleted ? "" : " AND NOT deleted:true") + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : ""); } diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts index 59682e51e..ea5388004 100644 --- a/src/server/GarbageCollector.ts +++ b/src/server/GarbageCollector.ts @@ -59,7 +59,9 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { } } -async function GarbageCollect() { +async function GarbageCollect(full: boolean = true) { + console.log("start GC"); + const start = Date.now(); // await new Promise(res => setTimeout(res, 3000)); const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users'); const users = await cursor.toArray(); @@ -68,7 +70,7 @@ async function GarbageCollect() { const files: { [name: string]: string[] } = {}; while (ids.length) { - const count = Math.min(ids.length, 100); + const count = Math.min(ids.length, 1000); const index = ids.length - count; const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); if (!fetchIds.length) { @@ -91,43 +93,58 @@ async function GarbageCollect() { cursor.close(); - const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }, { _id: 1 }); + const notToDelete = Array.from(visited); + const toDeleteCursor = await Database.Instance.query({ _id: { $nin: notToDelete } }, { _id: 1 }); const toDelete: string[] = (await toDeleteCursor.toArray()).map(doc => doc._id); toDeleteCursor.close(); - let i = 0; - let deleted = 0; - while (i < toDelete.length) { - const count = Math.min(toDelete.length, 5000); - const toDeleteDocs = toDelete.slice(i, i + count); - i += count; - const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments"); - deleted += result.deletedCount || 0; - } - // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); - console.log(`${deleted} documents deleted`); + if (!full) { + await Database.Instance.updateMany({ _id: { $nin: notToDelete } }, { $set: { "deleted": true } }); + await Database.Instance.updateMany({ _id: { $in: notToDelete } }, { $unset: { "deleted": true } }); + console.log(await Search.Instance.updateDocuments( + notToDelete.map(id => ({ + id, deleted: { set: null } + })) + .concat(toDelete.map(id => ({ + id, deleted: { set: true } + }))))); + console.log("Done with partial GC"); + console.log(`Took ${(Date.now() - start) / 1000} seconds`); + } else { + let i = 0; + let deleted = 0; + while (i < toDelete.length) { + const count = Math.min(toDelete.length, 5000); + const toDeleteDocs = toDelete.slice(i, i + count); + i += count; + const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments"); + deleted += result.deletedCount || 0; + } + // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); + console.log(`${deleted} documents deleted`); - await Search.Instance.deleteDocuments(toDelete); - console.log("Cleared search documents"); + await Search.Instance.deleteDocuments(toDelete); + console.log("Cleared search documents"); - const folder = "./src/server/public/files/"; - fs.readdir(folder, (_, fileList) => { - const filesToDelete = fileList.filter(file => { - const ext = path.extname(file); - let base = path.basename(file, ext); - const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext); - return file !== ".gitignore" && !existsInDb; - }); - console.log(`Deleting ${filesToDelete.length} files`); - filesToDelete.forEach(file => { - console.log(`Deleting file ${file}`); - try { - fs.unlinkSync(folder + file); - } catch { - console.warn(`Couldn't delete file ${file}`); - } + const folder = "./src/server/public/files/"; + fs.readdir(folder, (_, fileList) => { + const filesToDelete = fileList.filter(file => { + const ext = path.extname(file); + let base = path.basename(file, ext); + const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext); + return file !== ".gitignore" && !existsInDb; + }); + console.log(`Deleting ${filesToDelete.length} files`); + filesToDelete.forEach(file => { + console.log(`Deleting file ${file}`); + try { + fs.unlinkSync(folder + file); + } catch { + console.warn(`Couldn't delete file ${file}`); + } + }); + console.log(`Deleted ${filesToDelete.length} files`); }); - console.log(`Deleted ${filesToDelete.length} files`); - }); + } } -GarbageCollect(); +GarbageCollect(false); -- cgit v1.2.3-70-g09d2 From 189bb0ffd14fb0b8db0edbe813efe19f143a30fd Mon Sep 17 00:00:00 2001 From: monikahedman Date: Tue, 16 Jul 2019 14:46:44 -0400 Subject: shes fixed --- src/client/views/search/SearchBox.tsx | 1 + 1 file changed, 1 insertion(+) (limited to 'src/client/views/search') diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx index 661c9c10b..0b2d792e0 100644 --- a/src/client/views/search/SearchBox.tsx +++ b/src/client/views/search/SearchBox.tsx @@ -94,6 +94,7 @@ export class SearchBox extends React.Component { this._results = []; this._isSearch = []; this._visibleElements = []; + FilterBox.Instance.closeFilter(); //if there is no query there should be no result if (query === "") { -- cgit v1.2.3-70-g09d2 From ecbae6521dde8e7e973b8d5f4e826746ea21a665 Mon Sep 17 00:00:00 2001 From: monikahedman Date: Tue, 16 Jul 2019 14:48:48 -0400 Subject: fixed AGAIN --- src/client/views/search/FilterBox.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/client/views/search') diff --git a/src/client/views/search/FilterBox.tsx b/src/client/views/search/FilterBox.tsx index f28b074a7..706d1eb7f 100644 --- a/src/client/views/search/FilterBox.tsx +++ b/src/client/views/search/FilterBox.tsx @@ -170,7 +170,7 @@ export class FilterBox extends React.Component { return finalQuery; } - get fieldFiltersApplied() { return !(this._deletedDocsStatus && this._authorFieldStatus && this._titleFieldStatus); } + get fieldFiltersApplied() { return !(this._authorFieldStatus && this._titleFieldStatus); } //TODO: basically all of this //gets all of the collections of all the docviews that are selected -- cgit v1.2.3-70-g09d2 From 6353522fbd8dc961b9c172b15d33af842fdab323 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 16 Jul 2019 17:03:41 -0400 Subject: Added deduplication of results --- src/client/views/search/SearchBox.tsx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'src/client/views/search') diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx index 04db26aca..cb4aef961 100644 --- a/src/client/views/search/SearchBox.tsx +++ b/src/client/views/search/SearchBox.tsx @@ -24,6 +24,7 @@ export class SearchBox extends React.Component { @observable private _resultsOpen: boolean = false; @observable private _searchbarOpen: boolean = false; @observable private _results: Doc[] = []; + private _resultsSet = new Set(); @observable private _openNoResults: boolean = false; @observable private _visibleElements: JSX.Element[] = []; @@ -61,6 +62,7 @@ export class SearchBox extends React.Component { this._openNoResults = false; this._results = []; + this._resultsSet.clear(); this._visibleElements = []; this._numTotalResults = -1; this._endIndex = -1; @@ -92,6 +94,7 @@ export class SearchBox extends React.Component { let query = this._searchString; query = FilterBox.Instance.getFinalQuery(query); this._results = []; + this._resultsSet.clear(); this._isSearch = []; this._visibleElements = []; @@ -120,7 +123,7 @@ export class SearchBox extends React.Component { private get filterQuery() { const types = FilterBox.Instance.filterTypes; - const includeDeleted = false; + const includeDeleted = FilterBox.Instance.getDataStatus(); return "NOT baseProto_b:true" + (includeDeleted ? "" : " AND NOT deleted:true") + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : ""); } @@ -142,7 +145,13 @@ export class SearchBox extends React.Component { const docs = await Promise.all(res.docs.map(doc => Cast(doc.extendsDoc, Doc, doc as any))); let filteredDocs = FilterBox.Instance.filterDocsByType(docs); runInAction(() => { - this._results.push(...filteredDocs); + // this._results.push(...filteredDocs); + filteredDocs.forEach(doc => { + if (!this._resultsSet.has(doc)) { + this._results.push(doc); + this._resultsSet.add(doc); + } + }); }); this._curRequest = undefined; @@ -219,6 +228,7 @@ export class SearchBox extends React.Component { closeResults() { this._resultsOpen = false; this._results = []; + this._resultsSet.clear(); this._visibleElements = []; this._numTotalResults = -1; this._endIndex = -1; -- cgit v1.2.3-70-g09d2