diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/client/views/search/SearchBox.tsx | 15 | ||||
| -rw-r--r-- | src/server/GarbageCollector.ts | 87 |
2 files changed, 65 insertions, 37 deletions
diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx index 32319d897..d07df7e58 100644 --- a/src/client/views/search/SearchBox.tsx +++ b/src/client/views/search/SearchBox.tsx @@ -24,6 +24,7 @@ export class SearchBox extends React.Component { @observable private _resultsOpen: boolean = false; @observable private _searchbarOpen: boolean = false; @observable private _results: Doc[] = []; + private _resultsSet = new Set<Doc>(); @observable private _openNoResults: boolean = false; @observable private _visibleElements: JSX.Element[] = []; @@ -61,6 +62,7 @@ export class SearchBox extends React.Component { this._openNoResults = false; this._results = []; + this._resultsSet.clear(); this._visibleElements = []; this._numTotalResults = -1; this._endIndex = -1; @@ -92,6 +94,7 @@ export class SearchBox extends React.Component { let query = this._searchString; query = FilterBox.Instance.getFinalQuery(query); this._results = []; + this._resultsSet.clear(); this._isSearch = []; this._visibleElements = []; FilterBox.Instance.closeFilter(); @@ -121,7 +124,8 @@ export class SearchBox extends React.Component { private get filterQuery() { const types = FilterBox.Instance.filterTypes; - return "NOT baseProto_b:true" + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : ""); + const includeDeleted = FilterBox.Instance.getDataStatus(); + return "NOT baseProto_b:true" + (includeDeleted ? "" : " AND NOT deleted:true") + (types ? ` AND (${types.map(type => `({!join from=id to=proto_i}type_t:"${type}" AND NOT type_t:*) OR type_t:"${type}"`).join(" ")})` : ""); } @@ -142,7 +146,13 @@ export class SearchBox extends React.Component { const docs = await Promise.all(res.docs.map(doc => Cast(doc.extendsDoc, Doc, doc as any))); let filteredDocs = FilterBox.Instance.filterDocsByType(docs); runInAction(() => { - this._results.push(...filteredDocs); + // this._results.push(...filteredDocs); + filteredDocs.forEach(doc => { + if (!this._resultsSet.has(doc)) { + this._results.push(doc); + this._resultsSet.add(doc); + } + }); }); this._curRequest = undefined; @@ -219,6 +229,7 @@ export class SearchBox extends React.Component { closeResults() { this._resultsOpen = false; this._results = []; + this._resultsSet.clear(); this._visibleElements = []; this._numTotalResults = -1; this._endIndex = -1; diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts index 59682e51e..ea5388004 100644 --- a/src/server/GarbageCollector.ts +++ b/src/server/GarbageCollector.ts @@ -59,7 +59,9 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { } } -async function GarbageCollect() { +async function GarbageCollect(full: boolean = true) { + console.log("start GC"); + const start = Date.now(); // await new Promise(res => setTimeout(res, 3000)); const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users'); const users = await cursor.toArray(); @@ -68,7 +70,7 @@ async function GarbageCollect() { const files: { [name: string]: string[] } = {}; while (ids.length) { - const count = Math.min(ids.length, 100); + const count = Math.min(ids.length, 1000); const index = ids.length - count; const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); if (!fetchIds.length) { @@ -91,43 +93,58 @@ async function GarbageCollect() { cursor.close(); - const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }, { _id: 1 }); + const notToDelete = Array.from(visited); + const toDeleteCursor = await Database.Instance.query({ _id: { $nin: notToDelete } }, { _id: 1 }); const toDelete: string[] = (await toDeleteCursor.toArray()).map(doc => doc._id); toDeleteCursor.close(); - let i = 0; - let deleted = 0; - while (i < toDelete.length) { - const count = Math.min(toDelete.length, 5000); - const toDeleteDocs = toDelete.slice(i, i + count); - i += count; - const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments"); - deleted += result.deletedCount || 0; - } - // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); - console.log(`${deleted} documents deleted`); + if (!full) { + await Database.Instance.updateMany({ _id: { $nin: notToDelete } }, { $set: { "deleted": true } }); + await Database.Instance.updateMany({ _id: { $in: notToDelete } }, { $unset: { "deleted": true } }); + console.log(await Search.Instance.updateDocuments( + notToDelete.map<any>(id => ({ + id, deleted: { set: null } + })) + .concat(toDelete.map(id => ({ + id, deleted: { set: true } + }))))); + console.log("Done with partial GC"); + console.log(`Took ${(Date.now() - start) / 1000} seconds`); + } else { + let i = 0; + let deleted = 0; + while (i < toDelete.length) { + const count = Math.min(toDelete.length, 5000); + const toDeleteDocs = toDelete.slice(i, i + count); + i += count; + const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments"); + deleted += result.deletedCount || 0; + } + // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); + console.log(`${deleted} documents deleted`); - await Search.Instance.deleteDocuments(toDelete); - console.log("Cleared search documents"); + await Search.Instance.deleteDocuments(toDelete); + console.log("Cleared search documents"); - const folder = "./src/server/public/files/"; - fs.readdir(folder, (_, fileList) => { - const filesToDelete = fileList.filter(file => { - const ext = path.extname(file); - let base = path.basename(file, ext); - const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext); - return file !== ".gitignore" && !existsInDb; - }); - console.log(`Deleting ${filesToDelete.length} files`); - filesToDelete.forEach(file => { - console.log(`Deleting file ${file}`); - try { - fs.unlinkSync(folder + file); - } catch { - console.warn(`Couldn't delete file ${file}`); - } + const folder = "./src/server/public/files/"; + fs.readdir(folder, (_, fileList) => { + const filesToDelete = fileList.filter(file => { + const ext = path.extname(file); + let base = path.basename(file, ext); + const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext); + return file !== ".gitignore" && !existsInDb; + }); + console.log(`Deleting ${filesToDelete.length} files`); + filesToDelete.forEach(file => { + console.log(`Deleting file ${file}`); + try { + fs.unlinkSync(folder + file); + } catch { + console.warn(`Couldn't delete file ${file}`); + } + }); + console.log(`Deleted ${filesToDelete.length} files`); }); - console.log(`Deleted ${filesToDelete.length} files`); - }); + } } -GarbageCollect(); +GarbageCollect(false); |