diff options
Diffstat (limited to 'src/server/GarbageCollector.ts')
| -rw-r--r-- | src/server/GarbageCollector.ts | 70 |
1 files changed, 42 insertions, 28 deletions
diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts index 423c719c2..041f65592 100644 --- a/src/server/GarbageCollector.ts +++ b/src/server/GarbageCollector.ts @@ -1,11 +1,15 @@ +/* eslint-disable no-await-in-loop */ +/* eslint-disable no-continue */ +/* eslint-disable no-cond-assign */ +/* eslint-disable no-restricted-syntax */ import * as fs from 'fs'; import * as path from 'path'; import { Database } from './database'; import { Search } from './Search'; - function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { for (const key in doc) { + // eslint-disable-next-line no-prototype-builtins if (!doc.hasOwnProperty(key)) { continue; } @@ -13,22 +17,22 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { if (field === undefined || field === null) { continue; } - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + if (field.__type === 'proxy' || field.__type === 'prefetch_proxy') { ids.push(field.fieldId); - } else if (field.__type === "list") { + } else if (field.__type === 'list') { addDoc(field.fields, ids, files); - } else if (typeof field === "string") { - const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + } else if (typeof field === 'string') { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w-]*)"/g; let match: string[] | null; while ((match = re.exec(field)) !== null) { ids.push(match[1]); } - } else if (field.__type === "RichTextField") { + } else if (field.__type === 'RichTextField') { const re = /"href"\s*:\s*"(.*?)"/g; let match: string[] | null; while ((match = re.exec(field.Data)) !== null) { const urlString = match[1]; - const split = new URL(urlString).pathname.split("doc/"); + const split = new URL(urlString).pathname.split('doc/'); if (split.length > 1) { ids.push(split[split.length - 1]); } @@ -36,7 +40,7 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { const re2 = /"src"\s*:\s*"(.*?)"/g; while ((match = re2.exec(field.Data)) !== null) { const urlString = match[1]; - const pathname = new URL(urlString).pathname; + const { pathname } = new URL(urlString); const ext = path.extname(pathname); const fileName = path.basename(pathname, ext); let exts = files[fileName]; @@ -45,9 +49,9 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { } exts.push(ext); } - } else if (["audio", "image", "video", "pdf", "web", "map"].includes(field.__type)) { + } else if (['audio', 'image', 'video', 'pdf', 'web', 'map'].includes(field.__type)) { const url = new URL(field.url); - const pathname = url.pathname; + const { pathname } = url; const ext = path.extname(pathname); const fileName = path.basename(pathname, ext); let exts = files[fileName]; @@ -60,12 +64,12 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { } async function GarbageCollect(full: boolean = true) { - console.log("start GC"); + console.log('start GC'); const start = Date.now(); // await new Promise(res => setTimeout(res, 3000)); const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users'); const users = await cursor.toArray(); - const ids: string[] = [...users.map((user:any) => user.userDocumentId), ...users.map((user:any) => user.sharingDocumentId), ...users.map((user:any) => user.linkDatabaseId)]; + const ids: string[] = [...users.map((user: any) => user.userDocumentId), ...users.map((user: any) => user.sharingDocumentId), ...users.map((user: any) => user.linkDatabaseId)]; const visited = new Set<string>(); const files: { [name: string]: string[] } = {}; @@ -76,9 +80,11 @@ async function GarbageCollect(full: boolean = true) { if (!fetchIds.length) { continue; } - const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res)); + const docs = await new Promise<{ [key: string]: any }[]>(res => { + Database.Instance.getDocuments(fetchIds, res); + }); for (const doc of docs) { - const id = doc.id; + const { id } = doc; if (doc === undefined) { console.log(`Couldn't find field with Id ${id}`); continue; @@ -95,19 +101,27 @@ async function GarbageCollect(full: boolean = true) { const notToDelete = Array.from(visited); const toDeleteCursor = await Database.Instance.query({ _id: { $nin: notToDelete } }, { _id: 1 }); - const toDelete: string[] = (await toDeleteCursor.toArray()).map((doc:any) => doc._id); + const toDelete: string[] = (await toDeleteCursor.toArray()).map((doc: any) => doc._id); toDeleteCursor.close(); if (!full) { - await Database.Instance.updateMany({ _id: { $nin: notToDelete } }, { $set: { "deleted": true } }); - await Database.Instance.updateMany({ _id: { $in: notToDelete } }, { $unset: { "deleted": true } }); - console.log(await Search.updateDocuments( - notToDelete.map<any>(id => ({ - id, deleted: { set: null } - })) - .concat(toDelete.map(id => ({ - id, deleted: { set: true } - }))))); - console.log("Done with partial GC"); + await Database.Instance.updateMany({ _id: { $nin: notToDelete } }, { $set: { deleted: true } }); + await Database.Instance.updateMany({ _id: { $in: notToDelete } }, { $unset: { deleted: true } }); + console.log( + await Search.updateDocuments( + notToDelete + .map<any>(id => ({ + id, + deleted: { set: null }, + })) + .concat( + toDelete.map(id => ({ + id, + deleted: { set: true }, + })) + ) + ) + ); + console.log('Done with partial GC'); console.log(`Took ${(Date.now() - start) / 1000} seconds`); } else { let i = 0; @@ -123,15 +137,15 @@ async function GarbageCollect(full: boolean = true) { console.log(`${deleted} documents deleted`); await Search.deleteDocuments(toDelete); - console.log("Cleared search documents"); + console.log('Cleared search documents'); - const folder = "./src/server/public/files/"; + const folder = './src/server/public/files/'; fs.readdir(folder, (_, fileList) => { const filesToDelete = fileList.filter(file => { const ext = path.extname(file); let base = path.basename(file, ext); const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext); - return file !== ".gitignore" && !existsInDb; + return file !== '.gitignore' && !existsInDb; }); console.log(`Deleting ${filesToDelete.length} files`); filesToDelete.forEach(file => { |