aboutsummaryrefslogtreecommitdiff
path: root/src/server
diff options
context:
space:
mode:
Diffstat (limited to 'src/server')
-rw-r--r--src/server/GarbageCollector.ts87
-rw-r--r--src/server/Search.ts12
-rw-r--r--src/server/authentication/controllers/user_controller.ts3
-rw-r--r--src/server/database.ts11
-rw-r--r--src/server/index.ts15
-rw-r--r--src/server/updateProtos.ts15
6 files changed, 91 insertions, 52 deletions
diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts
index 59682e51e..ea5388004 100644
--- a/src/server/GarbageCollector.ts
+++ b/src/server/GarbageCollector.ts
@@ -59,7 +59,9 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) {
}
}
-async function GarbageCollect() {
+async function GarbageCollect(full: boolean = true) {
+ console.log("start GC");
+ const start = Date.now();
// await new Promise(res => setTimeout(res, 3000));
const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users');
const users = await cursor.toArray();
@@ -68,7 +70,7 @@ async function GarbageCollect() {
const files: { [name: string]: string[] } = {};
while (ids.length) {
- const count = Math.min(ids.length, 100);
+ const count = Math.min(ids.length, 1000);
const index = ids.length - count;
const fetchIds = ids.splice(index, count).filter(id => !visited.has(id));
if (!fetchIds.length) {
@@ -91,43 +93,58 @@ async function GarbageCollect() {
cursor.close();
- const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }, { _id: 1 });
+ const notToDelete = Array.from(visited);
+ const toDeleteCursor = await Database.Instance.query({ _id: { $nin: notToDelete } }, { _id: 1 });
const toDelete: string[] = (await toDeleteCursor.toArray()).map(doc => doc._id);
toDeleteCursor.close();
- let i = 0;
- let deleted = 0;
- while (i < toDelete.length) {
- const count = Math.min(toDelete.length, 5000);
- const toDeleteDocs = toDelete.slice(i, i + count);
- i += count;
- const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments");
- deleted += result.deletedCount || 0;
- }
- // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments");
- console.log(`${deleted} documents deleted`);
+ if (!full) {
+ await Database.Instance.updateMany({ _id: { $nin: notToDelete } }, { $set: { "deleted": true } });
+ await Database.Instance.updateMany({ _id: { $in: notToDelete } }, { $unset: { "deleted": true } });
+ console.log(await Search.Instance.updateDocuments(
+ notToDelete.map<any>(id => ({
+ id, deleted: { set: null }
+ }))
+ .concat(toDelete.map(id => ({
+ id, deleted: { set: true }
+ })))));
+ console.log("Done with partial GC");
+ console.log(`Took ${(Date.now() - start) / 1000} seconds`);
+ } else {
+ let i = 0;
+ let deleted = 0;
+ while (i < toDelete.length) {
+ const count = Math.min(toDelete.length, 5000);
+ const toDeleteDocs = toDelete.slice(i, i + count);
+ i += count;
+ const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments");
+ deleted += result.deletedCount || 0;
+ }
+ // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments");
+ console.log(`${deleted} documents deleted`);
- await Search.Instance.deleteDocuments(toDelete);
- console.log("Cleared search documents");
+ await Search.Instance.deleteDocuments(toDelete);
+ console.log("Cleared search documents");
- const folder = "./src/server/public/files/";
- fs.readdir(folder, (_, fileList) => {
- const filesToDelete = fileList.filter(file => {
- const ext = path.extname(file);
- let base = path.basename(file, ext);
- const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext);
- return file !== ".gitignore" && !existsInDb;
- });
- console.log(`Deleting ${filesToDelete.length} files`);
- filesToDelete.forEach(file => {
- console.log(`Deleting file ${file}`);
- try {
- fs.unlinkSync(folder + file);
- } catch {
- console.warn(`Couldn't delete file ${file}`);
- }
+ const folder = "./src/server/public/files/";
+ fs.readdir(folder, (_, fileList) => {
+ const filesToDelete = fileList.filter(file => {
+ const ext = path.extname(file);
+ let base = path.basename(file, ext);
+ const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext);
+ return file !== ".gitignore" && !existsInDb;
+ });
+ console.log(`Deleting ${filesToDelete.length} files`);
+ filesToDelete.forEach(file => {
+ console.log(`Deleting file ${file}`);
+ try {
+ fs.unlinkSync(folder + file);
+ } catch {
+ console.warn(`Couldn't delete file ${file}`);
+ }
+ });
+ console.log(`Deleted ${filesToDelete.length} files`);
});
- console.log(`Deleted ${filesToDelete.length} files`);
- });
+ }
}
-GarbageCollect();
+GarbageCollect(false);
diff --git a/src/server/Search.ts b/src/server/Search.ts
index 69e327d2d..723dc101b 100644
--- a/src/server/Search.ts
+++ b/src/server/Search.ts
@@ -30,20 +30,14 @@ export class Search {
}
}
- public async search(query: string, filterQuery: string = "", start: number = 0, rows: number = 10) {
+ public async search(query: any) {
try {
const searchResults = JSON.parse(await rp.get(this.url + "dash/select", {
- qs: {
- q: query,
- fq: filterQuery,
- fl: "id",
- start,
- rows,
- }
+ qs: query
}));
const { docs, numFound } = searchResults.response;
const ids = docs.map((field: any) => field.id);
- return { ids, numFound };
+ return { ids, numFound, highlighting: searchResults.highlighting };
} catch {
return { ids: [], numFound: -1 };
}
diff --git a/src/server/authentication/controllers/user_controller.ts b/src/server/authentication/controllers/user_controller.ts
index 0e431f1e6..f5c6e1610 100644
--- a/src/server/authentication/controllers/user_controller.ts
+++ b/src/server/authentication/controllers/user_controller.ts
@@ -77,8 +77,9 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => {
let tryRedirectToTarget = (req: Request, res: Response) => {
if (req.session && req.session.target) {
- res.redirect(req.session.target);
+ let target = req.session.target;
req.session.target = undefined;
+ res.redirect(target);
} else {
res.redirect(RouteStore.home);
}
diff --git a/src/server/database.ts b/src/server/database.ts
index 29a8ffafa..7f5331998 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -140,6 +140,17 @@ export class Database {
}
}
+ public updateMany(query: any, update: any, collectionName = "newDocuments") {
+ if (this.db) {
+ const db = this.db;
+ return new Promise<mongodb.WriteOpResult>(res => db.collection(collectionName).update(query, update, (_, result) => res(result)));
+ } else {
+ return new Promise<mongodb.WriteOpResult>(res => {
+ this.onConnect.push(() => this.updateMany(query, update, collectionName).then(res));
+ });
+ }
+ }
+
public print() {
console.log("db says hi!");
}
diff --git a/src/server/index.ts b/src/server/index.ts
index 06f8358e1..5703b032e 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -110,7 +110,7 @@ function addSecureRoute(method: Method,
if (req.user) {
handler(req.user, res, req);
} else {
- req.session!.target = `http://localhost:${port}${req.originalUrl}`;
+ req.session!.target = req.originalUrl;
onRejection(res, req);
}
};
@@ -144,12 +144,13 @@ app.get("/pull", (req, res) =>
// GETTERS
app.get("/search", async (req, res) => {
- const { query, filterQuery, start, rows } = req.query;
- if (query === undefined) {
+ const solrQuery: any = {};
+ ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]);
+ if (solrQuery.q === undefined) {
res.send([]);
return;
}
- let results = await Search.Instance.search(query, filterQuery, start, rows);
+ let results = await Search.Instance.search(solrQuery);
res.send(results);
});
@@ -178,11 +179,10 @@ app.get("/whosOnline", (req, res) => {
res.send(users);
});
-
app.get("/thumbnail/:filename", (req, res) => {
let filename = req.params.filename;
let noExt = filename.substring(0, filename.length - ".png".length);
- let pagenumber = parseInt(noExt[noExt.length - 1]);
+ let pagenumber = parseInt(noExt.split('-')[1]);
fs.exists(uploadDir + filename, (exists: boolean) => {
console.log(`${uploadDir + filename} ${exists ? "exists" : "does not exist"}`);
if (exists) {
@@ -190,13 +190,14 @@ app.get("/thumbnail/:filename", (req, res) => {
probe(input, (err: any, result: any) => {
if (err) {
console.log(err);
+ console.log(`error on ${filename}`);
return;
}
res.send({ path: "/files/" + filename, width: result.width, height: result.height });
});
}
else {
- LoadPage(uploadDir + filename.substring(0, filename.length - "-n.png".length) + ".pdf", pagenumber, res);
+ LoadPage(uploadDir + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res);
}
});
});
diff --git a/src/server/updateProtos.ts b/src/server/updateProtos.ts
new file mode 100644
index 000000000..90490eb45
--- /dev/null
+++ b/src/server/updateProtos.ts
@@ -0,0 +1,15 @@
+import { Database } from "./database";
+
+const protos =
+ ["text", "histogram", "image", "web", "collection", "kvp",
+ "video", "audio", "pdf", "icon", "import", "linkdoc"];
+
+(async function () {
+ await Promise.all(
+ protos.map(protoId => new Promise(res => Database.Instance.update(protoId, {
+ $set: { "fields.baseProto": true }
+ }, res)))
+ );
+
+ console.log("done");
+})(); \ No newline at end of file
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-22 18:05:38 +0000