aboutsummaryrefslogtreecommitdiff
path: root/src/server
diff options
context:
space:
mode:
Diffstat (limited to 'src/server')
-rw-r--r--src/server/GarbageCollector.ts133
-rw-r--r--src/server/Message.ts2
-rw-r--r--src/server/Search.ts21
-rw-r--r--src/server/authentication/controllers/user_controller.ts22
-rw-r--r--src/server/authentication/models/current_user_utils.ts25
-rw-r--r--src/server/database.ts22
-rw-r--r--src/server/index.ts28
-rw-r--r--src/server/updateSearch.ts38
8 files changed, 250 insertions, 41 deletions
diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts
new file mode 100644
index 000000000..59682e51e
--- /dev/null
+++ b/src/server/GarbageCollector.ts
@@ -0,0 +1,133 @@
+import { Database } from './database';
+
+import * as path from 'path';
+import * as fs from 'fs';
+import { Search } from './Search';
+
+function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) {
+ for (const key in doc) {
+ if (!doc.hasOwnProperty(key)) {
+ continue;
+ }
+ const field = doc[key];
+ if (field === undefined || field === null) {
+ continue;
+ }
+ if (field.__type === "proxy") {
+ ids.push(field.fieldId);
+ } else if (field.__type === "list") {
+ addDoc(field.fields, ids, files);
+ } else if (typeof field === "string") {
+ const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g;
+ let match: string[] | null;
+ while ((match = re.exec(field)) !== null) {
+ ids.push(match[1]);
+ }
+ } else if (field.__type === "RichTextField") {
+ const re = /"href"\s*:\s*"(.*?)"/g;
+ let match: string[] | null;
+ while ((match = re.exec(field.Data)) !== null) {
+ const urlString = match[1];
+ const split = new URL(urlString).pathname.split("doc/");
+ if (split.length > 1) {
+ ids.push(split[split.length - 1]);
+ }
+ }
+ const re2 = /"src"\s*:\s*"(.*?)"/g;
+ while ((match = re2.exec(field.Data)) !== null) {
+ const urlString = match[1];
+ const pathname = new URL(urlString).pathname;
+ const ext = path.extname(pathname);
+ const fileName = path.basename(pathname, ext);
+ let exts = files[fileName];
+ if (!exts) {
+ files[fileName] = exts = [];
+ }
+ exts.push(ext);
+ }
+ } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) {
+ const url = new URL(field.url);
+ const pathname = url.pathname;
+ const ext = path.extname(pathname);
+ const fileName = path.basename(pathname, ext);
+ let exts = files[fileName];
+ if (!exts) {
+ files[fileName] = exts = [];
+ }
+ exts.push(ext);
+ }
+ }
+}
+
+async function GarbageCollect() {
+ // await new Promise(res => setTimeout(res, 3000));
+ const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users');
+ const users = await cursor.toArray();
+ const ids: string[] = users.map(user => user.userDocumentId);
+ const visited = new Set<string>();
+ const files: { [name: string]: string[] } = {};
+
+ while (ids.length) {
+ const count = Math.min(ids.length, 100);
+ const index = ids.length - count;
+ const fetchIds = ids.splice(index, count).filter(id => !visited.has(id));
+ if (!fetchIds.length) {
+ continue;
+ }
+ const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments"));
+ for (const doc of docs) {
+ const id = doc.id;
+ if (doc === undefined) {
+ console.log(`Couldn't find field with Id ${id}`);
+ continue;
+ }
+ visited.add(id);
+ addDoc(doc.fields, ids, files);
+ }
+ console.log(`To Go: ${ids.length}, visited: ${visited.size}`);
+ }
+
+ console.log(`Done: ${visited.size}`);
+
+ cursor.close();
+
+ const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }, { _id: 1 });
+ const toDelete: string[] = (await toDeleteCursor.toArray()).map(doc => doc._id);
+ toDeleteCursor.close();
+ let i = 0;
+ let deleted = 0;
+ while (i < toDelete.length) {
+ const count = Math.min(toDelete.length, 5000);
+ const toDeleteDocs = toDelete.slice(i, i + count);
+ i += count;
+ const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments");
+ deleted += result.deletedCount || 0;
+ }
+ // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments");
+ console.log(`${deleted} documents deleted`);
+
+ await Search.Instance.deleteDocuments(toDelete);
+ console.log("Cleared search documents");
+
+ const folder = "./src/server/public/files/";
+ fs.readdir(folder, (_, fileList) => {
+ const filesToDelete = fileList.filter(file => {
+ const ext = path.extname(file);
+ let base = path.basename(file, ext);
+ const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext);
+ return file !== ".gitignore" && !existsInDb;
+ });
+ console.log(`Deleting ${filesToDelete.length} files`);
+ filesToDelete.forEach(file => {
+ console.log(`Deleting file ${file}`);
+ try {
+ fs.unlinkSync(folder + file);
+ } catch {
+ console.warn(`Couldn't delete file ${file}`);
+ }
+ });
+ console.log(`Deleted ${filesToDelete.length} files`);
+ });
+}
+
+GarbageCollect();
diff --git a/src/server/Message.ts b/src/server/Message.ts
index e9a8b0f0c..19e0a48aa 100644
--- a/src/server/Message.ts
+++ b/src/server/Message.ts
@@ -45,4 +45,6 @@ export namespace MessageStore {
export const GetRefFields = new Message<string[]>("Get Ref Fields");
export const UpdateField = new Message<Diff>("Update Ref Field");
export const CreateField = new Message<Reference>("Create Ref Field");
+ export const DeleteField = new Message<string>("Delete field");
+ export const DeleteFields = new Message<string[]>("Delete fields");
}
diff --git a/src/server/Search.ts b/src/server/Search.ts
index 5a22f7da7..ffba4ea8e 100644
--- a/src/server/Search.ts
+++ b/src/server/Search.ts
@@ -48,4 +48,25 @@ export class Search {
});
} catch { }
}
+
+ public deleteDocuments(docs: string[]) {
+ const promises: rp.RequestPromise[] = [];
+ const nToDelete = 1000;
+ let index = 0;
+ while (index < docs.length) {
+ const count = Math.min(docs.length - index, nToDelete);
+ const deleteIds = docs.slice(index, index + count);
+ index += count;
+ promises.push(rp.post(this.url + "dash/update", {
+ body: {
+ delete: {
+ query: deleteIds.map(id => `id:"${id}"`).join(" ")
+ }
+ },
+ json: true
+ }));
+ }
+
+ return Promise.all(promises);
+ }
} \ No newline at end of file
diff --git a/src/server/authentication/controllers/user_controller.ts b/src/server/authentication/controllers/user_controller.ts
index 1dacdf3fa..ca4fc171c 100644
--- a/src/server/authentication/controllers/user_controller.ts
+++ b/src/server/authentication/controllers/user_controller.ts
@@ -42,10 +42,6 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => {
const errors = req.validationErrors();
if (errors) {
- res.render("signup.pug", {
- title: "Sign Up",
- user: req.user,
- });
return res.redirect(RouteStore.signup);
}
@@ -66,16 +62,23 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => {
user.save((err) => {
if (err) { return next(err); }
req.logIn(user, (err) => {
- if (err) {
- return next(err);
- }
- res.redirect(RouteStore.home);
+ if (err) { return next(err); }
+ tryRedirectToTarget(req, res);
});
});
});
};
+let tryRedirectToTarget = (req: Request, res: Response) => {
+ if (req.session && req.session.target) {
+ res.redirect(req.session.target);
+ req.session.target = undefined;
+ } else {
+ res.redirect(RouteStore.home);
+ }
+};
+
/**
* GET /login
@@ -83,6 +86,7 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => {
*/
export let getLogin = (req: Request, res: Response) => {
if (req.user) {
+ req.session!.target = undefined;
return res.redirect(RouteStore.home);
}
res.render("login.pug", {
@@ -115,7 +119,7 @@ export let postLogin = (req: Request, res: Response, next: NextFunction) => {
}
req.logIn(user, (err) => {
if (err) { next(err); return; }
- res.redirect(RouteStore.home);
+ tryRedirectToTarget(req, res);
});
})(req, res, next);
};
diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts
index 30a6f108a..384c579de 100644
--- a/src/server/authentication/models/current_user_utils.ts
+++ b/src/server/authentication/models/current_user_utils.ts
@@ -10,7 +10,7 @@ import { CollectionView } from "../../../client/views/collections/CollectionView
import { Doc } from "../../../new_fields/Doc";
import { List } from "../../../new_fields/List";
import { listSpec } from "../../../new_fields/Schema";
-import { Cast, FieldValue } from "../../../new_fields/Types";
+import { Cast, FieldValue, StrCast } from "../../../new_fields/Types";
import { RouteStore } from "../../RouteStore";
export class CurrentUserUtils {
@@ -34,33 +34,42 @@ export class CurrentUserUtils {
doc.title = this.email;
this.updateUserDocument(doc);
doc.data = new List<Doc>();
+ doc.gridGap = 5;
+ doc.xMargin = 5;
+ doc.yMargin = 5;
+ doc.boxShadow = "0 0";
doc.excludeFromLibrary = true;
- doc.optionalRightCollection = Docs.StackingDocument([], { title: "New mobile uploads" });
- // doc.library = Docs.TreeDocument([doc], { title: `Library: ${CurrentUserUtils.email}` });
+ doc.optionalRightCollection = Docs.Create.StackingDocument([], { title: "New mobile uploads" });
+ // doc.library = Docs.Create.TreeDocument([doc], { title: `Library: ${CurrentUserUtils.email}` });
// (doc.library as Doc).excludeFromLibrary = true;
return doc;
}
static updateUserDocument(doc: Doc) {
if (doc.workspaces === undefined) {
- const workspaces = Docs.TreeDocument([], { title: "Workspaces", height: 100 });
+ const workspaces = Docs.Create.TreeDocument([], { title: "Workspaces", height: 100 });
workspaces.excludeFromLibrary = true;
workspaces.workspaceLibrary = true;
+ workspaces.boxShadow = "0 0";
doc.workspaces = workspaces;
}
if (doc.recentlyClosed === undefined) {
- const recentlyClosed = Docs.TreeDocument([], { title: "Recently Closed", height: 75 });
+ const recentlyClosed = Docs.Create.TreeDocument([], { title: "Recently Closed", height: 75 });
recentlyClosed.excludeFromLibrary = true;
+ recentlyClosed.boxShadow = "0 0";
doc.recentlyClosed = recentlyClosed;
}
if (doc.sidebar === undefined) {
- const sidebar = Docs.StackingDocument([doc.workspaces as Doc, doc, doc.recentlyClosed as Doc], { title: "Sidebar" });
+ const sidebar = Docs.Create.StackingDocument([doc.workspaces as Doc, doc, doc.recentlyClosed as Doc], { title: "Sidebar" });
sidebar.excludeFromLibrary = true;
sidebar.gridGap = 5;
sidebar.xMargin = 5;
sidebar.yMargin = 5;
+ Doc.GetProto(sidebar).backgroundColor = "#aca3a6";
+ sidebar.boxShadow = "1 1 3";
doc.sidebar = sidebar;
}
+ StrCast(doc.title).indexOf("@") !== -1 && (doc.title = StrCast(doc.title).split("@")[0] + "'s Library");
}
@@ -125,12 +134,12 @@ export class CurrentUserUtils {
// new AttributeTransformationModel(atmod, AggregateFunction.None),
// new AttributeTransformationModel(atmod, AggregateFunction.Count),
// new AttributeTransformationModel(atmod, AggregateFunction.Count));
- // schemaDocuments.push(Docs.HistogramDocument(histoOp, { width: 200, height: 200, title: attr.displayName! }));
+ // schemaDocuments.push(Docs.Create.HistogramDocument(histoOp, { width: 200, height: 200, title: attr.displayName! }));
// }
// })));
// return promises;
// }, [] as Promise<void>[]));
- // return CurrentUserUtils._northstarSchemas.push(Docs.TreeDocument(schemaDocuments, { width: 50, height: 100, title: schema.displayName! }));
+ // return CurrentUserUtils._northstarSchemas.push(Docs.Create.TreeDocument(schemaDocuments, { width: 50, height: 100, title: schema.displayName! }));
// });
// }
}
diff --git a/src/server/database.ts b/src/server/database.ts
index d240bd909..29a8ffafa 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -41,11 +41,17 @@ export class Database {
}
}
- public delete(id: string, collectionName = Database.DocumentsCollection) {
+ public delete(query: any, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
+ public delete(id: string, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
+ public delete(id: any, collectionName = Database.DocumentsCollection) {
+ if (typeof id === "string") {
+ id = { _id: id };
+ }
if (this.db) {
- this.db.collection(collectionName).remove({ id: id });
+ const db = this.db;
+ return new Promise(res => db.collection(collectionName).deleteMany(id, (err, result) => res(result)));
} else {
- this.onConnect.push(() => this.delete(id, collectionName));
+ return new Promise(res => this.onConnect.push(() => res(this.delete(id, collectionName))));
}
}
@@ -120,12 +126,16 @@ export class Database {
}
}
- public query(query: any, collectionName = "newDocuments"): Promise<mongodb.Cursor> {
+ public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise<mongodb.Cursor> {
if (this.db) {
- return Promise.resolve<mongodb.Cursor>(this.db.collection(collectionName).find(query));
+ let cursor = this.db.collection(collectionName).find(query);
+ if (projection) {
+ cursor = cursor.project(projection);
+ }
+ return Promise.resolve<mongodb.Cursor>(cursor);
} else {
return new Promise<mongodb.Cursor>(res => {
- this.onConnect.push(() => res(this.query(query)));
+ this.onConnect.push(() => res(this.query(query, projection, collectionName)));
});
}
}
diff --git a/src/server/index.ts b/src/server/index.ts
index 66fe3c990..e9ca256fa 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -25,7 +25,7 @@ import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLo
import { DashUserModel } from './authentication/models/user_model';
import { Client } from './Client';
import { Database } from './database';
-import { MessageStore, Transferable, Types, Diff } from "./Message";
+import { MessageStore, Transferable, Types, Diff, Message } from "./Message";
import { RouteStore } from './RouteStore';
const app = express();
const config = require('../../webpack.config');
@@ -103,14 +103,15 @@ enum Method {
*/
function addSecureRoute(method: Method,
handler: (user: DashUserModel, res: express.Response, req: express.Request) => void,
- onRejection: (res: express.Response) => any = (res) => res.redirect(RouteStore.logout),
+ onRejection: (res: express.Response, req: express.Request) => any = res => res.redirect(RouteStore.login),
...subscribers: string[]
) {
let abstracted = (req: express.Request, res: express.Response) => {
if (req.user) {
handler(req.user, res, req);
} else {
- onRejection(res);
+ req.session!.target = `http://localhost:${port}${req.originalUrl}`;
+ onRejection(res, req);
}
};
subscribers.forEach(route => {
@@ -222,7 +223,7 @@ addSecureRoute(
addSecureRoute(
Method.GET,
async (_, res) => {
- const cursor = await Database.Instance.query({}, "users");
+ const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users");
const results = await cursor.toArray();
res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId })));
},
@@ -468,6 +469,8 @@ server.on("connection", function (socket: Socket) {
Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField);
Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff));
+ Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id));
+ Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids));
Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField);
Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields);
});
@@ -594,6 +597,23 @@ function UpdateField(socket: Socket, diff: Diff) {
}
}
+function DeleteField(socket: Socket, id: string) {
+ Database.Instance.delete({ _id: id }, "newDocuments").then(() => {
+ socket.broadcast.emit(MessageStore.DeleteField.Message, id);
+ });
+
+ Search.Instance.deleteDocuments([id]);
+}
+
+function DeleteFields(socket: Socket, ids: string[]) {
+ Database.Instance.delete({ _id: { $in: ids } }, "newDocuments").then(() => {
+ socket.broadcast.emit(MessageStore.DeleteFields.Message, ids);
+ });
+
+ Search.Instance.deleteDocuments(ids);
+
+}
+
function CreateField(newValue: any) {
Database.Instance.insert(newValue, "newDocuments");
}
diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts
index f5de00978..906b795f1 100644
--- a/src/server/updateSearch.ts
+++ b/src/server/updateSearch.ts
@@ -7,7 +7,7 @@ const suffixMap: { [type: string]: (string | [string, string | ((json: any) => a
"number": "_n",
"string": "_t",
"boolean": "_b",
- "image": ["_t", "url"],
+ // "image": ["_t", "url"],
"video": ["_t", "url"],
"pdf": ["_t", "url"],
"audio": ["_t", "url"],
@@ -67,7 +67,7 @@ async function update() {
if ((numDocs % 50) === 0) {
console.log("updateDoc " + numDocs);
}
- console.log("doc " + numDocs);
+ // console.log("doc " + numDocs);
if (doc.__type !== "Doc") {
return;
}
@@ -88,22 +88,32 @@ async function update() {
}
if (dynfield) {
updates.push(update);
- console.log(updates.length);
+ // console.log(updates.length);
}
}
await cursor.forEach(updateDoc);
- for (let i = 0; i < updates.length; i++) {
- console.log(i);
- const result = await Search.Instance.updateDocument(updates[i]);
- try {
- console.log(JSON.parse(result).responseHeader.status);
- } catch {
- console.log("Error:");
- console.log(updates[i]);
- console.log(result);
- console.log("\n");
- }
+ console.log(`Updating ${updates.length} documents`);
+ const result = await Search.Instance.updateDocuments(updates);
+ try {
+ console.log(JSON.parse(result).responseHeader.status);
+ } catch {
+ console.log("Error:");
+ // console.log(updates[i]);
+ console.log(result);
+ console.log("\n");
}
+ // for (let i = 0; i < updates.length; i++) {
+ // console.log(i);
+ // const result = await Search.Instance.updateDocument(updates[i]);
+ // try {
+ // console.log(JSON.parse(result).responseHeader.status);
+ // } catch {
+ // console.log("Error:");
+ // console.log(updates[i]);
+ // console.log(result);
+ // console.log("\n");
+ // }
+ // }
// await Promise.all(updates.map(update => {
// return limit(() => Search.Instance.updateDocument(update));
// }));
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2026-02-26 03:36:22 +0000