Merge branch 'master' of https://github.com/browngraphicslab/Dash-Web

5 files changed, 136 insertions, 70 deletions

diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts
index fb4a107ad..0d95bb96c 100644
--- a/src/client/views/GlobalKeyHandler.ts
+++ b/src/client/views/GlobalKeyHandler.ts
@@ -67,6 +67,14 @@ export default class KeyManager {
                 }
                 MainView.Instance.toggleColorPicker(true);
                 break;
+            case "delete":
+            case "backspace":
+                SelectionManager.SelectedDocuments().map(docView => {
+                    let doc = docView.props.Document;
+                    let remove = docView.props.removeDocument;
+                    remove && remove(doc);
+                });
+                break;
         }
 
         return {
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index fe59f52c8..da126fb74 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -67,12 +67,6 @@ export class MainView extends React.Component {
         window.removeEventListener("keydown", KeyManager.Instance.handle);
         window.addEventListener("keydown", KeyManager.Instance.handle);
 
-        window.removeEventListener("pointerdown", this.pointerDown);
-        window.addEventListener("pointerdown", this.pointerDown);
-
-        window.removeEventListener("pointerup", this.pointerUp);
-        window.addEventListener("pointerup", this.pointerUp);
-
         reaction(() => {
             let workspaces = CurrentUserUtils.UserDocument.workspaces;
             let recent = CurrentUserUtils.UserDocument.recentlyClosed;
@@ -86,17 +80,12 @@ export class MainView extends React.Component {
             if (libraryHeight && Math.abs(CurrentUserUtils.UserDocument[HeightSym]() - libraryHeight) > 5) {
                 CurrentUserUtils.UserDocument.height = libraryHeight;
             }
-            (Cast(CurrentUserUtils.UserDocument.recentlyClosed, Doc) as Doc)!.allowClear = true;
+            (Cast(CurrentUserUtils.UserDocument.recentlyClosed, Doc) as Doc).allowClear = true;
         }, { fireImmediately: true });
     }
 
-    pointerDown = (e: PointerEvent) => this.isPointerDown = true;
-    pointerUp = (e: PointerEvent) => this.isPointerDown = false;
-
     componentWillUnMount() {
         window.removeEventListener("keydown", KeyManager.Instance.handle);
-        window.removeEventListener("pointerdown", this.pointerDown);
-        window.removeEventListener("pointerup", this.pointerUp);
     }
 
     constructor(props: Readonly<{}>) {
@@ -151,8 +140,8 @@ export class MainView extends React.Component {
         window.addEventListener("drop", (e) => e.preventDefault(), false); // drop event handler
         window.addEventListener("dragover", (e) => e.preventDefault(), false); // drag event handler
         // click interactions for the context menu
-        document.addEventListener("pointerdown", action(function (e: PointerEvent) {
-
+        document.addEventListener("pointerdown", action((e: PointerEvent) => {
+            this.isPointerDown = true;
             const targets = document.elementsFromPoint(e.x, e.y);
             if (targets && targets.length && targets[0].className.toString().indexOf("contextMenu") === -1) {
                 ContextMenu.Instance.closeMenu();
@@ -286,6 +275,7 @@ export class MainView extends React.Component {
     }
     @action
     onPointerUp = (e: PointerEvent) => {
+        this.isPointerDown = false;
         if (Math.abs(e.clientX - this._downsize) < 4) {
             if (this.flyoutWidth < 5) this.flyoutWidth = 250;
             else this.flyoutWidth = 0;
@@ -300,7 +290,7 @@ export class MainView extends React.Component {
         } else {
             CollectionDockingView.Instance.AddRightSplit(doc, undefined);
         }
-    };
+    }
     @computed
     get flyout() {
         let sidebar = CurrentUserUtils.UserDocument.sidebar;
@@ -325,7 +315,7 @@ export class MainView extends React.Component {
             ContainingCollectionView={undefined}
             zoomToScale={emptyFunction}
             getScale={returnOne}>
-        </DocumentView>
+        </DocumentView>;
     }
     @computed
     get mainContent() {
diff --git a/src/client/views/nodes/FormattedTextBox.tsx b/src/client/views/nodes/FormattedTextBox.tsx
index 68a4372d5..32bb97255 100644
--- a/src/client/views/nodes/FormattedTextBox.tsx
+++ b/src/client/views/nodes/FormattedTextBox.tsx
@@ -89,9 +89,8 @@ export class FormattedTextBox extends DocComponent<(FieldViewProps & FormattedTe
     }
     public static GetDocFromUrl(url: string) {
         if (url.startsWith(document.location.origin)) {
-            let start = url.indexOf(window.location.origin);
-            let path = url.substr(start, url.length - start);
-            let docid = path.replace(DocServer.prepend("/doc/"), "").split("?")[0];
+            const split = new URL(url).pathname.split("doc/");
+            const docid = split[split.length - 1];
             return docid;
         }
         return "";
diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts
new file mode 100644
index 000000000..3c32a94c2
--- /dev/null
+++ b/src/server/GarbageCollector.ts
@@ -0,0 +1,120 @@
+import { Database } from './database';
+
+import * as path from 'path';
+import * as fs from 'fs';
+
+function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) {
+    for (const key in doc) {
+        if (!doc.hasOwnProperty(key)) {
+            continue;
+        }
+        const field = doc[key];
+        if (field === undefined || field === null) {
+            continue;
+        }
+        if (field.__type === "proxy") {
+            ids.push(field.fieldId);
+        } else if (field.__type === "list") {
+            addDoc(field.fields, ids, files);
+        } else if (typeof field === "string") {
+            const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g;
+            let match: string[] | null;
+            while ((match = re.exec(field)) !== null) {
+                ids.push(match[1]);
+            }
+        } else if (field.__type === "RichTextField") {
+            const re = /"href"\s*:\s*"(.*?)"/g;
+            let match: string[] | null;
+            while ((match = re.exec(field.Data)) !== null) {
+                const urlString = match[1];
+                const split = new URL(urlString).pathname.split("doc/");
+                if (split.length > 1) {
+                    ids.push(split[split.length - 1]);
+                }
+            }
+            const re2 = /"src"\s*:\s*"(.*?)"/g;
+            while ((match = re2.exec(field.Data)) !== null) {
+                const urlString = match[1];
+                const pathname = new URL(urlString).pathname;
+                const ext = path.extname(pathname);
+                const fileName = path.basename(pathname, ext);
+                let exts = files[fileName];
+                if (!exts) {
+                    files[fileName] = exts = [];
+                }
+                exts.push(ext);
+            }
+        } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) {
+            const url = new URL(field.url);
+            const pathname = url.pathname;
+            const ext = path.extname(pathname);
+            const fileName = path.basename(pathname, ext);
+            let exts = files[fileName];
+            if (!exts) {
+                files[fileName] = exts = [];
+            }
+            exts.push(ext);
+        }
+    }
+}
+
+async function GarbageCollect() {
+    // await new Promise(res => setTimeout(res, 3000));
+    const cursor = await Database.Instance.query({}, 'users');
+    const users = await cursor.toArray();
+    const ids: string[] = users.map(user => user.userDocumentId);
+    const visited = new Set<string>();
+    const files: { [name: string]: string[] } = {};
+
+    while (ids.length) {
+        const count = Math.min(ids.length, 100);
+        const index = ids.length - count;
+        const fetchIds = ids.splice(index, count).filter(id => !visited.has(id));
+        if (!fetchIds.length) {
+            continue;
+        }
+        const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments"));
+        for (const doc of docs) {
+            const id = doc.id;
+            if (doc === undefined) {
+                console.log(`Couldn't find field with Id ${id}`);
+                continue;
+            }
+            visited.add(id);
+            addDoc(doc.fields, ids, files);
+        }
+        console.log(`To Go: ${ids.length}, visited: ${visited.size}`);
+    }
+
+    console.log(`Done: ${visited.size}`);
+
+    cursor.close();
+
+    const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } });
+    const toDelete = (await toDeleteCursor.toArray()).map(doc => doc._id);
+    toDeleteCursor.close();
+    const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments");
+    console.log(`${result.deletedCount} documents deleted`);
+
+    const folder = "./src/server/public/files/";
+    fs.readdir(folder, (_, fileList) => {
+        const filesToDelete = fileList.filter(file => {
+            const ext = path.extname(file);
+            let base = path.basename(file, ext);
+            const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext);
+            return file !== ".gitignore" && !existsInDb;
+        });
+        console.log(`Deleting ${filesToDelete.length} files`);
+        filesToDelete.forEach(file => {
+            console.log(`Deleting file ${file}`);
+            try {
+                fs.unlinkSync(folder + file);
+            } catch {
+                console.warn(`Couldn't delete file ${file}`);
+            }
+        });
+        console.log(`Deleted ${filesToDelete.length} files`);
+    });
+}
+
+GarbageCollect();
diff --git a/src/server/GarbageColletor.ts b/src/server/GarbageColletor.ts
deleted file mode 100644
index f26b0cec6..000000000
--- a/src/server/GarbageColletor.ts
+++ /dev/null
@@ -1,51 +0,0 @@
-import { Database } from './database';
-
-function addDoc(doc: any, ids: string[]) {
-    for (const key in doc) {
-        if (!doc.hasOwnProperty(key)) {
-            continue;
-        }
-        const field = doc[key];
-        if (!(field instanceof Object)) {
-            continue;
-        }
-        if (field.__type === "proxy") {
-            ids.push(field.fieldId);
-        } else if (field.__type === "list") {
-            addDoc(field.fields, ids);
-        }
-    }
-}
-
-async function GarbageCollect() {
-    // await new Promise(res => setTimeout(res, 3000));
-    const cursor = await Database.Instance.query({}, 'users');
-    const users = await cursor.toArray();
-    const ids: string[] = users.map(user => user.userDocumentId);
-    const visited = new Set<string>();
-
-    while (ids.length) {
-        const id = ids.pop()!;
-        if (visited.has(id)) continue;
-        const doc = await new Promise<{ [key: string]: any }>(res => Database.Instance.getDocument(id, res, "newDocuments"));
-        if (doc === undefined) {
-            console.log(`Couldn't find field with Id ${id}`);
-            continue;
-        }
-        visited.add(id);
-        addDoc(doc.fields, ids);
-        console.log(`To Go: ${ids.length}, visited: ${visited.size}`);
-    }
-
-    console.log(`Done: ${visited.size}`);
-
-    cursor.close();
-
-    const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } });
-    const toDelete = (await toDeleteCursor.toArray()).map(doc => doc._id);
-    toDeleteCursor.close();
-    const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments");
-    console.log(`${result.deletedCount} documents deleted`);
-}
-
-GarbageCollect();