From 9334c8d12df76df60eedcc093b986f42d1f7f67d Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 2 Jul 2019 14:03:36 -0400 Subject: Refactored some search stuff and fixed treeview flyout min X --- src/server/Search.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/server') diff --git a/src/server/Search.ts b/src/server/Search.ts index 2db2b242c..e2a74b737 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -27,11 +27,11 @@ export class Search { start: start } })); - const fields = searchResults.response.docs; - const ids = fields.map((field: any) => field.id); - return ids; + const { docs, numFound } = searchResults.response; + const ids = docs.map((field: any) => field.id); + return { ids, numFound }; } catch { - return []; + return { ids: [], numFound: -1 }; } } -- cgit v1.2.3-70-g09d2 From f951da80c668f2707311fb6ac87bee57f9856a2d Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Fri, 5 Jul 2019 13:22:19 -0400 Subject: Fixes #191, images being rotated incorrectly --- src/server/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/server') diff --git a/src/server/index.ts b/src/server/index.ts index a93940907..50b086339 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -295,9 +295,9 @@ app.post( const file = path.basename(files[name].path); const ext = path.extname(file); let resizers = [ - { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, - { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, - { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }).rotate().withMetadata(), suffix: "_s" }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }).rotate().withMetadata(), suffix: "_m" }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }).rotate().withMetadata(), suffix: "_l" }, ]; let isImage = false; if (pngTypes.includes(ext)) { -- cgit v1.2.3-70-g09d2 From 264b50473b15338b26c132c9c7acded93021c6f9 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Fri, 5 Jul 2019 13:59:25 -0400 Subject: Fix image rotation again --- src/client/views/nodes/ImageBox.tsx | 3 ++- src/server/index.ts | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'src/server') diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index dd4e4262d..ee25dca4e 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -182,7 +182,8 @@ export class ImageBox extends DocComponent(ImageD return url.href; } let ext = path.extname(url.href); - return url.href.replace(ext, this._curSuffix + ext); + const suffix = this.props.renderDepth <= 1 ? "_o" : this._curSuffix; + return url.href.replace(ext, suffix + ext); } @observable _smallRetryCount = 1; diff --git a/src/server/index.ts b/src/server/index.ts index 50b086339..bf946fc9f 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -295,9 +295,10 @@ app.post( const file = path.basename(files[name].path); const ext = path.extname(file); let resizers = [ - { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }).rotate().withMetadata(), suffix: "_s" }, - { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }).rotate().withMetadata(), suffix: "_m" }, - { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }).rotate().withMetadata(), suffix: "_l" }, + { resizer: sharp().rotate(), suffix: "_o" }, + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }).rotate(), suffix: "_s" }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }).rotate(), suffix: "_m" }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }).rotate(), suffix: "_l" }, ]; let isImage = false; if (pngTypes.includes(ext)) { -- cgit v1.2.3-70-g09d2 From 8b9fa667816378cc6cae2977f1c2d80c204b28d0 Mon Sep 17 00:00:00 2001 From: bob Date: Fri, 5 Jul 2019 14:45:15 -0400 Subject: added more options to sidebar --- src/client/views/Main.scss | 18 ++++++++++++++ src/client/views/MainView.tsx | 28 ++++++++++++++++++---- .../views/collections/CollectionDockingView.tsx | 4 ++++ .../views/collections/CollectionTreeView.tsx | 7 ++++-- src/new_fields/Doc.ts | 11 +++++---- .../authentication/models/current_user_utils.ts | 10 ++++++++ 6 files changed, 67 insertions(+), 11 deletions(-) (limited to 'src/server') diff --git a/src/client/views/Main.scss b/src/client/views/Main.scss index b85a8040a..f52e3b658 100644 --- a/src/client/views/Main.scss +++ b/src/client/views/Main.scss @@ -230,6 +230,8 @@ ul#add-options-list { .mainView-libraryFlyout { height: 100%; position: absolute; + display: flex; + flex-direction:column; } .mainView-libraryHandle { @@ -241,4 +243,20 @@ ul#add-options-list { position: absolute; z-index: 1; background: gray; +} + +.mainView-workspace { + height:200px; + position:relative; + display:flex; +} +.mainView-library { + height:75%; + position:relative; + display:flex; +} +.mainView-recentlyClosed { + height:25%; + position:relative; + display:flex; } \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 61ccf4c1d..92468e3fd 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -8,12 +8,12 @@ import * as React from 'react'; import { SketchPicker } from 'react-color'; import Measure from 'react-measure'; import * as request from 'request'; -import { Doc, DocListCast, Opt } from '../../new_fields/Doc'; +import { Doc, DocListCast, Opt, HeightSym } from '../../new_fields/Doc'; import { Id } from '../../new_fields/FieldSymbols'; import { InkTool } from '../../new_fields/InkField'; import { List } from '../../new_fields/List'; import { listSpec } from '../../new_fields/Schema'; -import { Cast, FieldValue } from '../../new_fields/Types'; +import { Cast, FieldValue, NumCast } from '../../new_fields/Types'; import { CurrentUserUtils } from '../../server/authentication/models/current_user_utils'; import { RouteStore } from '../../server/RouteStore'; import { emptyFunction, returnOne, returnTrue } from '../../Utils'; @@ -163,7 +163,9 @@ export class MainView extends React.Component { @action createNewWorkspace = async (id?: string) => { - const list = Cast(CurrentUserUtils.UserDocument.data, listSpec(Doc)); + let workspaces = Cast(CurrentUserUtils.UserDocument.workspaces, Doc); + if (!(workspaces instanceof Doc)) return; + const list = Cast((CurrentUserUtils.UserDocument.workspaces as Doc).data, listSpec(Doc)); if (list) { let freeformDoc = Docs.FreeformDocument([], { x: 0, y: 400, width: this.pwidth * .7, height: this.pheight, title: `WS collection ${list.length + 1}` }); var dockingLayout = { content: [{ type: 'row', content: [CollectionDockingView.makeDocumentConfig(freeformDoc, freeformDoc, 600)] }] }; @@ -285,8 +287,24 @@ export class MainView extends React.Component { }; @computed get flyout() { + let sidebar = CurrentUserUtils.UserDocument.sidebar; + let workspaces = CurrentUserUtils.UserDocument.workspaces; + let recent = CurrentUserUtils.UserDocument.recentlyClosed; + if (!(sidebar instanceof Doc)) return (null); + if (!(recent instanceof Doc)) return (null); + if (!(workspaces instanceof Doc)) return (null); + let workspacesDoc = workspaces as Doc; + let sidebarDoc = sidebar as Doc; + let recentDoc = recent as Doc; + let library = CurrentUserUtils.UserDocument; + let gridGap = NumCast(sidebar.gridGap, 10); + let yMargin = NumCast(sidebar.yMargin, 2 * gridGap); + let libraryHeight = this.getPHeight() - workspacesDoc[HeightSym]() - recentDoc[HeightSym]() - 2 * gridGap - 2 * yMargin; + if (library[HeightSym]() != libraryHeight) { + setTimeout(() => CurrentUserUtils.UserDocument.height = libraryHeight, 0); + } return - ; + } @computed get mainContent() { diff --git a/src/client/views/collections/CollectionDockingView.tsx b/src/client/views/collections/CollectionDockingView.tsx index c5f8fb728..e0270fab3 100644 --- a/src/client/views/collections/CollectionDockingView.tsx +++ b/src/client/views/collections/CollectionDockingView.tsx @@ -27,6 +27,7 @@ import { MainView } from '../MainView'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { library } from '@fortawesome/fontawesome-svg-core'; import { faFile } from '@fortawesome/free-solid-svg-icons'; +import { CurrentUserUtils } from '../../../server/authentication/models/current_user_utils'; library.add(faFile); @observer @@ -416,6 +417,9 @@ export class CollectionDockingView extends React.Component { // need to test if propagation has stopped because GoldenLayout forces a parallel react hierarchy to be created for its top-level layout - if (!e.isPropagationStopped() && this.props.Document.excludeFromLibrary) { // excludeFromLibrary means this is the user document + if (!e.isPropagationStopped() && this.props.Document.workspaceLibrary) { // excludeFromLibrary means this is the user document ContextMenu.Instance.addItem({ description: "Create Workspace", event: undoBatch(() => MainView.Instance.createNewWorkspace()) }); ContextMenu.Instance.addItem({ description: "Delete Workspace", event: undoBatch(() => this.remove(this.props.Document)) }); + e.stopPropagation(); + e.preventDefault(); + ContextMenu.Instance.displayMenu(e.pageX - 15, e.pageY - 15); } } @@ -531,7 +534,7 @@ export class CollectionTreeView extends CollectionSubView(Document) { TreeView.loadId = doc[Id]; Doc.AddDocToList(this.props.Document, this.props.fieldKey, doc, this.childDocs.length ? this.childDocs[0] : undefined, true); }} /> - {this.props.Document.excludeFromLibrary ? this.notifsButton : (null)} + {this.props.Document.workspaceLibrary ? this.notifsButton : (null)}
    { TreeView.GetChildElements(this.childDocs, this.props.Document[Id], this.props.Document, this.props.DataDoc, this.props.fieldKey, addDoc, this.remove, diff --git a/src/new_fields/Doc.ts b/src/new_fields/Doc.ts index c361e3032..092205f52 100644 --- a/src/new_fields/Doc.ts +++ b/src/new_fields/Doc.ts @@ -241,12 +241,15 @@ export namespace Doc { return Array.from(results); } - export function AddDocToList(target: Doc, key: string, doc: Doc, relativeTo?: Doc, before?: boolean) { + export function AddDocToList(target: Doc, key: string, doc: Doc, relativeTo?: Doc, before?: boolean, first?: boolean) { let list = Cast(target[key], listSpec(Doc)); if (list) { - let ind = relativeTo ? list.indexOf(relativeTo) : -1; - if (ind === -1) list.push(doc); - else list.splice(before ? ind : ind + 1, 0, doc); + if (first) list.splice(0, 0, doc); + else { + let ind = relativeTo ? list.indexOf(relativeTo) : -1; + if (ind === -1) list.push(doc); + else list.splice(before ? ind : ind + 1, 0, doc); + } } return true; } diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 816c5f269..3134ecaff 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -31,6 +31,16 @@ export class CurrentUserUtils { doc.viewType = CollectionViewType.Tree; doc.dropAction = "alias"; doc.layout = CollectionView.LayoutString(); + doc.workspaces = Docs.TreeDocument([], { title: "Workspaces", height: 100 }); + (doc.workspaces as Doc).excludeFromLibrary = true; + (doc.workspaces as Doc).workspaceLibrary = true; + doc.recentlyClosed = Docs.TreeDocument([], { title: "Recently Closed", height: 75 }); + (doc.workspaces as Doc).excludeFromLibrary = true; + doc.sidebar = Docs.StackingDocument([doc.workspaces as Doc, doc, doc.recentlyClosed as Doc], { title: "Sidebar" }); + (doc.sidebar as Doc).excludeFromLibrary = true; + (doc.sidebar as Doc).gridGap = 5; + (doc.sidebar as Doc).xMargin = 5; + (doc.sidebar as Doc).yMargin = 5; doc.title = this.email; doc.data = new List(); doc.excludeFromLibrary = true; -- cgit v1.2.3-70-g09d2 From c42d288a6ea658542f398ee48492ff35e14d400d Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Fri, 5 Jul 2019 16:32:59 -0400 Subject: user document stuff --- .../authentication/models/current_user_utils.ts | 46 ++++++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) (limited to 'src/server') diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 3134ecaff..30a6f108a 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -10,7 +10,7 @@ import { CollectionView } from "../../../client/views/collections/CollectionView import { Doc } from "../../../new_fields/Doc"; import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; -import { Cast } from "../../../new_fields/Types"; +import { Cast, FieldValue } from "../../../new_fields/Types"; import { RouteStore } from "../../RouteStore"; export class CurrentUserUtils { @@ -31,17 +31,8 @@ export class CurrentUserUtils { doc.viewType = CollectionViewType.Tree; doc.dropAction = "alias"; doc.layout = CollectionView.LayoutString(); - doc.workspaces = Docs.TreeDocument([], { title: "Workspaces", height: 100 }); - (doc.workspaces as Doc).excludeFromLibrary = true; - (doc.workspaces as Doc).workspaceLibrary = true; - doc.recentlyClosed = Docs.TreeDocument([], { title: "Recently Closed", height: 75 }); - (doc.workspaces as Doc).excludeFromLibrary = true; - doc.sidebar = Docs.StackingDocument([doc.workspaces as Doc, doc, doc.recentlyClosed as Doc], { title: "Sidebar" }); - (doc.sidebar as Doc).excludeFromLibrary = true; - (doc.sidebar as Doc).gridGap = 5; - (doc.sidebar as Doc).xMargin = 5; - (doc.sidebar as Doc).yMargin = 5; doc.title = this.email; + this.updateUserDocument(doc); doc.data = new List(); doc.excludeFromLibrary = true; doc.optionalRightCollection = Docs.StackingDocument([], { title: "New mobile uploads" }); @@ -50,6 +41,29 @@ export class CurrentUserUtils { return doc; } + static updateUserDocument(doc: Doc) { + if (doc.workspaces === undefined) { + const workspaces = Docs.TreeDocument([], { title: "Workspaces", height: 100 }); + workspaces.excludeFromLibrary = true; + workspaces.workspaceLibrary = true; + doc.workspaces = workspaces; + } + if (doc.recentlyClosed === undefined) { + const recentlyClosed = Docs.TreeDocument([], { title: "Recently Closed", height: 75 }); + recentlyClosed.excludeFromLibrary = true; + doc.recentlyClosed = recentlyClosed; + } + if (doc.sidebar === undefined) { + const sidebar = Docs.StackingDocument([doc.workspaces as Doc, doc, doc.recentlyClosed as Doc], { title: "Sidebar" }); + sidebar.excludeFromLibrary = true; + sidebar.gridGap = 5; + sidebar.xMargin = 5; + sidebar.yMargin = 5; + doc.sidebar = sidebar; + } + + } + public static async loadCurrentUser(): Promise { let userPromise = rp.get(DocServer.prepend(RouteStore.getCurrUser)).then(response => { if (response) { @@ -62,8 +76,14 @@ export class CurrentUserUtils { }); let userDocPromise = await rp.get(DocServer.prepend(RouteStore.getUserDocumentId)).then(id => { if (id) { - return DocServer.GetRefField(id).then(field => - runInAction(() => this.user_document = field instanceof Doc ? field : this.createUserDocument(id))); + return DocServer.GetRefField(id).then(async field => { + if (field instanceof Doc) { + await this.updateUserDocument(field); + runInAction(() => this.user_document = field); + } else { + runInAction(() => this.user_document = this.createUserDocument(id)); + } + }); } else { throw new Error("There should be a user id! Why does Dash think there isn't one?"); } -- cgit v1.2.3-70-g09d2 From 8a902049db8378e13a28b19b1ad34a36882fbcb1 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Fri, 5 Jul 2019 19:32:44 -0400 Subject: Added garbage collector --- src/client/views/MainView.tsx | 6 ++--- src/server/GarbageColletor.ts | 51 +++++++++++++++++++++++++++++++++++++++++++ src/server/database.ts | 14 ++++++++---- 3 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 src/server/GarbageColletor.ts (limited to 'src/server') diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index d709a1a5b..82b17f142 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -293,9 +293,9 @@ export class MainView extends React.Component { if (!(sidebar instanceof Doc)) return (null); if (!(recent instanceof Doc)) return (null); if (!(workspaces instanceof Doc)) return (null); - let workspacesDoc = workspaces as Doc; - let sidebarDoc = sidebar as Doc; - let recentDoc = recent as Doc; + let workspacesDoc = workspaces; + let sidebarDoc = sidebar; + let recentDoc = recent; let library = CurrentUserUtils.UserDocument; let gridGap = NumCast(sidebar.gridGap, 10); let yMargin = NumCast(sidebar.yMargin, 2 * gridGap); diff --git a/src/server/GarbageColletor.ts b/src/server/GarbageColletor.ts new file mode 100644 index 000000000..f26b0cec6 --- /dev/null +++ b/src/server/GarbageColletor.ts @@ -0,0 +1,51 @@ +import { Database } from './database'; + +function addDoc(doc: any, ids: string[]) { + for (const key in doc) { + if (!doc.hasOwnProperty(key)) { + continue; + } + const field = doc[key]; + if (!(field instanceof Object)) { + continue; + } + if (field.__type === "proxy") { + ids.push(field.fieldId); + } else if (field.__type === "list") { + addDoc(field.fields, ids); + } + } +} + +async function GarbageCollect() { + // await new Promise(res => setTimeout(res, 3000)); + const cursor = await Database.Instance.query({}, 'users'); + const users = await cursor.toArray(); + const ids: string[] = users.map(user => user.userDocumentId); + const visited = new Set(); + + while (ids.length) { + const id = ids.pop()!; + if (visited.has(id)) continue; + const doc = await new Promise<{ [key: string]: any }>(res => Database.Instance.getDocument(id, res, "newDocuments")); + if (doc === undefined) { + console.log(`Couldn't find field with Id ${id}`); + continue; + } + visited.add(id); + addDoc(doc.fields, ids); + console.log(`To Go: ${ids.length}, visited: ${visited.size}`); + } + + console.log(`Done: ${visited.size}`); + + cursor.close(); + + const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }); + const toDelete = (await toDeleteCursor.toArray()).map(doc => doc._id); + toDeleteCursor.close(); + const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); + console.log(`${result.deletedCount} documents deleted`); +} + +GarbageCollect(); diff --git a/src/server/database.ts b/src/server/database.ts index d240bd909..a17447629 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -41,11 +41,17 @@ export class Database { } } - public delete(id: string, collectionName = Database.DocumentsCollection) { + public delete(query: any, collectionName?: string): Promise; + public delete(id: string, collectionName?: string): Promise; + public delete(id: any, collectionName = Database.DocumentsCollection) { + if (typeof id === "string") { + id = { _id: id }; + } if (this.db) { - this.db.collection(collectionName).remove({ id: id }); + const db = this.db; + return new Promise(res => db.collection(collectionName).deleteMany(id, (err, result) => res(result))); } else { - this.onConnect.push(() => this.delete(id, collectionName)); + return new Promise(res => this.onConnect.push(() => res(this.delete(id, collectionName)))); } } @@ -125,7 +131,7 @@ export class Database { return Promise.resolve(this.db.collection(collectionName).find(query)); } else { return new Promise(res => { - this.onConnect.push(() => res(this.query(query))); + this.onConnect.push(() => res(this.query(query, collectionName))); }); } } -- cgit v1.2.3-70-g09d2 From 8780f8652ce2cfdf37dd89be050123da3dc4eac4 Mon Sep 17 00:00:00 2001 From: Bob Zeleznik Date: Sat, 6 Jul 2019 00:30:52 -0400 Subject: don't understand what could be happening.. --- src/client/views/DocumentDecorations.tsx | 6 ++-- src/client/views/MainView.tsx | 39 ++++++++++++++++------ .../views/collections/CollectionStackingView.tsx | 8 ++--- src/client/views/nodes/DocumentView.tsx | 4 +-- src/new_fields/util.ts | 2 +- .../authentication/models/current_user_utils.ts | 10 +++++- 6 files changed, 48 insertions(+), 21 deletions(-) (limited to 'src/server') diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index c7990647a..e565566c7 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -536,8 +536,8 @@ export class DocumentDecorations extends React.Component<{}, { value: string }> let scale = element.props.ScreenToLocalTransform().Scale * element.props.ContentScaling(); let actualdW = Math.max(width + (dW * scale), 20); let actualdH = Math.max(height + (dH * scale), 20); - doc.x = (doc.x || 0) + dX * (actualdW - width); - doc.y = (doc.y || 0) + dY * (actualdH - height); + // doc.x = (doc.x || 0) + dX * (actualdW - width); + // doc.y = (doc.y || 0) + dY * (actualdH - height); let proto = Doc.GetProto(element.props.Document); let fixedAspect = e.ctrlKey || (!BoolCast(proto.ignoreAspect, false) && nwidth && nheight); if (fixedAspect && (!nwidth || !nheight)) { @@ -567,7 +567,7 @@ export class DocumentDecorations extends React.Component<{}, { value: string }> } else { dW && (doc.width = actualdW); dH && (doc.height = actualdH); - Doc.SetInPlace(element.props.Document, "autoHeight", undefined, true); + //Doc.SetInPlace(element.props.Document, "autoHeight", undefined, true); } } }); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 82b17f142..f9a5b2bb7 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -1,7 +1,7 @@ import { IconName, library } from '@fortawesome/fontawesome-svg-core'; import { faArrowDown, faArrowUp, faBell, faCheck, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faImage, faMusic, faObjectGroup, faPenNib, faRedoAlt, faTable, faThumbtack, faTree, faUndoAlt } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { action, computed, configure, observable, runInAction } from 'mobx'; +import { action, computed, configure, observable, runInAction, reaction } from 'mobx'; import { observer } from 'mobx-react'; import "normalize.css"; import * as React from 'react'; @@ -72,6 +72,31 @@ export class MainView extends React.Component { window.removeEventListener("pointerup", this.pointerUp); window.addEventListener("pointerup", this.pointerUp); + + reaction(() => { + let workspaces = CurrentUserUtils.UserDocument.workspaces; + let recent = CurrentUserUtils.UserDocument.recentlyClosed; + let fakeLibrary = CurrentUserUtils.UserDocument.fakeLibrary; + if (!(recent instanceof Doc)) return 0; + if (!(workspaces instanceof Doc)) return 0; + if (!(fakeLibrary instanceof Doc)) return 0; + let workspacesDoc = workspaces; + let recentDoc = recent; + let fakeLibraryDoc = fakeLibrary; + // console.log("pheight = " + this.getPHeight() + " " + workspacesDoc[HeightSym]() + " " + recentDoc[HeightSym]()); + let libraryHeight = this.getPHeight() - workspacesDoc[HeightSym]() - recentDoc[HeightSym](); + return libraryHeight; + }, (libraryHeight: number) => { + let fakeLibrary = CurrentUserUtils.UserDocument.fakeLibrary; + let cuser: Doc = fakeLibrary as Doc;//CurrentUserUtils.UserDocument; + console.log("BEFORE:" + cuser.title + ".height =" + cuser[HeightSym]() + "& .plight = " + cuser.plight); + if (libraryHeight && Math.abs(cuser[HeightSym]() - libraryHeight) > 5) { + console.log("ASSIGN to height & plight:" + libraryHeight); + cuser.height = libraryHeight; + cuser.plight = libraryHeight; + } + console.log("AFTER:" + cuser.title + ".height =" + cuser[HeightSym]() + "& .plight= " + cuser.plight); + }, { fireImmediately: true }); } pointerDown = (e: PointerEvent) => this.isPointerDown = true; @@ -285,24 +310,18 @@ export class MainView extends React.Component { CollectionDockingView.Instance.AddRightSplit(doc, undefined); } }; + computing = false; @computed get flyout() { let sidebar = CurrentUserUtils.UserDocument.sidebar; let workspaces = CurrentUserUtils.UserDocument.workspaces; let recent = CurrentUserUtils.UserDocument.recentlyClosed; + let fakeLibrary = CurrentUserUtils.UserDocument.fakeLibrary; if (!(sidebar instanceof Doc)) return (null); if (!(recent instanceof Doc)) return (null); if (!(workspaces instanceof Doc)) return (null); - let workspacesDoc = workspaces; + if (!(fakeLibrary instanceof Doc)) return (null); let sidebarDoc = sidebar; - let recentDoc = recent; - let library = CurrentUserUtils.UserDocument; - let gridGap = NumCast(sidebar.gridGap, 10); - let yMargin = NumCast(sidebar.yMargin, 2 * gridGap); - let libraryHeight = this.getPHeight() - workspacesDoc[HeightSym]() - recentDoc[HeightSym]() - 2 * gridGap - 2 * yMargin; - if (Math.abs(library[HeightSym]() - libraryHeight) > 25) { - setTimeout(() => CurrentUserUtils.UserDocument.height = libraryHeight, 0); - } return doc) { () => { if (this.singleColumn) { let children = this.childDocs.filter(d => !d.isMinimized); - this.props.Document.height = children.reduce((height, d, i) => - height + this.singleColDocHeight(d) + (i === children.length - 1 ? this.yMargin : this.gridGap) - , this.yMargin); + let hgtbefore = this.props.Document.height; + // this.props.Document.height = children.reduce((height, d, i) => + // height + this.singleColDocHeight(d) + (i === children.length - 1 ? this.yMargin : this.gridGap) + // , this.yMargin); } }, { fireImmediately: true }); } diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index a84cac37f..8ecc77946 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -564,8 +564,8 @@ export class DocumentView extends DocComponent(Docu }); } - onPointerEnter = (e: React.PointerEvent): void => { this.props.Document.libraryBrush = true; }; - onPointerLeave = (e: React.PointerEvent): void => { this.props.Document.libraryBrush = false; }; + onPointerEnter = (e: React.PointerEvent): void => { }//this.props.Document.libraryBrush = true; }; + onPointerLeave = (e: React.PointerEvent): void => { }//this.props.Document.libraryBrush = false; }; isSelected = () => SelectionManager.IsSelected(this); @action select = (ctrlPressed: boolean) => { SelectionManager.SelectDoc(this, ctrlPressed); }; diff --git a/src/new_fields/util.ts b/src/new_fields/util.ts index abb777adf..c7bcdca34 100644 --- a/src/new_fields/util.ts +++ b/src/new_fields/util.ts @@ -2,7 +2,6 @@ import { UndoManager } from "../client/util/UndoManager"; import { Doc, Field } from "./Doc"; import { SerializationHelper } from "../client/util/SerializationHelper"; import { ProxyField } from "./Proxy"; -import { FieldValue } from "./Types"; import { RefField } from "./RefField"; import { ObjectField } from "./ObjectField"; import { action } from "mobx"; @@ -13,6 +12,7 @@ function _readOnlySetter(): never { throw new Error("Documents can't be modified in read-only mode"); } const _setterImpl = action(function (target: any, prop: string | symbol | number, value: any, receiver: any): boolean { + console.log("-set " + target[SelfProxy].title + "(" + target[SelfProxy][prop] + ")." + prop.toString() + " = " + value); if (SerializationHelper.IsSerializing()) { target[prop] = value; return true; diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 30a6f108a..3e7b407a8 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -34,6 +34,9 @@ export class CurrentUserUtils { doc.title = this.email; this.updateUserDocument(doc); doc.data = new List(); + doc.gridGap = 5; + doc.xMargin = 5; + doc.yMargin = 5; doc.excludeFromLibrary = true; doc.optionalRightCollection = Docs.StackingDocument([], { title: "New mobile uploads" }); // doc.library = Docs.TreeDocument([doc], { title: `Library: ${CurrentUserUtils.email}` }); @@ -53,8 +56,13 @@ export class CurrentUserUtils { recentlyClosed.excludeFromLibrary = true; doc.recentlyClosed = recentlyClosed; } + if (doc.fakeLibrary === undefined) { + const recentlyClosed = Docs.TreeDocument([], { title: "Fake Library", height: 200 }); + recentlyClosed.excludeFromLibrary = true; + doc.fakeLibrary = recentlyClosed; + } if (doc.sidebar === undefined) { - const sidebar = Docs.StackingDocument([doc.workspaces as Doc, doc, doc.recentlyClosed as Doc], { title: "Sidebar" }); + const sidebar = Docs.StackingDocument([doc.workspaces as Doc, doc.fakeLibrary as Doc, doc.recentlyClosed as Doc], { title: "Sidebar" }); sidebar.excludeFromLibrary = true; sidebar.gridGap = 5; sidebar.xMargin = 5; -- cgit v1.2.3-70-g09d2 From d07b56aec1b1698d8a49283aade3c12c82ae1d0b Mon Sep 17 00:00:00 2001 From: Bob Zeleznik Date: Sat, 6 Jul 2019 02:36:54 -0400 Subject: fixed sidebar problems with doc decorations. --- src/client/views/DocumentDecorations.tsx | 6 +++--- src/client/views/MainView.tsx | 25 ++++------------------ .../views/collections/CollectionStackingView.tsx | 21 +++++++++--------- src/client/views/nodes/DocumentView.tsx | 4 ++-- src/new_fields/util.ts | 2 +- .../authentication/models/current_user_utils.ts | 7 +----- 6 files changed, 21 insertions(+), 44 deletions(-) (limited to 'src/server') diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index e565566c7..c7990647a 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -536,8 +536,8 @@ export class DocumentDecorations extends React.Component<{}, { value: string }> let scale = element.props.ScreenToLocalTransform().Scale * element.props.ContentScaling(); let actualdW = Math.max(width + (dW * scale), 20); let actualdH = Math.max(height + (dH * scale), 20); - // doc.x = (doc.x || 0) + dX * (actualdW - width); - // doc.y = (doc.y || 0) + dY * (actualdH - height); + doc.x = (doc.x || 0) + dX * (actualdW - width); + doc.y = (doc.y || 0) + dY * (actualdH - height); let proto = Doc.GetProto(element.props.Document); let fixedAspect = e.ctrlKey || (!BoolCast(proto.ignoreAspect, false) && nwidth && nheight); if (fixedAspect && (!nwidth || !nheight)) { @@ -567,7 +567,7 @@ export class DocumentDecorations extends React.Component<{}, { value: string }> } else { dW && (doc.width = actualdW); dH && (doc.height = actualdH); - //Doc.SetInPlace(element.props.Document, "autoHeight", undefined, true); + Doc.SetInPlace(element.props.Document, "autoHeight", undefined, true); } } }); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index f9a5b2bb7..8e5bb3615 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -1,7 +1,7 @@ import { IconName, library } from '@fortawesome/fontawesome-svg-core'; import { faArrowDown, faArrowUp, faBell, faCheck, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faImage, faMusic, faObjectGroup, faPenNib, faRedoAlt, faTable, faThumbtack, faTree, faUndoAlt } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { action, computed, configure, observable, runInAction, reaction } from 'mobx'; +import { action, computed, configure, observable, runInAction, reaction, trace } from 'mobx'; import { observer } from 'mobx-react'; import "normalize.css"; import * as React from 'react'; @@ -76,26 +76,16 @@ export class MainView extends React.Component { reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; let recent = CurrentUserUtils.UserDocument.recentlyClosed; - let fakeLibrary = CurrentUserUtils.UserDocument.fakeLibrary; if (!(recent instanceof Doc)) return 0; if (!(workspaces instanceof Doc)) return 0; - if (!(fakeLibrary instanceof Doc)) return 0; let workspacesDoc = workspaces; let recentDoc = recent; - let fakeLibraryDoc = fakeLibrary; - // console.log("pheight = " + this.getPHeight() + " " + workspacesDoc[HeightSym]() + " " + recentDoc[HeightSym]()); - let libraryHeight = this.getPHeight() - workspacesDoc[HeightSym]() - recentDoc[HeightSym](); + let libraryHeight = this.getPHeight() - workspacesDoc[HeightSym]() - recentDoc[HeightSym]() + CurrentUserUtils.UserDocument[HeightSym]() * 0.00001; return libraryHeight; }, (libraryHeight: number) => { - let fakeLibrary = CurrentUserUtils.UserDocument.fakeLibrary; - let cuser: Doc = fakeLibrary as Doc;//CurrentUserUtils.UserDocument; - console.log("BEFORE:" + cuser.title + ".height =" + cuser[HeightSym]() + "& .plight = " + cuser.plight); - if (libraryHeight && Math.abs(cuser[HeightSym]() - libraryHeight) > 5) { - console.log("ASSIGN to height & plight:" + libraryHeight); - cuser.height = libraryHeight; - cuser.plight = libraryHeight; + if (libraryHeight && Math.abs(CurrentUserUtils.UserDocument[HeightSym]() - libraryHeight) > 5) { + CurrentUserUtils.UserDocument.height = libraryHeight; } - console.log("AFTER:" + cuser.title + ".height =" + cuser[HeightSym]() + "& .plight= " + cuser.plight); }, { fireImmediately: true }); } @@ -310,17 +300,10 @@ export class MainView extends React.Component { CollectionDockingView.Instance.AddRightSplit(doc, undefined); } }; - computing = false; @computed get flyout() { let sidebar = CurrentUserUtils.UserDocument.sidebar; - let workspaces = CurrentUserUtils.UserDocument.workspaces; - let recent = CurrentUserUtils.UserDocument.recentlyClosed; - let fakeLibrary = CurrentUserUtils.UserDocument.fakeLibrary; if (!(sidebar instanceof Doc)) return (null); - if (!(recent instanceof Doc)) return (null); - if (!(workspaces instanceof Doc)) return (null); - if (!(fakeLibrary instanceof Doc)) return (null); let sidebarDoc = sidebar; return doc) { () => { if (this.singleColumn) { let children = this.childDocs.filter(d => !d.isMinimized); - let hgtbefore = this.props.Document.height; - // this.props.Document.height = children.reduce((height, d, i) => - // height + this.singleColDocHeight(d) + (i === children.length - 1 ? this.yMargin : this.gridGap) - // , this.yMargin); + this.props.Document.height = children.reduce((height, d, i) => + height + this.singleColDocHeight(d) + (i === children.length - 1 ? this.yMargin : this.gridGap) + , this.yMargin); } }, { fireImmediately: true }); } @@ -54,10 +53,12 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { addDocument(doc); return true; } - getDocTransform(doc: Doc, dref: HTMLDivElement) { - let { scale, translateX, translateY } = Utils.GetScreenTransform(dref); + getDocTransform(doc: Doc, ind: number, width: number) { + let localY = this.childDocs.filter(d => !d.isMinimized).reduce((height, d, i) => + height + (i < ind ? this.singleColDocHeight(d) + this.gridGap : 0), this.yMargin); + let translate = this.props.ScreenToLocalTransform().inverse().transformPoint((this.props.PanelWidth() - width) / 2, localY); let outerXf = Utils.GetScreenTransform(this._masonryGridRef!); - let offset = this.props.ScreenToLocalTransform().transformDirection(outerXf.translateX - translateX, outerXf.translateY - translateY); + let offset = this.props.ScreenToLocalTransform().transformDirection(outerXf.translateX - translate[0], outerXf.translateY - translate[1]); return this.props.ScreenToLocalTransform().translate(offset[0], offset[1]).scale(NumCast(doc.width, 1) / this.columnWidth); } createRef = (ele: HTMLDivElement | null) => { @@ -70,13 +71,11 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { let children = this.childDocs.filter(d => !d.isMinimized); return children.map((d, i) => { let layoutDoc = Doc.expandTemplateLayout(d, this.props.DataDoc); - let dref = React.createRef(); - let dxf = () => this.getDocTransform(layoutDoc, dref.current!).scale(this.columnWidth / d[WidthSym]()); let width = () => d.nativeWidth ? Math.min(d[WidthSym](), this.columnWidth) : this.columnWidth; let height = () => this.singleColDocHeight(layoutDoc); + let dxf = () => this.getDocTransform(layoutDoc, i, width()).scale(this.columnWidth / d[WidthSym]()); return
    (Docu }); } - onPointerEnter = (e: React.PointerEvent): void => { }//this.props.Document.libraryBrush = true; }; - onPointerLeave = (e: React.PointerEvent): void => { }//this.props.Document.libraryBrush = false; }; + onPointerEnter = (e: React.PointerEvent): void => { this.props.Document.libraryBrush = true; }; + onPointerLeave = (e: React.PointerEvent): void => { this.props.Document.libraryBrush = false; }; isSelected = () => SelectionManager.IsSelected(this); @action select = (ctrlPressed: boolean) => { SelectionManager.SelectDoc(this, ctrlPressed); }; diff --git a/src/new_fields/util.ts b/src/new_fields/util.ts index c7bcdca34..b5e50d501 100644 --- a/src/new_fields/util.ts +++ b/src/new_fields/util.ts @@ -12,7 +12,7 @@ function _readOnlySetter(): never { throw new Error("Documents can't be modified in read-only mode"); } const _setterImpl = action(function (target: any, prop: string | symbol | number, value: any, receiver: any): boolean { - console.log("-set " + target[SelfProxy].title + "(" + target[SelfProxy][prop] + ")." + prop.toString() + " = " + value); + //console.log("-set " + target[SelfProxy].title + "(" + target[SelfProxy][prop] + ")." + prop.toString() + " = " + value); if (SerializationHelper.IsSerializing()) { target[prop] = value; return true; diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 3e7b407a8..e328d6e5c 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -56,13 +56,8 @@ export class CurrentUserUtils { recentlyClosed.excludeFromLibrary = true; doc.recentlyClosed = recentlyClosed; } - if (doc.fakeLibrary === undefined) { - const recentlyClosed = Docs.TreeDocument([], { title: "Fake Library", height: 200 }); - recentlyClosed.excludeFromLibrary = true; - doc.fakeLibrary = recentlyClosed; - } if (doc.sidebar === undefined) { - const sidebar = Docs.StackingDocument([doc.workspaces as Doc, doc.fakeLibrary as Doc, doc.recentlyClosed as Doc], { title: "Sidebar" }); + const sidebar = Docs.StackingDocument([doc.workspaces as Doc, doc, doc.recentlyClosed as Doc], { title: "Sidebar" }); sidebar.excludeFromLibrary = true; sidebar.gridGap = 5; sidebar.xMargin = 5; -- cgit v1.2.3-70-g09d2 From be993c7bfd441a3a7d581371337c94fe692f0c65 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Mon, 8 Jul 2019 18:06:38 -0400 Subject: Added some stuff to GC and sped it up --- src/client/views/nodes/FormattedTextBox.tsx | 5 +- src/server/GarbageCollector.ts | 108 ++++++++++++++++++++++++++++ src/server/GarbageColletor.ts | 51 ------------- 3 files changed, 110 insertions(+), 54 deletions(-) create mode 100644 src/server/GarbageCollector.ts delete mode 100644 src/server/GarbageColletor.ts (limited to 'src/server') diff --git a/src/client/views/nodes/FormattedTextBox.tsx b/src/client/views/nodes/FormattedTextBox.tsx index bf6f4c764..af11e1524 100644 --- a/src/client/views/nodes/FormattedTextBox.tsx +++ b/src/client/views/nodes/FormattedTextBox.tsx @@ -89,9 +89,8 @@ export class FormattedTextBox extends DocComponent<(FieldViewProps & FormattedTe } public static GetDocFromUrl(url: string) { if (url.startsWith(document.location.origin)) { - let start = url.indexOf(window.location.origin); - let path = url.substr(start, url.length - start); - let docid = path.replace(DocServer.prepend("/doc/"), "").split("?")[0]; + const split = new URL(url).pathname.split("doc/"); + const docid = split[split.length - 1]; return docid; } return ""; diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts new file mode 100644 index 000000000..fa182af40 --- /dev/null +++ b/src/server/GarbageCollector.ts @@ -0,0 +1,108 @@ +import { Database } from './database'; + +import * as path from 'path'; +import * as fs from 'fs'; + +function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { + for (const key in doc) { + if (!doc.hasOwnProperty(key)) { + continue; + } + const field = doc[key]; + if (field === undefined || field === null) { + continue; + } + if (field.__type === "proxy") { + ids.push(field.fieldId); + } else if (field.__type === "list") { + addDoc(field.fields, ids, files); + } else if (typeof field === "string") { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + let match: string[] | null; + while ((match = re.exec(field)) !== null) { + ids.push(match[1]); + } + } else if (field.__type === "RichTextField") { + const re = /"href"\s*:\s*"(.*?)"/g; + let match: string[] | null; + while ((match = re.exec(field.Data)) !== null) { + const urlString = match[1]; + const split = new URL(urlString).pathname.split("doc/"); + if (split.length > 1) { + ids.push(split[split.length - 1]); + } + } + } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { + const url = new URL(field.url); + const pathname = url.pathname; + const ext = path.extname(pathname); + const fileName = path.basename(pathname, ext); + let exts = files[fileName]; + if (!exts) { + files[fileName] = exts = []; + } + exts.push(ext); + } + } +} + +async function GarbageCollect() { + // await new Promise(res => setTimeout(res, 3000)); + const cursor = await Database.Instance.query({}, 'users'); + const users = await cursor.toArray(); + const ids: string[] = users.map(user => user.userDocumentId); + const visited = new Set(); + const files: { [name: string]: string[] } = {}; + + while (ids.length) { + const count = Math.min(ids.length, 100); + const index = ids.length - count; + const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); + if (!fetchIds.length) { + continue; + } + const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments")); + for (const doc of docs) { + const id = doc.id; + if (doc === undefined) { + console.log(`Couldn't find field with Id ${id}`); + continue; + } + visited.add(id); + addDoc(doc.fields, ids, files); + } + console.log(`To Go: ${ids.length}, visited: ${visited.size}`); + } + + console.log(`Done: ${visited.size}`); + + cursor.close(); + + const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }); + const toDelete = (await toDeleteCursor.toArray()).map(doc => doc._id); + toDeleteCursor.close(); + const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); + console.log(`${result.deletedCount} documents deleted`); + + const folder = "./src/server/public/files/"; + fs.readdir(folder, (_, fileList) => { + const filesToDelete = fileList.filter(file => { + const ext = path.extname(file); + let base = path.basename(file, ext); + const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext); + return file !== ".gitignore" && !existsInDb; + }); + console.log(`Deleting ${filesToDelete.length} files`); + filesToDelete.forEach(file => { + console.log(`Deleting file ${file}`); + try { + fs.unlinkSync(folder + file); + } catch { + console.warn(`Couldn't delete file ${file}`); + } + }); + console.log(`Deleted ${filesToDelete.length} files`); + }); +} + +GarbageCollect(); diff --git a/src/server/GarbageColletor.ts b/src/server/GarbageColletor.ts deleted file mode 100644 index f26b0cec6..000000000 --- a/src/server/GarbageColletor.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Database } from './database'; - -function addDoc(doc: any, ids: string[]) { - for (const key in doc) { - if (!doc.hasOwnProperty(key)) { - continue; - } - const field = doc[key]; - if (!(field instanceof Object)) { - continue; - } - if (field.__type === "proxy") { - ids.push(field.fieldId); - } else if (field.__type === "list") { - addDoc(field.fields, ids); - } - } -} - -async function GarbageCollect() { - // await new Promise(res => setTimeout(res, 3000)); - const cursor = await Database.Instance.query({}, 'users'); - const users = await cursor.toArray(); - const ids: string[] = users.map(user => user.userDocumentId); - const visited = new Set(); - - while (ids.length) { - const id = ids.pop()!; - if (visited.has(id)) continue; - const doc = await new Promise<{ [key: string]: any }>(res => Database.Instance.getDocument(id, res, "newDocuments")); - if (doc === undefined) { - console.log(`Couldn't find field with Id ${id}`); - continue; - } - visited.add(id); - addDoc(doc.fields, ids); - console.log(`To Go: ${ids.length}, visited: ${visited.size}`); - } - - console.log(`Done: ${visited.size}`); - - cursor.close(); - - const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }); - const toDelete = (await toDeleteCursor.toArray()).map(doc => doc._id); - toDeleteCursor.close(); - const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); - console.log(`${result.deletedCount} documents deleted`); -} - -GarbageCollect(); -- cgit v1.2.3-70-g09d2 From b10115fd6203bd5777c4fe2e0b2f24bb07793d3c Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Mon, 8 Jul 2019 19:04:05 -0400 Subject: Added embedded images to GC --- src/server/GarbageCollector.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'src/server') diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts index fa182af40..3c32a94c2 100644 --- a/src/server/GarbageCollector.ts +++ b/src/server/GarbageCollector.ts @@ -32,6 +32,18 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { ids.push(split[split.length - 1]); } } + const re2 = /"src"\s*:\s*"(.*?)"/g; + while ((match = re2.exec(field.Data)) !== null) { + const urlString = match[1]; + const pathname = new URL(urlString).pathname; + const ext = path.extname(pathname); + const fileName = path.basename(pathname, ext); + let exts = files[fileName]; + if (!exts) { + files[fileName] = exts = []; + } + exts.push(ext); + } } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { const url = new URL(field.url); const pathname = url.pathname; -- cgit v1.2.3-70-g09d2 From 3dfd1de5f4d0c72dfbc9b981b8a27f98bbe7dbf2 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 9 Jul 2019 13:53:09 -0400 Subject: Added Search to GC and cleaned up some stuff --- src/server/GarbageCollector.ts | 10 +++++++--- src/server/Search.ts | 33 +++++++++++++++++++++++++++++++++ src/server/database.ts | 10 +++++++--- src/server/index.ts | 2 +- src/server/updateSearch.ts | 38 ++++++++++++++++++++++++-------------- 5 files changed, 72 insertions(+), 21 deletions(-) (limited to 'src/server') diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts index 3c32a94c2..268239481 100644 --- a/src/server/GarbageCollector.ts +++ b/src/server/GarbageCollector.ts @@ -2,6 +2,7 @@ import { Database } from './database'; import * as path from 'path'; import * as fs from 'fs'; +import { Search } from './Search'; function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { for (const key in doc) { @@ -60,7 +61,7 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { async function GarbageCollect() { // await new Promise(res => setTimeout(res, 3000)); - const cursor = await Database.Instance.query({}, 'users'); + const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users'); const users = await cursor.toArray(); const ids: string[] = users.map(user => user.userDocumentId); const visited = new Set(); @@ -90,12 +91,15 @@ async function GarbageCollect() { cursor.close(); - const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }); - const toDelete = (await toDeleteCursor.toArray()).map(doc => doc._id); + const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }, { _id: 1 }); + const toDelete: string[] = (await toDeleteCursor.toArray()).map(doc => doc._id); toDeleteCursor.close(); const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); console.log(`${result.deletedCount} documents deleted`); + await Search.Instance.deleteDocuments(toDelete); + console.log("Cleared search documents"); + const folder = "./src/server/public/files/"; fs.readdir(folder, (_, fileList) => { const filesToDelete = fileList.filter(file => { diff --git a/src/server/Search.ts b/src/server/Search.ts index e2a74b737..8591f8857 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -18,6 +18,18 @@ export class Search { } } + public async updateDocuments(documents: any[]) { + try { + const res = await rp.post(this.url + "dash/update", { + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(documents) + }); + return res; + } catch (e) { + // console.warn("Search error: " + e + document); + } + } + public async search(query: string, start: number = 0) { try { const searchResults = JSON.parse(await rp.get(this.url + "dash/select", { @@ -47,4 +59,25 @@ export class Search { }); } catch { } } + + public deleteDocuments(docs: string[]) { + const promises: rp.RequestPromise[] = []; + const nToDelete = 1000; + let index = 0; + while (index < docs.length) { + const count = Math.min(docs.length - index, nToDelete); + const deleteIds = docs.slice(index, index + count); + index += count; + promises.push(rp.post(this.url + "dash/update", { + body: { + delete: { + query: deleteIds.map(id => `id:"${id}"`).join(" ") + } + }, + json: true + })); + } + + return Promise.all(promises); + } } \ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index a17447629..29a8ffafa 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -126,12 +126,16 @@ export class Database { } } - public query(query: any, collectionName = "newDocuments"): Promise { + public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise { if (this.db) { - return Promise.resolve(this.db.collection(collectionName).find(query)); + let cursor = this.db.collection(collectionName).find(query); + if (projection) { + cursor = cursor.project(projection); + } + return Promise.resolve(cursor); } else { return new Promise(res => { - this.onConnect.push(() => res(this.query(query, collectionName))); + this.onConnect.push(() => res(this.query(query, projection, collectionName))); }); } } diff --git a/src/server/index.ts b/src/server/index.ts index bf946fc9f..c9ec11bfa 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -218,7 +218,7 @@ addSecureRoute( addSecureRoute( Method.GET, async (_, res) => { - const cursor = await Database.Instance.query({}, "users"); + const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); const results = await cursor.toArray(); res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); }, diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts index f5de00978..906b795f1 100644 --- a/src/server/updateSearch.ts +++ b/src/server/updateSearch.ts @@ -7,7 +7,7 @@ const suffixMap: { [type: string]: (string | [string, string | ((json: any) => a "number": "_n", "string": "_t", "boolean": "_b", - "image": ["_t", "url"], + // "image": ["_t", "url"], "video": ["_t", "url"], "pdf": ["_t", "url"], "audio": ["_t", "url"], @@ -67,7 +67,7 @@ async function update() { if ((numDocs % 50) === 0) { console.log("updateDoc " + numDocs); } - console.log("doc " + numDocs); + // console.log("doc " + numDocs); if (doc.__type !== "Doc") { return; } @@ -88,22 +88,32 @@ async function update() { } if (dynfield) { updates.push(update); - console.log(updates.length); + // console.log(updates.length); } } await cursor.forEach(updateDoc); - for (let i = 0; i < updates.length; i++) { - console.log(i); - const result = await Search.Instance.updateDocument(updates[i]); - try { - console.log(JSON.parse(result).responseHeader.status); - } catch { - console.log("Error:"); - console.log(updates[i]); - console.log(result); - console.log("\n"); - } + console.log(`Updating ${updates.length} documents`); + const result = await Search.Instance.updateDocuments(updates); + try { + console.log(JSON.parse(result).responseHeader.status); + } catch { + console.log("Error:"); + // console.log(updates[i]); + console.log(result); + console.log("\n"); } + // for (let i = 0; i < updates.length; i++) { + // console.log(i); + // const result = await Search.Instance.updateDocument(updates[i]); + // try { + // console.log(JSON.parse(result).responseHeader.status); + // } catch { + // console.log("Error:"); + // console.log(updates[i]); + // console.log(result); + // console.log("\n"); + // } + // } // await Promise.all(updates.map(update => { // return limit(() => Search.Instance.updateDocument(update)); // })); -- cgit v1.2.3-70-g09d2 From 15914700e307a372a1e304955a5244184108a6fa Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 9 Jul 2019 19:52:09 -0400 Subject: logging in now directs you to original target url and restored logout button functionality --- src/client/views/MainView.tsx | 2 +- .../authentication/controllers/user_controller.ts | 22 +++++++++++++--------- src/server/index.ts | 10 +++++++--- 3 files changed, 21 insertions(+), 13 deletions(-) (limited to 'src/server') diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index b8fc3f47b..b37ba1cb0 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -413,7 +413,7 @@ export class MainView extends React.Component { return [ this.isSearchVisible ?
    : null,
    -
    +
    ]; } diff --git a/src/server/authentication/controllers/user_controller.ts b/src/server/authentication/controllers/user_controller.ts index 1dacdf3fa..ca4fc171c 100644 --- a/src/server/authentication/controllers/user_controller.ts +++ b/src/server/authentication/controllers/user_controller.ts @@ -42,10 +42,6 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { const errors = req.validationErrors(); if (errors) { - res.render("signup.pug", { - title: "Sign Up", - user: req.user, - }); return res.redirect(RouteStore.signup); } @@ -66,16 +62,23 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { user.save((err) => { if (err) { return next(err); } req.logIn(user, (err) => { - if (err) { - return next(err); - } - res.redirect(RouteStore.home); + if (err) { return next(err); } + tryRedirectToTarget(req, res); }); }); }); }; +let tryRedirectToTarget = (req: Request, res: Response) => { + if (req.session && req.session.target) { + res.redirect(req.session.target); + req.session.target = undefined; + } else { + res.redirect(RouteStore.home); + } +}; + /** * GET /login @@ -83,6 +86,7 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { */ export let getLogin = (req: Request, res: Response) => { if (req.user) { + req.session!.target = undefined; return res.redirect(RouteStore.home); } res.render("login.pug", { @@ -115,7 +119,7 @@ export let postLogin = (req: Request, res: Response, next: NextFunction) => { } req.logIn(user, (err) => { if (err) { next(err); return; } - res.redirect(RouteStore.home); + tryRedirectToTarget(req, res); }); })(req, res, next); }; diff --git a/src/server/index.ts b/src/server/index.ts index c9ec11bfa..5ecbfa41a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -40,6 +40,7 @@ import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); import { Response } from 'express-serve-static-core'; +import { DocServer } from '../client/DocServer'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -69,7 +70,7 @@ app.use(session({ secret: "64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc", resave: true, cookie: { maxAge: 7 * 24 * 60 * 60 * 1000 }, - saveUninitialized: true, + saveUninitialized: false, store: new MongoStore({ url: 'mongodb://localhost:27017/Dash' }) })); @@ -82,6 +83,7 @@ app.use(passport.initialize()); app.use(passport.session()); app.use((req, res, next) => { res.locals.user = req.user; + // res.locals.target = req.session!.target; next(); }); @@ -103,14 +105,16 @@ enum Method { */ function addSecureRoute(method: Method, handler: (user: DashUserModel, res: express.Response, req: express.Request) => void, - onRejection: (res: express.Response) => any = (res) => res.redirect(RouteStore.logout), + onRejection: (res: express.Response, req: express.Request) => any = res => res.redirect(RouteStore.login), ...subscribers: string[] ) { let abstracted = (req: express.Request, res: express.Response) => { if (req.user) { handler(req.user, res, req); } else { - onRejection(res); + let target = `http://localhost:${port}${req.originalUrl}`; + req.session!.target = target; + onRejection(res, req); } }; subscribers.forEach(route => { -- cgit v1.2.3-70-g09d2 From e9189e389adbbbc6c13281b4fa4ea2a5c0076275 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 9 Jul 2019 19:57:29 -0400 Subject: clean up --- src/server/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src/server') diff --git a/src/server/index.ts b/src/server/index.ts index 5ecbfa41a..554bdad17 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -70,7 +70,7 @@ app.use(session({ secret: "64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc", resave: true, cookie: { maxAge: 7 * 24 * 60 * 60 * 1000 }, - saveUninitialized: false, + saveUninitialized: true, store: new MongoStore({ url: 'mongodb://localhost:27017/Dash' }) })); @@ -112,8 +112,7 @@ function addSecureRoute(method: Method, if (req.user) { handler(req.user, res, req); } else { - let target = `http://localhost:${port}${req.originalUrl}`; - req.session!.target = target; + req.session!.target = `http://localhost:${port}${req.originalUrl}`; onRejection(res, req); } }; -- cgit v1.2.3-70-g09d2 From dc807afc7b2b0bbeb22378b4e1603eaec688615e Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 9 Jul 2019 19:58:55 -0400 Subject: last cleanup --- src/server/index.ts | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/server') diff --git a/src/server/index.ts b/src/server/index.ts index 554bdad17..2073046ce 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -40,7 +40,6 @@ import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); import { Response } from 'express-serve-static-core'; -import { DocServer } from '../client/DocServer'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -83,7 +82,6 @@ app.use(passport.initialize()); app.use(passport.session()); app.use((req, res, next) => { res.locals.user = req.user; - // res.locals.target = req.session!.target; next(); }); -- cgit v1.2.3-70-g09d2 From dfd3427bf23c3c8df5c78fd3e993588800771269 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Wed, 10 Jul 2019 11:57:11 -0400 Subject: Started adding delete fields to DocServer --- src/client/DocServer.ts | 8 ++++++++ src/server/Message.ts | 2 ++ 2 files changed, 10 insertions(+) (limited to 'src/server') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index f477b4999..785a0acf5 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -38,6 +38,14 @@ export namespace DocServer { Utils.Emit(_socket, MessageStore.DeleteAll, {}); } + export function DeleteDocument(id: string) { + Utils.Emit(_socket, MessageStore.DeleteField, id); + } + + export function DeleteDocuments(ids: string[]) { + Utils.Emit(_socket, MessageStore.DeleteFields, ids); + } + export async function GetRefField(id: string): Promise> { let cached = _cache[id]; if (cached === undefined) { diff --git a/src/server/Message.ts b/src/server/Message.ts index e9a8b0f0c..19e0a48aa 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -45,4 +45,6 @@ export namespace MessageStore { export const GetRefFields = new Message("Get Ref Fields"); export const UpdateField = new Message("Update Ref Field"); export const CreateField = new Message("Create Ref Field"); + export const DeleteField = new Message("Delete field"); + export const DeleteFields = new Message("Delete fields"); } -- cgit v1.2.3-70-g09d2 From fd8fcfe74fff78bc67b6302f917c53e69d598712 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Wed, 10 Jul 2019 12:35:01 -0400 Subject: Added rest of delete fields (untested) --- src/client/DocServer.ts | 18 ++++++++++++++++++ src/server/index.ts | 21 ++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) (limited to 'src/server') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index b7b03f04c..443f70bb1 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -168,16 +168,34 @@ export namespace DocServer { } } + function _respondToDeleteImpl(ids: string | string[]) { + function deleteId(id: string) { + delete _cache[id]; + } + if (typeof ids === "string") { + deleteId(ids); + } else if (Array.isArray(ids)) { + ids.map(deleteId); + } + } + let _respondToUpdate = _respondToUpdateImpl; + let _respondToDelete = _respondToDeleteImpl; function respondToUpdate(diff: any) { _respondToUpdate(diff); } + function respondToDelete(ids: string | string[]) { + _respondToDelete(ids); + } + function connected() { _socket.emit(MessageStore.Bar.Message, GUID); } Utils.AddServerHandler(_socket, MessageStore.Foo, connected); Utils.AddServerHandler(_socket, MessageStore.UpdateField, respondToUpdate); + Utils.AddServerHandler(_socket, MessageStore.DeleteField, respondToDelete); + Utils.AddServerHandler(_socket, MessageStore.DeleteFields, respondToDelete); } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 2073046ce..21adff9e5 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -25,7 +25,7 @@ import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLo import { DashUserModel } from './authentication/models/user_model'; import { Client } from './Client'; import { Database } from './database'; -import { MessageStore, Transferable, Types, Diff } from "./Message"; +import { MessageStore, Transferable, Types, Diff, Message } from "./Message"; import { RouteStore } from './RouteStore'; const app = express(); const config = require('../../webpack.config'); @@ -465,6 +465,8 @@ server.on("connection", function (socket: Socket) { Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); + Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); + Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); }); @@ -591,6 +593,23 @@ function UpdateField(socket: Socket, diff: Diff) { } } +function DeleteField(socket: Socket, id: string) { + Database.Instance.delete({ _id: id }, "newDocuments").then(() => { + socket.broadcast.emit(MessageStore.DeleteField.Message, id); + }); + + Search.Instance.deleteDocuments([id]); +} + +function DeleteFields(socket: Socket, ids: string[]) { + Database.Instance.delete({ _id: { $in: ids } }, "newDocuments").then(() => { + socket.broadcast.emit(MessageStore.DeleteFields.Message, ids); + }); + + Search.Instance.deleteDocuments(ids); + +} + function CreateField(newValue: any) { Database.Instance.insert(newValue, "newDocuments"); } -- cgit v1.2.3-70-g09d2