diff options
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/GarbageCollector.ts | 78 | ||||
| -rw-r--r-- | src/server/RouteStore.ts | 3 | ||||
| -rw-r--r-- | src/server/Search.ts | 12 | ||||
| -rw-r--r-- | src/server/authentication/controllers/user_controller.ts | 22 | ||||
| -rw-r--r-- | src/server/authentication/models/current_user_utils.ts | 40 | ||||
| -rw-r--r-- | src/server/authentication/models/user_model.ts | 4 | ||||
| -rw-r--r-- | src/server/database.ts | 11 | ||||
| -rw-r--r-- | src/server/index.ts | 110 | ||||
| -rw-r--r-- | src/server/updateProtos.ts | 15 |
9 files changed, 193 insertions, 102 deletions
diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts index 268239481..ea5388004 100644 --- a/src/server/GarbageCollector.ts +++ b/src/server/GarbageCollector.ts @@ -59,7 +59,9 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { } } -async function GarbageCollect() { +async function GarbageCollect(full: boolean = true) { + console.log("start GC"); + const start = Date.now(); // await new Promise(res => setTimeout(res, 3000)); const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users'); const users = await cursor.toArray(); @@ -68,7 +70,7 @@ async function GarbageCollect() { const files: { [name: string]: string[] } = {}; while (ids.length) { - const count = Math.min(ids.length, 100); + const count = Math.min(ids.length, 1000); const index = ids.length - count; const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); if (!fetchIds.length) { @@ -91,34 +93,58 @@ async function GarbageCollect() { cursor.close(); - const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }, { _id: 1 }); + const notToDelete = Array.from(visited); + const toDeleteCursor = await Database.Instance.query({ _id: { $nin: notToDelete } }, { _id: 1 }); const toDelete: string[] = (await toDeleteCursor.toArray()).map(doc => doc._id); toDeleteCursor.close(); - const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); - console.log(`${result.deletedCount} documents deleted`); + if (!full) { + await Database.Instance.updateMany({ _id: { $nin: notToDelete } }, { $set: { "deleted": true } }); + await Database.Instance.updateMany({ _id: { $in: notToDelete } }, { $unset: { "deleted": true } }); + console.log(await Search.Instance.updateDocuments( + notToDelete.map<any>(id => ({ + id, deleted: { set: null } + })) + .concat(toDelete.map(id => ({ + id, deleted: { set: true } + }))))); + console.log("Done with partial GC"); + console.log(`Took ${(Date.now() - start) / 1000} seconds`); + } else { + let i = 0; + let deleted = 0; + while (i < toDelete.length) { + const count = Math.min(toDelete.length, 5000); + const toDeleteDocs = toDelete.slice(i, i + count); + i += count; + const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments"); + deleted += result.deletedCount || 0; + } + // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); + console.log(`${deleted} documents deleted`); - await Search.Instance.deleteDocuments(toDelete); - console.log("Cleared search documents"); + await Search.Instance.deleteDocuments(toDelete); + console.log("Cleared search documents"); - const folder = "./src/server/public/files/"; - fs.readdir(folder, (_, fileList) => { - const filesToDelete = fileList.filter(file => { - const ext = path.extname(file); - let base = path.basename(file, ext); - const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext); - return file !== ".gitignore" && !existsInDb; - }); - console.log(`Deleting ${filesToDelete.length} files`); - filesToDelete.forEach(file => { - console.log(`Deleting file ${file}`); - try { - fs.unlinkSync(folder + file); - } catch { - console.warn(`Couldn't delete file ${file}`); - } + const folder = "./src/server/public/files/"; + fs.readdir(folder, (_, fileList) => { + const filesToDelete = fileList.filter(file => { + const ext = path.extname(file); + let base = path.basename(file, ext); + const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext); + return file !== ".gitignore" && !existsInDb; + }); + console.log(`Deleting ${filesToDelete.length} files`); + filesToDelete.forEach(file => { + console.log(`Deleting file ${file}`); + try { + fs.unlinkSync(folder + file); + } catch { + console.warn(`Couldn't delete file ${file}`); + } + }); + console.log(`Deleted ${filesToDelete.length} files`); }); - console.log(`Deleted ${filesToDelete.length} files`); - }); + } } -GarbageCollect(); +GarbageCollect(false); diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 5c13495ff..e30015e39 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -29,4 +29,7 @@ export enum RouteStore { forgot = "/forgotpassword", reset = "/reset/:token", + // APIS + cognitiveServices = "/cognitiveservices" + }
\ No newline at end of file diff --git a/src/server/Search.ts b/src/server/Search.ts index 8591f8857..723dc101b 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -26,22 +26,18 @@ export class Search { }); return res; } catch (e) { - // console.warn("Search error: " + e + document); + // console.warn("Search error: ", e, documents); } } - public async search(query: string, start: number = 0) { + public async search(query: any) { try { const searchResults = JSON.parse(await rp.get(this.url + "dash/select", { - qs: { - q: query, - fl: "id", - start: start - } + qs: query })); const { docs, numFound } = searchResults.response; const ids = docs.map((field: any) => field.id); - return { ids, numFound }; + return { ids, numFound, highlighting: searchResults.highlighting }; } catch { return { ids: [], numFound: -1 }; } diff --git a/src/server/authentication/controllers/user_controller.ts b/src/server/authentication/controllers/user_controller.ts index ca4fc171c..f5c6e1610 100644 --- a/src/server/authentication/controllers/user_controller.ts +++ b/src/server/authentication/controllers/user_controller.ts @@ -12,6 +12,9 @@ import * as nodemailer from 'nodemailer'; import c = require("crypto"); import { RouteStore } from "../../RouteStore"; import { Utils } from "../../../Utils"; +import { Schema } from "mongoose"; +import { Opt } from "../../../new_fields/Doc"; +import { MailOptions } from "nodemailer/lib/stream-transport"; /** * GET /signup @@ -45,21 +48,23 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { return res.redirect(RouteStore.signup); } - const email = req.body.email; + const email = req.body.email as String; const password = req.body.password; - const user = new User({ + const model = { email, password, userDocumentId: Utils.GenerateGuid() - }); + } as Partial<DashUserModel>; + + const user = new User(model); User.findOne({ email }, (err, existingUser) => { if (err) { return next(err); } if (existingUser) { return res.redirect(RouteStore.login); } - user.save((err) => { + user.save((err: any) => { if (err) { return next(err); } req.logIn(user, (err) => { if (err) { return next(err); } @@ -72,8 +77,9 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { let tryRedirectToTarget = (req: Request, res: Response) => { if (req.session && req.session.target) { - res.redirect(req.session.target); + let target = req.session.target; req.session.target = undefined; + res.redirect(target); } else { res.redirect(RouteStore.home); } @@ -188,8 +194,8 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio 'Please click on the following link, or paste this into your browser to complete the process:\n\n' + 'http://' + req.headers.host + '/reset/' + token + '\n\n' + 'If you did not request this, please ignore this email and your password will remain unchanged.\n' - }; - smtpTransport.sendMail(mailOptions, function (err) { + } as MailOptions; + smtpTransport.sendMail(mailOptions, function (err: Error | null) { // req.flash('info', 'An e-mail has been sent to ' + user.email + ' with further instructions.'); done(null, err, 'done'); }); @@ -259,7 +265,7 @@ export let postReset = function (req: Request, res: Response) { subject: 'Your password has been changed', text: 'Hello,\n\n' + 'This is a confirmation that the password for your account ' + user.email + ' has just been changed.\n' - }; + } as MailOptions; smtpTransport.sendMail(mailOptions, function (err) { done(null, err); }); diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index e328d6e5c..1c52a3f11 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -10,8 +10,9 @@ import { CollectionView } from "../../../client/views/collections/CollectionView import { Doc } from "../../../new_fields/Doc"; import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; -import { Cast, FieldValue } from "../../../new_fields/Types"; +import { Cast, FieldValue, StrCast } from "../../../new_fields/Types"; import { RouteStore } from "../../RouteStore"; +import { Utils } from "../../../Utils"; export class CurrentUserUtils { private static curr_email: string; @@ -37,47 +38,57 @@ export class CurrentUserUtils { doc.gridGap = 5; doc.xMargin = 5; doc.yMargin = 5; + doc.boxShadow = "0 0"; doc.excludeFromLibrary = true; - doc.optionalRightCollection = Docs.StackingDocument([], { title: "New mobile uploads" }); - // doc.library = Docs.TreeDocument([doc], { title: `Library: ${CurrentUserUtils.email}` }); + doc.optionalRightCollection = Docs.Create.StackingDocument([], { title: "New mobile uploads" }); + // doc.library = Docs.Create.TreeDocument([doc], { title: `Library: ${CurrentUserUtils.email}` }); // (doc.library as Doc).excludeFromLibrary = true; return doc; } static updateUserDocument(doc: Doc) { if (doc.workspaces === undefined) { - const workspaces = Docs.TreeDocument([], { title: "Workspaces", height: 100 }); + const workspaces = Docs.Create.TreeDocument([], { title: "Workspaces", height: 100 }); workspaces.excludeFromLibrary = true; workspaces.workspaceLibrary = true; + workspaces.boxShadow = "0 0"; doc.workspaces = workspaces; } if (doc.recentlyClosed === undefined) { - const recentlyClosed = Docs.TreeDocument([], { title: "Recently Closed", height: 75 }); + const recentlyClosed = Docs.Create.TreeDocument([], { title: "Recently Closed", height: 75 }); recentlyClosed.excludeFromLibrary = true; + recentlyClosed.boxShadow = "0 0"; doc.recentlyClosed = recentlyClosed; } if (doc.sidebar === undefined) { - const sidebar = Docs.StackingDocument([doc.workspaces as Doc, doc, doc.recentlyClosed as Doc], { title: "Sidebar" }); + const sidebar = Docs.Create.StackingDocument([doc.workspaces as Doc, doc, doc.recentlyClosed as Doc], { title: "Sidebar" }); sidebar.excludeFromLibrary = true; sidebar.gridGap = 5; sidebar.xMargin = 5; sidebar.yMargin = 5; + Doc.GetProto(sidebar).backgroundColor = "#aca3a6"; + sidebar.boxShadow = "1 1 3"; doc.sidebar = sidebar; } + StrCast(doc.title).indexOf("@") !== -1 && (doc.title = StrCast(doc.title).split("@")[0] + "'s Library"); } - public static async loadCurrentUser(): Promise<any> { - let userPromise = rp.get(DocServer.prepend(RouteStore.getCurrUser)).then(response => { + public static loadCurrentUser() { + return rp.get(Utils.prepend(RouteStore.getCurrUser)).then(response => { if (response) { - let obj = JSON.parse(response); - CurrentUserUtils.curr_id = obj.id as string; - CurrentUserUtils.curr_email = obj.email as string; + const result: { id: string, email: string } = JSON.parse(response); + return result; } else { throw new Error("There should be a user! Why does Dash think there isn't one?"); } }); - let userDocPromise = await rp.get(DocServer.prepend(RouteStore.getUserDocumentId)).then(id => { + } + + public static async loadUserDocument({ id, email }: { id: string, email: string }) { + this.curr_id = id; + this.curr_email = email; + await rp.get(Utils.prepend(RouteStore.getUserDocumentId)).then(id => { if (id) { return DocServer.GetRefField(id).then(async field => { if (field instanceof Doc) { @@ -102,7 +113,6 @@ export class CurrentUserUtils { } catch (e) { } - return Promise.all([userPromise, userDocPromise]); } /* Northstar catalog ... really just for testing so this should eventually go away */ @@ -128,12 +138,12 @@ export class CurrentUserUtils { // new AttributeTransformationModel(atmod, AggregateFunction.None), // new AttributeTransformationModel(atmod, AggregateFunction.Count), // new AttributeTransformationModel(atmod, AggregateFunction.Count)); - // schemaDocuments.push(Docs.HistogramDocument(histoOp, { width: 200, height: 200, title: attr.displayName! })); + // schemaDocuments.push(Docs.Create.HistogramDocument(histoOp, { width: 200, height: 200, title: attr.displayName! })); // } // }))); // return promises; // }, [] as Promise<void>[])); - // return CurrentUserUtils._northstarSchemas.push(Docs.TreeDocument(schemaDocuments, { width: 50, height: 100, title: schema.displayName! })); + // return CurrentUserUtils._northstarSchemas.push(Docs.Create.TreeDocument(schemaDocuments, { width: 50, height: 100, title: schema.displayName! })); // }); // } } diff --git a/src/server/authentication/models/user_model.ts b/src/server/authentication/models/user_model.ts index ee85e1c05..45fbf23b1 100644 --- a/src/server/authentication/models/user_model.ts +++ b/src/server/authentication/models/user_model.ts @@ -16,7 +16,7 @@ mongoose.connection.on('disconnected', function () { console.log('connection closed'); }); export type DashUserModel = mongoose.Document & { - email: string, + email: String, password: string, passwordResetToken?: string, passwordResetExpires?: Date, @@ -42,7 +42,7 @@ export type AuthToken = { }; const userSchema = new mongoose.Schema({ - email: { type: String, unique: true }, + email: String, password: String, passwordResetToken: String, passwordResetExpires: Date, diff --git a/src/server/database.ts b/src/server/database.ts index 29a8ffafa..7f5331998 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -140,6 +140,17 @@ export class Database { } } + public updateMany(query: any, update: any, collectionName = "newDocuments") { + if (this.db) { + const db = this.db; + return new Promise<mongodb.WriteOpResult>(res => db.collection(collectionName).update(query, update, (_, result) => res(result))); + } else { + return new Promise<mongodb.WriteOpResult>(res => { + this.onConnect.push(() => this.updateMany(query, update, collectionName).then(res)); + }); + } + } + public print() { console.log("db says hi!"); } diff --git a/src/server/index.ts b/src/server/index.ts index 64a2a6899..80dbd8a79 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -60,7 +60,7 @@ clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode" fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8"); const mongoUrl = 'mongodb://localhost:27017/Dash'; -mongoose.connect(mongoUrl); +mongoose.connection.readyState === 0 && mongoose.connect(mongoUrl); mongoose.connection.on('connected', () => console.log("connected")); // SESSION MANAGEMENT AND AUTHENTICATION MIDDLEWARE @@ -112,7 +112,7 @@ function addSecureRoute(method: Method, if (req.user) { handler(req.user, res, req); } else { - req.session!.target = `http://localhost:${port}${req.originalUrl}`; + req.session!.target = req.originalUrl; onRejection(res, req); } }; @@ -147,15 +147,45 @@ const solrURL = "http://localhost:8983/solr/#/dash"; // GETTERS app.get("/search", async (req, res) => { - let query = req.query.query || "hello"; - let results = await Search.Instance.search(query); + const solrQuery: any = {}; + ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); + if (solrQuery.q === undefined) { + res.send([]); + return; + } + let results = await Search.Instance.search(solrQuery); res.send(results); }); +function msToTime(duration: number) { + let milliseconds = Math.floor((duration % 1000) / 100), + seconds = Math.floor((duration / 1000) % 60), + minutes = Math.floor((duration / (1000 * 60)) % 60), + hours = Math.floor((duration / (1000 * 60 * 60)) % 24); + + let hoursS = (hours < 10) ? "0" + hours : hours; + let minutesS = (minutes < 10) ? "0" + minutes : minutes; + let secondsS = (seconds < 10) ? "0" + seconds : seconds; + + return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; +} + +app.get("/whosOnline", (req, res) => { + let users: any = { active: {}, inactive: {} }; + const now = Date.now(); + + for (const user in timeMap) { + const time = timeMap[user]; + const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; + users[key][user] = `Last active ${msToTime(now - time)} ago`; + } + + res.send(users); +}); app.get("/thumbnail/:filename", (req, res) => { let filename = req.params.filename; let noExt = filename.substring(0, filename.length - ".png".length); - let pagenumber = parseInt(noExt[noExt.length - 1]); + let pagenumber = parseInt(noExt.split('-')[1]); fs.exists(uploadDir + filename, (exists: boolean) => { console.log(`${uploadDir + filename} ${exists ? "exists" : "does not exist"}`); if (exists) { @@ -163,13 +193,14 @@ app.get("/thumbnail/:filename", (req, res) => { probe(input, (err: any, result: any) => { if (err) { console.log(err); + console.log(`error on ${filename}`); return; } res.send({ path: "/files/" + filename, width: result.width, height: result.height }); }); } else { - LoadPage(uploadDir + filename.substring(0, filename.length - "-n.png".length) + ".pdf", pagenumber, res); + LoadPage(uploadDir + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); } }); }); @@ -256,6 +287,20 @@ addSecureRoute( RouteStore.getCurrUser ); +addSecureRoute(Method.GET, (user, res, req) => { + let requested = req.params.requestedservice; + switch (requested) { + case "face": + res.send(process.env.FACE); + break; + case "vision": + res.send(process.env.VISION); + break; + default: + res.send(undefined); + } +}, undefined, `${RouteStore.cognitiveServices}/:requestedservice`); + class NodeCanvasFactory { create = (width: number, height: number) => { var canvas = createCanvas(width, height); @@ -316,38 +361,6 @@ app.post( }); isImage = true; } - else if (pdfTypes.includes(ext)) { - // Pdfjs.getDocument(uploadDir + file).promise - // .then((pdf: Pdfjs.PDFDocumentProxy) => { - // let numPages = pdf.numPages; - // let factory = new NodeCanvasFactory(); - // for (let pageNum = 0; pageNum < numPages; pageNum++) { - // console.log(pageNum); - // pdf.getPage(pageNum + 1).then((page: Pdfjs.PDFPageProxy) => { - // console.log("reading " + pageNum); - // let viewport = page.getViewport(1); - // let canvasAndContext = factory.create(viewport.width, viewport.height); - // let renderContext = { - // canvasContext: canvasAndContext.context, - // viewport: viewport, - // canvasFactory: factory - // } - // console.log("read " + pageNum); - - // page.render(renderContext).promise - // .then(() => { - // console.log("saving " + pageNum); - // let stream = canvasAndContext.canvas.createPNGStream(); - // let out = fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + `-${pageNum + 1}.PNG`); - // stream.pipe(out); - // out.on("finish", () => console.log(`Success! Saved to ${uploadDir + file.substring(0, file.length - ext.length) + `-${pageNum + 1}.PNG`}`)); - // }, (reason: string) => { - // console.error(reason + ` ${pageNum}`); - // }); - // }); - // } - // }); - } if (isImage) { resizers.forEach(resizer => { fs.createReadStream(uploadDir + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); @@ -421,7 +434,7 @@ app.get(RouteStore.reset, getReset); app.post(RouteStore.reset, postReset); app.use(RouteStore.corsProxy, (req, res) => - req.pipe(request(req.url.substring(1))).pipe(res)); + req.pipe(request(decodeURIComponent(req.url.substring(1)))).pipe(res)); app.get(RouteStore.delete, (req, res) => { if (release) { @@ -453,12 +466,21 @@ interface Map { } let clients: Map = {}; +let socketMap = new Map<SocketIO.Socket, string>(); +let timeMap: { [id: string]: number } = {}; + server.on("connection", function (socket: Socket) { - console.log("a user has connected"); + socket.use((packet, next) => { + let id = socketMap.get(socket); + if (id) { + timeMap[id] = Date.now(); + } + next(); + }); Utils.Emit(socket, MessageStore.Foo, "handshooken"); - Utils.AddServerHandler(socket, MessageStore.Bar, barReceived); + Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); Utils.AddServerHandler(socket, MessageStore.SetField, (args) => setField(socket, args)); Utils.AddServerHandlerCallback(socket, MessageStore.GetField, getField); Utils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields); @@ -488,8 +510,10 @@ async function deleteAll() { await Search.Instance.clear(); } -function barReceived(guid: String) { - clients[guid.toString()] = new Client(guid.toString()); +function barReceived(socket: SocketIO.Socket, guid: string) { + clients[guid] = new Client(guid.toString()); + console.log(`User ${guid} has connected`); + socketMap.set(socket, guid); } function getField([id, callback]: [string, (result?: Transferable) => void]) { diff --git a/src/server/updateProtos.ts b/src/server/updateProtos.ts new file mode 100644 index 000000000..90490eb45 --- /dev/null +++ b/src/server/updateProtos.ts @@ -0,0 +1,15 @@ +import { Database } from "./database"; + +const protos = + ["text", "histogram", "image", "web", "collection", "kvp", + "video", "audio", "pdf", "icon", "import", "linkdoc"]; + +(async function () { + await Promise.all( + protos.map(protoId => new Promise(res => Database.Instance.update(protoId, { + $set: { "fields.baseProto": true } + }, res))) + ); + + console.log("done"); +})();
\ No newline at end of file |