diff options
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/RouteStore.ts | 1 | ||||
| -rw-r--r-- | src/server/database.ts | 124 | ||||
| -rw-r--r-- | src/server/index.ts | 48 | ||||
| -rw-r--r-- | src/server/updateSearch.ts | 101 |
4 files changed, 226 insertions, 48 deletions
diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index fdf5b6a5c..c4af5cdaa 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -11,6 +11,7 @@ export enum RouteStore { // UPLOAD AND STATIC FILE SERVING public = "/public", upload = "/upload", + dataUriToImage = "/uploadURI", images = "/images", // USER AND WORKSPACES diff --git a/src/server/database.ts b/src/server/database.ts index 69005d2d3..70b3efced 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -8,9 +8,13 @@ export class Database { private url = 'mongodb://localhost:27017/Dash'; private currentWrites: { [id: string]: Promise<void> } = {}; private db?: mongodb.Db; + private onConnect: (() => void)[] = []; constructor() { - this.MongoClient.connect(this.url, (err, client) => this.db = client.db()); + this.MongoClient.connect(this.url, (err, client) => { + this.db = client.db(); + this.onConnect.forEach(fn => fn()); + }); } public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) { @@ -32,66 +36,98 @@ export class Database { }; newProm = prom ? prom.then(run) : run(); this.currentWrites[id] = newProm; + } else { + this.onConnect.push(() => this.update(id, value, callback, upsert, collectionName)); } } public delete(id: string, collectionName = Database.DocumentsCollection) { - this.db && this.db.collection(collectionName).remove({ id: id }); + if (this.db) { + this.db.collection(collectionName).remove({ id: id }); + } else { + this.onConnect.push(() => this.delete(id, collectionName)); + } } public deleteAll(collectionName = Database.DocumentsCollection): Promise<any> { - return new Promise(res => - this.db && this.db.collection(collectionName).deleteMany({}, res)); + return new Promise(res => { + if (this.db) { + this.db.collection(collectionName).deleteMany({}, res); + } else { + this.onConnect.push(() => this.db && this.db.collection(collectionName).deleteMany({}, res)); + } + }); } public insert(value: any, collectionName = Database.DocumentsCollection) { - if (!this.db) { return; } - if ("id" in value) { - value._id = value.id; - delete value.id; - } - const id = value._id; - const collection = this.db.collection(collectionName); - const prom = this.currentWrites[id]; - let newProm: Promise<void>; - const run = (): Promise<void> => { - return new Promise<void>(resolve => { - collection.insertOne(value, (err, res) => { - if (this.currentWrites[id] === newProm) { - delete this.currentWrites[id]; - } - resolve(); + if (this.db) { + if ("id" in value) { + value._id = value.id; + delete value.id; + } + const id = value._id; + const collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise<void>; + const run = (): Promise<void> => { + return new Promise<void>(resolve => { + collection.insertOne(value, (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + }); }); - }); - }; - newProm = prom ? prom.then(run) : run(); - this.currentWrites[id] = newProm; + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; + } else { + this.onConnect.push(() => this.insert(value, collectionName)); + } } public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = Database.DocumentsCollection) { - this.db && this.db.collection(collectionName).findOne({ _id: id }, (err, result) => { - if (result) { - result.id = result._id; - delete result._id; - fn(result); - } else { - fn(undefined); - } - }); + if (this.db) { + this.db.collection(collectionName).findOne({ _id: id }, (err, result) => { + if (result) { + result.id = result._id; + delete result._id; + fn(result); + } else { + fn(undefined); + } + }); + } else { + this.onConnect.push(() => this.getDocument(id, fn, collectionName)); + } } public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = Database.DocumentsCollection) { - this.db && this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => { - if (err) { - console.log(err.message); - console.log(err.errmsg); - } - fn(docs.map(doc => { - doc.id = doc._id; - delete doc._id; - return doc; - })); - }); + if (this.db) { + this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => { + if (err) { + console.log(err.message); + console.log(err.errmsg); + } + fn(docs.map(doc => { + doc.id = doc._id; + delete doc._id; + return doc; + })); + }); + } else { + this.onConnect.push(() => this.getDocuments(ids, fn, collectionName)); + } + } + + public query(query: any): Promise<mongodb.Cursor> { + if (this.db) { + return Promise.resolve<mongodb.Cursor>(this.db.collection('newDocuments').find(query)); + } else { + return new Promise<mongodb.Cursor>(res => { + this.onConnect.push(() => res(this.query(query))); + }); + } } public print() { diff --git a/src/server/index.ts b/src/server/index.ts index 83c0bc222..deb3c1bd6 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -7,6 +7,7 @@ import * as expressValidator from 'express-validator'; import * as formidable from 'formidable'; import * as fs from 'fs'; import * as sharp from 'sharp'; +const imageDataUri = require('image-data-uri'); import * as mobileDetect from 'mobile-detect'; import { ObservableMap } from 'mobx'; import * as passport from 'passport'; @@ -59,7 +60,7 @@ app.use(session({ app.use(flash()); app.use(expressFlash()); -app.use(bodyParser.json()); +app.use(bodyParser.json({ limit: "10mb" })); app.use(bodyParser.urlencoded({ extended: true })); app.use(expressValidator()); app.use(passport.initialize()); @@ -216,6 +217,45 @@ addSecureRoute( RouteStore.upload ); +addSecureRoute( + Method.POST, + (user, res, req) => { + const uri = req.body.uri; + const filename = req.body.name; + if (!uri || !filename) { + res.status(401).send("incorrect parameters specified"); + return; + } + imageDataUri.outputFile(uri, uploadDir + filename).then((savedName: string) => { + const ext = path.extname(savedName); + let resizers = [ + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, + ]; + let isImage = false; + if (pngTypes.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.png(); + }); + isImage = true; + } else if (jpgTypes.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.jpeg(); + }); + isImage = true; + } + if (isImage) { + resizers.forEach(resizer => { + fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + filename + resizer.suffix + ext)); + }); + } + res.send("/files/" + filename + ext); + }); + }, + undefined, + RouteStore.dataUriToImage +); // AUTHENTICATION // Sign Up @@ -327,7 +367,7 @@ const suffixMap: { [type: string]: (string | [string, string | ((json: any) => a "number": "_n", "string": "_t", // "boolean": "_b", - "image": ["_t", "url"], + // "image": ["_t", "url"], "video": ["_t", "url"], "pdf": ["_t", "url"], "audio": ["_t", "url"], @@ -390,8 +430,8 @@ function UpdateField(socket: Socket, diff: Diff) { Object.values(suffixMap).forEach(suf => update[key + getSuffix(suf)] = { set: null }); let term = ToSearchTerm(val); if (term !== undefined) { - // let { suffix, value } = term; - // update[key + suffix] = { set: value }; + let { suffix, value } = term; + update[key + suffix] = { set: value }; } } if (dynfield) { diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts new file mode 100644 index 000000000..de1fd25e1 --- /dev/null +++ b/src/server/updateSearch.ts @@ -0,0 +1,101 @@ +import { Database } from "./database"; +import { Cursor } from "mongodb"; +import { Search } from "./Search"; +import pLimit from 'p-limit'; + +const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { + "number": "_n", + "string": "_t", + // "boolean": "_b", + // "image": ["_t", "url"], + "video": ["_t", "url"], + "pdf": ["_t", "url"], + "audio": ["_t", "url"], + "web": ["_t", "url"], + "date": ["_d", value => new Date(value.date).toISOString()], + "proxy": ["_i", "fieldId"], + "list": ["_l", list => { + const results = []; + for (const value of list.fields) { + const term = ToSearchTerm(value); + if (term) { + results.push(term.value); + } + } + return results.length ? results : null; + }] +}; + +function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { + if (val === null || val === undefined) { + return; + } + const type = val.__type || typeof val; + let suffix = suffixMap[type]; + if (!suffix) { + return; + } + + if (Array.isArray(suffix)) { + const accessor = suffix[1]; + if (typeof accessor === "function") { + val = accessor(val); + } else { + val = val[accessor]; + } + suffix = suffix[0]; + } + + return { suffix, value: val }; +} + +function getSuffix(value: string | [string, any]): string { + return typeof value === "string" ? value : value[0]; +} + +const limit = pLimit(5); +async function update() { + // await new Promise(res => setTimeout(res, 5)); + console.log("update"); + await Search.Instance.clear(); + const cursor = await Database.Instance.query({}); + console.log("Cleared"); + const updates: any[] = []; + let numDocs = 0; + function updateDoc(doc: any) { + numDocs++; + if ((numDocs % 50) === 0) { + console.log("updateDoc " + numDocs); + } + console.log("doc " + numDocs); + if (doc.__type !== "Doc") { + return; + } + const fields = doc.fields; + if (!fields) { + return; + } + const update: any = { id: doc._id }; + let dynfield = false; + for (const key in fields) { + const value = fields[key]; + const term = ToSearchTerm(value); + if (term !== undefined) { + let { suffix, value } = term; + update[key + suffix] = value; + dynfield = true; + } + } + if (dynfield) { + updates.push(update); + console.log(updates.length); + } + } + await cursor.forEach(updateDoc); + await Promise.all(updates.map(update => { + return limit(() => Search.Instance.updateDocument(update)); + })); + cursor.close(); +} + +update();
\ No newline at end of file |