From 99bbb42cfa5718f543213f7a64463bd31c73d781 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Mon, 20 May 2019 14:36:43 -0400 Subject: Added route to save data uri as image --- src/server/RouteStore.ts | 1 + src/server/index.ts | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) (limited to 'src/server') diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index fdf5b6a5c..c4af5cdaa 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -11,6 +11,7 @@ export enum RouteStore { // UPLOAD AND STATIC FILE SERVING public = "/public", upload = "/upload", + dataUriToImage = "/uploadURI", images = "/images", // USER AND WORKSPACES diff --git a/src/server/index.ts b/src/server/index.ts index af99b116a..a12751fe3 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -7,6 +7,7 @@ import * as expressValidator from 'express-validator'; import * as formidable from 'formidable'; import * as fs from 'fs'; import * as sharp from 'sharp'; +const imageDataUri = require('image-data-uri'); import * as mobileDetect from 'mobile-detect'; import { ObservableMap } from 'mobx'; import * as passport from 'passport'; @@ -216,6 +217,45 @@ addSecureRoute( RouteStore.upload ); +addSecureRoute( + Method.POST, + (user, res, req) => { + const uri = req.query.uri; + const filename = req.query.name; + if (!uri || !filename) { + res.status(400).send("incorrect parameters specified"); + return; + } + imageDataUri.outputFile(uri, uploadDir + filename).then((savedName: string) => { + const ext = path.extname(savedName); + let resizers = [ + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, + ]; + let isImage = false; + if (pngTypes.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.png(); + }); + isImage = true; + } else if (jpgTypes.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.jpeg(); + }); + isImage = true; + } + if (isImage) { + resizers.forEach(resizer => { + fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + filename + resizer.suffix + ext)); + }); + } + res.send("/files/" + filename + ext); + }); + }, + undefined, + RouteStore.dataUriToImage +); // AUTHENTICATION // Sign Up -- cgit v1.2.3-70-g09d2 From e02e8bee7ee36490b83edb0a416978216b0f4a17 Mon Sep 17 00:00:00 2001 From: bob Date: Mon, 20 May 2019 15:46:21 -0400 Subject: hooking up image data url stuff. --- src/client/views/SearchBox.tsx | 17 +++++++++++++++++ src/client/views/nodes/PDFBox.tsx | 10 +++++++--- src/server/index.ts | 8 +++++--- 3 files changed, 29 insertions(+), 6 deletions(-) (limited to 'src/server') diff --git a/src/client/views/SearchBox.tsx b/src/client/views/SearchBox.tsx index 8efd8d266..0ec1a6758 100644 --- a/src/client/views/SearchBox.tsx +++ b/src/client/views/SearchBox.tsx @@ -20,6 +20,7 @@ import { Id } from '../../new_fields/FieldSymbols'; import { DocumentManager } from '../util/DocumentManager'; import { SetupDrag } from '../util/DragManager'; import { Docs } from '../documents/Documents'; +import { RouteStore } from '../../server/RouteStore'; library.add(faSearch); library.add(faObjectGroup); @@ -70,6 +71,22 @@ export class SearchBox extends React.Component { } return docs; } + public static async convertDataUri(imageUri: string, returnedFilename: string) { + try { + let posting = DocServer.prepend(RouteStore.dataUriToImage); + const returnedUri = await rp.post(posting, { + body: { + uri: imageUri, + name: returnedFilename + }, + json: true, + }); + return returnedUri; + + } catch (e) { + console.log(e); + } + } @action handleClickFilter = (e: Event): void => { diff --git a/src/client/views/nodes/PDFBox.tsx b/src/client/views/nodes/PDFBox.tsx index e71ac4924..1017f5235 100644 --- a/src/client/views/nodes/PDFBox.tsx +++ b/src/client/views/nodes/PDFBox.tsx @@ -21,6 +21,8 @@ import { positionSchema } from "./DocumentView"; import { pageSchema } from "./ImageBox"; import { ImageField, PdfField } from "../../../new_fields/URLField"; import { InkingControl } from "../InkingControl"; +import { SearchBox } from "../SearchBox"; +import { Id } from "../../../new_fields/FieldSymbols"; /** ALSO LOOK AT: Annotation.tsx, Sticky.tsx * This method renders PDF and puts all kinds of functionalities such as annotation, highlighting, @@ -250,9 +252,11 @@ export class PDFBox extends DocComponent(PdfDocumen let nheight = FieldValue(this.Document.nativeHeight, 0); htmlToImage.toPng(this._mainDiv.current!, { width: nwidth, height: nheight, quality: 1 }) .then(action((dataUrl: string) => { - this.props.Document.thumbnail = new ImageField(new URL(dataUrl)); - this.props.Document.thumbnailPage = FieldValue(this.Document.curPage, -1); - this._renderAsSvg = true; + SearchBox.convertDataUri(dataUrl, this.Document[Id] + "_" + this.curPage).then((returnedFilename) => { + this.props.Document.thumbnail = new ImageField(new URL(dataUrl)); + this.props.Document.thumbnailPage = FieldValue(this.Document.curPage, -1); + this._renderAsSvg = true; + }) })) .catch(function (error: any) { console.error('oops, something went wrong!', error); diff --git a/src/server/index.ts b/src/server/index.ts index a12751fe3..6843afbe4 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -114,6 +114,8 @@ function addSecureRoute(method: Method, app.use(express.static(__dirname + RouteStore.public)); app.use(RouteStore.images, express.static(__dirname + RouteStore.public)); +app.use(express.json()); + app.get("/pull", (req, res) => exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', (err, stdout, stderr) => { if (err) { @@ -220,10 +222,10 @@ addSecureRoute( addSecureRoute( Method.POST, (user, res, req) => { - const uri = req.query.uri; - const filename = req.query.name; + const uri = req.body.uri; + const filename = req.body.name; if (!uri || !filename) { - res.status(400).send("incorrect parameters specified"); + res.status(401).send("incorrect parameters specified"); return; } imageDataUri.outputFile(uri, uploadDir + filename).then((savedName: string) => { -- cgit v1.2.3-70-g09d2 From ba57c927bbb7b7ba80e65e8ada9520533f1270b6 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Mon, 20 May 2019 15:49:57 -0400 Subject: uncommented search thing --- src/server/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/server') diff --git a/src/server/index.ts b/src/server/index.ts index a12751fe3..0952e20aa 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -429,8 +429,8 @@ function UpdateField(socket: Socket, diff: Diff) { Object.values(suffixMap).forEach(suf => update[key + getSuffix(suf)] = { set: null }); let term = ToSearchTerm(val); if (term !== undefined) { - // let { suffix, value } = term; - // update[key + suffix] = { set: value }; + let { suffix, value } = term; + update[key + suffix] = { set: value }; } } if (dynfield) { -- cgit v1.2.3-70-g09d2 From fe3a7e6fbdcb10d764813d551ae052fa837f76ff Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Mon, 20 May 2019 15:56:20 -0400 Subject: Fixed pdf uploader --- src/server/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src/server') diff --git a/src/server/index.ts b/src/server/index.ts index 5cdd62726..574241f62 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -60,7 +60,7 @@ app.use(session({ app.use(flash()); app.use(expressFlash()); -app.use(bodyParser.json()); +app.use(bodyParser.json({ limit: "10mb" })); app.use(bodyParser.urlencoded({ extended: true })); app.use(expressValidator()); app.use(passport.initialize()); @@ -114,8 +114,6 @@ function addSecureRoute(method: Method, app.use(express.static(__dirname + RouteStore.public)); app.use(RouteStore.images, express.static(__dirname + RouteStore.public)); -app.use(express.json()); - app.get("/pull", (req, res) => exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', (err, stdout, stderr) => { if (err) { -- cgit v1.2.3-70-g09d2 From 7cde8383e45ccf8293ec9cba2beddc64bceda2aa Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Mon, 20 May 2019 17:08:40 -0400 Subject: Added update search script Added debug current file launch config Made database not ignore requests before it is initialized. --- .vscode/launch.json | 24 +++++---- package.json | 1 + src/server/database.ts | 124 +++++++++++++++++++++++++++++---------------- src/server/updateSearch.ts | 101 ++++++++++++++++++++++++++++++++++++ 4 files changed, 196 insertions(+), 54 deletions(-) create mode 100644 src/server/updateSearch.ts (limited to 'src/server') diff --git a/.vscode/launch.json b/.vscode/launch.json index 452acc823..d245be2ad 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -31,6 +31,20 @@ "localRoot": "${workspaceFolder}", "remoteRoot": "." }, + { + "type": "node", + "request": "launch", + "name": "Current TS File", + "runtimeExecutable": "npx", + "runtimeArgs": [ + "ts-node-dev", + "--nolazy", + "--inspect", + "--", + "${relativeFile}" + ], + "port": 9229 + }, { "type": "node", "request": "launch", @@ -65,15 +79,5 @@ "internalConsoleOptions": "openOnSessionStart", "protocol": "inspector" }, - { - "type": "node", - "request": "launch", - "name": "Launch via NPM", - "runtimeExecutable": "npm", - "runtimeArgs": [ - "start" - ], - "port": 9229 - } ] } \ No newline at end of file diff --git a/package.json b/package.json index 40402b44c..790535728 100644 --- a/package.json +++ b/package.json @@ -133,6 +133,7 @@ "nodemon": "^1.18.10", "normalize.css": "^8.0.1", "npm": "^6.9.0", + "p-limit": "^2.2.0", "passport": "^0.4.0", "passport-local": "^1.0.0", "prosemirror-commands": "^1.0.7", diff --git a/src/server/database.ts b/src/server/database.ts index 69005d2d3..70b3efced 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -8,9 +8,13 @@ export class Database { private url = 'mongodb://localhost:27017/Dash'; private currentWrites: { [id: string]: Promise } = {}; private db?: mongodb.Db; + private onConnect: (() => void)[] = []; constructor() { - this.MongoClient.connect(this.url, (err, client) => this.db = client.db()); + this.MongoClient.connect(this.url, (err, client) => { + this.db = client.db(); + this.onConnect.forEach(fn => fn()); + }); } public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) { @@ -32,66 +36,98 @@ export class Database { }; newProm = prom ? prom.then(run) : run(); this.currentWrites[id] = newProm; + } else { + this.onConnect.push(() => this.update(id, value, callback, upsert, collectionName)); } } public delete(id: string, collectionName = Database.DocumentsCollection) { - this.db && this.db.collection(collectionName).remove({ id: id }); + if (this.db) { + this.db.collection(collectionName).remove({ id: id }); + } else { + this.onConnect.push(() => this.delete(id, collectionName)); + } } public deleteAll(collectionName = Database.DocumentsCollection): Promise { - return new Promise(res => - this.db && this.db.collection(collectionName).deleteMany({}, res)); + return new Promise(res => { + if (this.db) { + this.db.collection(collectionName).deleteMany({}, res); + } else { + this.onConnect.push(() => this.db && this.db.collection(collectionName).deleteMany({}, res)); + } + }); } public insert(value: any, collectionName = Database.DocumentsCollection) { - if (!this.db) { return; } - if ("id" in value) { - value._id = value.id; - delete value.id; - } - const id = value._id; - const collection = this.db.collection(collectionName); - const prom = this.currentWrites[id]; - let newProm: Promise; - const run = (): Promise => { - return new Promise(resolve => { - collection.insertOne(value, (err, res) => { - if (this.currentWrites[id] === newProm) { - delete this.currentWrites[id]; - } - resolve(); + if (this.db) { + if ("id" in value) { + value._id = value.id; + delete value.id; + } + const id = value._id; + const collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise; + const run = (): Promise => { + return new Promise(resolve => { + collection.insertOne(value, (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + }); }); - }); - }; - newProm = prom ? prom.then(run) : run(); - this.currentWrites[id] = newProm; + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; + } else { + this.onConnect.push(() => this.insert(value, collectionName)); + } } public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = Database.DocumentsCollection) { - this.db && this.db.collection(collectionName).findOne({ _id: id }, (err, result) => { - if (result) { - result.id = result._id; - delete result._id; - fn(result); - } else { - fn(undefined); - } - }); + if (this.db) { + this.db.collection(collectionName).findOne({ _id: id }, (err, result) => { + if (result) { + result.id = result._id; + delete result._id; + fn(result); + } else { + fn(undefined); + } + }); + } else { + this.onConnect.push(() => this.getDocument(id, fn, collectionName)); + } } public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = Database.DocumentsCollection) { - this.db && this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => { - if (err) { - console.log(err.message); - console.log(err.errmsg); - } - fn(docs.map(doc => { - doc.id = doc._id; - delete doc._id; - return doc; - })); - }); + if (this.db) { + this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => { + if (err) { + console.log(err.message); + console.log(err.errmsg); + } + fn(docs.map(doc => { + doc.id = doc._id; + delete doc._id; + return doc; + })); + }); + } else { + this.onConnect.push(() => this.getDocuments(ids, fn, collectionName)); + } + } + + public query(query: any): Promise { + if (this.db) { + return Promise.resolve(this.db.collection('newDocuments').find(query)); + } else { + return new Promise(res => { + this.onConnect.push(() => res(this.query(query))); + }); + } } public print() { diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts new file mode 100644 index 000000000..da33e3747 --- /dev/null +++ b/src/server/updateSearch.ts @@ -0,0 +1,101 @@ +import { Database } from "./database"; +import { Cursor } from "mongodb"; +import { Search } from "./Search"; +import pLimit from 'p-limit'; + +const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { + "number": "_n", + "string": "_t", + // "boolean": "_b", + "image": ["_t", "url"], + "video": ["_t", "url"], + "pdf": ["_t", "url"], + "audio": ["_t", "url"], + "web": ["_t", "url"], + "date": ["_d", value => new Date(value.date).toISOString()], + "proxy": ["_i", "fieldId"], + "list": ["_l", list => { + const results = []; + for (const value of list.fields) { + const term = ToSearchTerm(value); + if (term) { + results.push(term.value); + } + } + return results.length ? results : null; + }] +}; + +function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { + if (val === null || val === undefined) { + return; + } + const type = val.__type || typeof val; + let suffix = suffixMap[type]; + if (!suffix) { + return; + } + + if (Array.isArray(suffix)) { + const accessor = suffix[1]; + if (typeof accessor === "function") { + val = accessor(val); + } else { + val = val[accessor]; + } + suffix = suffix[0]; + } + + return { suffix, value: val }; +} + +function getSuffix(value: string | [string, any]): string { + return typeof value === "string" ? value : value[0]; +} + +const limit = pLimit(5); +async function update() { + // await new Promise(res => setTimeout(res, 5)); + console.log("update"); + await Search.Instance.clear(); + const cursor = await Database.Instance.query({}); + console.log("Cleared"); + const updates: any[] = []; + let numDocs = 0; + function updateDoc(doc: any) { + numDocs++; + if ((numDocs % 50) === 0) { + console.log("updateDoc " + numDocs); + } + console.log("doc " + numDocs); + if (doc.__type !== "Doc") { + return; + } + const fields = doc.fields; + if (!fields) { + return; + } + const update: any = { id: doc._id }; + let dynfield = false; + for (const key in fields) { + const value = fields[key]; + const term = ToSearchTerm(value); + if (term !== undefined) { + let { suffix, value } = term; + update[key + suffix] = value; + dynfield = true; + } + } + if (dynfield) { + updates.push(update); + console.log(updates.length); + } + } + await cursor.forEach(updateDoc); + await Promise.all(updates.map(update => { + return limit(() => Search.Instance.updateDocument(update)); + })); + cursor.close(); +} + +update(); \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 2e2dafc6d805e54e4119e28943a3800fe08c0ca9 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Mon, 20 May 2019 17:33:59 -0400 Subject: Disabled images in search --- src/server/index.ts | 2 +- src/server/updateSearch.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/server') diff --git a/src/server/index.ts b/src/server/index.ts index 574241f62..5a534afae 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -366,7 +366,7 @@ const suffixMap: { [type: string]: (string | [string, string | ((json: any) => a "number": "_n", "string": "_t", // "boolean": "_b", - "image": ["_t", "url"], + // "image": ["_t", "url"], "video": ["_t", "url"], "pdf": ["_t", "url"], "audio": ["_t", "url"], diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts index da33e3747..de1fd25e1 100644 --- a/src/server/updateSearch.ts +++ b/src/server/updateSearch.ts @@ -7,7 +7,7 @@ const suffixMap: { [type: string]: (string | [string, string | ((json: any) => a "number": "_n", "string": "_t", // "boolean": "_b", - "image": ["_t", "url"], + // "image": ["_t", "url"], "video": ["_t", "url"], "pdf": ["_t", "url"], "audio": ["_t", "url"], -- cgit v1.2.3-70-g09d2