diff options
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/GarbageCollector.ts | 124 | ||||
| -rw-r--r-- | src/server/RouteStore.ts | 1 | ||||
| -rw-r--r-- | src/server/Search.ts | 48 | ||||
| -rw-r--r-- | src/server/authentication/controllers/user_controller.ts | 22 | ||||
| -rw-r--r-- | src/server/authentication/models/current_user_utils.ts | 44 | ||||
| -rw-r--r-- | src/server/database.ts | 22 | ||||
| -rw-r--r-- | src/server/index.ts | 188 | ||||
| -rw-r--r-- | src/server/updateSearch.ts | 34 |
8 files changed, 433 insertions, 50 deletions
diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts new file mode 100644 index 000000000..268239481 --- /dev/null +++ b/src/server/GarbageCollector.ts @@ -0,0 +1,124 @@ +import { Database } from './database'; + +import * as path from 'path'; +import * as fs from 'fs'; +import { Search } from './Search'; + +function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { + for (const key in doc) { + if (!doc.hasOwnProperty(key)) { + continue; + } + const field = doc[key]; + if (field === undefined || field === null) { + continue; + } + if (field.__type === "proxy") { + ids.push(field.fieldId); + } else if (field.__type === "list") { + addDoc(field.fields, ids, files); + } else if (typeof field === "string") { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + let match: string[] | null; + while ((match = re.exec(field)) !== null) { + ids.push(match[1]); + } + } else if (field.__type === "RichTextField") { + const re = /"href"\s*:\s*"(.*?)"/g; + let match: string[] | null; + while ((match = re.exec(field.Data)) !== null) { + const urlString = match[1]; + const split = new URL(urlString).pathname.split("doc/"); + if (split.length > 1) { + ids.push(split[split.length - 1]); + } + } + const re2 = /"src"\s*:\s*"(.*?)"/g; + while ((match = re2.exec(field.Data)) !== null) { + const urlString = match[1]; + const pathname = new URL(urlString).pathname; + const ext = path.extname(pathname); + const fileName = path.basename(pathname, ext); + let exts = files[fileName]; + if (!exts) { + files[fileName] = exts = []; + } + exts.push(ext); + } + } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { + const url = new URL(field.url); + const pathname = url.pathname; + const ext = path.extname(pathname); + const fileName = path.basename(pathname, ext); + let exts = files[fileName]; + if (!exts) { + files[fileName] = exts = []; + } + exts.push(ext); + } + } +} + +async function GarbageCollect() { + // await new Promise(res => setTimeout(res, 3000)); + const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users'); + const users = await cursor.toArray(); + const ids: string[] = users.map(user => user.userDocumentId); + const visited = new Set<string>(); + const files: { [name: string]: string[] } = {}; + + while (ids.length) { + const count = Math.min(ids.length, 100); + const index = ids.length - count; + const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); + if (!fetchIds.length) { + continue; + } + const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments")); + for (const doc of docs) { + const id = doc.id; + if (doc === undefined) { + console.log(`Couldn't find field with Id ${id}`); + continue; + } + visited.add(id); + addDoc(doc.fields, ids, files); + } + console.log(`To Go: ${ids.length}, visited: ${visited.size}`); + } + + console.log(`Done: ${visited.size}`); + + cursor.close(); + + const toDeleteCursor = await Database.Instance.query({ _id: { $nin: Array.from(visited) } }, { _id: 1 }); + const toDelete: string[] = (await toDeleteCursor.toArray()).map(doc => doc._id); + toDeleteCursor.close(); + const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); + console.log(`${result.deletedCount} documents deleted`); + + await Search.Instance.deleteDocuments(toDelete); + console.log("Cleared search documents"); + + const folder = "./src/server/public/files/"; + fs.readdir(folder, (_, fileList) => { + const filesToDelete = fileList.filter(file => { + const ext = path.extname(file); + let base = path.basename(file, ext); + const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext); + return file !== ".gitignore" && !existsInDb; + }); + console.log(`Deleting ${filesToDelete.length} files`); + filesToDelete.forEach(file => { + console.log(`Deleting file ${file}`); + try { + fs.unlinkSync(folder + file); + } catch { + console.warn(`Couldn't delete file ${file}`); + } + }); + console.log(`Deleted ${filesToDelete.length} files`); + }); +} + +GarbageCollect(); diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index c4af5cdaa..5c13495ff 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -16,6 +16,7 @@ export enum RouteStore { // USER AND WORKSPACES getCurrUser = "/getCurrentUser", + getUsers = "/getUsers", getUserDocumentId = "/getUserDocumentId", updateCursor = "/updateCursor", diff --git a/src/server/Search.ts b/src/server/Search.ts index 5ca5578a7..8591f8857 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -14,23 +14,36 @@ export class Search { }); return res; } catch (e) { - console.warn("Search error: " + e + document); + // console.warn("Search error: " + e + document); } } - public async search(query: string) { + public async updateDocuments(documents: any[]) { + try { + const res = await rp.post(this.url + "dash/update", { + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(documents) + }); + return res; + } catch (e) { + // console.warn("Search error: " + e + document); + } + } + + public async search(query: string, start: number = 0) { try { const searchResults = JSON.parse(await rp.get(this.url + "dash/select", { qs: { q: query, - fl: "id" + fl: "id", + start: start } })); - const fields = searchResults.response.docs; - const ids = fields.map((field: any) => field.id); - return ids; + const { docs, numFound } = searchResults.response; + const ids = docs.map((field: any) => field.id); + return { ids, numFound }; } catch { - return []; + return { ids: [], numFound: -1 }; } } @@ -46,4 +59,25 @@ export class Search { }); } catch { } } + + public deleteDocuments(docs: string[]) { + const promises: rp.RequestPromise[] = []; + const nToDelete = 1000; + let index = 0; + while (index < docs.length) { + const count = Math.min(docs.length - index, nToDelete); + const deleteIds = docs.slice(index, index + count); + index += count; + promises.push(rp.post(this.url + "dash/update", { + body: { + delete: { + query: deleteIds.map(id => `id:"${id}"`).join(" ") + } + }, + json: true + })); + } + + return Promise.all(promises); + } }
\ No newline at end of file diff --git a/src/server/authentication/controllers/user_controller.ts b/src/server/authentication/controllers/user_controller.ts index 1dacdf3fa..ca4fc171c 100644 --- a/src/server/authentication/controllers/user_controller.ts +++ b/src/server/authentication/controllers/user_controller.ts @@ -42,10 +42,6 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { const errors = req.validationErrors(); if (errors) { - res.render("signup.pug", { - title: "Sign Up", - user: req.user, - }); return res.redirect(RouteStore.signup); } @@ -66,16 +62,23 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { user.save((err) => { if (err) { return next(err); } req.logIn(user, (err) => { - if (err) { - return next(err); - } - res.redirect(RouteStore.home); + if (err) { return next(err); } + tryRedirectToTarget(req, res); }); }); }); }; +let tryRedirectToTarget = (req: Request, res: Response) => { + if (req.session && req.session.target) { + res.redirect(req.session.target); + req.session.target = undefined; + } else { + res.redirect(RouteStore.home); + } +}; + /** * GET /login @@ -83,6 +86,7 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { */ export let getLogin = (req: Request, res: Response) => { if (req.user) { + req.session!.target = undefined; return res.redirect(RouteStore.home); } res.render("login.pug", { @@ -115,7 +119,7 @@ export let postLogin = (req: Request, res: Response, next: NextFunction) => { } req.logIn(user, (err) => { if (err) { next(err); return; } - res.redirect(RouteStore.home); + tryRedirectToTarget(req, res); }); })(req, res, next); }; diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 95c20d2db..763693dd6 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -10,7 +10,7 @@ import { CollectionView } from "../../../client/views/collections/CollectionView import { Doc } from "../../../new_fields/Doc"; import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; -import { Cast } from "../../../new_fields/Types"; +import { Cast, FieldValue } from "../../../new_fields/Types"; import { RouteStore } from "../../RouteStore"; export class CurrentUserUtils { @@ -29,9 +29,14 @@ export class CurrentUserUtils { private static createUserDocument(id: string): Doc { let doc = new Doc(id, true); doc.viewType = CollectionViewType.Tree; + doc.dropAction = "alias"; doc.layout = CollectionView.LayoutString(); doc.title = this.email; + this.updateUserDocument(doc); doc.data = new List<Doc>(); + doc.gridGap = 5; + doc.xMargin = 5; + doc.yMargin = 5; doc.excludeFromLibrary = true; doc.optionalRightCollection = Docs.Create.StackingDocument([], { title: "New mobile uploads" }); // doc.library = Docs.Create.TreeDocument([doc], { title: `Library: ${CurrentUserUtils.email}` }); @@ -39,8 +44,31 @@ export class CurrentUserUtils { return doc; } + static updateUserDocument(doc: Doc) { + if (doc.workspaces === undefined) { + const workspaces = Docs.Create.TreeDocument([], { title: "Workspaces", height: 100 }); + workspaces.excludeFromLibrary = true; + workspaces.workspaceLibrary = true; + doc.workspaces = workspaces; + } + if (doc.recentlyClosed === undefined) { + const recentlyClosed = Docs.Create.TreeDocument([], { title: "Recently Closed", height: 75 }); + recentlyClosed.excludeFromLibrary = true; + doc.recentlyClosed = recentlyClosed; + } + if (doc.sidebar === undefined) { + const sidebar = Docs.Create.StackingDocument([doc.workspaces as Doc, doc, doc.recentlyClosed as Doc], { title: "Sidebar" }); + sidebar.excludeFromLibrary = true; + sidebar.gridGap = 5; + sidebar.xMargin = 5; + sidebar.yMargin = 5; + doc.sidebar = sidebar; + } + + } + public static async loadCurrentUser(): Promise<any> { - let userPromise = rp.get(DocServer.Util.prepend(RouteStore.getCurrUser)).then(response => { + let userPromise = rp.get(DocServer.prepend(RouteStore.getCurrUser)).then(response => { if (response) { let obj = JSON.parse(response); CurrentUserUtils.curr_id = obj.id as string; @@ -49,10 +77,16 @@ export class CurrentUserUtils { throw new Error("There should be a user! Why does Dash think there isn't one?"); } }); - let userDocPromise = await rp.get(DocServer.Util.prepend(RouteStore.getUserDocumentId)).then(id => { + let userDocPromise = await rp.get(DocServer.prepend(RouteStore.getUserDocumentId)).then(id => { if (id) { - return DocServer.getRefField(id).then(field => - runInAction(() => this.user_document = field instanceof Doc ? field : this.createUserDocument(id))); + return DocServer.GetRefField(id).then(async field => { + if (field instanceof Doc) { + await this.updateUserDocument(field); + runInAction(() => this.user_document = field); + } else { + runInAction(() => this.user_document = this.createUserDocument(id)); + } + }); } else { throw new Error("There should be a user id! Why does Dash think there isn't one?"); } diff --git a/src/server/database.ts b/src/server/database.ts index 70b3efced..29a8ffafa 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -41,11 +41,17 @@ export class Database { } } - public delete(id: string, collectionName = Database.DocumentsCollection) { + public delete(query: any, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>; + public delete(id: string, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>; + public delete(id: any, collectionName = Database.DocumentsCollection) { + if (typeof id === "string") { + id = { _id: id }; + } if (this.db) { - this.db.collection(collectionName).remove({ id: id }); + const db = this.db; + return new Promise(res => db.collection(collectionName).deleteMany(id, (err, result) => res(result))); } else { - this.onConnect.push(() => this.delete(id, collectionName)); + return new Promise(res => this.onConnect.push(() => res(this.delete(id, collectionName)))); } } @@ -120,12 +126,16 @@ export class Database { } } - public query(query: any): Promise<mongodb.Cursor> { + public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise<mongodb.Cursor> { if (this.db) { - return Promise.resolve<mongodb.Cursor>(this.db.collection('newDocuments').find(query)); + let cursor = this.db.collection(collectionName).find(query); + if (projection) { + cursor = cursor.project(projection); + } + return Promise.resolve<mongodb.Cursor>(cursor); } else { return new Promise<mongodb.Cursor>(res => { - this.onConnect.push(() => res(this.query(query))); + this.onConnect.push(() => res(this.query(query, projection, collectionName))); }); } } diff --git a/src/server/index.ts b/src/server/index.ts index d9fe9d93d..287ba6058 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,3 +1,4 @@ +require('dotenv').config(); import * as bodyParser from 'body-parser'; import { exec } from 'child_process'; import * as cookieParser from 'cookie-parser'; @@ -7,9 +8,9 @@ import * as expressValidator from 'express-validator'; import * as formidable from 'formidable'; import * as fs from 'fs'; import * as sharp from 'sharp'; +import * as Pdfjs from 'pdfjs-dist'; const imageDataUri = require('image-data-uri'); import * as mobileDetect from 'mobile-detect'; -import { ObservableMap } from 'mobx'; import * as passport from 'passport'; import * as path from 'path'; import * as request from 'request'; @@ -28,6 +29,7 @@ import { MessageStore, Transferable, Types, Diff } from "./Message"; import { RouteStore } from './RouteStore'; const app = express(); const config = require('../../webpack.config'); +import { createCanvas, loadImage, Canvas } from "canvas"; const compiler = webpack(config); const port = 1050; // default port to listen const serverPort = 4321; @@ -37,11 +39,24 @@ import c = require("crypto"); import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); +import { Response } from 'express-serve-static-core'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); +const probe = require("probe-image-size"); const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); +const release = process.env.RELEASE === "true"; +if (process.env.RELEASE === "true") { + console.log("Running server in release mode"); +} else { + console.log("Running server in debug mode"); +} +console.log(process.env.PWD); +let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8"); +clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(release))}`; +fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8"); + const mongoUrl = 'mongodb://localhost:27017/Dash'; mongoose.connect(mongoUrl); mongoose.connection.on('connected', () => console.log("connected")); @@ -88,14 +103,15 @@ enum Method { */ function addSecureRoute(method: Method, handler: (user: DashUserModel, res: express.Response, req: express.Request) => void, - onRejection: (res: express.Response) => any = (res) => res.redirect(RouteStore.logout), + onRejection: (res: express.Response, req: express.Request) => any = res => res.redirect(RouteStore.login), ...subscribers: string[] ) { let abstracted = (req: express.Request, res: express.Response) => { if (req.user) { handler(req.user, res, req); } else { - onRejection(res); + req.session!.target = `http://localhost:${port}${req.originalUrl}`; + onRejection(res, req); } }; subscribers.forEach(route => { @@ -133,6 +149,64 @@ app.get("/search", async (req, res) => { res.send(results); }); +app.get("/thumbnail/:filename", (req, res) => { + let filename = req.params.filename; + let noExt = filename.substring(0, filename.length - ".png".length); + let pagenumber = parseInt(noExt[noExt.length - 1]); + fs.exists(uploadDir + filename, (exists: boolean) => { + console.log(`${uploadDir + filename} ${exists ? "exists" : "does not exist"}`); + if (exists) { + let input = fs.createReadStream(uploadDir + filename); + probe(input, (err: any, result: any) => { + if (err) { + console.log(err); + return; + } + res.send({ path: "/files/" + filename, width: result.width, height: result.height }); + }); + } + else { + LoadPage(uploadDir + filename.substring(0, filename.length - "-n.png".length) + ".pdf", pagenumber, res); + } + }); +}); + +function LoadPage(file: string, pageNumber: number, res: Response) { + console.log(file); + Pdfjs.getDocument(file).promise + .then((pdf: Pdfjs.PDFDocumentProxy) => { + let factory = new NodeCanvasFactory(); + console.log(pageNumber); + pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { + console.log("reading " + page); + let viewport = page.getViewport(1); + let canvasAndContext = factory.create(viewport.width, viewport.height); + let renderContext = { + canvasContext: canvasAndContext.context, + viewport: viewport, + canvasFactory: factory + }; + console.log("read " + pageNumber); + + page.render(renderContext).promise + .then(() => { + console.log("saving " + pageNumber); + let stream = canvasAndContext.canvas.createPNGStream(); + let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; + let out = fs.createWriteStream(pngFile); + stream.pipe(out); + out.on("finish", () => { + console.log(`Success! Saved to ${pngFile}`); + let name = path.basename(pngFile); + res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); + }); + }, (reason: string) => { + console.error(reason + ` ${pageNumber}`); + }); + }); + }); +} + // anyone attempting to navigate to localhost at this port will // first have to login addSecureRoute( @@ -144,6 +218,17 @@ addSecureRoute( addSecureRoute( Method.GET, + async (_, res) => { + const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); + const results = await cursor.toArray(); + res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); + }, + undefined, + RouteStore.getUsers +); + +addSecureRoute( + Method.GET, (user, res, req) => { let detector = new mobileDetect(req.headers['user-agent'] || ""); let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; @@ -168,7 +253,31 @@ addSecureRoute( RouteStore.getCurrUser ); +class NodeCanvasFactory { + create = (width: number, height: number) => { + var canvas = createCanvas(width, height); + var context = canvas.getContext('2d'); + return { + canvas: canvas, + context: context, + }; + } + + reset = (canvasAndContext: any, width: number, height: number) => { + canvasAndContext.canvas.width = width; + canvasAndContext.canvas.height = height; + } + + destroy = (canvasAndContext: any) => { + canvasAndContext.canvas.width = 0; + canvasAndContext.canvas.height = 0; + canvasAndContext.canvas = null; + canvasAndContext.context = null; + } +} + const pngTypes = [".png", ".PNG"]; +const pdfTypes = [".pdf", ".PDF"]; const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; const uploadDir = __dirname + "/public/files/"; // SETTERS @@ -187,9 +296,10 @@ app.post( const file = path.basename(files[name].path); const ext = path.extname(file); let resizers = [ - { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, - { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, - { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, + { resizer: sharp().rotate(), suffix: "_o" }, + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }).rotate(), suffix: "_s" }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }).rotate(), suffix: "_m" }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }).rotate(), suffix: "_l" }, ]; let isImage = false; if (pngTypes.includes(ext)) { @@ -203,6 +313,38 @@ app.post( }); isImage = true; } + else if (pdfTypes.includes(ext)) { + // Pdfjs.getDocument(uploadDir + file).promise + // .then((pdf: Pdfjs.PDFDocumentProxy) => { + // let numPages = pdf.numPages; + // let factory = new NodeCanvasFactory(); + // for (let pageNum = 0; pageNum < numPages; pageNum++) { + // console.log(pageNum); + // pdf.getPage(pageNum + 1).then((page: Pdfjs.PDFPageProxy) => { + // console.log("reading " + pageNum); + // let viewport = page.getViewport(1); + // let canvasAndContext = factory.create(viewport.width, viewport.height); + // let renderContext = { + // canvasContext: canvasAndContext.context, + // viewport: viewport, + // canvasFactory: factory + // } + // console.log("read " + pageNum); + + // page.render(renderContext).promise + // .then(() => { + // console.log("saving " + pageNum); + // let stream = canvasAndContext.canvas.createPNGStream(); + // let out = fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + `-${pageNum + 1}.PNG`); + // stream.pipe(out); + // out.on("finish", () => console.log(`Success! Saved to ${uploadDir + file.substring(0, file.length - ext.length) + `-${pageNum + 1}.PNG`}`)); + // }, (reason: string) => { + // console.error(reason + ` ${pageNum}`); + // }); + // }); + // } + // }); + } if (isImage) { resizers.forEach(resizer => { fs.createReadStream(uploadDir + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); @@ -278,11 +420,21 @@ app.post(RouteStore.reset, postReset); app.use(RouteStore.corsProxy, (req, res) => req.pipe(request(req.url.substring(1))).pipe(res)); -app.get(RouteStore.delete, (req, res) => - deleteFields().then(() => res.redirect(RouteStore.home))); +app.get(RouteStore.delete, (req, res) => { + if (release) { + res.send("no"); + return; + } + deleteFields().then(() => res.redirect(RouteStore.home)); +}); -app.get(RouteStore.deleteAll, (req, res) => - deleteAll().then(() => res.redirect(RouteStore.home))); +app.get(RouteStore.deleteAll, (req, res) => { + if (release) { + res.send("no"); + return; + } + deleteAll().then(() => res.redirect(RouteStore.home)); +}); app.use(wdm(compiler, { publicPath: config.output.publicPath })); @@ -303,14 +455,16 @@ server.on("connection", function (socket: Socket) { Utils.emit(socket, MessageStore.Foo, "handshooken"); - Utils.addServerHandler(socket, MessageStore.Bar, barReceived); - Utils.addServerHandler(socket, MessageStore.SetField, (args) => setField(socket, args)); + Utils.AddServerHandler(socket, MessageStore.Bar, barReceived); + Utils.AddServerHandler(socket, MessageStore.SetField, (args) => setField(socket, args)); Utils.AddServerHandlerCallback(socket, MessageStore.GetField, getField); Utils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields); - Utils.addServerHandler(socket, MessageStore.DeleteAll, deleteFields); + if (!release) { + Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); + } - Utils.addServerHandler(socket, MessageStore.CreateField, CreateField); - Utils.addServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); + Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); + Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); }); @@ -364,8 +518,8 @@ function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => v const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", "string": "_t", - // "boolean": "_b", - // "image": ["_t", "url"], + "boolean": "_b", + "image": ["_t", "url"], "video": ["_t", "url"], "pdf": ["_t", "url"], "audio": ["_t", "url"], diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts index de1fd25e1..906b795f1 100644 --- a/src/server/updateSearch.ts +++ b/src/server/updateSearch.ts @@ -6,7 +6,7 @@ import pLimit from 'p-limit'; const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", "string": "_t", - // "boolean": "_b", + "boolean": "_b", // "image": ["_t", "url"], "video": ["_t", "url"], "pdf": ["_t", "url"], @@ -67,7 +67,7 @@ async function update() { if ((numDocs % 50) === 0) { console.log("updateDoc " + numDocs); } - console.log("doc " + numDocs); + // console.log("doc " + numDocs); if (doc.__type !== "Doc") { return; } @@ -88,13 +88,35 @@ async function update() { } if (dynfield) { updates.push(update); - console.log(updates.length); + // console.log(updates.length); } } await cursor.forEach(updateDoc); - await Promise.all(updates.map(update => { - return limit(() => Search.Instance.updateDocument(update)); - })); + console.log(`Updating ${updates.length} documents`); + const result = await Search.Instance.updateDocuments(updates); + try { + console.log(JSON.parse(result).responseHeader.status); + } catch { + console.log("Error:"); + // console.log(updates[i]); + console.log(result); + console.log("\n"); + } + // for (let i = 0; i < updates.length; i++) { + // console.log(i); + // const result = await Search.Instance.updateDocument(updates[i]); + // try { + // console.log(JSON.parse(result).responseHeader.status); + // } catch { + // console.log("Error:"); + // console.log(updates[i]); + // console.log(result); + // console.log("\n"); + // } + // } + // await Promise.all(updates.map(update => { + // return limit(() => Search.Instance.updateDocument(update)); + // })); cursor.close(); } |