diff options
Diffstat (limited to 'src/server/index.ts')
| -rw-r--r-- | src/server/index.ts | 281 |
1 files changed, 243 insertions, 38 deletions
diff --git a/src/server/index.ts b/src/server/index.ts index 51fafbdc3..10a84c823 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -25,8 +25,9 @@ import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLo import { DashUserModel } from './authentication/models/user_model'; import { Client } from './Client'; import { Database } from './database'; -import { MessageStore, Transferable, Types, Diff, Message } from "./Message"; +import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message"; import { RouteStore } from './RouteStore'; +import v4 = require('uuid/v4'); const app = express(); const config = require('../../webpack.config'); import { createCanvas, loadImage, Canvas } from "canvas"; @@ -39,12 +40,20 @@ import c = require("crypto"); import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); +import * as Archiver from 'archiver'; +import * as AdmZip from 'adm-zip'; +import * as YoutubeApi from './youtubeApi/youtubeApiSample.js'; import { Response } from 'express-serve-static-core'; +import { DocComponent } from '../client/views/DocComponent'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); +var SolrNode = require('solr-node'); +var shell = require('shelljs'); const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); +let youtubeApiKey: string; +YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); const release = process.env.RELEASE === "true"; if (process.env.RELEASE === "true") { @@ -110,7 +119,7 @@ function addSecureRoute(method: Method, if (req.user) { handler(req.user, res, req); } else { - req.session!.target = `${req.headers.host}${req.originalUrl}`; + req.session!.target = req.originalUrl; onRejection(res, req); } }; @@ -139,7 +148,18 @@ app.get("/pull", (req, res) => res.redirect("/"); })); +app.get("/version", (req, res) => { + exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout, stderr) => { + if (err) { + res.send(err.message); + return; + } + res.send(stdout); + }); +}); + // SEARCH +const solrURL = "http://localhost:8983/solr/#/dash"; // GETTERS @@ -167,6 +187,186 @@ function msToTime(duration: number) { return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } +async function getDocs(id: string) { + const files = new Set<string>(); + const docs: { [id: string]: any } = {}; + const fn = (doc: any): string[] => { + const id = doc.id; + if (typeof id === "string" && id.endsWith("Proto")) { + //Skip protos + return []; + } + const ids: string[] = []; + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + ids.push(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + ids.push(field.captures.fieldId); + } + } else if (field.__type === "list") { + ids.push(...fn(field)); + } else if (typeof field === "string") { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + let match: string[] | null; + while ((match = re.exec(field)) !== null) { + ids.push(match[1]); + } + } else if (field.__type === "RichTextField") { + const re = /"href"\s*:\s*"(.*?)"/g; + let match: string[] | null; + while ((match = re.exec(field.Data)) !== null) { + const urlString = match[1]; + const split = new URL(urlString).pathname.split("doc/"); + if (split.length > 1) { + ids.push(split[split.length - 1]); + } + } + const re2 = /"src"\s*:\s*"(.*?)"/g; + while ((match = re2.exec(field.Data)) !== null) { + const urlString = match[1]; + const pathname = new URL(urlString).pathname; + files.add(pathname); + } + } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { + const url = new URL(field.url); + const pathname = url.pathname; + files.add(pathname); + } + } + + if (doc.id) { + docs[doc.id] = doc; + } + return ids; + }; + await Database.Instance.visit([id], fn); + return { id, docs, files }; +} +app.get("/serializeDoc/:docId", async (req, res) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); +}); + +app.get("/downloadId/:docId", async (req, res) => { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + const zip = Archiver('zip'); + zip.pipe(res); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); + }); + zip.finalize(); +}); + +app.post("/uploadDoc", (req, res) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); + } + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); + } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); + } + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } + } + }; + form.parse(req, async (err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path_2 = files[name].path; + const zip = new AdmZip(path_2); + zip.getEntries().forEach(entry => { + if (!entry.entryName.startsWith("files/")) return; + let dirname = path.dirname(entry.entryName) + "/"; + let extname = path.extname(entry.entryName); + let basename = path.basename(entry.entryName).split(".")[0]; + // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); + zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); + dirname = "/" + dirname; + + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); + }); + const json = zip.getEntry("doc.json"); + let docs: any; + try { + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + fs.unlink(path_2, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + }); +}); + app.get("/whosOnline", (req, res) => { let users: any = { active: {}, inactive: {} }; const now = Date.now(); @@ -179,7 +379,6 @@ app.get("/whosOnline", (req, res) => { res.send(users); }); - app.get("/thumbnail/:filename", (req, res) => { let filename = req.params.filename; let noExt = filename.substring(0, filename.length - ".png".length); @@ -285,6 +484,17 @@ addSecureRoute( RouteStore.getCurrUser ); +const ServicesApiKeyMap = new Map<string, string | undefined>([ + ["face", process.env.FACE], + ["vision", process.env.VISION], + ["handwriting", process.env.HANDWRITING] +]); + +addSecureRoute(Method.GET, (user, res, req) => { + let service = req.params.requestedservice; + res.send(ServicesApiKeyMap.get(service)); +}, undefined, `${RouteStore.cognitiveServices}/:requestedservice`); + class NodeCanvasFactory { create = (width: number, height: number) => { var canvas = createCanvas(width, height); @@ -345,38 +555,6 @@ app.post( }); isImage = true; } - else if (pdfTypes.includes(ext)) { - // Pdfjs.getDocument(uploadDir + file).promise - // .then((pdf: Pdfjs.PDFDocumentProxy) => { - // let numPages = pdf.numPages; - // let factory = new NodeCanvasFactory(); - // for (let pageNum = 0; pageNum < numPages; pageNum++) { - // console.log(pageNum); - // pdf.getPage(pageNum + 1).then((page: Pdfjs.PDFPageProxy) => { - // console.log("reading " + pageNum); - // let viewport = page.getViewport(1); - // let canvasAndContext = factory.create(viewport.width, viewport.height); - // let renderContext = { - // canvasContext: canvasAndContext.context, - // viewport: viewport, - // canvasFactory: factory - // } - // console.log("read " + pageNum); - - // page.render(renderContext).promise - // .then(() => { - // console.log("saving " + pageNum); - // let stream = canvasAndContext.canvas.createPNGStream(); - // let out = fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + `-${pageNum + 1}.PNG`); - // stream.pipe(out); - // out.on("finish", () => console.log(`Success! Saved to ${uploadDir + file.substring(0, file.length - ext.length) + `-${pageNum + 1}.PNG`}`)); - // }, (reason: string) => { - // console.error(reason + ` ${pageNum}`); - // }); - // }); - // } - // }); - } if (isImage) { resizers.forEach(resizer => { fs.createReadStream(uploadDir + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); @@ -449,8 +627,22 @@ app.post(RouteStore.forgot, postForgot); app.get(RouteStore.reset, getReset); app.post(RouteStore.reset, postReset); -app.use(RouteStore.corsProxy, (req, res) => - req.pipe(request(req.url.substring(1))).pipe(res)); +const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; +app.use(RouteStore.corsProxy, (req, res) => { + req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { + const headers = Object.keys(res.headers); + headers.forEach(headerName => { + const header = res.headers[headerName]; + if (Array.isArray(header)) { + res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); + } else if (header) { + if (headerCharRegex.test(header as any)) { + delete res.headers[headerName]; + } + } + }); + }).pipe(res); +}); app.get(RouteStore.delete, (req, res) => { if (release) { @@ -505,6 +697,7 @@ server.on("connection", function (socket: Socket) { } Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); + Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery); Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); @@ -559,6 +752,17 @@ function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => v Database.Instance.getDocuments(ids, callback, "newDocuments"); } +function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { + switch (query.type) { + case YoutubeQueryType.Channels: + YoutubeApi.authorizedGetChannel(youtubeApiKey); + break; + case YoutubeQueryType.SearchVideo: + YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback); + case YoutubeQueryType.VideoDetails: + YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback); + } +} const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", @@ -658,4 +862,5 @@ function CreateField(newValue: any) { } server.listen(serverPort); -console.log(`listening on port ${serverPort}`);
\ No newline at end of file +console.log(`listening on port ${serverPort}`); + |