From 7844debd9862b15d2b36bf9ce1efb3274616a313 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Wed, 24 Jul 2019 18:53:21 -0400 Subject: Context Menu and Some Preview --- src/client/views/collections/CollectionTreeView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/client/views/collections') diff --git a/src/client/views/collections/CollectionTreeView.tsx b/src/client/views/collections/CollectionTreeView.tsx index 006de0c70..c0347ad7d 100644 --- a/src/client/views/collections/CollectionTreeView.tsx +++ b/src/client/views/collections/CollectionTreeView.tsx @@ -64,7 +64,7 @@ library.add(faArrowsAltH); /** * Component that takes in a document prop and a boolean whether it's collapsed or not. */ -class TreeView extends React.Component { +export class TreeView extends React.Component { private _header?: React.RefObject = React.createRef(); private _treedropDisposer?: DragManager.DragDropDisposer; private _dref = React.createRef(); -- cgit v1.2.3-70-g09d2 From c8074da503a5f4d252417c652709f7a393d7d158 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Mon, 29 Jul 2019 20:07:27 -0400 Subject: Partially fixed add tab --- src/client/views/collections/CollectionDockingView.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/views/collections/CollectionDockingView.tsx b/src/client/views/collections/CollectionDockingView.tsx index ab4d1aa62..3a402b6b2 100644 --- a/src/client/views/collections/CollectionDockingView.tsx +++ b/src/client/views/collections/CollectionDockingView.tsx @@ -210,14 +210,14 @@ export class CollectionDockingView extends React.Component Date: Mon, 29 Jul 2019 20:48:02 -0400 Subject: Changed how AddTab works to work better --- .../views/collections/CollectionDockingView.tsx | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/views/collections/CollectionDockingView.tsx b/src/client/views/collections/CollectionDockingView.tsx index 3a402b6b2..588102f01 100644 --- a/src/client/views/collections/CollectionDockingView.tsx +++ b/src/client/views/collections/CollectionDockingView.tsx @@ -210,20 +210,22 @@ export class CollectionDockingView extends React.Component Date: Tue, 30 Jul 2019 09:37:38 -0400 Subject: removed TreeView export --- src/client/views/collections/CollectionTreeView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/client/views/collections') diff --git a/src/client/views/collections/CollectionTreeView.tsx b/src/client/views/collections/CollectionTreeView.tsx index 6cc065342..b1e6eada0 100644 --- a/src/client/views/collections/CollectionTreeView.tsx +++ b/src/client/views/collections/CollectionTreeView.tsx @@ -66,7 +66,7 @@ library.add(faPlus, faMinus); /** * Component that takes in a document prop and a boolean whether it's collapsed or not. */ -export class TreeView extends React.Component { +class TreeView extends React.Component { private _header?: React.RefObject = React.createRef(); private _treedropDisposer?: DragManager.DragDropDisposer; private _dref = React.createRef(); -- cgit v1.2.3-70-g09d2 From 5b455e2aaf119c7db1fe9ef22d71a3accf55a8e2 Mon Sep 17 00:00:00 2001 From: bob Date: Tue, 30 Jul 2019 10:27:42 -0400 Subject: tweaks. --- src/client/apis/youtube/YoutubeBox.tsx | 22 +++++++++------------- src/client/views/collections/CollectionView.tsx | 2 +- .../views/collections/CollectionViewChromes.tsx | 1 + 3 files changed, 11 insertions(+), 14 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/apis/youtube/YoutubeBox.tsx b/src/client/apis/youtube/YoutubeBox.tsx index 019d191bc..7f9a3ad70 100644 --- a/src/client/apis/youtube/YoutubeBox.tsx +++ b/src/client/apis/youtube/YoutubeBox.tsx @@ -1,20 +1,16 @@ -import "../../views/nodes/WebBox.scss"; -import React = require("react"); -import { FieldViewProps, FieldView } from "../../views/nodes/FieldView"; -import { HtmlField } from "../../../new_fields/HtmlField"; -import { WebField } from "../../../new_fields/URLField"; +import { action, observable, runInAction } from 'mobx'; import { observer } from "mobx-react"; -import { computed, reaction, IReactionDisposer, observable, action, runInAction } from 'mobx'; -import { DocumentDecorations } from "../../views/DocumentDecorations"; -import { InkingControl } from "../../views/InkingControl"; +import { Doc, DocListCastAsync } from "../../../new_fields/Doc"; +import { Cast, NumCast, StrCast } from "../../../new_fields/Types"; import { Utils } from "../../../Utils"; import { DocServer } from "../../DocServer"; -import { NumCast, Cast, StrCast } from "../../../new_fields/Types"; -import "./YoutubeBox.scss"; import { Docs } from "../../documents/Documents"; -import { Doc, DocListCastAsync } from "../../../new_fields/Doc"; -import { listSpec } from "../../../new_fields/Schema"; -import { List } from "../../../new_fields/List"; +import { DocumentDecorations } from "../../views/DocumentDecorations"; +import { InkingControl } from "../../views/InkingControl"; +import { FieldView, FieldViewProps } from "../../views/nodes/FieldView"; +import "../../views/nodes/WebBox.scss"; +import "./YoutubeBox.scss"; +import React = require("react"); interface VideoTemplate { thumbnailUrl: string; diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index b7ac8768f..212cc5477 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -85,7 +85,7 @@ export class CollectionView extends React.Component { } else { return [ - (), + (), this.SubViewHelper(type, renderProps) ]; } diff --git a/src/client/views/collections/CollectionViewChromes.tsx b/src/client/views/collections/CollectionViewChromes.tsx index 2bffe3cc0..38aafd3cc 100644 --- a/src/client/views/collections/CollectionViewChromes.tsx +++ b/src/client/views/collections/CollectionViewChromes.tsx @@ -208,6 +208,7 @@ export class CollectionViewBaseChrome extends React.Component { }} onPointerDown={this.openViewSpecs} />
Date: Tue, 30 Jul 2019 13:43:02 -0400 Subject: fixed snapshots for youtube videos. --- src/client/views/MainView.tsx | 3 +- .../views/collections/CollectionVideoView.scss | 9 ++ .../views/collections/CollectionVideoView.tsx | 24 +++-- src/client/views/nodes/VideoBox.tsx | 101 +++++++++++---------- 4 files changed, 80 insertions(+), 57 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index ababbe949..88a636784 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -1,5 +1,5 @@ import { IconName, library } from '@fortawesome/fontawesome-svg-core'; -import { faArrowDown, faCloudUploadAlt, faArrowUp, faClone, faCheck, faPlay, faCaretUp, faLongArrowAltRight, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faPortrait, faMusic, faObjectGroup, faPenNib, faRedoAlt, faTable, faThumbtack, faTree, faUndoAlt, faCat, faBolt } from '@fortawesome/free-solid-svg-icons'; +import { faArrowDown, faCloudUploadAlt, faArrowUp, faClone, faCheck, faPlay, faPause, faCaretUp, faLongArrowAltRight, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faPortrait, faMusic, faObjectGroup, faPenNib, faRedoAlt, faTable, faThumbtack, faTree, faUndoAlt, faCat, faBolt } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, computed, configure, observable, runInAction, reaction, trace } from 'mobx'; import { observer } from 'mobx-react'; @@ -126,6 +126,7 @@ export class MainView extends React.Component { library.add(faMusic); library.add(faTree); library.add(faPlay); + library.add(faPause); library.add(faClone); library.add(faCut); library.add(faCommentAlt); diff --git a/src/client/views/collections/CollectionVideoView.scss b/src/client/views/collections/CollectionVideoView.scss index 9d2c23d3e..509851ebb 100644 --- a/src/client/views/collections/CollectionVideoView.scss +++ b/src/client/views/collections/CollectionVideoView.scss @@ -6,6 +6,7 @@ top: 0; left:0; z-index: -1; + display:inline-table; } .collectionVideoView-time{ color : white; @@ -15,6 +16,14 @@ background-color: rgba(50, 50, 50, 0.2); transform-origin: left top; } +.collectionVideoView-snapshot{ + color : white; + top :25px; + right : 25px; + position: absolute; + background-color: rgba(50, 50, 50, 0.2); + transform-origin: left top; +} .collectionVideoView-play { width: 25px; height: 20px; diff --git a/src/client/views/collections/CollectionVideoView.tsx b/src/client/views/collections/CollectionVideoView.tsx index a264cc402..5185d9d0e 100644 --- a/src/client/views/collections/CollectionVideoView.tsx +++ b/src/client/views/collections/CollectionVideoView.tsx @@ -9,6 +9,7 @@ import "./CollectionVideoView.scss"; import React = require("react"); import { InkingControl } from "../InkingControl"; import { InkTool } from "../../../new_fields/InkField"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; @observer @@ -21,18 +22,20 @@ export class CollectionVideoView extends React.Component { private get uIButtons() { let scaling = Math.min(1.8, this.props.ScreenToLocalTransform().Scale); let curTime = NumCast(this.props.Document.curPage); - return ([
+ return ([
{"" + Math.round(curTime)} {" " + Math.round((curTime - Math.trunc(curTime)) * 100)}
, +
+ +
, VideoBox._showControls ? (null) : [ -
- {this._videoBox && this._videoBox.Playing ? "\"" : ">"} +
+
, -
+
F -
- +
]]); } @@ -56,6 +59,15 @@ export class CollectionVideoView extends React.Component { } } + @action + onSnapshot = (e: React.PointerEvent) => { + if (this._videoBox) { + this._videoBox.Snapshot(); + e.stopPropagation(); + e.preventDefault(); + } + } + _isclick = 0; @action onResetDown = (e: React.PointerEvent) => { diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx index d2657227a..1f8636826 100644 --- a/src/client/views/nodes/VideoBox.tsx +++ b/src/client/views/nodes/VideoBox.tsx @@ -89,6 +89,56 @@ export class VideoBox extends DocComponent(VideoD this._youtubePlayer && this.props.addDocTab(this.props.Document, this.props.DataDoc, "inTab"); } + @action public Snapshot() { + let width = NumCast(this.props.Document.width); + let height = NumCast(this.props.Document.height); + var canvas = document.createElement('canvas'); + canvas.width = 640; + canvas.height = 640 * NumCast(this.props.Document.nativeHeight) / NumCast(this.props.Document.nativeWidth); + var ctx = canvas.getContext('2d');//draw image to canvas. scale to target dimensions + if (ctx) { + ctx.rect(0, 0, canvas.width, canvas.height); + ctx.fillStyle = "blue"; + ctx.fill(); + this._videoRef && ctx.drawImage(this._videoRef, 0, 0, canvas.width, canvas.height); + } + + if (!this._videoRef) { // can't find a way to take snapshots of videos + let b = Docs.Create.ButtonDocument({ + x: NumCast(this.props.Document.x) + width, y: NumCast(this.props.Document.y), + width: 150, height: 50, title: NumCast(this.props.Document.curPage).toString() + }); + const script = CompileScript(`(self as any).curPage = ${NumCast(this.props.Document.curPage)}`, { + params: { this: Doc.name }, + capturedVariables: { self: this.props.Document }, + typecheck: false, + editable: true, + }); + if (script.compiled) { + b.onClick = new ScriptField(script); + this.props.ContainingCollectionView && this.props.ContainingCollectionView.props.addDocument && this.props.ContainingCollectionView.props.addDocument(b, false); + } else { + console.log(script.errors.map(error => error.messageText).join("\n")); + } + } else { + //convert to desired file format + var dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png' + // if you want to preview the captured image, + let filename = encodeURIComponent("snapshot" + this.props.Document.title + "_" + this.props.Document.curPage).replace(/\./g, ""); + VideoBox.convertDataUri(dataUrl, filename).then(returnedFilename => { + if (returnedFilename) { + let url = Utils.prepend(returnedFilename); + let imageSummary = Docs.Create.ImageDocument(url, { + x: NumCast(this.props.Document.x) + width, y: NumCast(this.props.Document.y), + width: 150, height: height / width * 150, title: "--snapshot" + NumCast(this.props.Document.curPage) + " image-" + }); + this.props.ContainingCollectionView && this.props.ContainingCollectionView.props.addDocument && this.props.ContainingCollectionView.props.addDocument(imageSummary, false); + DocUtils.MakeLink(imageSummary, this.props.Document); + } + }); + } + } + @action updateTimecode = () => { this.player && (this.props.Document.curPage = this.player.currentTime); @@ -152,56 +202,7 @@ export class VideoBox extends DocComponent(VideoD let subitems: ContextMenuProps[] = []; subitems.push({ description: "Copy path", event: () => { Utils.CopyText(url); }, icon: "expand-arrows-alt" }); subitems.push({ description: "Toggle Show Controls", event: action(() => VideoBox._showControls = !VideoBox._showControls), icon: "expand-arrows-alt" }); - let width = NumCast(this.props.Document.width); - let height = NumCast(this.props.Document.height); - subitems.push({ - description: "Take Snapshot", event: async () => { - var canvas = document.createElement('canvas'); - canvas.width = 640; - canvas.height = 640 * NumCast(this.props.Document.nativeHeight) / NumCast(this.props.Document.nativeWidth); - var ctx = canvas.getContext('2d');//draw image to canvas. scale to target dimensions - if (ctx) { - ctx.rect(0, 0, canvas.width, canvas.height); - ctx.fillStyle = "blue"; - ctx.fill(); - this._videoRef && ctx.drawImage(this._videoRef, 0, 0, canvas.width, canvas.height); - } - - if (!this._videoRef) { // can't find a way to take snapshots of videos - let b = Docs.Create.ButtonDocument({ - x: NumCast(this.props.Document.x) + width, y: NumCast(this.props.Document.y), width: 150, height: 50, title: NumCast(this.props.Document.curPage).toString() - }); - const script = CompileScript(`this.props.Document.curPage = ${NumCast(this.props.Document.curPage)}`, { - params: { this: Doc.name }, - typecheck: false, - editable: true, - }); - if (!script.compiled) { - console.log(script.errors.map(error => error.messageText).join("\n")); - return; - } - b.onClick = new ScriptField(script); - this.props.ContainingCollectionView && this.props.ContainingCollectionView.props.addDocument && this.props.ContainingCollectionView.props.addDocument(b, false); - } else { - //convert to desired file format - var dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png' - // if you want to preview the captured image, - let filename = encodeURIComponent("snapshot" + this.props.Document.title + "_" + this.props.Document.curPage).replace(/\./g, ""); - VideoBox.convertDataUri(dataUrl, filename).then(returnedFilename => { - if (returnedFilename) { - let url = Utils.prepend(returnedFilename); - let imageSummary = Docs.Create.ImageDocument(url, { - x: NumCast(this.props.Document.x) + width, y: NumCast(this.props.Document.y), - width: 150, height: height / width * 150, title: "--snapshot" + NumCast(this.props.Document.curPage) + " image-" - }); - this.props.ContainingCollectionView && this.props.ContainingCollectionView.props.addDocument && this.props.ContainingCollectionView.props.addDocument(imageSummary, false); - DocUtils.MakeLink(imageSummary, this.props.Document); - } - }); - } - }, - icon: "expand-arrows-alt" - }); + subitems.push({ description: "Take Snapshot", event: () => this.Snapshot(), icon: "expand-arrows-alt" }); ContextMenu.Instance.addItem({ description: "Video Funcs...", subitems: subitems, icon: "video" }); } } -- cgit v1.2.3-70-g09d2 From 26086ee95a9a16486d637aa43c96638b6154379f Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 30 Jul 2019 16:02:42 -0400 Subject: Added document export and import --- package.json | 4 + .../collectionFreeForm/CollectionFreeFormView.tsx | 39 +++++- src/client/views/nodes/DocumentView.tsx | 10 ++ src/scraping/buxton/scraper.py | 2 + src/server/database.ts | 37 +++++- src/server/index.ts | 142 +++++++++++++++++---- 6 files changed, 202 insertions(+), 32 deletions(-) (limited to 'src/client/views/collections') diff --git a/package.json b/package.json index 37052fde3..b29355738 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,9 @@ "@hig/theme-context": "^2.1.3", "@hig/theme-data": "^2.3.3", "@trendmicro/react-dropdown": "^1.3.0", + "@types/adm-zip": "^0.4.32", "@types/animejs": "^2.0.2", + "@types/archiver": "^3.0.0", "@types/async": "^2.4.1", "@types/bcrypt-nodejs": "0.0.30", "@types/bluebird": "^3.5.25", @@ -105,6 +107,8 @@ "@types/uuid": "^3.4.4", "@types/webpack": "^4.4.25", "@types/youtube": "0.0.38", + "adm-zip": "^0.4.13", + "archiver": "^3.0.3", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 8dac785e1..cbab14976 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -5,7 +5,7 @@ import { Id } from "../../../../new_fields/FieldSymbols"; import { InkField, StrokeData } from "../../../../new_fields/InkField"; import { createSchema, makeInterface } from "../../../../new_fields/Schema"; import { BoolCast, Cast, FieldValue, NumCast, StrCast } from "../../../../new_fields/Types"; -import { emptyFunction, returnOne } from "../../../../Utils"; +import { emptyFunction, returnOne, Utils } from "../../../../Utils"; import { DocumentManager } from "../../../util/DocumentManager"; import { DragManager } from "../../../util/DragManager"; import { HistoryUtil } from "../../../util/History"; @@ -34,12 +34,14 @@ import { CompileScript } from "../../../util/Scripting"; import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; import { library } from "@fortawesome/fontawesome-svg-core"; import { faEye } from "@fortawesome/free-regular-svg-icons"; -import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass } from "@fortawesome/free-solid-svg-icons"; +import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload } from "@fortawesome/free-solid-svg-icons"; import { undo } from "prosemirror-history"; import { number } from "prop-types"; import { ContextMenu } from "../../ContextMenu"; +import { RouteStore } from "../../../../server/RouteStore"; +import { DocServer } from "../../../DocServer"; -library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass); +library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload); export const panZoomSchema = createSchema({ panX: "number", @@ -516,7 +518,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY)); } - onContextMenu = () => { + onContextMenu = (e: React.MouseEvent) => { let layoutItems: ContextMenuProps[] = []; layoutItems.push({ description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`, @@ -561,6 +563,35 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData); }, icon: "paint-brush" }); + ContextMenu.Instance.addItem({ + description: "Import document", icon: "upload", event: () => { + const input = document.createElement("input"); + input.type = "file"; + input.accept = ".zip"; + input.onchange = async _e => { + const files = input.files; + if (!files) return; + const file = files[0]; + let formData = new FormData(); + formData.append('file', file); + formData.append('remap', "true"); + const upload = Utils.prepend("/uploadDoc"); + const response = await fetch(upload, { method: "POST", body: formData }); + const json = await response.json(); + if (json === "error") { + return; + } + const doc = await DocServer.GetRefField(json); + if (!doc || !(doc instanceof Doc)) { + return; + } + const [x, y] = this.props.ScreenToLocalTransform().transformPoint(e.pageX, e.pageY); + doc.x = x, doc.y = y; + this.addDocument(doc, false); + }; + input.click(); + } + }); } diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4b5cf3a43..58e2443c2 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -45,6 +45,7 @@ const JsxParser = require('react-jsx-parser').default; //TODO Why does this need library.add(fa.faTrash); library.add(fa.faShare); +library.add(fa.faDownload); library.add(fa.faExpandArrowsAlt); library.add(fa.faCompressArrowsAlt); library.add(fa.faLayerGroup); @@ -597,6 +598,15 @@ export class DocumentView extends DocComponent(Docu copies.push({ description: "Copy ID", event: () => Utils.CopyText(this.props.Document[Id]), icon: "fingerprint" }); cm.addItem({ description: "Copy...", subitems: copies, icon: "copy" }); } + cm.addItem({ + description: "Download document", icon: "download", event: () => { + const a = document.createElement("a"); + const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); + a.href = url; + a.download = `DocExport-${this.props.Document[Id]}.zip`; + a.click(); + } + }); cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" }); type User = { email: string, userDocumentId: string }; let usersMenu: ContextMenuProps[] = []; diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 1ff0e3b31..8ff7cb223 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -236,6 +236,8 @@ def parse_document(file_name: str): view_guids.append(write_image(pure_name, image)) copyfile(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_o.", 1)) + copyfile(dir_path + "/" + image, dir_path + + "/" + image) os.rename(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") diff --git a/src/server/database.ts b/src/server/database.ts index acb6ce751..a7254fb0c 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -17,7 +17,7 @@ export class Database { }); } - public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) { + public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { if (this.db) { let collection = this.db.collection(collectionName); const prom = this.currentWrites[id]; @@ -30,7 +30,7 @@ export class Database { delete this.currentWrites[id]; } resolve(); - callback(); + callback(err, res); }); }); }; @@ -41,6 +41,30 @@ export class Database { } } + public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { + if (this.db) { + let collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise; + const run = (): Promise => { + return new Promise(resolve => { + collection.replaceOne({ _id: id }, value, { upsert } + , (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + callback(err, res); + }); + }); + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; + } else { + this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName)); + } + } + public delete(query: any, collectionName?: string): Promise; public delete(id: string, collectionName?: string): Promise; public delete(id: any, collectionName = Database.DocumentsCollection) { @@ -126,7 +150,7 @@ export class Database { } } - public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments") { + public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise { if (this.db) { const visited = new Set(); while (ids.length) { @@ -145,7 +169,12 @@ export class Database { } } else { - this.onConnect.push(() => this.visit(ids, fn, collectionName)); + return new Promise(res => { + this.onConnect.push(() => { + this.visit(ids, fn, collectionName); + res(); + }); + }); } } diff --git a/src/server/index.ts b/src/server/index.ts index 230c574cf..2fa5132d0 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -27,6 +27,7 @@ import { Client } from './Client'; import { Database } from './database'; import { MessageStore, Transferable, Types, Diff, Message } from "./Message"; import { RouteStore } from './RouteStore'; +import v4 = require('uuid/v4'); const app = express(); const config = require('../../webpack.config'); import { createCanvas, loadImage, Canvas } from "canvas"; @@ -39,7 +40,10 @@ import c = require("crypto"); import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); +import * as Archiver from 'archiver'; +import * as AdmZip from 'adm-zip'; import { Response } from 'express-serve-static-core'; +import { DocComponent } from '../client/views/DocComponent'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -177,16 +181,21 @@ function msToTime(duration: number) { return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } -app.get("/serializeDoc/:docId", async (req, res) => { - const files: { [name: string]: string[] } = {}; +async function getDocs(id: string) { + const files = new Set(); const docs: { [id: string]: any } = {}; const fn = (doc: any): string[] => { + const id = doc.id; + if (typeof id === "string" && id.endsWith("Proto")) { + //Skip protos + return []; + } const ids: string[] = []; - for (const key in doc) { - if (!doc.hasOwnProperty(key)) { + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { continue; } - const field = doc[key]; + const field = doc.fields[key]; if (field === undefined || field === null) { continue; } @@ -194,7 +203,7 @@ app.get("/serializeDoc/:docId", async (req, res) => { if (field.__type === "proxy" || field.__type === "prefetch_proxy") { ids.push(field.fieldId); } else if (field.__type === "list") { - ids.push(...fn(field.fields)); + ids.push(...fn(field)); } else if (typeof field === "string") { const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; let match: string[] | null; @@ -215,35 +224,120 @@ app.get("/serializeDoc/:docId", async (req, res) => { while ((match = re2.exec(field.Data)) !== null) { const urlString = match[1]; const pathname = new URL(urlString).pathname; - const ext = path.extname(pathname); - const fileName = path.basename(pathname, ext); - let exts = files[fileName]; - if (!exts) { - files[fileName] = exts = []; - } - exts.push(ext); + files.add(pathname); } } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { const url = new URL(field.url); const pathname = url.pathname; - const ext = path.extname(pathname); - const fileName = path.basename(pathname, ext); - let exts = files[fileName]; - if (!exts) { - files[fileName] = exts = []; - } - exts.push(ext); + files.add(pathname); } } - docs[doc.id] = doc; + if (doc.id) { + docs[doc.id] = doc; + } return ids; }; - Database.Instance.visit([req.params.docId], fn); + await Database.Instance.visit([id], fn); + return { id, docs, files }; +} +app.get("/serializeDoc/:docId", async (req, res) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); }); -app.get("/downloadId/:docId", (req, res) => { - res.download(`/serializeDoc/${req.params.docId}`, `DocumentExport.zip`); +app.get("/downloadId/:docId", async (req, res) => { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + const zip = Archiver('zip'); + zip.pipe(res); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); + }); + zip.finalize(); +}); + +app.post("/uploadDoc", (req, res) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); + } + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); + } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } + } + }; + form.parse(req, async (err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path = files[name].path; + const zip = new AdmZip(path); + zip.getEntries().forEach(entry => { + if (!entry.name.startsWith("files/")) return; + zip.extractEntryTo(entry.name, __dirname + RouteStore.public, true, false); + }); + const json = zip.getEntry("doc.json"); + let docs: any; + try { + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + fs.unlink(path, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + }); }); app.get("/whosOnline", (req, res) => { -- cgit v1.2.3-70-g09d2