aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--package.json4
-rw-r--r--src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx39
-rw-r--r--src/client/views/nodes/DocumentView.tsx10
-rw-r--r--src/scraping/buxton/scraper.py2
-rw-r--r--src/server/database.ts56
-rw-r--r--src/server/index.ts163
6 files changed, 268 insertions, 6 deletions
diff --git a/package.json b/package.json
index 23fd9b275..3001fd35e 100644
--- a/package.json
+++ b/package.json
@@ -51,7 +51,9 @@
"@hig/theme-context": "^2.1.3",
"@hig/theme-data": "^2.3.3",
"@trendmicro/react-dropdown": "^1.3.0",
+ "@types/adm-zip": "^0.4.32",
"@types/animejs": "^2.0.2",
+ "@types/archiver": "^3.0.0",
"@types/async": "^2.4.1",
"@types/bcrypt-nodejs": "0.0.30",
"@types/bluebird": "^3.5.25",
@@ -105,6 +107,8 @@
"@types/uuid": "^3.4.4",
"@types/webpack": "^4.4.25",
"@types/youtube": "0.0.38",
+ "adm-zip": "^0.4.13",
+ "archiver": "^3.0.3",
"async": "^2.6.2",
"babel-runtime": "^6.26.0",
"bcrypt-nodejs": "0.0.3",
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 8dac785e1..cbab14976 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -5,7 +5,7 @@ import { Id } from "../../../../new_fields/FieldSymbols";
import { InkField, StrokeData } from "../../../../new_fields/InkField";
import { createSchema, makeInterface } from "../../../../new_fields/Schema";
import { BoolCast, Cast, FieldValue, NumCast, StrCast } from "../../../../new_fields/Types";
-import { emptyFunction, returnOne } from "../../../../Utils";
+import { emptyFunction, returnOne, Utils } from "../../../../Utils";
import { DocumentManager } from "../../../util/DocumentManager";
import { DragManager } from "../../../util/DragManager";
import { HistoryUtil } from "../../../util/History";
@@ -34,12 +34,14 @@ import { CompileScript } from "../../../util/Scripting";
import { CognitiveServices } from "../../../cognitive_services/CognitiveServices";
import { library } from "@fortawesome/fontawesome-svg-core";
import { faEye } from "@fortawesome/free-regular-svg-icons";
-import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass } from "@fortawesome/free-solid-svg-icons";
+import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload } from "@fortawesome/free-solid-svg-icons";
import { undo } from "prosemirror-history";
import { number } from "prop-types";
import { ContextMenu } from "../../ContextMenu";
+import { RouteStore } from "../../../../server/RouteStore";
+import { DocServer } from "../../../DocServer";
-library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass);
+library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload);
export const panZoomSchema = createSchema({
panX: "number",
@@ -516,7 +518,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY));
}
- onContextMenu = () => {
+ onContextMenu = (e: React.MouseEvent) => {
let layoutItems: ContextMenuProps[] = [];
layoutItems.push({
description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`,
@@ -561,6 +563,35 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData);
}, icon: "paint-brush"
});
+ ContextMenu.Instance.addItem({
+ description: "Import document", icon: "upload", event: () => {
+ const input = document.createElement("input");
+ input.type = "file";
+ input.accept = ".zip";
+ input.onchange = async _e => {
+ const files = input.files;
+ if (!files) return;
+ const file = files[0];
+ let formData = new FormData();
+ formData.append('file', file);
+ formData.append('remap', "true");
+ const upload = Utils.prepend("/uploadDoc");
+ const response = await fetch(upload, { method: "POST", body: formData });
+ const json = await response.json();
+ if (json === "error") {
+ return;
+ }
+ const doc = await DocServer.GetRefField(json);
+ if (!doc || !(doc instanceof Doc)) {
+ return;
+ }
+ const [x, y] = this.props.ScreenToLocalTransform().transformPoint(e.pageX, e.pageY);
+ doc.x = x, doc.y = y;
+ this.addDocument(doc, false);
+ };
+ input.click();
+ }
+ });
}
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index dc56c1c8f..51662274d 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -47,6 +47,7 @@ const JsxParser = require('react-jsx-parser').default; //TODO Why does this need
library.add(fa.faTrash);
library.add(fa.faShare);
+library.add(fa.faDownload);
library.add(fa.faExpandArrowsAlt);
library.add(fa.faCompressArrowsAlt);
library.add(fa.faLayerGroup);
@@ -605,6 +606,15 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
copies.push({ description: "Copy ID", event: () => Utils.CopyText(this.props.Document[Id]), icon: "fingerprint" });
cm.addItem({ description: "Copy...", subitems: copies, icon: "copy" });
}
+ cm.addItem({
+ description: "Download document", icon: "download", event: () => {
+ const a = document.createElement("a");
+ const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`);
+ a.href = url;
+ a.download = `DocExport-${this.props.Document[Id]}.zip`;
+ a.click();
+ }
+ });
cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" });
type User = { email: string, userDocumentId: string };
let usersMenu: ContextMenuProps[] = [];
diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py
index 1ff0e3b31..8ff7cb223 100644
--- a/src/scraping/buxton/scraper.py
+++ b/src/scraping/buxton/scraper.py
@@ -236,6 +236,8 @@ def parse_document(file_name: str):
view_guids.append(write_image(pure_name, image))
copyfile(dir_path + "/" + image, dir_path +
"/" + image.replace(".", "_o.", 1))
+ copyfile(dir_path + "/" + image, dir_path +
+ "/" + image)
os.rename(dir_path + "/" + image, dir_path +
"/" + image.replace(".", "_m.", 1))
print(f"extracted {count} images...")
diff --git a/src/server/database.ts b/src/server/database.ts
index 7f5331998..a7254fb0c 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -17,7 +17,7 @@ export class Database {
});
}
- public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) {
+ public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) {
if (this.db) {
let collection = this.db.collection(collectionName);
const prom = this.currentWrites[id];
@@ -30,7 +30,7 @@ export class Database {
delete this.currentWrites[id];
}
resolve();
- callback();
+ callback(err, res);
});
});
};
@@ -41,6 +41,30 @@ export class Database {
}
}
+ public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) {
+ if (this.db) {
+ let collection = this.db.collection(collectionName);
+ const prom = this.currentWrites[id];
+ let newProm: Promise<void>;
+ const run = (): Promise<void> => {
+ return new Promise<void>(resolve => {
+ collection.replaceOne({ _id: id }, value, { upsert }
+ , (err, res) => {
+ if (this.currentWrites[id] === newProm) {
+ delete this.currentWrites[id];
+ }
+ resolve();
+ callback(err, res);
+ });
+ });
+ };
+ newProm = prom ? prom.then(run) : run();
+ this.currentWrites[id] = newProm;
+ } else {
+ this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName));
+ }
+ }
+
public delete(query: any, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
public delete(id: string, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
public delete(id: any, collectionName = Database.DocumentsCollection) {
@@ -126,6 +150,34 @@ export class Database {
}
}
+ public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise<void> {
+ if (this.db) {
+ const visited = new Set<string>();
+ while (ids.length) {
+ const count = Math.min(ids.length, 1000);
+ const index = ids.length - count;
+ const fetchIds = ids.splice(index, count).filter(id => !visited.has(id));
+ if (!fetchIds.length) {
+ continue;
+ }
+ const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments"));
+ for (const doc of docs) {
+ const id = doc.id;
+ visited.add(id);
+ ids.push(...fn(doc));
+ }
+ }
+
+ } else {
+ return new Promise(res => {
+ this.onConnect.push(() => {
+ this.visit(ids, fn, collectionName);
+ res();
+ });
+ });
+ }
+ }
+
public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise<mongodb.Cursor> {
if (this.db) {
let cursor = this.db.collection(collectionName).find(query);
diff --git a/src/server/index.ts b/src/server/index.ts
index 080b50ada..281fc1eb2 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -27,6 +27,7 @@ import { Client } from './Client';
import { Database } from './database';
import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message";
import { RouteStore } from './RouteStore';
+import v4 = require('uuid/v4');
const app = express();
const config = require('../../webpack.config');
import { createCanvas, loadImage, Canvas } from "canvas";
@@ -39,8 +40,11 @@ import c = require("crypto");
import { Search } from './Search';
import { debug } from 'util';
import _ = require('lodash');
+import * as Archiver from 'archiver';
+import * as AdmZip from 'adm-zip';
import * as YoutubeApi from './youtubeApi/youtubeApiSample.js';
import { Response } from 'express-serve-static-core';
+import { DocComponent } from '../client/views/DocComponent';
const MongoStore = require('connect-mongo')(session);
const mongoose = require('mongoose');
const probe = require("probe-image-size");
@@ -180,6 +184,165 @@ function msToTime(duration: number) {
return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds;
}
+async function getDocs(id: string) {
+ const files = new Set<string>();
+ const docs: { [id: string]: any } = {};
+ const fn = (doc: any): string[] => {
+ const id = doc.id;
+ if (typeof id === "string" && id.endsWith("Proto")) {
+ //Skip protos
+ return [];
+ }
+ const ids: string[] = [];
+ for (const key in doc.fields) {
+ if (!doc.fields.hasOwnProperty(key)) {
+ continue;
+ }
+ const field = doc.fields[key];
+ if (field === undefined || field === null) {
+ continue;
+ }
+
+ if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+ ids.push(field.fieldId);
+ } else if (field.__type === "list") {
+ ids.push(...fn(field));
+ } else if (typeof field === "string") {
+ const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g;
+ let match: string[] | null;
+ while ((match = re.exec(field)) !== null) {
+ ids.push(match[1]);
+ }
+ } else if (field.__type === "RichTextField") {
+ const re = /"href"\s*:\s*"(.*?)"/g;
+ let match: string[] | null;
+ while ((match = re.exec(field.Data)) !== null) {
+ const urlString = match[1];
+ const split = new URL(urlString).pathname.split("doc/");
+ if (split.length > 1) {
+ ids.push(split[split.length - 1]);
+ }
+ }
+ const re2 = /"src"\s*:\s*"(.*?)"/g;
+ while ((match = re2.exec(field.Data)) !== null) {
+ const urlString = match[1];
+ const pathname = new URL(urlString).pathname;
+ files.add(pathname);
+ }
+ } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) {
+ const url = new URL(field.url);
+ const pathname = url.pathname;
+ files.add(pathname);
+ }
+ }
+
+ if (doc.id) {
+ docs[doc.id] = doc;
+ }
+ return ids;
+ };
+ await Database.Instance.visit([id], fn);
+ return { id, docs, files };
+}
+app.get("/serializeDoc/:docId", async (req, res) => {
+ const { docs, files } = await getDocs(req.params.docId);
+ res.send({ docs, files: Array.from(files) });
+});
+
+app.get("/downloadId/:docId", async (req, res) => {
+ res.set('Content-disposition', `attachment;`);
+ res.set('Content-Type', "application/zip");
+ const { id, docs, files } = await getDocs(req.params.docId);
+ const docString = JSON.stringify({ id, docs });
+ const zip = Archiver('zip');
+ zip.pipe(res);
+ zip.append(docString, { name: "doc.json" });
+ files.forEach(val => {
+ zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) });
+ });
+ zip.finalize();
+});
+
+app.post("/uploadDoc", (req, res) => {
+ let form = new formidable.IncomingForm();
+ form.keepExtensions = true;
+ // let path = req.body.path;
+ const ids: { [id: string]: string } = {};
+ let remap = true;
+ const getId = (id: string): string => {
+ if (!remap) return id;
+ if (id.endsWith("Proto")) return id;
+ if (id in ids) {
+ return ids[id];
+ } else {
+ return ids[id] = v4();
+ }
+ };
+ const mapFn = (doc: any) => {
+ if (doc.id) {
+ doc.id = getId(doc.id);
+ }
+ for (const key in doc.fields) {
+ if (!doc.fields.hasOwnProperty(key)) {
+ continue;
+ }
+ const field = doc.fields[key];
+ if (field === undefined || field === null) {
+ continue;
+ }
+
+ if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+ field.fieldId = getId(field.fieldId);
+ } else if (field.__type === "list") {
+ mapFn(field);
+ } else if (typeof field === "string") {
+ const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g;
+ doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => {
+ return `${p1}${getId(p2)}"`;
+ });
+ } else if (field.__type === "RichTextField") {
+ const re = /("href"\s*:\s*")(.*?)"/g;
+ field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => {
+ return `${p1}${getId(p2)}"`;
+ });
+ }
+ }
+ };
+ form.parse(req, async (err, fields, files) => {
+ remap = fields.remap !== "false";
+ let id: string = "";
+ try {
+ for (const name in files) {
+ const path = files[name].path;
+ const zip = new AdmZip(path);
+ zip.getEntries().forEach(entry => {
+ if (!entry.name.startsWith("files/")) return;
+ zip.extractEntryTo(entry.name, __dirname + RouteStore.public, true, false);
+ });
+ const json = zip.getEntry("doc.json");
+ let docs: any;
+ try {
+ let data = JSON.parse(json.getData().toString("utf8"));
+ docs = data.docs;
+ id = data.id;
+ docs = Object.keys(docs).map(key => docs[key]);
+ docs.forEach(mapFn);
+ await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => {
+ err && console.log(err);
+ res();
+ }, true, "newDocuments"))));
+ } catch (e) { console.log(e); }
+ fs.unlink(path, () => { });
+ }
+ if (id) {
+ res.send(JSON.stringify(getId(id)));
+ } else {
+ res.send(JSON.stringify("error"));
+ }
+ } catch (e) { console.log(e); }
+ });
+});
+
app.get("/whosOnline", (req, res) => {
let users: any = { active: {}, inactive: {} };
const now = Date.now();