aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--package.json4
-rw-r--r--src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx39
-rw-r--r--src/client/views/nodes/DocumentView.tsx10
-rw-r--r--src/scraping/buxton/scraper.py2
-rw-r--r--src/server/database.ts37
-rw-r--r--src/server/index.ts142
6 files changed, 202 insertions, 32 deletions
diff --git a/package.json b/package.json
index 37052fde3..b29355738 100644
--- a/package.json
+++ b/package.json
@@ -51,7 +51,9 @@
"@hig/theme-context": "^2.1.3",
"@hig/theme-data": "^2.3.3",
"@trendmicro/react-dropdown": "^1.3.0",
+ "@types/adm-zip": "^0.4.32",
"@types/animejs": "^2.0.2",
+ "@types/archiver": "^3.0.0",
"@types/async": "^2.4.1",
"@types/bcrypt-nodejs": "0.0.30",
"@types/bluebird": "^3.5.25",
@@ -105,6 +107,8 @@
"@types/uuid": "^3.4.4",
"@types/webpack": "^4.4.25",
"@types/youtube": "0.0.38",
+ "adm-zip": "^0.4.13",
+ "archiver": "^3.0.3",
"async": "^2.6.2",
"babel-runtime": "^6.26.0",
"bcrypt-nodejs": "0.0.3",
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 8dac785e1..cbab14976 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -5,7 +5,7 @@ import { Id } from "../../../../new_fields/FieldSymbols";
import { InkField, StrokeData } from "../../../../new_fields/InkField";
import { createSchema, makeInterface } from "../../../../new_fields/Schema";
import { BoolCast, Cast, FieldValue, NumCast, StrCast } from "../../../../new_fields/Types";
-import { emptyFunction, returnOne } from "../../../../Utils";
+import { emptyFunction, returnOne, Utils } from "../../../../Utils";
import { DocumentManager } from "../../../util/DocumentManager";
import { DragManager } from "../../../util/DragManager";
import { HistoryUtil } from "../../../util/History";
@@ -34,12 +34,14 @@ import { CompileScript } from "../../../util/Scripting";
import { CognitiveServices } from "../../../cognitive_services/CognitiveServices";
import { library } from "@fortawesome/fontawesome-svg-core";
import { faEye } from "@fortawesome/free-regular-svg-icons";
-import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass } from "@fortawesome/free-solid-svg-icons";
+import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload } from "@fortawesome/free-solid-svg-icons";
import { undo } from "prosemirror-history";
import { number } from "prop-types";
import { ContextMenu } from "../../ContextMenu";
+import { RouteStore } from "../../../../server/RouteStore";
+import { DocServer } from "../../../DocServer";
-library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass);
+library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload);
export const panZoomSchema = createSchema({
panX: "number",
@@ -516,7 +518,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY));
}
- onContextMenu = () => {
+ onContextMenu = (e: React.MouseEvent) => {
let layoutItems: ContextMenuProps[] = [];
layoutItems.push({
description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`,
@@ -561,6 +563,35 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) {
CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData);
}, icon: "paint-brush"
});
+ ContextMenu.Instance.addItem({
+ description: "Import document", icon: "upload", event: () => {
+ const input = document.createElement("input");
+ input.type = "file";
+ input.accept = ".zip";
+ input.onchange = async _e => {
+ const files = input.files;
+ if (!files) return;
+ const file = files[0];
+ let formData = new FormData();
+ formData.append('file', file);
+ formData.append('remap', "true");
+ const upload = Utils.prepend("/uploadDoc");
+ const response = await fetch(upload, { method: "POST", body: formData });
+ const json = await response.json();
+ if (json === "error") {
+ return;
+ }
+ const doc = await DocServer.GetRefField(json);
+ if (!doc || !(doc instanceof Doc)) {
+ return;
+ }
+ const [x, y] = this.props.ScreenToLocalTransform().transformPoint(e.pageX, e.pageY);
+ doc.x = x, doc.y = y;
+ this.addDocument(doc, false);
+ };
+ input.click();
+ }
+ });
}
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 4b5cf3a43..58e2443c2 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -45,6 +45,7 @@ const JsxParser = require('react-jsx-parser').default; //TODO Why does this need
library.add(fa.faTrash);
library.add(fa.faShare);
+library.add(fa.faDownload);
library.add(fa.faExpandArrowsAlt);
library.add(fa.faCompressArrowsAlt);
library.add(fa.faLayerGroup);
@@ -597,6 +598,15 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
copies.push({ description: "Copy ID", event: () => Utils.CopyText(this.props.Document[Id]), icon: "fingerprint" });
cm.addItem({ description: "Copy...", subitems: copies, icon: "copy" });
}
+ cm.addItem({
+ description: "Download document", icon: "download", event: () => {
+ const a = document.createElement("a");
+ const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`);
+ a.href = url;
+ a.download = `DocExport-${this.props.Document[Id]}.zip`;
+ a.click();
+ }
+ });
cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" });
type User = { email: string, userDocumentId: string };
let usersMenu: ContextMenuProps[] = [];
diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py
index 1ff0e3b31..8ff7cb223 100644
--- a/src/scraping/buxton/scraper.py
+++ b/src/scraping/buxton/scraper.py
@@ -236,6 +236,8 @@ def parse_document(file_name: str):
view_guids.append(write_image(pure_name, image))
copyfile(dir_path + "/" + image, dir_path +
"/" + image.replace(".", "_o.", 1))
+ copyfile(dir_path + "/" + image, dir_path +
+ "/" + image)
os.rename(dir_path + "/" + image, dir_path +
"/" + image.replace(".", "_m.", 1))
print(f"extracted {count} images...")
diff --git a/src/server/database.ts b/src/server/database.ts
index acb6ce751..a7254fb0c 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -17,7 +17,7 @@ export class Database {
});
}
- public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) {
+ public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) {
if (this.db) {
let collection = this.db.collection(collectionName);
const prom = this.currentWrites[id];
@@ -30,7 +30,7 @@ export class Database {
delete this.currentWrites[id];
}
resolve();
- callback();
+ callback(err, res);
});
});
};
@@ -41,6 +41,30 @@ export class Database {
}
}
+ public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) {
+ if (this.db) {
+ let collection = this.db.collection(collectionName);
+ const prom = this.currentWrites[id];
+ let newProm: Promise<void>;
+ const run = (): Promise<void> => {
+ return new Promise<void>(resolve => {
+ collection.replaceOne({ _id: id }, value, { upsert }
+ , (err, res) => {
+ if (this.currentWrites[id] === newProm) {
+ delete this.currentWrites[id];
+ }
+ resolve();
+ callback(err, res);
+ });
+ });
+ };
+ newProm = prom ? prom.then(run) : run();
+ this.currentWrites[id] = newProm;
+ } else {
+ this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName));
+ }
+ }
+
public delete(query: any, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
public delete(id: string, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
public delete(id: any, collectionName = Database.DocumentsCollection) {
@@ -126,7 +150,7 @@ export class Database {
}
}
- public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments") {
+ public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise<void> {
if (this.db) {
const visited = new Set<string>();
while (ids.length) {
@@ -145,7 +169,12 @@ export class Database {
}
} else {
- this.onConnect.push(() => this.visit(ids, fn, collectionName));
+ return new Promise(res => {
+ this.onConnect.push(() => {
+ this.visit(ids, fn, collectionName);
+ res();
+ });
+ });
}
}
diff --git a/src/server/index.ts b/src/server/index.ts
index 230c574cf..2fa5132d0 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -27,6 +27,7 @@ import { Client } from './Client';
import { Database } from './database';
import { MessageStore, Transferable, Types, Diff, Message } from "./Message";
import { RouteStore } from './RouteStore';
+import v4 = require('uuid/v4');
const app = express();
const config = require('../../webpack.config');
import { createCanvas, loadImage, Canvas } from "canvas";
@@ -39,7 +40,10 @@ import c = require("crypto");
import { Search } from './Search';
import { debug } from 'util';
import _ = require('lodash');
+import * as Archiver from 'archiver';
+import * as AdmZip from 'adm-zip';
import { Response } from 'express-serve-static-core';
+import { DocComponent } from '../client/views/DocComponent';
const MongoStore = require('connect-mongo')(session);
const mongoose = require('mongoose');
const probe = require("probe-image-size");
@@ -177,16 +181,21 @@ function msToTime(duration: number) {
return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds;
}
-app.get("/serializeDoc/:docId", async (req, res) => {
- const files: { [name: string]: string[] } = {};
+async function getDocs(id: string) {
+ const files = new Set<string>();
const docs: { [id: string]: any } = {};
const fn = (doc: any): string[] => {
+ const id = doc.id;
+ if (typeof id === "string" && id.endsWith("Proto")) {
+ //Skip protos
+ return [];
+ }
const ids: string[] = [];
- for (const key in doc) {
- if (!doc.hasOwnProperty(key)) {
+ for (const key in doc.fields) {
+ if (!doc.fields.hasOwnProperty(key)) {
continue;
}
- const field = doc[key];
+ const field = doc.fields[key];
if (field === undefined || field === null) {
continue;
}
@@ -194,7 +203,7 @@ app.get("/serializeDoc/:docId", async (req, res) => {
if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
ids.push(field.fieldId);
} else if (field.__type === "list") {
- ids.push(...fn(field.fields));
+ ids.push(...fn(field));
} else if (typeof field === "string") {
const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g;
let match: string[] | null;
@@ -215,35 +224,120 @@ app.get("/serializeDoc/:docId", async (req, res) => {
while ((match = re2.exec(field.Data)) !== null) {
const urlString = match[1];
const pathname = new URL(urlString).pathname;
- const ext = path.extname(pathname);
- const fileName = path.basename(pathname, ext);
- let exts = files[fileName];
- if (!exts) {
- files[fileName] = exts = [];
- }
- exts.push(ext);
+ files.add(pathname);
}
} else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) {
const url = new URL(field.url);
const pathname = url.pathname;
- const ext = path.extname(pathname);
- const fileName = path.basename(pathname, ext);
- let exts = files[fileName];
- if (!exts) {
- files[fileName] = exts = [];
- }
- exts.push(ext);
+ files.add(pathname);
}
}
- docs[doc.id] = doc;
+ if (doc.id) {
+ docs[doc.id] = doc;
+ }
return ids;
};
- Database.Instance.visit([req.params.docId], fn);
+ await Database.Instance.visit([id], fn);
+ return { id, docs, files };
+}
+app.get("/serializeDoc/:docId", async (req, res) => {
+ const { docs, files } = await getDocs(req.params.docId);
+ res.send({ docs, files: Array.from(files) });
});
-app.get("/downloadId/:docId", (req, res) => {
- res.download(`/serializeDoc/${req.params.docId}`, `DocumentExport.zip`);
+app.get("/downloadId/:docId", async (req, res) => {
+ res.set('Content-disposition', `attachment;`);
+ res.set('Content-Type', "application/zip");
+ const { id, docs, files } = await getDocs(req.params.docId);
+ const docString = JSON.stringify({ id, docs });
+ const zip = Archiver('zip');
+ zip.pipe(res);
+ zip.append(docString, { name: "doc.json" });
+ files.forEach(val => {
+ zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) });
+ });
+ zip.finalize();
+});
+
+app.post("/uploadDoc", (req, res) => {
+ let form = new formidable.IncomingForm();
+ form.keepExtensions = true;
+ // let path = req.body.path;
+ const ids: { [id: string]: string } = {};
+ let remap = true;
+ const getId = (id: string): string => {
+ if (!remap) return id;
+ if (id.endsWith("Proto")) return id;
+ if (id in ids) {
+ return ids[id];
+ } else {
+ return ids[id] = v4();
+ }
+ };
+ const mapFn = (doc: any) => {
+ if (doc.id) {
+ doc.id = getId(doc.id);
+ }
+ for (const key in doc.fields) {
+ if (!doc.fields.hasOwnProperty(key)) {
+ continue;
+ }
+ const field = doc.fields[key];
+ if (field === undefined || field === null) {
+ continue;
+ }
+
+ if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+ field.fieldId = getId(field.fieldId);
+ } else if (field.__type === "list") {
+ mapFn(field);
+ } else if (typeof field === "string") {
+ const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g;
+ doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => {
+ return `${p1}${getId(p2)}"`;
+ });
+ } else if (field.__type === "RichTextField") {
+ const re = /("href"\s*:\s*")(.*?)"/g;
+ field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => {
+ return `${p1}${getId(p2)}"`;
+ });
+ }
+ }
+ };
+ form.parse(req, async (err, fields, files) => {
+ remap = fields.remap !== "false";
+ let id: string = "";
+ try {
+ for (const name in files) {
+ const path = files[name].path;
+ const zip = new AdmZip(path);
+ zip.getEntries().forEach(entry => {
+ if (!entry.name.startsWith("files/")) return;
+ zip.extractEntryTo(entry.name, __dirname + RouteStore.public, true, false);
+ });
+ const json = zip.getEntry("doc.json");
+ let docs: any;
+ try {
+ let data = JSON.parse(json.getData().toString("utf8"));
+ docs = data.docs;
+ id = data.id;
+ docs = Object.keys(docs).map(key => docs[key]);
+ docs.forEach(mapFn);
+ await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => {
+ err && console.log(err);
+ res();
+ }, true, "newDocuments"))));
+ } catch (e) { console.log(e); }
+ fs.unlink(path, () => { });
+ }
+ if (id) {
+ res.send(JSON.stringify(getId(id)));
+ } else {
+ res.send(JSON.stringify("error"));
+ }
+ } catch (e) { console.log(e); }
+ });
});
app.get("/whosOnline", (req, res) => {
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 15:24:17 +0000