From 27cafb6eebd1c6229c3377187a5c0043db25ba0a Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 28 Jul 2019 22:58:32 -0400 Subject: templating fixes, context menu undo streamlining --- src/scraping/buxton/scraper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/scraping/buxton/scraper.py') diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 182b22a1a..1ff0e3b31 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -139,7 +139,7 @@ def write_text_doc(content): data_doc = { "_id": data_doc_guid, "fields": { - "proto": protofy("commonImportProto"), + "proto": protofy("textProto"), "data": { "Data": '{"doc":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"' + content + '"}]}]},"selection":{"type":"text","anchor":1,"head":1}' + '}', "__type": "RichTextField" @@ -235,7 +235,7 @@ def parse_document(file_name: str): count += 1 view_guids.append(write_image(pure_name, image)) copyfile(dir_path + "/" + image, dir_path + - "/" + image.replace(".", "_o.", 1)) + "/" + image.replace(".", "_o.", 1)) os.rename(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") -- cgit v1.2.3-70-g09d2 From 26086ee95a9a16486d637aa43c96638b6154379f Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 30 Jul 2019 16:02:42 -0400 Subject: Added document export and import --- package.json | 4 + .../collectionFreeForm/CollectionFreeFormView.tsx | 39 +++++- src/client/views/nodes/DocumentView.tsx | 10 ++ src/scraping/buxton/scraper.py | 2 + src/server/database.ts | 37 +++++- src/server/index.ts | 142 +++++++++++++++++---- 6 files changed, 202 insertions(+), 32 deletions(-) (limited to 'src/scraping/buxton/scraper.py') diff --git a/package.json b/package.json index 37052fde3..b29355738 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,9 @@ "@hig/theme-context": "^2.1.3", "@hig/theme-data": "^2.3.3", "@trendmicro/react-dropdown": "^1.3.0", + "@types/adm-zip": "^0.4.32", "@types/animejs": "^2.0.2", + "@types/archiver": "^3.0.0", "@types/async": "^2.4.1", "@types/bcrypt-nodejs": "0.0.30", "@types/bluebird": "^3.5.25", @@ -105,6 +107,8 @@ "@types/uuid": "^3.4.4", "@types/webpack": "^4.4.25", "@types/youtube": "0.0.38", + "adm-zip": "^0.4.13", + "archiver": "^3.0.3", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 8dac785e1..cbab14976 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -5,7 +5,7 @@ import { Id } from "../../../../new_fields/FieldSymbols"; import { InkField, StrokeData } from "../../../../new_fields/InkField"; import { createSchema, makeInterface } from "../../../../new_fields/Schema"; import { BoolCast, Cast, FieldValue, NumCast, StrCast } from "../../../../new_fields/Types"; -import { emptyFunction, returnOne } from "../../../../Utils"; +import { emptyFunction, returnOne, Utils } from "../../../../Utils"; import { DocumentManager } from "../../../util/DocumentManager"; import { DragManager } from "../../../util/DragManager"; import { HistoryUtil } from "../../../util/History"; @@ -34,12 +34,14 @@ import { CompileScript } from "../../../util/Scripting"; import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; import { library } from "@fortawesome/fontawesome-svg-core"; import { faEye } from "@fortawesome/free-regular-svg-icons"; -import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass } from "@fortawesome/free-solid-svg-icons"; +import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload } from "@fortawesome/free-solid-svg-icons"; import { undo } from "prosemirror-history"; import { number } from "prop-types"; import { ContextMenu } from "../../ContextMenu"; +import { RouteStore } from "../../../../server/RouteStore"; +import { DocServer } from "../../../DocServer"; -library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass); +library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload); export const panZoomSchema = createSchema({ panX: "number", @@ -516,7 +518,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY)); } - onContextMenu = () => { + onContextMenu = (e: React.MouseEvent) => { let layoutItems: ContextMenuProps[] = []; layoutItems.push({ description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`, @@ -561,6 +563,35 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData); }, icon: "paint-brush" }); + ContextMenu.Instance.addItem({ + description: "Import document", icon: "upload", event: () => { + const input = document.createElement("input"); + input.type = "file"; + input.accept = ".zip"; + input.onchange = async _e => { + const files = input.files; + if (!files) return; + const file = files[0]; + let formData = new FormData(); + formData.append('file', file); + formData.append('remap', "true"); + const upload = Utils.prepend("/uploadDoc"); + const response = await fetch(upload, { method: "POST", body: formData }); + const json = await response.json(); + if (json === "error") { + return; + } + const doc = await DocServer.GetRefField(json); + if (!doc || !(doc instanceof Doc)) { + return; + } + const [x, y] = this.props.ScreenToLocalTransform().transformPoint(e.pageX, e.pageY); + doc.x = x, doc.y = y; + this.addDocument(doc, false); + }; + input.click(); + } + }); } diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4b5cf3a43..58e2443c2 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -45,6 +45,7 @@ const JsxParser = require('react-jsx-parser').default; //TODO Why does this need library.add(fa.faTrash); library.add(fa.faShare); +library.add(fa.faDownload); library.add(fa.faExpandArrowsAlt); library.add(fa.faCompressArrowsAlt); library.add(fa.faLayerGroup); @@ -597,6 +598,15 @@ export class DocumentView extends DocComponent(Docu copies.push({ description: "Copy ID", event: () => Utils.CopyText(this.props.Document[Id]), icon: "fingerprint" }); cm.addItem({ description: "Copy...", subitems: copies, icon: "copy" }); } + cm.addItem({ + description: "Download document", icon: "download", event: () => { + const a = document.createElement("a"); + const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); + a.href = url; + a.download = `DocExport-${this.props.Document[Id]}.zip`; + a.click(); + } + }); cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" }); type User = { email: string, userDocumentId: string }; let usersMenu: ContextMenuProps[] = []; diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 1ff0e3b31..8ff7cb223 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -236,6 +236,8 @@ def parse_document(file_name: str): view_guids.append(write_image(pure_name, image)) copyfile(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_o.", 1)) + copyfile(dir_path + "/" + image, dir_path + + "/" + image) os.rename(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") diff --git a/src/server/database.ts b/src/server/database.ts index acb6ce751..a7254fb0c 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -17,7 +17,7 @@ export class Database { }); } - public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) { + public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { if (this.db) { let collection = this.db.collection(collectionName); const prom = this.currentWrites[id]; @@ -30,7 +30,7 @@ export class Database { delete this.currentWrites[id]; } resolve(); - callback(); + callback(err, res); }); }); }; @@ -41,6 +41,30 @@ export class Database { } } + public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { + if (this.db) { + let collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise; + const run = (): Promise => { + return new Promise(resolve => { + collection.replaceOne({ _id: id }, value, { upsert } + , (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + callback(err, res); + }); + }); + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; + } else { + this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName)); + } + } + public delete(query: any, collectionName?: string): Promise; public delete(id: string, collectionName?: string): Promise; public delete(id: any, collectionName = Database.DocumentsCollection) { @@ -126,7 +150,7 @@ export class Database { } } - public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments") { + public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise { if (this.db) { const visited = new Set(); while (ids.length) { @@ -145,7 +169,12 @@ export class Database { } } else { - this.onConnect.push(() => this.visit(ids, fn, collectionName)); + return new Promise(res => { + this.onConnect.push(() => { + this.visit(ids, fn, collectionName); + res(); + }); + }); } } diff --git a/src/server/index.ts b/src/server/index.ts index 230c574cf..2fa5132d0 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -27,6 +27,7 @@ import { Client } from './Client'; import { Database } from './database'; import { MessageStore, Transferable, Types, Diff, Message } from "./Message"; import { RouteStore } from './RouteStore'; +import v4 = require('uuid/v4'); const app = express(); const config = require('../../webpack.config'); import { createCanvas, loadImage, Canvas } from "canvas"; @@ -39,7 +40,10 @@ import c = require("crypto"); import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); +import * as Archiver from 'archiver'; +import * as AdmZip from 'adm-zip'; import { Response } from 'express-serve-static-core'; +import { DocComponent } from '../client/views/DocComponent'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -177,16 +181,21 @@ function msToTime(duration: number) { return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } -app.get("/serializeDoc/:docId", async (req, res) => { - const files: { [name: string]: string[] } = {}; +async function getDocs(id: string) { + const files = new Set(); const docs: { [id: string]: any } = {}; const fn = (doc: any): string[] => { + const id = doc.id; + if (typeof id === "string" && id.endsWith("Proto")) { + //Skip protos + return []; + } const ids: string[] = []; - for (const key in doc) { - if (!doc.hasOwnProperty(key)) { + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { continue; } - const field = doc[key]; + const field = doc.fields[key]; if (field === undefined || field === null) { continue; } @@ -194,7 +203,7 @@ app.get("/serializeDoc/:docId", async (req, res) => { if (field.__type === "proxy" || field.__type === "prefetch_proxy") { ids.push(field.fieldId); } else if (field.__type === "list") { - ids.push(...fn(field.fields)); + ids.push(...fn(field)); } else if (typeof field === "string") { const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; let match: string[] | null; @@ -215,35 +224,120 @@ app.get("/serializeDoc/:docId", async (req, res) => { while ((match = re2.exec(field.Data)) !== null) { const urlString = match[1]; const pathname = new URL(urlString).pathname; - const ext = path.extname(pathname); - const fileName = path.basename(pathname, ext); - let exts = files[fileName]; - if (!exts) { - files[fileName] = exts = []; - } - exts.push(ext); + files.add(pathname); } } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { const url = new URL(field.url); const pathname = url.pathname; - const ext = path.extname(pathname); - const fileName = path.basename(pathname, ext); - let exts = files[fileName]; - if (!exts) { - files[fileName] = exts = []; - } - exts.push(ext); + files.add(pathname); } } - docs[doc.id] = doc; + if (doc.id) { + docs[doc.id] = doc; + } return ids; }; - Database.Instance.visit([req.params.docId], fn); + await Database.Instance.visit([id], fn); + return { id, docs, files }; +} +app.get("/serializeDoc/:docId", async (req, res) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); }); -app.get("/downloadId/:docId", (req, res) => { - res.download(`/serializeDoc/${req.params.docId}`, `DocumentExport.zip`); +app.get("/downloadId/:docId", async (req, res) => { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + const zip = Archiver('zip'); + zip.pipe(res); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); + }); + zip.finalize(); +}); + +app.post("/uploadDoc", (req, res) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); + } + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); + } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } + } + }; + form.parse(req, async (err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path = files[name].path; + const zip = new AdmZip(path); + zip.getEntries().forEach(entry => { + if (!entry.name.startsWith("files/")) return; + zip.extractEntryTo(entry.name, __dirname + RouteStore.public, true, false); + }); + const json = zip.getEntry("doc.json"); + let docs: any; + try { + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + fs.unlink(path, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + }); }); app.get("/whosOnline", (req, res) => { -- cgit v1.2.3-70-g09d2 From 8ca17d379ce7d3cc751408553b6819223d31a3e0 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 30 Jul 2019 16:22:54 -0400 Subject: Fixed import thing --- src/scraping/buxton/scraper.py | 2 -- src/server/index.ts | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'src/scraping/buxton/scraper.py') diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 8ff7cb223..f0f45d8f9 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -237,8 +237,6 @@ def parse_document(file_name: str): copyfile(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_o.", 1)) copyfile(dir_path + "/" + image, dir_path + - "/" + image) - os.rename(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") diff --git a/src/server/index.ts b/src/server/index.ts index 281fc1eb2..1912cf5c1 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -205,6 +205,10 @@ async function getDocs(id: string) { if (field.__type === "proxy" || field.__type === "prefetch_proxy") { ids.push(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + ids.push(field.captures.fieldId); + } } else if (field.__type === "list") { ids.push(...fn(field)); } else if (typeof field === "string") { @@ -293,6 +297,10 @@ app.post("/uploadDoc", (req, res) => { if (field.__type === "proxy" || field.__type === "prefetch_proxy") { field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); + } } else if (field.__type === "list") { mapFn(field); } else if (typeof field === "string") { -- cgit v1.2.3-70-g09d2 From d8c065935a0cb5a89b6d76407b7729f8584424eb Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 6 Aug 2019 22:03:59 -0400 Subject: pivot view --- src/client/views/Main.tsx | 2 + src/client/views/collections/CollectionView.tsx | 1 + .../collectionFreeForm/CollectionFreeFormView.tsx | 32 +++++++++++++++ src/scraping/buxton/scraper.py | 15 +++---- src/scraping/buxton/scripts/initialization.txt | 46 ++++++++++++++++++++++ src/scraping/buxton/scripts/layout.txt | 1 + src/server/index.ts | 29 +++++++++----- src/server/remapUrl.ts | 2 + 8 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 src/scraping/buxton/scripts/initialization.txt create mode 100644 src/scraping/buxton/scripts/layout.txt (limited to 'src/scraping/buxton/scraper.py') diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 1cf13aa74..daa778ed3 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -7,6 +7,7 @@ import { Cast } from "../../new_fields/Types"; import { Doc, DocListCastAsync } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { DocServer } from "../DocServer"; +import { PivotView } from "./collections/collectionFreeForm/CollectionFreeFormView"; let swapDocs = async () => { let oldDoc = await Cast(CurrentUserUtils.UserDocument.linkManagerDoc, Doc); @@ -33,6 +34,7 @@ let swapDocs = async () => { DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email); await Docs.Prototypes.initialize(); await CurrentUserUtils.loadUserDocument(info); + await PivotView.loadLayouts(); // updates old user documents to prevent chrome on tree view. (await Cast(CurrentUserUtils.UserDocument.workspaces, Doc))!.chromeStatus = "disabled"; (await Cast(CurrentUserUtils.UserDocument.recentlyClosed, Doc))!.chromeStatus = "disabled"; diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index f59fee985..59572b7e7 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -97,6 +97,7 @@ export class CollectionView extends React.Component { switch (this.props.Document.viewType) { case CollectionViewType.Freeform: { subItems.push({ description: "Custom", icon: "fingerprint", event: CollectionFreeFormView.AddCustomLayout(this.props.Document, this.props.fieldKey) }); + subItems.push({ description: "Pivot", icon: "fingerprint", event: () => CollectionFreeFormView.SetPivotLayout(this.props.Document) }); break; } } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 7ba13e961..17c4e83b0 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -37,6 +37,7 @@ import "./CollectionFreeFormView.scss"; import { MarqueeView } from "./MarqueeView"; import React = require("react"); import { DocumentType, Docs } from "../../../documents/Documents"; +import { RouteStore } from "../../../../server/RouteStore"; library.add(faEye as any, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBraille, faChalkboard); @@ -48,6 +49,20 @@ export const panZoomSchema = createSchema({ arrangeInit: ScriptField, }); +export namespace PivotView { + + export let arrangeInit: string; + export let arrangeScript: string; + + export async function loadLayouts() { + let response = await fetch(Utils.prepend("/layoutscripts")); + let scripts = JSON.parse(await response.text()); + arrangeInit = scripts[0]; + arrangeScript = scripts[1]; + } + +} + type PanZoomDocument = makeInterface<[typeof panZoomSchema, typeof positionSchema, typeof pageSchema]>; const PanZoomDocument = makeInterface(panZoomSchema, positionSchema, pageSchema); @@ -787,6 +802,23 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { }; } + public static SetPivotLayout = (target: Doc) => { + let setSpecifiedLayoutField = (originalText: string, key: string, params: Record, requiredType?: string) => { + const script = CompileScript(originalText, { + params, + requiredType, + typecheck: false + }); + if (!script.compiled) { + console.log(script.errors.map(error => error.messageText).join("\n")); + return; + } + target[key] = new ScriptField(script); + }; + setSpecifiedLayoutField(PivotView.arrangeInit, "arrangeInit", { collection: "Doc", docs: "Doc[]" }, undefined); + setSpecifiedLayoutField(PivotView.arrangeScript, "arrangeScript", { doc: "Doc", index: "number", collection: "Doc", state: "any", docs: "Doc[]" }, "{x: number, y: number, width?: number, height?: number}"); + } + render() { const easing = () => this.props.Document.panTransformType === "Ease"; Doc.UpdateDocumentExtensionForField(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey); diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index f0f45d8f9..1c19118fd 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -17,6 +17,7 @@ dist = "../../server/public/files" db = MongoClient("localhost", 27017)["Dash"] target_collection = db.newDocuments +target_doc_title = "Workspace 1" schema_guids = [] common_proto_id = "" @@ -69,7 +70,7 @@ def text_doc_map(string_list): return listify(proxify_guids(list(map(guid_map, string_list)))) -def write_schema(parse_results, display_fields, storage_key): +def write_collection(parse_results, display_fields, storage_key, viewType=2): view_guids = parse_results["child_guids"] data_doc = parse_results["schema"] @@ -90,7 +91,7 @@ def write_schema(parse_results, display_fields, storage_key): "zoomBasis": 1, "zIndex": 2, "libraryBrush": False, - "viewType": 2 + "viewType": viewType }, "__type": "Doc" } @@ -237,7 +238,7 @@ def parse_document(file_name: str): copyfile(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_o.", 1)) copyfile(dir_path + "/" + image, dir_path + - "/" + image.replace(".", "_m.", 1)) + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") def sanitize(line): return re.sub("[\n\t]+", "", line).replace(u"\u00A0", " ").replace( @@ -381,22 +382,22 @@ candidates = 0 for file_name in os.listdir(source): if file_name.endswith('.docx'): candidates += 1 - schema_guids.append(write_schema( + schema_guids.append(write_collection( parse_document(file_name), ["title", "data"], "image_data")) print("writing parent schema...") -parent_guid = write_schema({ +parent_guid = write_collection({ "schema": { "_id": guid(), "fields": {}, "__type": "Doc" }, "child_guids": schema_guids -}, ["title", "short_description", "original_price"], "data") +}, ["title", "short_description", "original_price"], "data", 1) print("appending parent schema to main workspace...\n") target_collection.update_one( - {"fields.title": "WS collection 1"}, + {"fields.title": target_doc_title}, {"$push": {"fields.data.fields": {"fieldId": parent_guid, "__type": "proxy"}}} ) diff --git a/src/scraping/buxton/scripts/initialization.txt b/src/scraping/buxton/scripts/initialization.txt new file mode 100644 index 000000000..53f3f0d53 --- /dev/null +++ b/src/scraping/buxton/scripts/initialization.txt @@ -0,0 +1,46 @@ +const field = collection.pivotField || "title"; +const width = collection.pivotWidth || 200; + +const groups = new Map; + +for (const doc of docs) { + const val = doc[field]; + if (val === undefined) continue; + + const l = groups.get(val); + if (l) { + l.push(doc); + } else { + groups.set(val, [doc]); + } + +} + +let minSize = Infinity; + +groups.forEach((val, key) => { + minSize = Math.min(minSize, val.length); +}); + +const numCols = collection.pivotNumColumns || Math.ceil(Math.sqrt(minSize)); + +const docMap = new Map; +const groupNames = []; + +let x = 0; +groups.forEach((val, key) => { + let y = 0; + let xCount = 0; + groupNames.push({type:"text", text:String(key), x, y:width + 50, width: width * 1.25 * numCols, height:100, fontSize:collection.pivotFontSize}); + for (const doc of val) { + docMap.set(doc, {x: x + xCount * width * 1.25, y:-y, width, height:width}); + xCount++; + if (xCount >= numCols) { + xCount = 0; + y += width * 1.25; + } + } + x += width * 1.25 * (numCols + 1); +}); + +return {state:{ map: docMap}, views:groupNames }; \ No newline at end of file diff --git a/src/scraping/buxton/scripts/layout.txt b/src/scraping/buxton/scripts/layout.txt new file mode 100644 index 000000000..46b6dbaac --- /dev/null +++ b/src/scraping/buxton/scripts/layout.txt @@ -0,0 +1 @@ +return state.map.get(doc) \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 1e811c1b2..d735a6d8c 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -148,24 +148,33 @@ app.get("/pull", (req, res) => res.redirect("/"); })); -app.get("/buxton/:clear", (req, res) => { - if (req.params.clear === "true") { - deleteFields().then(() => upload_buxton_docs(res)); - } else { - upload_buxton_docs(res); - } -}); - -let upload_buxton_docs = (res: Response) => { +app.get("/buxton", (req, res) => { let buxton_scraping = path.join(__dirname, '../scraping/buxton'); exec('python scraper.py', { cwd: buxton_scraping }, (err, stdout, sterr) => { if (err) { res.send(err.message); return; } + console.log(stdout); res.redirect("/"); }); -}; +}); + +app.get('/layoutscripts', (req, res) => { + let scripts: string[] = []; + let handler = (err: NodeJS.ErrnoException | null, data: Buffer) => { + if (err) { + console.log(err.message); + return; + } + scripts.push(data.toString()); + if (scripts.length === 2) { + res.send(JSON.stringify(scripts)); + } + }; + fs.readFile(path.join(__dirname, '../scraping/buxton/scripts/initialization.txt'), handler); + fs.readFile(path.join(__dirname, '../scraping/buxton/scripts/layout.txt'), handler); +}); app.get("/version", (req, res) => { exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout, stderr) => { diff --git a/src/server/remapUrl.ts b/src/server/remapUrl.ts index 69c766d56..5218a239a 100644 --- a/src/server/remapUrl.ts +++ b/src/server/remapUrl.ts @@ -2,6 +2,8 @@ import { Database } from "./database"; import { Search } from "./Search"; import * as path from 'path'; +//npx ts-node src/server/remapUrl.ts + const suffixMap: { [type: string]: true } = { "video": true, "pdf": true, -- cgit v1.2.3-70-g09d2 From ebddce0975fa7e224e022cc075eee71abeacbe1d Mon Sep 17 00:00:00 2001 From: Bob Zeleznik Date: Wed, 7 Aug 2019 23:27:21 -0400 Subject: changed pdfs to render regions as collections. changed libraryBrush implementation. --- src/client/views/SearchItem.tsx | 6 ++-- .../views/collections/CollectionDockingView.tsx | 2 +- .../views/collections/CollectionSchemaView.tsx | 2 +- .../views/collections/CollectionTreeView.tsx | 8 ++--- src/client/views/nodes/DocumentView.tsx | 37 ++++++++++----------- src/client/views/nodes/FormattedTextBox.tsx | 2 +- src/client/views/pdf/PDFViewer.tsx | 3 +- src/client/views/pdf/Page.tsx | 9 ++--- .../views/presentationview/PresentationElement.tsx | 4 +-- src/client/views/search/SearchItem.tsx | 38 +++++++--------------- src/new_fields/Doc.ts | 18 ++++++++++ src/new_fields/util.ts | 1 - src/scraping/buxton/scraper.py | 9 ++--- 13 files changed, 65 insertions(+), 74 deletions(-) (limited to 'src/scraping/buxton/scraper.py') diff --git a/src/client/views/SearchItem.tsx b/src/client/views/SearchItem.tsx index 13e4b88f7..fd4b2420d 100644 --- a/src/client/views/SearchItem.tsx +++ b/src/client/views/SearchItem.tsx @@ -37,12 +37,10 @@ export class SearchItem extends React.Component { return ; } onPointerEnter = (e: React.PointerEvent) => { - this.props.doc.libraryBrush = true; - Doc.SetOnPrototype(this.props.doc, "protoBrush", true); + Doc.BrushDoc(this.props.doc); } onPointerLeave = (e: React.PointerEvent) => { - this.props.doc.libraryBrush = false; - Doc.SetOnPrototype(this.props.doc, "protoBrush", false); + Doc.UnBrushDoc(this.props.doc); } collectionRef = React.createRef(); diff --git a/src/client/views/collections/CollectionDockingView.tsx b/src/client/views/collections/CollectionDockingView.tsx index f559480ed..3af92cbe3 100644 --- a/src/client/views/collections/CollectionDockingView.tsx +++ b/src/client/views/collections/CollectionDockingView.tsx @@ -405,7 +405,7 @@ export class CollectionDockingView extends React.Component, dragSpan); + }}>, dragSpan); ReactDOM.render( CollectionDockingView.Instance.AddTab(stack, doc, dataDoc)} />, upDiv); tab.reactComponents = [dragSpan, upDiv]; tab.element.append(dragSpan); diff --git a/src/client/views/collections/CollectionSchemaView.tsx b/src/client/views/collections/CollectionSchemaView.tsx index 75787c0a8..ebfa737be 100644 --- a/src/client/views/collections/CollectionSchemaView.tsx +++ b/src/client/views/collections/CollectionSchemaView.tsx @@ -50,7 +50,7 @@ const columnTypes: Map = new Map([ ["title", ColumnType.String], ["x", ColumnType.Number], ["y", ColumnType.Number], ["width", ColumnType.Number], ["height", ColumnType.Number], ["nativeWidth", ColumnType.Number], ["nativeHeight", ColumnType.Number], ["isPrototype", ColumnType.Boolean], - ["page", ColumnType.Number], ["curPage", ColumnType.Number], ["libraryBrush", ColumnType.Boolean], ["zIndex", ColumnType.Number] + ["page", ColumnType.Number], ["curPage", ColumnType.Number], ["zIndex", ColumnType.Number] ]); @observer diff --git a/src/client/views/collections/CollectionTreeView.tsx b/src/client/views/collections/CollectionTreeView.tsx index 571967743..24bd24d11 100644 --- a/src/client/views/collections/CollectionTreeView.tsx +++ b/src/client/views/collections/CollectionTreeView.tsx @@ -127,19 +127,19 @@ class TreeView extends React.Component { onPointerDown = (e: React.PointerEvent) => e.stopPropagation(); onPointerEnter = (e: React.PointerEvent): void => { - this.props.active() && (this.props.document.libraryBrush = true); + this.props.active() && Doc.BrushDoc(this.dataDoc); if (e.buttons === 1 && SelectionManager.GetIsDragging()) { this._header!.current!.className = "treeViewItem-header"; document.addEventListener("pointermove", this.onDragMove, true); } } onPointerLeave = (e: React.PointerEvent): void => { - this.props.document.libraryBrush = false; + Doc.UnBrushDoc(this.dataDoc); this._header!.current!.className = "treeViewItem-header"; document.removeEventListener("pointermove", this.onDragMove, true); } onDragMove = (e: PointerEvent): void => { - this.props.document.libraryBrush = false; + Doc.UnBrushDoc(this.dataDoc); let x = this.props.ScreenToLocalTransform().transformPoint(e.clientX, e.clientY); let rect = this._header!.current!.getBoundingClientRect(); let bounds = this.props.ScreenToLocalTransform().transformPoint(rect.left, rect.top + rect.height / 2); @@ -359,7 +359,7 @@ class TreeView extends React.Component { return <>
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index a7b4f33db..e911dc47c 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -303,7 +303,7 @@ export class DocumentView extends DocComponent(Docu fullScreenAlias.showCaption = true; this.props.addDocTab(fullScreenAlias, this.dataDoc, "inTab"); SelectionManager.DeselectAll(); - this.props.Document.libraryBrush = false; + Doc.UnBrushDoc(this.props.Document); } else if (CurrentUserUtils.MainDocId !== this.props.Document[Id] && (Math.abs(e.clientX - this._downX) < Utils.DRAG_THRESHOLD && @@ -445,13 +445,11 @@ export class DocumentView extends DocComponent(Docu targetDoc.targetContext = de.data.targetContext; let annotations = await DocListCastAsync(annotationDoc.annotations); if (annotations) { - annotations.forEach(anno => { - anno.target = targetDoc; - }); + annotations.forEach(anno => anno.target = targetDoc); } - let pdfDoc = await Cast(annotationDoc.pdfDoc, Doc); - if (pdfDoc) { - DocUtils.MakeLink(annotationDoc, targetDoc, this.props.ContainingCollectionView!.props.Document, `Annotation from ${StrCast(pdfDoc.title)}`, "", StrCast(pdfDoc.title)); + let annotDoc = await Cast(annotationDoc.annotationOn, Doc); + if (annotDoc) { + DocUtils.MakeLink(annotationDoc, targetDoc, this.props.ContainingCollectionView!.props.Document, `Annotation from ${StrCast(annotDoc.title)}`, "", StrCast(annotDoc.title)); } } if (de.data instanceof DragManager.LinkDragData) { @@ -647,8 +645,8 @@ export class DocumentView extends DocComponent(Docu }); } - onPointerEnter = (e: React.PointerEvent): void => { this.props.Document.libraryBrush = true; }; - onPointerLeave = (e: React.PointerEvent): void => { this.props.Document.libraryBrush = false; }; + onPointerEnter = (e: React.PointerEvent): void => { Doc.BrushDoc(this.props.Document); }; + onPointerLeave = (e: React.PointerEvent): void => { Doc.UnBrushDoc(this.props.Document); }; isSelected = () => SelectionManager.IsSelected(this); @action select = (ctrlPressed: boolean) => { SelectionManager.SelectDoc(this, ctrlPressed); }; @@ -695,22 +693,23 @@ export class DocumentView extends DocComponent(Docu }); } let showTextTitle = showTitle && StrCast(this.layoutDoc.layout).startsWith(" { annoDoc.color = color; annoDoc.type = AnnotationTypes.Region; annoDocs.push(annoDoc); - annoDoc.isBackground = true; + annoDoc.isButton = true; anno.remove(); this.props.addDocument && this.props.addDocument(annoDoc, false); mainAnnoDoc = annoDoc; @@ -208,7 +208,6 @@ export class PDFViewer extends React.Component { mainAnnoDocProto.y = Math.max(minY, 0); mainAnnoDocProto.annotations = new List(annoDocs); } - mainAnnoDocProto.pdfDoc = this.props.Document; mainAnnoDocProto.title = "Annotation on " + StrCast(this.props.Document.title); mainAnnoDocProto.annotationOn = this.props.Document; if (sourceDoc && createLink) { diff --git a/src/client/views/pdf/Page.tsx b/src/client/views/pdf/Page.tsx index 4986f44d5..6bd98cbaa 100644 --- a/src/client/views/pdf/Page.tsx +++ b/src/client/views/pdf/Page.tsx @@ -112,8 +112,8 @@ export default class Page extends React.Component { if (!BoolCast(annotationDoc.linkedToDoc)) { let annotations = await DocListCastAsync(annotationDoc.annotations); annotations && annotations.forEach(anno => anno.target = targetDoc); - let pdfDoc = await Cast(annotationDoc.pdfDoc, Doc); - pdfDoc && DocUtils.MakeLink(annotationDoc, targetDoc, dragData.targetContext, `Annotation from ${StrCast(pdfDoc.title)}`, "", StrCast(pdfDoc.title)) + let parentDoc = await Cast(annotationDoc.annotationOn, Doc); + parentDoc && DocUtils.MakeLink(annotationDoc, targetDoc, dragData.targetContext, `Annotation from ${StrCast(parentDoc.title)}`, "", StrCast(parentDoc.title)) } } }, @@ -144,10 +144,7 @@ export default class Page extends React.Component { onPointerDown = (e: React.PointerEvent): void => { // if alt+left click, drag and annotate if (NumCast(this.props.Document.scale, 1) !== 1) return; - if (e.altKey && e.button === 0) { - e.stopPropagation(); - } - else if (e.button === 0) { + if (!e.altKey && e.button === 0) { PDFMenu.Instance.StartDrag = this.startDrag; PDFMenu.Instance.Highlight = this.highlight; PDFMenu.Instance.Snippet = this.createSnippet; diff --git a/src/client/views/presentationview/PresentationElement.tsx b/src/client/views/presentationview/PresentationElement.tsx index e2d8daea9..d98b66324 100644 --- a/src/client/views/presentationview/PresentationElement.tsx +++ b/src/client/views/presentationview/PresentationElement.tsx @@ -706,7 +706,7 @@ export default class PresentationElement extends React.Component { - this.props.document.libraryBrush = false; + Doc.UnBrushDoc(this.props.document); let x = this.ScreenToLocalListTransform(e.clientX, e.clientY); let rect = this.header!.getBoundingClientRect(); let bounds = this.ScreenToLocalListTransform(rect.left, rect.top + rect.height / 2); @@ -889,7 +889,7 @@ export default class PresentationElement extends React.Component { p.gotoDocument(p.index, NumCast(this.props.mainDocument.selectedDoc)); e.stopPropagation(); }}> diff --git a/src/client/views/search/SearchItem.tsx b/src/client/views/search/SearchItem.tsx index c4af30f5c..8201aa374 100644 --- a/src/client/views/search/SearchItem.tsx +++ b/src/client/views/search/SearchItem.tsx @@ -105,23 +105,11 @@ export interface LinkMenuProps { @observer export class LinkContextMenu extends React.Component { - highlightDoc = (doc: Doc) => { - return () => { - doc.libraryBrush = true; - }; - } + highlightDoc = (doc: Doc) => () => Doc.BrushDoc(doc); - unHighlightDoc = (doc: Doc) => { - return () => { - doc.libraryBrush = false; - }; - } + unHighlightDoc = (doc: Doc) => () => Doc.UnBrushDoc(doc); - getOnClick(col: Doc) { - return () => { - CollectionDockingView.Instance.AddRightSplit(col, undefined); - }; - } + getOnClick = (col: Doc) => () => CollectionDockingView.Instance.AddRightSplit(col, undefined); render() { return ( @@ -286,14 +274,12 @@ export class SearchItem extends React.Component { let doc1 = Cast(this.props.doc.anchor1, Doc, null); let doc2 = Cast(this.props.doc.anchor2, Doc, null); - doc1 && (doc1.libraryBrush = true); - doc2 && (doc2.libraryBrush = true); + Doc.BrushDoc(doc1); + Doc.BrushDoc(doc2); } } else { - let docViews: DocumentView[] = DocumentManager.Instance.getAllDocumentViews(this.props.doc); - docViews.forEach(element => { - element.props.Document.libraryBrush = true; - }); + DocumentManager.Instance.getAllDocumentViews(this.props.doc).forEach(element => + Doc.BrushDoc(element.props.Document)); } } @@ -303,14 +289,12 @@ export class SearchItem extends React.Component { let doc1 = Cast(this.props.doc.anchor1, Doc, null); let doc2 = Cast(this.props.doc.anchor2, Doc, null); - doc1 && (doc1.libraryBrush = false); - doc2 && (doc2.libraryBrush = false); + Doc.UnBrushDoc(doc1); + Doc.UnBrushDoc(doc2); } } else { - let docViews: DocumentView[] = DocumentManager.Instance.getAllDocumentViews(this.props.doc); - docViews.forEach(element => { - element.props.Document.libraryBrush = false; - }); + DocumentManager.Instance.getAllDocumentViews(this.props.doc). + forEach(element => Doc.UnBrushDoc(element.props.Document)); } } diff --git a/src/new_fields/Doc.ts b/src/new_fields/Doc.ts index c01f4e8cf..b3d1dc109 100644 --- a/src/new_fields/Doc.ts +++ b/src/new_fields/Doc.ts @@ -542,4 +542,22 @@ export namespace Doc { } }); } + + export class DocBrush { + @observable BrushedDoc: Doc[] = []; + } + const manager = new DocBrush(); + export function IsBrushed(doc: Doc) { + return manager.BrushedDoc.some(d => Doc.AreProtosEqual(d, doc)); + } + export function IsBrushedDegree(doc: Doc) { + return manager.BrushedDoc.some(d => d === doc) ? 2 : Doc.IsBrushed(doc) ? 1 : 0; + } + export function BrushDoc(doc: Doc) { + if (manager.BrushedDoc.indexOf(doc) === -1) runInAction(() => manager.BrushedDoc.push(doc)); + } + export function UnBrushDoc(doc: Doc) { + let index = manager.BrushedDoc.indexOf(doc); + if (index !== -1) runInAction(() => manager.BrushedDoc.splice(index, 1)); + } } \ No newline at end of file diff --git a/src/new_fields/util.ts b/src/new_fields/util.ts index c6f693f7f..48ae9f216 100644 --- a/src/new_fields/util.ts +++ b/src/new_fields/util.ts @@ -6,7 +6,6 @@ import { RefField } from "./RefField"; import { ObjectField } from "./ObjectField"; import { action } from "mobx"; import { Parent, OnUpdate, Update, Id, SelfProxy, Self } from "./FieldSymbols"; -import { ComputedField } from "./ScriptField"; function _readOnlySetter(): never { throw new Error("Documents can't be modified in read-only mode"); diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index f0f45d8f9..29cb8a256 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -88,8 +88,7 @@ def write_schema(parse_results, display_fields, storage_key): "panX": 0, "panY": 0, "zoomBasis": 1, - "zIndex": 2, - "libraryBrush": False, + "zIndex": 2 "viewType": 2 }, "__type": "Doc" @@ -130,8 +129,7 @@ def write_text_doc(content): "x": 10, "y": 10, "width": 400, - "zIndex": 2, - "libraryBrush": False + "zIndex": 2 }, "__type": "Doc" } @@ -183,8 +181,7 @@ def write_image(folder, name): "x": 10, "y": 10, "width": min(800, native_width), - "zIndex": 2, - "libraryBrush": False + "zIndex": 2 }, "__type": "Doc" } -- cgit v1.2.3-70-g09d2