From d6cef0815815c2587b9cc791e6a37c742aba1b45 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 25 Jul 2019 03:31:34 -0400 Subject: cognitive services refactor, buxton python script fixes, covered up imagebox context menu bug --- src/scraping/buxton/scraper.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'src/scraping/buxton/scraper.py') diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 700269727..14490cfe4 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -15,6 +15,7 @@ source = "./source" dist = "../../server/public/files" db = MongoClient("localhost", 27017)["Dash"] +target_collection = db.newDocuments schema_guids = [] @@ -84,7 +85,7 @@ def write_schema(parse_results, display_fields, storage_key): "height": 600, "panX": 0, "panY": 0, - "zoomBasis": 0.5, + "zoomBasis": 1, "zIndex": 2, "libraryBrush": False, "viewType": 2 @@ -106,8 +107,8 @@ def write_schema(parse_results, display_fields, storage_key): fields["isPrototype"] = True fields["page"] = -1 - db.newDocuments.insert_one(data_doc) - db.newDocuments.insert_one(view_doc) + target_collection.insert_one(data_doc) + target_collection.insert_one(view_doc) data_doc_guid = data_doc["_id"] print(f"inserted view document ({view_doc_guid})") @@ -158,8 +159,8 @@ def write_text_doc(content): "__type": "Doc" } - db.newDocuments.insert_one(view_doc) - db.newDocuments.insert_one(data_doc) + target_collection.insert_one(view_doc) + target_collection.insert_one(data_doc) return view_doc_guid @@ -209,8 +210,8 @@ def write_image(folder, name): "__type": "Doc" } - db.newDocuments.insert_one(view_doc) - db.newDocuments.insert_one(data_doc) + target_collection.insert_one(view_doc) + target_collection.insert_one(data_doc) return view_doc_guid @@ -372,7 +373,7 @@ parent_guid = write_schema({ }, ["title", "short_description", "original_price"], "data") print("appending parent schema to main workspace...\n") -db.newDocuments.update_one( +target_collection.update_one( {"fields.title": "WS collection 1"}, {"$push": {"fields.data.fields": {"fieldId": parent_guid, "__type": "proxy"}}} ) -- cgit v1.2.3-70-g09d2 From e1c7add158ce245ce6cb557177986b31fe107dd8 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 25 Jul 2019 18:56:46 -0400 Subject: scraping common proto tweak --- src/scraping/buxton/scraper.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) (limited to 'src/scraping/buxton/scraper.py') diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 14490cfe4..48b8fe3fa 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -17,6 +17,7 @@ dist = "../../server/public/files" db = MongoClient("localhost", 27017)["Dash"] target_collection = db.newDocuments schema_guids = [] +common_proto_id = "" def extract_links(fileName): @@ -93,7 +94,7 @@ def write_schema(parse_results, display_fields, storage_key): "__type": "Doc" } - fields["proto"] = protofy("collectionProto") + fields["proto"] = protofy(common_proto_id) fields[storage_key] = listify(proxify_guids(view_guids)) fields["schemaColumns"] = listify(display_fields) fields["backgroundColor"] = "white" @@ -137,7 +138,7 @@ def write_text_doc(content): data_doc = { "_id": data_doc_guid, "fields": { - "proto": protofy("textProto"), + "proto": protofy("commonImportProto"), "data": { "Data": '{"doc":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"' + content + '"}]}]},"selection":{"type":"text","anchor":1,"head":1}' + '}', "__type": "RichTextField" @@ -348,6 +349,22 @@ def proxify_guids(guids): return list(map(lambda guid: {"fieldId": guid, "__type": "proxy"}, guids)) +def write_common_proto(): + id = guid() + common_proto = { + "_id": id, + "fields": { + "proto": protofy("collectionProto"), + "title": "Common Import Proto", + }, + "__type": "Doc" + } + + target_collection.insert_one(common_proto) + + return id + + if os.path.exists(dist): shutil.rmtree(dist) while os.path.exists(dist): @@ -355,6 +372,8 @@ while os.path.exists(dist): os.mkdir(dist) mkdir_if_absent(source) +common_proto_id = write_common_proto() + candidates = 0 for file_name in os.listdir(source): if file_name.endswith('.docx'): -- cgit v1.2.3-70-g09d2 From c1377587f27044d89ec84befa9953de627d49873 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Sat, 27 Jul 2019 14:38:53 -0400 Subject: Fixed up buxton scraper to get fullscreen images and remove bad images --- src/scraping/buxton/scraper.py | 3 +++ .../buxton/source/Bill_Notes_Bill_Notes_CyKey.docx | Bin 1675500 -> 1561425 bytes .../buxton/source/Bill_Notes_Braun_T3.docx | Bin 1671968 -> 1510917 bytes .../buxton/source/Bill_Notes_CasioC801.docx | Bin 574664 -> 413861 bytes .../buxton/source/Bill_Notes_Casio_Mini.docx | Bin 581069 -> 467304 bytes .../source/Bill_Notes_FingerWorks_Prototype.docx | Bin 585090 -> 423384 bytes .../source/Bill_Notes_Fingerworks_TouchStream.docx | Bin 1722555 -> 1558473 bytes src/scraping/buxton/source/Bill_Notes_FrogPad.docx | Bin 840173 -> 679241 bytes .../buxton/source/Bill_Notes_Gavilan_SC.docx | Bin 1695290 -> 1531689 bytes .../source/Bill_Notes_Grandjean_Stenotype.docx | Bin 2094142 -> 1933004 bytes src/scraping/buxton/source/Bill_Notes_Matias.docx | Bin 590407 -> 476141 bytes .../buxton/source/Bill_Notes_MousePen.docx | Bin 505322 -> 344083 bytes src/scraping/buxton/source/Bill_Notes_NewO.docx | Bin 2264571 -> 2150143 bytes src/scraping/buxton/source/Bill_Notes_OLPC.docx | Bin 6883659 -> 6721592 bytes src/scraping/buxton/source/Bill_Notes_PARCkbd.docx | Bin 631959 -> 517484 bytes .../source/Bill_Notes_Philco_Mystery_Control.docx | Bin 1994439 -> 1880816 bytes .../buxton/source/Bill_Notes_TASA_Kbd.docx | Bin 461199 -> 347612 bytes src/scraping/buxton/source/Bill_Notes_The_Tap.docx | Bin 711321 -> 597382 bytes 18 files changed, 3 insertions(+) (limited to 'src/scraping/buxton/scraper.py') diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 48b8fe3fa..182b22a1a 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -1,4 +1,5 @@ import os +from shutil import copyfile import docx2txt from docx import Document from docx.opc.constants import RELATIONSHIP_TYPE as RT @@ -233,6 +234,8 @@ def parse_document(file_name: str): for image in os.listdir(dir_path): count += 1 view_guids.append(write_image(pure_name, image)) + copyfile(dir_path + "/" + image, dir_path + + "/" + image.replace(".", "_o.", 1)) os.rename(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") diff --git a/src/scraping/buxton/source/Bill_Notes_Bill_Notes_CyKey.docx b/src/scraping/buxton/source/Bill_Notes_Bill_Notes_CyKey.docx index 06094b4d3..649d636e3 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_Bill_Notes_CyKey.docx and b/src/scraping/buxton/source/Bill_Notes_Bill_Notes_CyKey.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_Braun_T3.docx b/src/scraping/buxton/source/Bill_Notes_Braun_T3.docx index 356697092..b00080e08 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_Braun_T3.docx and b/src/scraping/buxton/source/Bill_Notes_Braun_T3.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_CasioC801.docx b/src/scraping/buxton/source/Bill_Notes_CasioC801.docx index cd89fb97b..510a006e0 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_CasioC801.docx and b/src/scraping/buxton/source/Bill_Notes_CasioC801.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_Casio_Mini.docx b/src/scraping/buxton/source/Bill_Notes_Casio_Mini.docx index a503cddfc..cea9e7b69 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_Casio_Mini.docx and b/src/scraping/buxton/source/Bill_Notes_Casio_Mini.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_FingerWorks_Prototype.docx b/src/scraping/buxton/source/Bill_Notes_FingerWorks_Prototype.docx index 4d13a8cf5..f53402a06 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_FingerWorks_Prototype.docx and b/src/scraping/buxton/source/Bill_Notes_FingerWorks_Prototype.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_Fingerworks_TouchStream.docx b/src/scraping/buxton/source/Bill_Notes_Fingerworks_TouchStream.docx index 578a1be08..0eec89949 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_Fingerworks_TouchStream.docx and b/src/scraping/buxton/source/Bill_Notes_Fingerworks_TouchStream.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_FrogPad.docx b/src/scraping/buxton/source/Bill_Notes_FrogPad.docx index d01e1bf5c..ba80c1959 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_FrogPad.docx and b/src/scraping/buxton/source/Bill_Notes_FrogPad.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_Gavilan_SC.docx b/src/scraping/buxton/source/Bill_Notes_Gavilan_SC.docx index 7bd28b376..8558a4e13 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_Gavilan_SC.docx and b/src/scraping/buxton/source/Bill_Notes_Gavilan_SC.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_Grandjean_Stenotype.docx b/src/scraping/buxton/source/Bill_Notes_Grandjean_Stenotype.docx index 0615c4953..09e17f971 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_Grandjean_Stenotype.docx and b/src/scraping/buxton/source/Bill_Notes_Grandjean_Stenotype.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_Matias.docx b/src/scraping/buxton/source/Bill_Notes_Matias.docx index 547603256..d2d014bbe 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_Matias.docx and b/src/scraping/buxton/source/Bill_Notes_Matias.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_MousePen.docx b/src/scraping/buxton/source/Bill_Notes_MousePen.docx index 4e1056636..cd0b3eab3 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_MousePen.docx and b/src/scraping/buxton/source/Bill_Notes_MousePen.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_NewO.docx b/src/scraping/buxton/source/Bill_Notes_NewO.docx index a514926d2..2f4a04e81 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_NewO.docx and b/src/scraping/buxton/source/Bill_Notes_NewO.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_OLPC.docx b/src/scraping/buxton/source/Bill_Notes_OLPC.docx index bfca0a9bb..7a636e2d6 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_OLPC.docx and b/src/scraping/buxton/source/Bill_Notes_OLPC.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_PARCkbd.docx b/src/scraping/buxton/source/Bill_Notes_PARCkbd.docx index c0cf6ba9a..3038de363 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_PARCkbd.docx and b/src/scraping/buxton/source/Bill_Notes_PARCkbd.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_Philco_Mystery_Control.docx b/src/scraping/buxton/source/Bill_Notes_Philco_Mystery_Control.docx index ad06903f3..af72fa662 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_Philco_Mystery_Control.docx and b/src/scraping/buxton/source/Bill_Notes_Philco_Mystery_Control.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_TASA_Kbd.docx b/src/scraping/buxton/source/Bill_Notes_TASA_Kbd.docx index e4c659de9..5c2eb8d7f 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_TASA_Kbd.docx and b/src/scraping/buxton/source/Bill_Notes_TASA_Kbd.docx differ diff --git a/src/scraping/buxton/source/Bill_Notes_The_Tap.docx b/src/scraping/buxton/source/Bill_Notes_The_Tap.docx index 8ceebc71e..c9ee2eaea 100644 Binary files a/src/scraping/buxton/source/Bill_Notes_The_Tap.docx and b/src/scraping/buxton/source/Bill_Notes_The_Tap.docx differ -- cgit v1.2.3-70-g09d2 From 27cafb6eebd1c6229c3377187a5c0043db25ba0a Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 28 Jul 2019 22:58:32 -0400 Subject: templating fixes, context menu undo streamlining --- src/.DS_Store | Bin 6148 -> 6148 bytes src/client/documents/Documents.ts | 5 ++--- src/client/views/ContextMenuItem.tsx | 4 ++-- src/client/views/collections/CollectionTreeView.tsx | 10 +++++----- src/client/views/collections/CollectionView.tsx | 14 +++++++------- .../collectionFreeForm/CollectionFreeFormView.tsx | 2 +- src/client/views/nodes/KeyValueBox.tsx | 17 ++++++++++++----- src/scraping/buxton/scraper.py | 4 ++-- 8 files changed, 31 insertions(+), 25 deletions(-) (limited to 'src/scraping/buxton/scraper.py') diff --git a/src/.DS_Store b/src/.DS_Store index 071dafa1e..c544bc837 100644 Binary files a/src/.DS_Store and b/src/.DS_Store differ diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 3859f2255..30a114e33 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -163,7 +163,6 @@ export namespace Docs { [DocumentType.LINKDOC, { data: new List(), layout: { view: EmptyBox }, - options: {} }], [DocumentType.BUTTON, { layout: { view: ButtonBox }, @@ -414,8 +413,8 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.COL), new List(documents), { schemaColumns: new List(["title"]), ...options, viewType: CollectionViewType.Tree }); } - export function StackingDocument(documents: Array, options: DocumentOptions) { - return InstanceFromProto(Prototypes.get(DocumentType.COL), new List(documents), { schemaColumns: new List(["title"]), ...options, viewType: CollectionViewType.Stacking }); + export function StackingDocument(documents: Array, options: DocumentOptions, viewType: CollectionViewType = CollectionViewType.Stacking) { + return InstanceFromProto(Prototypes.get(DocumentType.COL), new List(documents), { schemaColumns: new List(["title"]), ...options, viewType: viewType }); } export function ButtonDocument(options?: DocumentOptions) { diff --git a/src/client/views/ContextMenuItem.tsx b/src/client/views/ContextMenuItem.tsx index 4ff0fe388..a1787e78f 100644 --- a/src/client/views/ContextMenuItem.tsx +++ b/src/client/views/ContextMenuItem.tsx @@ -37,13 +37,13 @@ export class ContextMenuItem extends React.Component) => { + handleEvent = async (e: React.MouseEvent) => { if ("event" in this.props) { let batch: UndoManager.Batch | undefined; if (this.props.undoable !== false) { batch = UndoManager.StartBatch(`Context menu event: ${this.props.description}`); } - this.props.event(); + await this.props.event(); batch && batch.end(); this.props.closeMenu && this.props.closeMenu(); } diff --git a/src/client/views/collections/CollectionTreeView.tsx b/src/client/views/collections/CollectionTreeView.tsx index 1c7fe4bee..3eab109bc 100644 --- a/src/client/views/collections/CollectionTreeView.tsx +++ b/src/client/views/collections/CollectionTreeView.tsx @@ -237,10 +237,10 @@ class TreeView extends React.Component { if (DocumentManager.Instance.getDocumentViews(this.dataDoc).length) { ContextMenu.Instance.addItem({ description: "Focus", event: () => DocumentManager.Instance.getDocumentViews(this.dataDoc).map(view => view.props.focus(this.props.document, true)), icon: "camera" }); } - ContextMenu.Instance.addItem({ description: "Delete Item", event: undoBatch(() => this.props.deleteDoc(this.props.document)), icon: "trash-alt" }); + ContextMenu.Instance.addItem({ description: "Delete Item", event: () => this.props.deleteDoc(this.props.document), icon: "trash-alt" }); } else { - ContextMenu.Instance.addItem({ description: "Open as Workspace", event: undoBatch(() => MainView.Instance.openWorkspace(this.dataDoc)), icon: "caret-square-right" }); - ContextMenu.Instance.addItem({ description: "Delete Workspace", event: undoBatch(() => this.props.deleteDoc(this.props.document)), icon: "trash-alt" }); + ContextMenu.Instance.addItem({ description: "Open as Workspace", event: () => MainView.Instance.openWorkspace(this.dataDoc), icon: "caret-square-right" }); + ContextMenu.Instance.addItem({ description: "Delete Workspace", event: () => this.props.deleteDoc(this.props.document), icon: "trash-alt" }); } ContextMenu.Instance.addItem({ description: "Open Fields", event: () => { let kvp = Docs.Create.KVPDocument(this.props.document, { width: 300, height: 300 }); this.props.addDocTab(kvp, this.props.dataDoc ? this.props.dataDoc : kvp, "onRight"); }, icon: "layer-group" }); ContextMenu.Instance.displayMenu(e.pageX > 156 ? e.pageX - 156 : 0, e.pageY - 15); @@ -520,8 +520,8 @@ export class CollectionTreeView extends CollectionSubView(Document) { onContextMenu = (e: React.MouseEvent): void => { // need to test if propagation has stopped because GoldenLayout forces a parallel react hierarchy to be created for its top-level layout if (!e.isPropagationStopped() && this.props.Document.workspaceLibrary) { // excludeFromLibrary means this is the user document - ContextMenu.Instance.addItem({ description: "Create Workspace", event: undoBatch(() => MainView.Instance.createNewWorkspace()), icon: "plus" }); - ContextMenu.Instance.addItem({ description: "Delete Workspace", event: undoBatch(() => this.remove(this.props.Document)), icon: "minus" }); + ContextMenu.Instance.addItem({ description: "Create Workspace", event: () => MainView.Instance.createNewWorkspace(), icon: "plus" }); + ContextMenu.Instance.addItem({ description: "Delete Workspace", event: () => this.remove(this.props.Document), icon: "minus" }); e.stopPropagation(); e.preventDefault(); ContextMenu.Instance.displayMenu(e.pageX - 15, e.pageY - 15); diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index b4f29755c..e3f5d1eb8 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -53,14 +53,14 @@ export class CollectionView extends React.Component { onContextMenu = (e: React.MouseEvent): void => { if (!this.isAnnotationOverlay && !e.isPropagationStopped() && this.props.Document[Id] !== CurrentUserUtils.MainDocId) { // need to test this because GoldenLayout causes a parallel hierarchy in the React DOM for its children and the main document view7 let subItems: ContextMenuProps[] = []; - subItems.push({ description: "Freeform", event: undoBatch(() => this.props.Document.viewType = CollectionViewType.Freeform), icon: "signature" }); + subItems.push({ description: "Freeform", event: () => this.props.Document.viewType = CollectionViewType.Freeform, icon: "signature" }); if (CollectionBaseView.InSafeMode()) { - ContextMenu.Instance.addItem({ description: "Test Freeform", event: undoBatch(() => this.props.Document.viewType = CollectionViewType.Invalid), icon: "project-diagram" }); + ContextMenu.Instance.addItem({ description: "Test Freeform", event: () => this.props.Document.viewType = CollectionViewType.Invalid, icon: "project-diagram" }); } - subItems.push({ description: "Schema", event: undoBatch(() => this.props.Document.viewType = CollectionViewType.Schema), icon: "th-list" }); - subItems.push({ description: "Treeview", event: undoBatch(() => this.props.Document.viewType = CollectionViewType.Tree), icon: "tree" }); - subItems.push({ description: "Stacking", event: undoBatch(() => this.props.Document.viewType = CollectionViewType.Stacking), icon: "ellipsis-v" }); - subItems.push({ description: "Masonry", event: undoBatch(() => this.props.Document.viewType = CollectionViewType.Masonry), icon: "columns" }); + subItems.push({ description: "Schema", event: () => this.props.Document.viewType = CollectionViewType.Schema, icon: "th-list" }); + subItems.push({ description: "Treeview", event: () => this.props.Document.viewType = CollectionViewType.Tree, icon: "tree" }); + subItems.push({ description: "Stacking", event: () => this.props.Document.viewType = CollectionViewType.Stacking, icon: "ellipsis-v" }); + subItems.push({ description: "Masonry", event: () => this.props.Document.viewType = CollectionViewType.Masonry, icon: "columns" }); switch (this.props.Document.viewType) { case CollectionViewType.Freeform: { subItems.push({ description: "Custom", icon: "fingerprint", event: CollectionFreeFormView.AddCustomLayout(this.props.Document, this.props.fieldKey) }); @@ -68,7 +68,7 @@ export class CollectionView extends React.Component { } } ContextMenu.Instance.addItem({ description: "View Modes...", subitems: subItems, icon: "eye" }); - ContextMenu.Instance.addItem({ description: "Apply Template", event: undoBatch(() => this.props.addDocTab && this.props.addDocTab(Doc.ApplyTemplate(this.props.Document)!, undefined, "onRight")), icon: "project-diagram" }); + ContextMenu.Instance.addItem({ description: "Apply Template", event: () => this.props.addDocTab && this.props.addDocTab(Doc.ApplyTemplate(this.props.Document)!, undefined, "onRight"), icon: "project-diagram" }); } } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index d07fc7f80..a6152e1b7 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -496,7 +496,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { let layoutItems: ContextMenuProps[] = []; layoutItems.push({ description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`, - event: undoBatch(async () => this.props.Document.fitToBox = !this.fitToBox), + event: async () => this.props.Document.fitToBox = !this.fitToBox, icon: !this.fitToBox ? "expand-arrows-alt" : "compress-arrows-alt" }); layoutItems.push({ diff --git a/src/client/views/nodes/KeyValueBox.tsx b/src/client/views/nodes/KeyValueBox.tsx index 77824b4ff..b0fcc4202 100644 --- a/src/client/views/nodes/KeyValueBox.tsx +++ b/src/client/views/nodes/KeyValueBox.tsx @@ -20,6 +20,7 @@ import { RichTextField } from "../../../new_fields/RichTextField"; import { ImageField } from "../../../new_fields/URLField"; import { SelectionManager } from "../../util/SelectionManager"; import { listSpec } from "../../../new_fields/Schema"; +import { CollectionViewType } from "../collections/CollectionBaseView"; export type KVPScript = { script: CompiledScript; @@ -195,6 +196,9 @@ export class KeyValueBox extends React.Component { } let fieldTemplate = await this.inferType(sourceDoc[metaKey], metaKey); + if (!fieldTemplate) { + return; + } let previousViewType = fieldTemplate.viewType; Doc.MakeTemplate(fieldTemplate, metaKey, Doc.GetProto(parentStackingDoc)); previousViewType && (fieldTemplate.viewType = previousViewType); @@ -211,14 +215,17 @@ export class KeyValueBox extends React.Component { return Docs.Create.StackingDocument([], options); } let first = await Cast(data[0], Doc); - if (!first) { + if (!first || !first.data) { return Docs.Create.StackingDocument([], options); } - switch (first.type) { - case "image": - return Docs.Create.StackingDocument([], options); - case "text": + switch (first.data.constructor) { + case RichTextField: return Docs.Create.TreeDocument([], options); + case ImageField: + return Docs.Create.StackingDocument([], options, CollectionViewType.Masonry); + default: + console.log(`Template for ${first.data.constructor} not supported!`); + return undefined; } } else if (data instanceof ImageField) { return Docs.Create.ImageDocument("https://image.flaticon.com/icons/png/512/23/23765.png", options); diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 182b22a1a..1ff0e3b31 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -139,7 +139,7 @@ def write_text_doc(content): data_doc = { "_id": data_doc_guid, "fields": { - "proto": protofy("commonImportProto"), + "proto": protofy("textProto"), "data": { "Data": '{"doc":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"' + content + '"}]}]},"selection":{"type":"text","anchor":1,"head":1}' + '}', "__type": "RichTextField" @@ -235,7 +235,7 @@ def parse_document(file_name: str): count += 1 view_guids.append(write_image(pure_name, image)) copyfile(dir_path + "/" + image, dir_path + - "/" + image.replace(".", "_o.", 1)) + "/" + image.replace(".", "_o.", 1)) os.rename(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") -- cgit v1.2.3-70-g09d2 From 26086ee95a9a16486d637aa43c96638b6154379f Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 30 Jul 2019 16:02:42 -0400 Subject: Added document export and import --- package.json | 4 + .../collectionFreeForm/CollectionFreeFormView.tsx | 39 +++++- src/client/views/nodes/DocumentView.tsx | 10 ++ src/scraping/buxton/scraper.py | 2 + src/server/database.ts | 37 +++++- src/server/index.ts | 142 +++++++++++++++++---- 6 files changed, 202 insertions(+), 32 deletions(-) (limited to 'src/scraping/buxton/scraper.py') diff --git a/package.json b/package.json index 37052fde3..b29355738 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,9 @@ "@hig/theme-context": "^2.1.3", "@hig/theme-data": "^2.3.3", "@trendmicro/react-dropdown": "^1.3.0", + "@types/adm-zip": "^0.4.32", "@types/animejs": "^2.0.2", + "@types/archiver": "^3.0.0", "@types/async": "^2.4.1", "@types/bcrypt-nodejs": "0.0.30", "@types/bluebird": "^3.5.25", @@ -105,6 +107,8 @@ "@types/uuid": "^3.4.4", "@types/webpack": "^4.4.25", "@types/youtube": "0.0.38", + "adm-zip": "^0.4.13", + "archiver": "^3.0.3", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 8dac785e1..cbab14976 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -5,7 +5,7 @@ import { Id } from "../../../../new_fields/FieldSymbols"; import { InkField, StrokeData } from "../../../../new_fields/InkField"; import { createSchema, makeInterface } from "../../../../new_fields/Schema"; import { BoolCast, Cast, FieldValue, NumCast, StrCast } from "../../../../new_fields/Types"; -import { emptyFunction, returnOne } from "../../../../Utils"; +import { emptyFunction, returnOne, Utils } from "../../../../Utils"; import { DocumentManager } from "../../../util/DocumentManager"; import { DragManager } from "../../../util/DragManager"; import { HistoryUtil } from "../../../util/History"; @@ -34,12 +34,14 @@ import { CompileScript } from "../../../util/Scripting"; import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; import { library } from "@fortawesome/fontawesome-svg-core"; import { faEye } from "@fortawesome/free-regular-svg-icons"; -import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass } from "@fortawesome/free-solid-svg-icons"; +import { faTable, faPaintBrush, faAsterisk, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload } from "@fortawesome/free-solid-svg-icons"; import { undo } from "prosemirror-history"; import { number } from "prop-types"; import { ContextMenu } from "../../ContextMenu"; +import { RouteStore } from "../../../../server/RouteStore"; +import { DocServer } from "../../../DocServer"; -library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass); +library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload); export const panZoomSchema = createSchema({ panX: "number", @@ -516,7 +518,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY)); } - onContextMenu = () => { + onContextMenu = (e: React.MouseEvent) => { let layoutItems: ContextMenuProps[] = []; layoutItems.push({ description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`, @@ -561,6 +563,35 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { CognitiveServices.Inking.Manager.analyzer(this.fieldExtensionDoc, relevantKeys, data.inkData); }, icon: "paint-brush" }); + ContextMenu.Instance.addItem({ + description: "Import document", icon: "upload", event: () => { + const input = document.createElement("input"); + input.type = "file"; + input.accept = ".zip"; + input.onchange = async _e => { + const files = input.files; + if (!files) return; + const file = files[0]; + let formData = new FormData(); + formData.append('file', file); + formData.append('remap', "true"); + const upload = Utils.prepend("/uploadDoc"); + const response = await fetch(upload, { method: "POST", body: formData }); + const json = await response.json(); + if (json === "error") { + return; + } + const doc = await DocServer.GetRefField(json); + if (!doc || !(doc instanceof Doc)) { + return; + } + const [x, y] = this.props.ScreenToLocalTransform().transformPoint(e.pageX, e.pageY); + doc.x = x, doc.y = y; + this.addDocument(doc, false); + }; + input.click(); + } + }); } diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4b5cf3a43..58e2443c2 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -45,6 +45,7 @@ const JsxParser = require('react-jsx-parser').default; //TODO Why does this need library.add(fa.faTrash); library.add(fa.faShare); +library.add(fa.faDownload); library.add(fa.faExpandArrowsAlt); library.add(fa.faCompressArrowsAlt); library.add(fa.faLayerGroup); @@ -597,6 +598,15 @@ export class DocumentView extends DocComponent(Docu copies.push({ description: "Copy ID", event: () => Utils.CopyText(this.props.Document[Id]), icon: "fingerprint" }); cm.addItem({ description: "Copy...", subitems: copies, icon: "copy" }); } + cm.addItem({ + description: "Download document", icon: "download", event: () => { + const a = document.createElement("a"); + const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); + a.href = url; + a.download = `DocExport-${this.props.Document[Id]}.zip`; + a.click(); + } + }); cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" }); type User = { email: string, userDocumentId: string }; let usersMenu: ContextMenuProps[] = []; diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 1ff0e3b31..8ff7cb223 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -236,6 +236,8 @@ def parse_document(file_name: str): view_guids.append(write_image(pure_name, image)) copyfile(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_o.", 1)) + copyfile(dir_path + "/" + image, dir_path + + "/" + image) os.rename(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") diff --git a/src/server/database.ts b/src/server/database.ts index acb6ce751..a7254fb0c 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -17,7 +17,7 @@ export class Database { }); } - public update(id: string, value: any, callback: () => void, upsert = true, collectionName = Database.DocumentsCollection) { + public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { if (this.db) { let collection = this.db.collection(collectionName); const prom = this.currentWrites[id]; @@ -30,7 +30,7 @@ export class Database { delete this.currentWrites[id]; } resolve(); - callback(); + callback(err, res); }); }); }; @@ -41,6 +41,30 @@ export class Database { } } + public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { + if (this.db) { + let collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise; + const run = (): Promise => { + return new Promise(resolve => { + collection.replaceOne({ _id: id }, value, { upsert } + , (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + callback(err, res); + }); + }); + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; + } else { + this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName)); + } + } + public delete(query: any, collectionName?: string): Promise; public delete(id: string, collectionName?: string): Promise; public delete(id: any, collectionName = Database.DocumentsCollection) { @@ -126,7 +150,7 @@ export class Database { } } - public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments") { + public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise { if (this.db) { const visited = new Set(); while (ids.length) { @@ -145,7 +169,12 @@ export class Database { } } else { - this.onConnect.push(() => this.visit(ids, fn, collectionName)); + return new Promise(res => { + this.onConnect.push(() => { + this.visit(ids, fn, collectionName); + res(); + }); + }); } } diff --git a/src/server/index.ts b/src/server/index.ts index 230c574cf..2fa5132d0 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -27,6 +27,7 @@ import { Client } from './Client'; import { Database } from './database'; import { MessageStore, Transferable, Types, Diff, Message } from "./Message"; import { RouteStore } from './RouteStore'; +import v4 = require('uuid/v4'); const app = express(); const config = require('../../webpack.config'); import { createCanvas, loadImage, Canvas } from "canvas"; @@ -39,7 +40,10 @@ import c = require("crypto"); import { Search } from './Search'; import { debug } from 'util'; import _ = require('lodash'); +import * as Archiver from 'archiver'; +import * as AdmZip from 'adm-zip'; import { Response } from 'express-serve-static-core'; +import { DocComponent } from '../client/views/DocComponent'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -177,16 +181,21 @@ function msToTime(duration: number) { return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } -app.get("/serializeDoc/:docId", async (req, res) => { - const files: { [name: string]: string[] } = {}; +async function getDocs(id: string) { + const files = new Set(); const docs: { [id: string]: any } = {}; const fn = (doc: any): string[] => { + const id = doc.id; + if (typeof id === "string" && id.endsWith("Proto")) { + //Skip protos + return []; + } const ids: string[] = []; - for (const key in doc) { - if (!doc.hasOwnProperty(key)) { + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { continue; } - const field = doc[key]; + const field = doc.fields[key]; if (field === undefined || field === null) { continue; } @@ -194,7 +203,7 @@ app.get("/serializeDoc/:docId", async (req, res) => { if (field.__type === "proxy" || field.__type === "prefetch_proxy") { ids.push(field.fieldId); } else if (field.__type === "list") { - ids.push(...fn(field.fields)); + ids.push(...fn(field)); } else if (typeof field === "string") { const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; let match: string[] | null; @@ -215,35 +224,120 @@ app.get("/serializeDoc/:docId", async (req, res) => { while ((match = re2.exec(field.Data)) !== null) { const urlString = match[1]; const pathname = new URL(urlString).pathname; - const ext = path.extname(pathname); - const fileName = path.basename(pathname, ext); - let exts = files[fileName]; - if (!exts) { - files[fileName] = exts = []; - } - exts.push(ext); + files.add(pathname); } } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { const url = new URL(field.url); const pathname = url.pathname; - const ext = path.extname(pathname); - const fileName = path.basename(pathname, ext); - let exts = files[fileName]; - if (!exts) { - files[fileName] = exts = []; - } - exts.push(ext); + files.add(pathname); } } - docs[doc.id] = doc; + if (doc.id) { + docs[doc.id] = doc; + } return ids; }; - Database.Instance.visit([req.params.docId], fn); + await Database.Instance.visit([id], fn); + return { id, docs, files }; +} +app.get("/serializeDoc/:docId", async (req, res) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); }); -app.get("/downloadId/:docId", (req, res) => { - res.download(`/serializeDoc/${req.params.docId}`, `DocumentExport.zip`); +app.get("/downloadId/:docId", async (req, res) => { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + const zip = Archiver('zip'); + zip.pipe(res); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); + }); + zip.finalize(); +}); + +app.post("/uploadDoc", (req, res) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); + } + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); + } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } + } + }; + form.parse(req, async (err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path = files[name].path; + const zip = new AdmZip(path); + zip.getEntries().forEach(entry => { + if (!entry.name.startsWith("files/")) return; + zip.extractEntryTo(entry.name, __dirname + RouteStore.public, true, false); + }); + const json = zip.getEntry("doc.json"); + let docs: any; + try { + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + fs.unlink(path, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + }); }); app.get("/whosOnline", (req, res) => { -- cgit v1.2.3-70-g09d2 From 8ca17d379ce7d3cc751408553b6819223d31a3e0 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 30 Jul 2019 16:22:54 -0400 Subject: Fixed import thing --- src/scraping/buxton/scraper.py | 2 -- src/server/index.ts | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'src/scraping/buxton/scraper.py') diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 8ff7cb223..f0f45d8f9 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -237,8 +237,6 @@ def parse_document(file_name: str): copyfile(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_o.", 1)) copyfile(dir_path + "/" + image, dir_path + - "/" + image) - os.rename(dir_path + "/" + image, dir_path + "/" + image.replace(".", "_m.", 1)) print(f"extracted {count} images...") diff --git a/src/server/index.ts b/src/server/index.ts index 281fc1eb2..1912cf5c1 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -205,6 +205,10 @@ async function getDocs(id: string) { if (field.__type === "proxy" || field.__type === "prefetch_proxy") { ids.push(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + ids.push(field.captures.fieldId); + } } else if (field.__type === "list") { ids.push(...fn(field)); } else if (typeof field === "string") { @@ -293,6 +297,10 @@ app.post("/uploadDoc", (req, res) => { if (field.__type === "proxy" || field.__type === "prefetch_proxy") { field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); + } } else if (field.__type === "list") { mapFn(field); } else if (typeof field === "string") { -- cgit v1.2.3-70-g09d2