From d6cef0815815c2587b9cc791e6a37c742aba1b45 Mon Sep 17 00:00:00 2001
From: Sam Wilkins <samwilkins333@gmail.com>
Date: Thu, 25 Jul 2019 03:31:34 -0400
Subject: cognitive services refactor, buxton python script fixes, covered up
 imagebox context menu bug

---
 src/scraping/buxton/scraper.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

(limited to 'src/scraping/buxton/scraper.py')

diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py
index 700269727..14490cfe4 100644
--- a/src/scraping/buxton/scraper.py
+++ b/src/scraping/buxton/scraper.py
@@ -15,6 +15,7 @@ source = "./source"
 dist = "../../server/public/files"
 
 db = MongoClient("localhost", 27017)["Dash"]
+target_collection = db.newDocuments
 schema_guids = []
 
 
@@ -84,7 +85,7 @@ def write_schema(parse_results, display_fields, storage_key):
             "height": 600,
             "panX": 0,
             "panY": 0,
-            "zoomBasis": 0.5,
+            "zoomBasis": 1,
             "zIndex": 2,
             "libraryBrush": False,
             "viewType": 2
@@ -106,8 +107,8 @@ def write_schema(parse_results, display_fields, storage_key):
     fields["isPrototype"] = True
     fields["page"] = -1
 
-    db.newDocuments.insert_one(data_doc)
-    db.newDocuments.insert_one(view_doc)
+    target_collection.insert_one(data_doc)
+    target_collection.insert_one(view_doc)
 
     data_doc_guid = data_doc["_id"]
     print(f"inserted view document ({view_doc_guid})")
@@ -158,8 +159,8 @@ def write_text_doc(content):
         "__type": "Doc"
     }
 
-    db.newDocuments.insert_one(view_doc)
-    db.newDocuments.insert_one(data_doc)
+    target_collection.insert_one(view_doc)
+    target_collection.insert_one(data_doc)
 
     return view_doc_guid
 
@@ -209,8 +210,8 @@ def write_image(folder, name):
         "__type": "Doc"
     }
 
-    db.newDocuments.insert_one(view_doc)
-    db.newDocuments.insert_one(data_doc)
+    target_collection.insert_one(view_doc)
+    target_collection.insert_one(data_doc)
 
     return view_doc_guid
 
@@ -372,7 +373,7 @@ parent_guid = write_schema({
 }, ["title", "short_description", "original_price"], "data")
 
 print("appending parent schema to main workspace...\n")
-db.newDocuments.update_one(
+target_collection.update_one(
     {"fields.title": "WS collection 1"},
     {"$push": {"fields.data.fields": {"fieldId": parent_guid, "__type": "proxy"}}}
 )
-- 
cgit v1.2.3-70-g09d2


From e1c7add158ce245ce6cb557177986b31fe107dd8 Mon Sep 17 00:00:00 2001
From: Sam Wilkins <samwilkins333@gmail.com>
Date: Thu, 25 Jul 2019 18:56:46 -0400
Subject: scraping common proto tweak

---
 src/scraping/buxton/scraper.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

(limited to 'src/scraping/buxton/scraper.py')

diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py
index 14490cfe4..48b8fe3fa 100644
--- a/src/scraping/buxton/scraper.py
+++ b/src/scraping/buxton/scraper.py
@@ -17,6 +17,7 @@ dist = "../../server/public/files"
 db = MongoClient("localhost", 27017)["Dash"]
 target_collection = db.newDocuments
 schema_guids = []
+common_proto_id = ""
 
 
 def extract_links(fileName):
@@ -93,7 +94,7 @@ def write_schema(parse_results, display_fields, storage_key):
         "__type": "Doc"
     }
 
-    fields["proto"] = protofy("collectionProto")
+    fields["proto"] = protofy(common_proto_id)
     fields[storage_key] = listify(proxify_guids(view_guids))
     fields["schemaColumns"] = listify(display_fields)
     fields["backgroundColor"] = "white"
@@ -137,7 +138,7 @@ def write_text_doc(content):
     data_doc = {
         "_id": data_doc_guid,
         "fields": {
-            "proto": protofy("textProto"),
+            "proto": protofy("commonImportProto"),
             "data": {
                 "Data": '{"doc":{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"' + content + '"}]}]},"selection":{"type":"text","anchor":1,"head":1}' + '}',
                 "__type": "RichTextField"
@@ -348,6 +349,22 @@ def proxify_guids(guids):
     return list(map(lambda guid: {"fieldId": guid, "__type": "proxy"}, guids))
 
 
+def write_common_proto():
+    id = guid()
+    common_proto = {
+        "_id": id,
+        "fields": {
+            "proto": protofy("collectionProto"),
+            "title": "Common Import Proto",
+        },
+        "__type": "Doc"
+    }
+
+    target_collection.insert_one(common_proto)
+
+    return id
+
+
 if os.path.exists(dist):
     shutil.rmtree(dist)
 while os.path.exists(dist):
@@ -355,6 +372,8 @@ while os.path.exists(dist):
 os.mkdir(dist)
 mkdir_if_absent(source)
 
+common_proto_id = write_common_proto()
+
 candidates = 0
 for file_name in os.listdir(source):
     if file_name.endswith('.docx'):
-- 
cgit v1.2.3-70-g09d2