From c75ffd4900acea74c55b6bf275a5e8082c15d573 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 26 Apr 2020 19:29:50 -0700 Subject: formatted textbox disposers refactor, paragraph chunked rich text initialization and buxton importer updates --- src/scraping/buxton/final/BuxtonImporter.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'src/scraping') diff --git a/src/scraping/buxton/final/BuxtonImporter.ts b/src/scraping/buxton/final/BuxtonImporter.ts index 122415460..64b988610 100644 --- a/src/scraping/buxton/final/BuxtonImporter.ts +++ b/src/scraping/buxton/final/BuxtonImporter.ts @@ -16,6 +16,7 @@ interface DocumentContents { hyperlinks: string[]; captions: string[]; embeddedFileNames: string[]; + longDescriptionParagraphs: string[]; } export interface DeviceDocument { @@ -186,10 +187,6 @@ const RegexMap = new Map>([ exp: /Short Description:\s+(.*)Bill Buxton[’']s Notes/, transformer: Utilities.correctSentences }], - ["longDescription", { - exp: /Bill Buxton[’']s Notes(.*)Device Details/, - transformer: Utilities.correctSentences - }], ]); const sourceDir = path.resolve(__dirname, "source"); @@ -267,7 +264,12 @@ async function extractFileContents(pathToDocument: string): Promise node.text()); + const captionTargets = document.find(tableCellXPath).map(node => node.text().trim()); + + const paragraphs = document.find('//*[name()="w:p"]').map(node => Utilities.correctSentences(node.text()).transformed!); + const start = paragraphs.indexOf(paragraphs.find(el => /Bill Buxton[’']s Notes/.test(el))!) + 1; + const end = paragraphs.indexOf("Device Details"); + const longDescriptionParagraphs = paragraphs.slice(start, end); const { length } = captionTargets; strictEqual(length > 3, true, "No captions written."); @@ -290,7 +292,7 @@ async function extractFileContents(pathToDocument: string): Promise { } function analyze(fileName: string, contents: DocumentContents): AnalysisResult { - const { body, imageData, captions, hyperlinks, embeddedFileNames } = contents; + const { body, imageData, captions, hyperlinks, embeddedFileNames, longDescriptionParagraphs } = contents; const device: any = { hyperlinks, captions, @@ -376,6 +378,7 @@ function analyze(fileName: string, contents: DocumentContents): AnalysisResult { return { errors }; } + device.longDescription = longDescriptionParagraphs.join("\n\n"); return { device }; } -- cgit v1.2.3-70-g09d2 From fc470b25759e8f051dc527066f9bebcaf5e7707d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 26 Apr 2020 23:55:23 -0700 Subject: various buxton fixes --- src/client/documents/Documents.ts | 1 - .../views/collections/CollectionTreeView.tsx | 24 ++++++++-------- src/client/views/nodes/DocumentContentsView.tsx | 8 +++--- src/client/views/nodes/FormattedTextBox.tsx | 9 +----- src/scraping/buxton/final/BuxtonImporter.ts | 32 ++++++++++++---------- 5 files changed, 36 insertions(+), 38 deletions(-) (limited to 'src/scraping') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 1651a6d55..5e0890e76 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -408,7 +408,6 @@ export namespace Docs { const doc = StackingDocument(deviceImages, { title: device.title, _LODdisable: true }); const deviceProto = Doc.GetProto(doc); deviceProto.hero = new ImageField(constructed[0].url); - deviceProto.fontFamily = "Arial"; Docs.Get.FromJson({ data: device, appendToExisting: { targetDoc: deviceProto } }); Doc.AddDocToList(parentProto, "data", doc); } else if (errors) { diff --git a/src/client/views/collections/CollectionTreeView.tsx b/src/client/views/collections/CollectionTreeView.tsx index 362d43ee7..dcb5e116c 100644 --- a/src/client/views/collections/CollectionTreeView.tsx +++ b/src/client/views/collections/CollectionTreeView.tsx @@ -733,19 +733,21 @@ export class CollectionTreeView extends CollectionSubView { const style: { [key: string]: any } = {}; const divKeys = OmitKeys(this.props, ["children", "htmltag", "RootDoc", "Document", "key", "onInput", "onClick", "__proto__"]).omit; Object.keys(divKeys).map((prop: string) => { - let p = (this.props as any)[prop] as string; + const p = (this.props as any)[prop] as string; const replacer = (match: any, expr: string, offset: any, string: any) => { // bcz: this executes a script to convert a propery expression string: { script } into a value return ScriptField.MakeFunction(expr, { self: Doc.name, this: Doc.name })?.script.run({ self: this.props.RootDoc, this: this.props.Document }).result as string || ""; }; @@ -178,9 +178,9 @@ export class DocumentContentsView extends React.Component 1 ? splits[0] + splits[1].replace(/{([^{}]|(?R))*}/, replacer4) : ""; // might have been more elegant if javascript supported recursive patterns diff --git a/src/client/views/nodes/FormattedTextBox.tsx b/src/client/views/nodes/FormattedTextBox.tsx index 8d4b90c41..d98172823 100644 --- a/src/client/views/nodes/FormattedTextBox.tsx +++ b/src/client/views/nodes/FormattedTextBox.tsx @@ -860,15 +860,8 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<(FieldViewProp }); const startupText = !rtfField && this._editorView && Field.toString(this.dataDoc[fieldKey] as Field); if (startupText) { - const paragraphSegments = startupText.split("\n\n"); const { state: { tr }, dispatch } = this._editorView; - if (paragraphSegments.length) { - for (const paragraph of paragraphSegments) { - dispatch(tr.insertText(paragraph)); - } - } else { - dispatch(tr.insertText(startupText)); - } + dispatch(tr.insertText(startupText)); } } diff --git a/src/scraping/buxton/final/BuxtonImporter.ts b/src/scraping/buxton/final/BuxtonImporter.ts index 64b988610..713207a07 100644 --- a/src/scraping/buxton/final/BuxtonImporter.ts +++ b/src/scraping/buxton/final/BuxtonImporter.ts @@ -16,7 +16,7 @@ interface DocumentContents { hyperlinks: string[]; captions: string[]; embeddedFileNames: string[]; - longDescriptionParagraphs: string[]; + longDescription: string; } export interface DeviceDocument { @@ -269,7 +269,7 @@ async function extractFileContents(pathToDocument: string): Promise Utilities.correctSentences(node.text()).transformed!); const start = paragraphs.indexOf(paragraphs.find(el => /Bill Buxton[’']s Notes/.test(el))!) + 1; const end = paragraphs.indexOf("Device Details"); - const longDescriptionParagraphs = paragraphs.slice(start, end); + const longDescription = paragraphs.slice(start, end).filter(paragraph => paragraph.length).join("\n\n"); const { length } = captionTargets; strictEqual(length > 3, true, "No captions written."); @@ -292,7 +292,7 @@ async function extractFileContents(pathToDocument: string): Promise { const imageEntries = allEntries.filter(name => imageEntry.test(name)); const imageUrls: ImageData[] = []; - for (const mediaPath of imageEntries) { - const getImageStream = () => new Promise((resolve, reject) => { - zip.stream(mediaPath, (error: any, stream: any) => error ? reject(error) : resolve(stream)); - }); + const valid: any[] = []; + + const getImageStream = (mediaPath: string) => new Promise((resolve, reject) => { + zip.stream(mediaPath, (error: any, stream: any) => error ? reject(error) : resolve(stream)); + }); + for (const mediaPath of imageEntries) { const { width, height, type } = await new Promise(async resolve => { const sizeStream = (createImageSizeStream() as PassThrough).on('size', (dimensions: Dimensions) => { readStream.destroy(); resolve(dimensions); }).on("error", () => readStream.destroy()); - const readStream = await getImageStream(); + const readStream = await getImageStream(mediaPath); readStream.pipe(sizeStream); }); - if (Math.abs(width - height) < 10) { - continue; + + if (Math.abs(width - height) > 10) { + valid.push({ width, height, type, mediaPath }); } + } + for (const { type, width, height, mediaPath } of valid) { const generatedFileName = `upload_${Utils.GenerateGuid()}.${type.toLowerCase()}`; - await DashUploadUtils.outputResizedImages(getImageStream, generatedFileName, imageDir); - + await DashUploadUtils.outputResizedImages(() => getImageStream(mediaPath), generatedFileName, imageDir); imageUrls.push({ url: `/files/images/buxton/${generatedFileName}`, nativeWidth: width, @@ -339,11 +343,12 @@ async function writeImages(zip: any): Promise { } function analyze(fileName: string, contents: DocumentContents): AnalysisResult { - const { body, imageData, captions, hyperlinks, embeddedFileNames, longDescriptionParagraphs } = contents; + const { body, imageData, captions, hyperlinks, embeddedFileNames, longDescription } = contents; const device: any = { hyperlinks, captions, embeddedFileNames, + longDescription, __images: imageData }; const errors: { [key: string]: string } = { fileName }; @@ -378,7 +383,6 @@ function analyze(fileName: string, contents: DocumentContents): AnalysisResult { return { errors }; } - device.longDescription = longDescriptionParagraphs.join("\n\n"); return { device }; } -- cgit v1.2.3-70-g09d2 From 1660defc561c904217ed5be34cd6e0fe64736fe1 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 30 Apr 2020 19:06:42 -0700 Subject: commented Buxton importer --- src/scraping/buxton/final/BuxtonImporter.ts | 212 +++++++++++++++++---- .../authentication/models/current_user_utils.ts | 1 - 2 files changed, 179 insertions(+), 34 deletions(-) (limited to 'src/scraping') diff --git a/src/scraping/buxton/final/BuxtonImporter.ts b/src/scraping/buxton/final/BuxtonImporter.ts index 713207a07..21363f848 100644 --- a/src/scraping/buxton/final/BuxtonImporter.ts +++ b/src/scraping/buxton/final/BuxtonImporter.ts @@ -10,6 +10,10 @@ import { parseXml } from "libxmljs"; import { strictEqual } from "assert"; import { Readable, PassThrough } from "stream"; +/** + * This is an arbitrary bundle of data that gets populated + * in extractFileContents + */ interface DocumentContents { body: string; imageData: ImageData[]; @@ -19,6 +23,10 @@ interface DocumentContents { longDescription: string; } +/** + * A rough schema for everything that Bill has + * included for each document + */ export interface DeviceDocument { title: string; shortDescription: string; @@ -33,36 +41,65 @@ export interface DeviceDocument { attribute: string; __images: ImageData[]; hyperlinks: string[]; - captions: string[]; - embeddedFileNames: string[]; + captions: string[]; // from the table column + embeddedFileNames: string[]; // from the table column } +/** + * A layer of abstraction around a single parsing + * attempt. The error is not a TypeScript error, but + * rather an invalidly formatted value for a given key. + */ export interface AnalysisResult { device?: DeviceDocument; - errors?: { [key: string]: string }; + invalid?: { [deviceProperty: string]: string }; } +/** + * A mini API that takes in a string and returns + * either the given T or an error indicating that the + * transformation was rejected. + */ type Transformer = (raw: string) => TransformResult; interface TransformResult { transformed?: T; error?: string; } +/** + * Simple bundle counting successful and failed imports + */ export interface ImportResults { deviceCount: number; errorCount: number; } +/** + * Definitions for callback functions. Such instances are + * just invoked by when a single document has been parsed + * or the entire import is over. As of this writing, these + * callbacks are supplied by WebSocket.ts and used to inform + * the client of these events. + */ type ResultCallback = (result: AnalysisResult) => void; type TerminatorCallback = (result: ImportResults) => void; -interface Processor { - exp: RegExp; - matchIndex?: number; - transformer?: Transformer; - required?: boolean; +/** + * Defines everything needed to define how a single key should be + * formatted within the plain body text. The association between + * keys and their format definitions is stored FormatMap + */ +interface ValueFormatDefinition { + exp: RegExp; // the expression that the key's value should match + matchIndex?: number; // defaults to 0, but can be overridden to account for grouping in @param exp + transformer?: Transformer; // if desirable, how to transform the Regex match + required?: boolean; // defaults to true, confirms that for a whole document to be counted successful, + // all of its required values should be present and properly formatted } +/** + * The basic data we extract from each image in the document + */ interface ImageData { url: string; nativeWidth: number; @@ -71,6 +108,10 @@ interface ImageData { namespace Utilities { + /** + * Numeric 'try parse', fits with the Transformer API + * @param raw the serialized number + */ export function numberValue(raw: string): TransformResult { const transformed = Number(raw); if (isNaN(transformed)) { @@ -79,18 +120,32 @@ namespace Utilities { return { transformed }; } + /** + * A simple tokenizer that splits along 'and' and commas, and removes duplicates + * Helpful mainly for attribute and primary key lists + * @param raw the string to tokenize + */ export function collectUniqueTokens(raw: string): TransformResult { const pieces = raw.replace(/,|\s+and\s+/g, " ").split(/\s+/).filter(piece => piece.length); const unique = new Set(pieces.map(token => token.toLowerCase().trim())); return { transformed: Array.from(unique).map(capitalize).sort() }; } + /** + * Tries to correct XML text parsing artifact where some sentences lose their separating space, + * and others gain excess whitespace + * @param raw + */ export function correctSentences(raw: string): TransformResult { raw = raw.replace(/\./g, ". ").replace(/\:/g, ": ").replace(/\,/g, ", ").replace(/\?/g, "? ").trimRight(); raw = raw.replace(/\s{2,}/g, " "); return { transformed: raw }; } + /** + * Simple capitalization + * @param word to capitalize + */ export function capitalize(word: string): string { const clean = word.trim(); if (!clean.length) { @@ -99,6 +154,12 @@ namespace Utilities { return word.charAt(0).toUpperCase() + word.slice(1); } + /** + * Streams the requeted file at the relative path to the + * root of the zip, then parses it with a library + * @param zip the zip instance data source + * @param relativePath the path to a .xml file within the zip to parse + */ export async function readAndParseXml(zip: any, relativePath: string) { console.log(`Text streaming ${relativePath}`); const contents = await new Promise((resolve, reject) => { @@ -111,13 +172,17 @@ namespace Utilities { stream.on('end', () => resolve(body)); }); }); - return parseXml(contents); } - } -const RegexMap = new Map>([ +/** + * Defines how device values should be formatted. As you can see, the formatting is + * not super consistent and has changed over time as edge cases have been found, but this + * at least imposes some constraints, and will notify you if a document doesn't match the specifications + * in this map. + */ +const FormatMap = new Map>([ ["title", { exp: /contact\s+(.*)Short Description:/ }], @@ -189,17 +254,25 @@ const RegexMap = new Map>([ }], ]); -const sourceDir = path.resolve(__dirname, "source"); -const outDir = path.resolve(__dirname, "json"); -const imageDir = path.resolve(__dirname, "../../../server/public/files/images/buxton"); -const successOut = "buxton.json"; -const failOut = "incomplete.json"; -const deviceKeys = Array.from(RegexMap.keys()); - +const sourceDir = path.resolve(__dirname, "source"); // where the Word documents are assumed to be stored +const outDir = path.resolve(__dirname, "json"); // where the JSON output of these device documents will be written +const imageDir = path.resolve(__dirname, "../../../server/public/files/images/buxton"); // where, in the server, these images will be written +const successOut = "buxton.json"; // the JSON list representing properly formatted documents +const failOut = "incomplete.json"; // the JSON list representing improperly formatted documents +const deviceKeys = Array.from(FormatMap.keys()); // a way to iterate through all keys of the DeviceDocument interface + +/** + * Starts by REMOVING ALL EXISTING BUXTON RESOURCES. This might need to be + * changed going forward + * @param emitter the callback when each document is completed + * @param terminator the callback when the entire import is completed + */ export default async function executeImport(emitter: ResultCallback, terminator: TerminatorCallback) { try { + // get all Word documents in the source directory const contents = readdirSync(sourceDir); const wordDocuments = contents.filter(file => /.*\.docx?$/.test(file)).map(file => `${sourceDir}/${file}`); + // removal takes place here [outDir, imageDir].forEach(dir => { rimraf.sync(dir); mkdirSync(dir); @@ -216,19 +289,28 @@ export default async function executeImport(emitter: ResultCallback, terminator: } } +/** + * Parse every Word document in the directory, notifying any callers as needed + * at each iteration via the emitter. + * @param wordDocuments the string list of Word document names to parse + * @param emitter the callback when each document is completed + * @param terminator the callback when the entire import is completed + */ async function parseFiles(wordDocuments: string[], emitter: ResultCallback, terminator: TerminatorCallback): Promise { + // execute parent-most parse function const results: AnalysisResult[] = []; for (const filePath of wordDocuments) { - const fileName = path.basename(filePath).replace("Bill_Notes_", ""); + const fileName = path.basename(filePath).replace("Bill_Notes_", ""); // not strictly needed, but cleaner console.log(cyan(`\nExtracting contents from ${fileName}...`)); const result = analyze(fileName, await extractFileContents(filePath)); emitter(result); results.push(result); } + // collect information about errors and successes const masterDevices: DeviceDocument[] = []; const masterErrors: { [key: string]: string }[] = []; - results.forEach(({ device, errors }) => { + results.forEach(({ device, invalid: errors }) => { if (device) { masterDevices.push(device); } else if (errors) { @@ -236,24 +318,45 @@ async function parseFiles(wordDocuments: string[], emitter: ResultCallback, term } }); + // something went wrong, since errors and successes should sum to total inputs const total = wordDocuments.length; if (masterDevices.length + masterErrors.length !== total) { throw new Error(`Encountered a ${masterDevices.length} to ${masterErrors.length} mismatch in device / error split!`); } + // write the external JSON representations of this import console.log(); await writeOutputFile(successOut, masterDevices, total, true); await writeOutputFile(failOut, masterErrors, total, false); console.log(); + // notify the caller that the import has finished terminator({ deviceCount: masterDevices.length, errorCount: masterErrors.length }); return masterDevices; } +/** + * XPath definitions for desired XML targets in respective hierarchies. + * + * For table cells, can be read as: "find me anything that looks like in XML, whose + * parent looks like , whose parent looks like " + * + * + * + * + * + * These are found by trial and error, and using an online XML parser / prettifier + * to inspect the structure, since the Node XML library does not expose the parsed + * structure very well for searching, say in the debug console. + */ const tableCellXPath = '//*[name()="w:tbl"]/*[name()="w:tr"]/*[name()="w:tc"]'; const hyperlinkXPath = '//*[name()="Relationship" and contains(@Type, "hyperlink")]'; +/** + * The meat of the script, images and text content are extracted here + * @param pathToDocument the path to the document relative to the root of the zip + */ async function extractFileContents(pathToDocument: string): Promise { console.log('Extracting text...'); const zip = new StreamZip({ file: pathToDocument, storeEntries: true }); @@ -261,22 +364,30 @@ async function extractFileContents(pathToDocument: string): Promise node.text().trim()); + // preserve paragraph formatting and line breaks that would otherwise get lost in the plain text parsing + // of the XML hierarchy const paragraphs = document.find('//*[name()="w:p"]').map(node => Utilities.correctSentences(node.text()).transformed!); const start = paragraphs.indexOf(paragraphs.find(el => /Bill Buxton[’']s Notes/.test(el))!) + 1; const end = paragraphs.indexOf("Device Details"); const longDescription = paragraphs.slice(start, end).filter(paragraph => paragraph.length).join("\n\n"); - const { length } = captionTargets; + // extract captions from the table cells + const tableRowsFlattened = document.find(tableCellXPath).map(node => node.text().trim()); + const { length } = tableRowsFlattened; strictEqual(length > 3, true, "No captions written."); strictEqual(length % 3 === 0, true, "Improper caption formatting."); - for (let i = 3; i < captionTargets.length; i += 3) { - const row = captionTargets.slice(i, i + 3); + // break the flat list of strings into groups of three, since there + // currently are three columns in the table. Thus, each group represents + // a row in the table, where the first row has no text content since it's + // the image, the second has the file name and the third has the caption + for (let i = 3; i < tableRowsFlattened.length; i += 3) { + const row = tableRowsFlattened.slice(i, i + 3); embeddedFileNames.push(row[1]); captions.push(row[2]); } @@ -286,23 +397,34 @@ async function extractFileContents(pathToDocument: string): Promise el.attrs()[2].value()); console.log("Text extracted."); + // write out the images for this document console.log("Beginning image extraction..."); const imageData = await writeImages(zip); console.log(`Extracted ${imageData.length} images.`); + // cleanup zip.close(); return { body, longDescription, imageData, captions, embeddedFileNames, hyperlinks }; } +// zip relative path from root expression / filter used to isolate only media assets const imageEntry = /^word\/media\/\w+\.(jpeg|jpg|png|gif)/; -interface Dimensions { +/** + * Image dimensions and file suffix, + */ +interface ImageAttrs { width: number; height: number; type: string; } +/** + * For each image, stream the file, get its size, check if it's an icon + * (if it is, ignore it) + * @param zip the zip instance data source + */ async function writeImages(zip: any): Promise { const allEntries = Object.values(zip.entries()).map(({ name }) => name); const imageEntries = allEntries.filter(name => imageEntry.test(name)); @@ -315,8 +437,8 @@ async function writeImages(zip: any): Promise { }); for (const mediaPath of imageEntries) { - const { width, height, type } = await new Promise(async resolve => { - const sizeStream = (createImageSizeStream() as PassThrough).on('size', (dimensions: Dimensions) => { + const { width, height, type } = await new Promise(async resolve => { + const sizeStream = (createImageSizeStream() as PassThrough).on('size', (dimensions: ImageAttrs) => { readStream.destroy(); resolve(dimensions); }).on("error", () => readStream.destroy()); @@ -324,11 +446,14 @@ async function writeImages(zip: any): Promise { readStream.pipe(sizeStream); }); + // if it's not an icon, by this rough heuristic, i.e. is it not square if (Math.abs(width - height) > 10) { valid.push({ width, height, type, mediaPath }); } } + // for each valid image, output the _o, _l, _m, and _s files + // THIS IS WHERE THE SCRIPT SPENDS MOST OF ITS TIME for (const { type, width, height, mediaPath } of valid) { const generatedFileName = `upload_${Utils.GenerateGuid()}.${type.toLowerCase()}`; await DashUploadUtils.outputResizedImages(() => getImageStream(mediaPath), generatedFileName, imageDir); @@ -342,6 +467,14 @@ async function writeImages(zip: any): Promise { return imageUrls; } +/** + * Takes the results of extractFileContents, which relative to this is sort of the + * external media / preliminary text processing, and now tests the given file name to + * with those value definitions to make sure the body of the document contains all + * required fields, properly formatted + * @param fileName the file whose body to inspect + * @param contents the data already computed / parsed by extractFileContents + */ function analyze(fileName: string, contents: DocumentContents): AnalysisResult { const { body, imageData, captions, hyperlinks, embeddedFileNames, longDescription } = contents; const device: any = { @@ -354,43 +487,56 @@ function analyze(fileName: string, contents: DocumentContents): AnalysisResult { const errors: { [key: string]: string } = { fileName }; for (const key of deviceKeys) { - const { exp, transformer, matchIndex, required } = RegexMap.get(key)!; + const { exp, transformer, matchIndex, required } = FormatMap.get(key)!; const matches = exp.exec(body); let captured: string; - if (matches && (captured = matches[matchIndex ?? 1])) { - captured = captured.replace(/\s{2,}/g, " "); + // if we matched and we got the specific match we're after + if (matches && (captured = matches[matchIndex ?? 1])) { // matchIndex defaults to 1 + captured = captured.replace(/\s{2,}/g, " "); // remove excess whitespace + // if supplied, apply the required transformation (recall this is specified in FormatMap) if (transformer) { const { error, transformed } = transformer(captured); if (error) { + // we hit a snag trying to transform the valid match + // still counts as a fundamental error errors[key] = `__ERR__${key.toUpperCase()}__TRANSFORM__: ${error}`; continue; } captured = transformed; } - device[key] = captured; } else if (required ?? true) { + // the field was either implicitly or explicitly required, and failed to match the definition in + // FormatMap errors[key] = `ERR__${key.toUpperCase()}__: outer match ${matches === null ? "wasn't" : "was"} captured.`; continue; } } + // print errors - this can be removed const errorKeys = Object.keys(errors); if (errorKeys.length > 1) { console.log(red(`@ ${cyan(fileName.toUpperCase())}...`)); errorKeys.forEach(key => key !== "filename" && console.log(red(errors[key]))); - return { errors }; + return { invalid: errors }; } return { device }; } +/** + * A utility function that writes the JSON results for this import out to the desired path + * @param relativePath where to write the JSON file + * @param data valid device document objects, or errors + * @param total used for more informative printing + * @param success whether or not the caller is writing the successful parses or the failures + */ async function writeOutputFile(relativePath: string, data: any[], total: number, success: boolean) { console.log(yellow(`Encountered ${data.length} ${success ? "valid" : "invalid"} documents out of ${total} candidates. Writing ${relativePath}...`)); return new Promise((resolve, reject) => { const destination = path.resolve(outDir, relativePath); - const contents = JSON.stringify(data, undefined, 4); + const contents = JSON.stringify(data, undefined, 4); // format the JSON writeFile(destination, contents, err => err ? reject(err) : resolve()); }); } \ No newline at end of file diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 663343f47..d7cc1e6bf 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -9,7 +9,6 @@ import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; import { ScriptField, ComputedField } from "../../../new_fields/ScriptField"; import { Cast, PromiseValue, StrCast, NumCast } from "../../../new_fields/Types"; -import { Utils } from "../../../Utils"; import { nullAudio, ImageField } from "../../../new_fields/URLField"; import { DragManager } from "../../../client/util/DragManager"; import { InkingControl } from "../../../client/views/InkingControl"; -- cgit v1.2.3-70-g09d2 From b8a62e6404a695e57ab1305fd13be23e8d935360 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 3 May 2020 15:05:04 -0700 Subject: cleanup --- .../apis/google_docs/GooglePhotosClientUtils.ts | 28 +++++++++------------- src/scraping/buxton/final/BuxtonImporter.ts | 28 ++++++++++++---------- src/server/DashUploadUtils.ts | 7 +----- 3 files changed, 28 insertions(+), 35 deletions(-) (limited to 'src/scraping') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index e3f801c46..ff471853a 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -76,7 +76,6 @@ export namespace GooglePhotos { } export const CollectionToAlbum = async (options: AlbumCreationOptions): Promise> => { - await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); const { collection, title, descriptionKey, tag } = options; const dataDocument = Doc.GetProto(collection); const images = ((await DocListCastAsync(dataDocument.data)) || []).filter(doc => Cast(doc.data, ImageField)); @@ -157,24 +156,20 @@ export namespace GooglePhotos { images && images.forEach(image => tagMapping.set(image[Id], ContentCategories.NONE)); const values = Object.values(ContentCategories); for (const value of values) { - if (value !== ContentCategories.NONE) { - const results = await ContentSearch({ included: [value] }); - if (results.mediaItems) { - const ids = results.mediaItems.map(item => item.id); - for (const id of ids) { - const image = await Cast(idMapping[id], Doc); - if (image) { - const key = image[Id]; - const tags = tagMapping.get(key)!; - if (!tags.includes(value)) { - tagMapping.set(key, tags + delimiter + value); - } - } - } + if (value === ContentCategories.NONE) { + continue; + } + for (const id of (await ContentSearch({ included: [value] }))?.mediaItems?.map(({ id }) => id)) { + const image = await Cast(idMapping[id], Doc); + if (!image) { + continue; } + const key = image[Id]; + const tags = tagMapping.get(key); + !tags?.includes(value) && tagMapping.set(key, tags + delimiter + value); } } - images && images.forEach(image => { + images?.forEach(image => { const concatenated = tagMapping.get(image[Id])!; const tags = concatenated.split(delimiter); if (tags.length > 1) { @@ -184,7 +179,6 @@ export namespace GooglePhotos { image.googlePhotosTags = ContentCategories.NONE; } }); - }; interface DateRange { diff --git a/src/scraping/buxton/final/BuxtonImporter.ts b/src/scraping/buxton/final/BuxtonImporter.ts index 21363f848..94302c7b3 100644 --- a/src/scraping/buxton/final/BuxtonImporter.ts +++ b/src/scraping/buxton/final/BuxtonImporter.ts @@ -350,8 +350,11 @@ async function parseFiles(wordDocuments: string[], emitter: ResultCallback, term * to inspect the structure, since the Node XML library does not expose the parsed * structure very well for searching, say in the debug console. */ -const tableCellXPath = '//*[name()="w:tbl"]/*[name()="w:tr"]/*[name()="w:tc"]'; -const hyperlinkXPath = '//*[name()="Relationship" and contains(@Type, "hyperlink")]'; +const xPaths = { + paragraphs: '//*[name()="w:p"]', + tableCells: '//*[name()="w:tbl"]/*[name()="w:tr"]/*[name()="w:tc"]', + hyperlinks: '//*[name()="Relationship" and contains(@Type, "hyperlink")]' +}; /** * The meat of the script, images and text content are extracted here @@ -371,30 +374,31 @@ async function extractFileContents(pathToDocument: string): Promise Utilities.correctSentences(node.text()).transformed!); + const paragraphs = document.find(xPaths.paragraphs).map(node => Utilities.correctSentences(node.text()).transformed!); const start = paragraphs.indexOf(paragraphs.find(el => /Bill Buxton[’']s Notes/.test(el))!) + 1; const end = paragraphs.indexOf("Device Details"); const longDescription = paragraphs.slice(start, end).filter(paragraph => paragraph.length).join("\n\n"); // extract captions from the table cells - const tableRowsFlattened = document.find(tableCellXPath).map(node => node.text().trim()); + const tableRowsFlattened = document.find(xPaths.tableCells).map(node => node.text().trim()); const { length } = tableRowsFlattened; - strictEqual(length > 3, true, "No captions written."); - strictEqual(length % 3 === 0, true, "Improper caption formatting."); + const numCols = 3; + strictEqual(length > numCols, true, "No captions written."); // first row has the headers, not content + strictEqual(length % numCols === 0, true, "Improper caption formatting."); - // break the flat list of strings into groups of three, since there - // currently are three columns in the table. Thus, each group represents + // break the flat list of strings into groups of numColumns. Thus, each group represents // a row in the table, where the first row has no text content since it's - // the image, the second has the file name and the third has the caption - for (let i = 3; i < tableRowsFlattened.length; i += 3) { - const row = tableRowsFlattened.slice(i, i + 3); + // the image, the second has the file name and the third has the caption (maybe additional columns + // have been added or reordered since this was written, but follow the same appraoch) + for (let i = numCols; i < tableRowsFlattened.length; i += numCols) { + const row = tableRowsFlattened.slice(i, i + numCols); embeddedFileNames.push(row[1]); captions.push(row[2]); } // extract all hyperlinks embedded in the document const rels = await Utilities.readAndParseXml(zip, "word/_rels/document.xml.rels"); - const hyperlinks = rels.find(hyperlinkXPath).map(el => el.attrs()[2].value()); + const hyperlinks = rels.find(xPaths.hyperlinks).map(el => el.attrs()[2].value()); console.log("Text extracted."); // write out the images for this document diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 3f903a861..8567631cd 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -325,12 +325,7 @@ export namespace DashUploadUtils { const outputPath = path.resolve(outputDirectory, writtenFiles[suffix] = InjectSize(outputFileName, suffix)); await new Promise(async (resolve, reject) => { const source = streamProvider(); - let readStream: Stream; - if (source instanceof Promise) { - readStream = await source; - } else { - readStream = source; - } + let readStream: Stream = source instanceof Promise ? await source : source; if (resizer) { readStream = readStream.pipe(resizer.withMetadata()); } -- cgit v1.2.3-70-g09d2 From aa1eb6ba4217fb48ab10539ca0373b4a1f649192 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 5 May 2020 02:48:03 -0700 Subject: database, delete and google authentication simplifications and improvements, as well as formatted text box updates data state at field key not just data --- src/client/apis/GoogleAuthenticationManager.scss | 7 ++ src/client/apis/GoogleAuthenticationManager.tsx | 125 ++++++++++++++------- .../apis/google_docs/GoogleApiClientUtils.ts | 2 +- src/client/views/GlobalKeyHandler.ts | 2 + src/client/views/MainView.tsx | 5 +- src/client/views/MainViewModal.tsx | 2 +- src/client/views/OverlayView.tsx | 6 +- .../views/nodes/formattedText/FormattedTextBox.tsx | 19 ++-- src/scraping/buxton/scraper.py | 2 +- src/server/ApiManagers/DeleteManager.ts | 85 +++++--------- src/server/ApiManagers/GeneralGoogleManager.ts | 19 +++- src/server/ApiManagers/GooglePhotosManager.ts | 2 +- src/server/ApiManagers/SessionManager.ts | 2 +- src/server/ApiManagers/UploadManager.ts | 2 +- src/server/ApiManagers/UserManager.ts | 2 - src/server/DashSession/DashSessionAgent.ts | 2 +- src/server/GarbageCollector.ts | 6 +- src/server/IDatabase.ts | 4 +- src/server/MemoryDatabase.ts | 23 +++- src/server/Websocket/Websocket.ts | 37 ++---- src/server/apis/google/GoogleApiServerUtils.ts | 47 +------- .../authentication/models/current_user_utils.ts | 36 +++--- src/server/authentication/models/user_model.ts | 4 +- src/server/database.ts | 118 ++++++++++--------- src/server/remapUrl.ts | 4 +- 25 files changed, 285 insertions(+), 278 deletions(-) (limited to 'src/scraping') diff --git a/src/client/apis/GoogleAuthenticationManager.scss b/src/client/apis/GoogleAuthenticationManager.scss index 13bde822d..bd30dd94f 100644 --- a/src/client/apis/GoogleAuthenticationManager.scss +++ b/src/client/apis/GoogleAuthenticationManager.scss @@ -16,4 +16,11 @@ font-style: italic; margin-top: 15px; } + + .disconnect { + font-size: 10px; + margin-top: 20px; + color: red; + cursor: grab; + } } \ No newline at end of file diff --git a/src/client/apis/GoogleAuthenticationManager.tsx b/src/client/apis/GoogleAuthenticationManager.tsx index 417dc3c3b..018c980d8 100644 --- a/src/client/apis/GoogleAuthenticationManager.tsx +++ b/src/client/apis/GoogleAuthenticationManager.tsx @@ -1,10 +1,11 @@ -import { observable, action, reaction, runInAction } from "mobx"; +import { observable, action, reaction, runInAction, IReactionDisposer } from "mobx"; import { observer } from "mobx-react"; import * as React from "react"; import MainViewModal from "../views/MainViewModal"; import { Opt } from "../../new_fields/Doc"; import { Networking } from "../Network"; import "./GoogleAuthenticationManager.scss"; +import { Scripting } from "../util/Scripting"; const AuthenticationUrl = "https://accounts.google.com/o/oauth2/v2/auth"; const prompt = "Paste authorization code here..."; @@ -15,64 +16,89 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { private authenticationLink: Opt = undefined; @observable private openState = false; @observable private authenticationCode: Opt = undefined; - @observable private clickedState = false; + @observable private showPasteTargetState = false; @observable private success: Opt = undefined; @observable private displayLauncher = true; - @observable private avatar: Opt = undefined; - @observable private username: Opt = undefined; + @observable private credentials: any; + private disposer: Opt; private set isOpen(value: boolean) { runInAction(() => this.openState = value); } - private set hasBeenClicked(value: boolean) { - runInAction(() => this.clickedState = value); + private set shouldShowPasteTarget(value: boolean) { + runInAction(() => this.showPasteTargetState = value); } - public fetchOrGenerateAccessToken = async () => { - const response = await Networking.FetchFromServer("/readGoogleAccessToken"); + public cancel() { + this.openState && this.resetState(0, 0); + } + + public fetchOrGenerateAccessToken = async (displayIfFound = false) => { + let response: any = await Networking.FetchFromServer("/readGoogleAccessToken"); + // if this is an authentication url, activate the UI to register the new access token if (new RegExp(AuthenticationUrl).test(response)) { this.isOpen = true; this.authenticationLink = response; return new Promise(async resolve => { - const disposer = reaction( + this.disposer?.(); + this.disposer = reaction( () => this.authenticationCode, async authenticationCode => { - if (authenticationCode) { - disposer(); - const { access_token, avatar, name } = await Networking.PostToServer("/writeGoogleAccessToken", { authenticationCode }); + if (authenticationCode && /\d{1}\/[\w-]{55}/.test(authenticationCode)) { + this.disposer?.(); + const response = await Networking.PostToServer("/writeGoogleAccessToken", { authenticationCode }); runInAction(() => { - this.avatar = avatar; - this.username = name; - this.hasBeenClicked = false; - this.success = false; + this.success = true; + this.credentials = response; }); - this.beginFadeout(); - resolve(access_token); + this.resetState(); + resolve(response.access_token); } } ); }); } - // otherwise, we already have a valid, stored access token - return response; + + // otherwise, we already have a valid, stored access token and user info + response = JSON.parse(response); + if (displayIfFound) { + runInAction(() => { + this.success = true; + this.credentials = response; + }); + this.resetState(-1, -1); + this.isOpen = true; + } + return response.access_token; } - beginFadeout = action(() => { - this.success = true; + resetState = action((visibleForMS: number = 3000, fadesOutInMS: number = 500) => { + if (!visibleForMS && !fadesOutInMS) { + runInAction(() => { + this.isOpen = false; + this.success = undefined; + this.displayLauncher = true; + this.credentials = undefined; + this.shouldShowPasteTarget = false; + this.authenticationCode = undefined; + }); + return; + } this.authenticationCode = undefined; this.displayLauncher = false; - this.hasBeenClicked = false; - setTimeout(action(() => { - this.isOpen = false; + this.shouldShowPasteTarget = false; + if (visibleForMS > 0 && fadesOutInMS > 0) { setTimeout(action(() => { - this.success = undefined; - this.displayLauncher = true; - this.avatar = undefined; - this.username = undefined; - }), 500); - }), 3000); + this.isOpen = false; + setTimeout(action(() => { + this.success = undefined; + this.displayLauncher = true; + this.credentials = undefined; + }), fadesOutInMS); + }), visibleForMS); + } }); constructor(props: {}) { @@ -83,27 +109,38 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { private get renderPrompt() { return (
+ {this.displayLauncher ? : (null)} - {this.clickedState ? this.authenticationCode = e.currentTarget.value)} placeholder={prompt} /> : (null)} - {this.avatar ? : (null)} - {this.username ? Welcome to Dash, {this.username} - : (null)} + {this.credentials ? + <> + + Welcome to Dash, {this.credentials.userInfo.name} + +
{ + await Networking.FetchFromServer("/revokeGoogleAccessToken"); + this.resetState(0, 0); + }} + >Disconnect Account
+ : (null)}
); } @@ -125,4 +162,6 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { ); } -} \ No newline at end of file +} + +Scripting.addGlobal("GoogleAuthenticationManager", GoogleAuthenticationManager); \ No newline at end of file diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index fa67ddbef..2f3cac8d3 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -95,7 +95,7 @@ export namespace GoogleApiClientUtils { export type ExtractResult = { text: string, paragraphs: DeconstructedParagraph[] }; export const extractText = (document: docs_v1.Schema$Document, removeNewlines = false): ExtractResult => { const paragraphs = extractParagraphs(document); - let text = paragraphs.map(paragraph => paragraph.contents.filter(content => !("inlineObjectId" in content)).map(run => run as docs_v1.Schema$TextRun).join("")).join(""); + let text = paragraphs.map(paragraph => paragraph.contents.filter(content => !("inlineObjectId" in content)).map(run => (run as docs_v1.Schema$TextRun).content).join("")).join(""); text = text.substring(0, text.length - 1); removeNewlines && text.replace(/\n/g, ""); return { text, paragraphs }; diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index 185222541..6cca4d69f 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -12,6 +12,7 @@ import { ScriptField } from "../../new_fields/ScriptField"; import { InkingControl } from "./InkingControl"; import { InkTool } from "../../new_fields/InkField"; import { DocumentView } from "./nodes/DocumentView"; +import GoogleAuthenticationManager from "../apis/GoogleAuthenticationManager"; const modifiers = ["control", "meta", "shift", "alt"]; type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo | Promise; @@ -79,6 +80,7 @@ export default class KeyManager { SelectionManager.DeselectAll(); DictationManager.Controls.stop(); // RecommendationsBox.Instance.closeMenu(); + GoogleAuthenticationManager.Instance.cancel(); SharingManager.Instance.close(); break; case "delete": diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index d60a9d64a..1a285d4ec 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -1,5 +1,5 @@ import { library } from '@fortawesome/fontawesome-svg-core'; -import { faTerminal, faToggleOn, faFile as fileSolid, faLocationArrow, faSearch, faFileDownload, faStop, faCalculator, faWindowMaximize, faAddressCard, faQuestionCircle, faArrowLeft, faArrowRight, faArrowDown, faArrowUp, faBolt, faBullseye, faCaretUp, faCat, faCheck, faChevronRight, faClipboard, faClone, faCloudUploadAlt, faCommentAlt, faCompressArrowsAlt, faCut, faEllipsisV, faEraser, faExclamation, faFileAlt, faFileAudio, faFilePdf, faFilm, faFilter, faFont, faGlobeAsia, faHighlighter, faLongArrowAltRight, faMicrophone, faMousePointer, faMusic, faObjectGroup, faPause, faPen, faPenNib, faPhone, faPlay, faPortrait, faRedoAlt, faStamp, faStickyNote, faThumbtack, faTree, faTv, faUndoAlt, faVideo } from '@fortawesome/free-solid-svg-icons'; +import { faTerminal, faToggleOn, faFile as fileSolid, faExternalLinkAlt, faLocationArrow, faSearch, faFileDownload, faStop, faCalculator, faWindowMaximize, faAddressCard, faQuestionCircle, faArrowLeft, faArrowRight, faArrowDown, faArrowUp, faBolt, faBullseye, faCaretUp, faCat, faCheck, faChevronRight, faClipboard, faClone, faCloudUploadAlt, faCommentAlt, faCompressArrowsAlt, faCut, faEllipsisV, faEraser, faExclamation, faFileAlt, faFileAudio, faFilePdf, faFilm, faFilter, faFont, faGlobeAsia, faHighlighter, faLongArrowAltRight, faMicrophone, faMousePointer, faMusic, faObjectGroup, faPause, faPen, faPenNib, faPhone, faPlay, faPortrait, faRedoAlt, faStamp, faStickyNote, faThumbtack, faTree, faTv, faUndoAlt, faVideo } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, computed, configure, observable, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; @@ -163,6 +163,7 @@ export class MainView extends React.Component { library.add(faPhone); library.add(faClipboard); library.add(faStamp); + library.add(faExternalLinkAlt); this.initEventListeners(); this.initAuthenticationRouters(); } @@ -583,7 +584,7 @@ export class MainView extends React.Component { {SnappingManager.horizSnapLines().map(l => )} {SnappingManager.vertSnapLines().map(l => )} - + ; } render() { diff --git a/src/client/views/MainViewModal.tsx b/src/client/views/MainViewModal.tsx index 9198fe3e3..a7bd5882d 100644 --- a/src/client/views/MainViewModal.tsx +++ b/src/client/views/MainViewModal.tsx @@ -4,7 +4,7 @@ import "./MainViewModal.scss"; export interface MainViewOverlayProps { isDisplayed: boolean; interactive: boolean; - contents: string | JSX.Element; + contents: string | JSX.Element | null; dialogueBoxStyle?: React.CSSProperties; overlayStyle?: React.CSSProperties; dialogueBoxDisplayedOpacity?: number; diff --git a/src/client/views/OverlayView.tsx b/src/client/views/OverlayView.tsx index 20aa14f84..afb6bfb7d 100644 --- a/src/client/views/OverlayView.tsx +++ b/src/client/views/OverlayView.tsx @@ -1,7 +1,7 @@ import { action, computed, observable } from "mobx"; import { observer } from "mobx-react"; import * as React from "react"; -import { Doc, DocListCast } from "../../new_fields/Doc"; +import { Doc, DocListCast, Opt } from "../../new_fields/Doc"; import { Id } from "../../new_fields/FieldSymbols"; import { NumCast } from "../../new_fields/Types"; import { emptyFunction, emptyPath, returnEmptyString, returnFalse, returnOne, returnTrue, returnZero, Utils } from "../../Utils"; @@ -214,4 +214,6 @@ export class OverlayView extends React.Component { } } // bcz: ugh ... want to be able to pass ScriptingRepl as tag argument, but that doesn't seem to work.. runtime error -Scripting.addGlobal(function addOverlayWindow(Tag: string, options: OverlayElementOptions) { const x = ; OverlayView.Instance.addWindow(x, options); }); \ No newline at end of file +Scripting.addGlobal(function addOverlayWindow(type: string, options: OverlayElementOptions) { + OverlayView.Instance.addWindow(, options); +}); \ No newline at end of file diff --git a/src/client/views/nodes/formattedText/FormattedTextBox.tsx b/src/client/views/nodes/formattedText/FormattedTextBox.tsx index 658a55f51..180cb043b 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBox.tsx +++ b/src/client/views/nodes/formattedText/FormattedTextBox.tsx @@ -59,6 +59,7 @@ import "./FormattedTextBox.scss"; import { FormattedTextBoxComment, formattedTextBoxCommentPlugin } from './FormattedTextBoxComment'; import React = require("react"); import { ScriptField } from '../../../../new_fields/ScriptField'; +import GoogleAuthenticationManager from '../../../apis/GoogleAuthenticationManager'; library.add(faEdit); library.add(faSmile, faTextHeight, faUpload); @@ -784,7 +785,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<(FieldViewProp let pullSuccess = false; if (exportState !== undefined) { pullSuccess = true; - dataDoc.data = new RichTextField(JSON.stringify(exportState.state.toJSON())); + dataDoc[this.props.fieldKey] = new RichTextField(JSON.stringify(exportState.state.toJSON())); setTimeout(() => { if (this._editorView) { const state = this._editorView.state; @@ -802,13 +803,15 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<(FieldViewProp } checkState = (exportState: Opt, dataDoc: Doc) => { - if (exportState && this._editorView) { - const equalContent = isEqual(this._editorView.state.doc, exportState.state.doc); - const equalTitles = dataDoc.title === exportState.title; - const unchanged = equalContent && equalTitles; - dataDoc.unchanged = unchanged; - DocumentButtonBar.Instance.setPullState(unchanged); - } + GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken().then(() => { + if (exportState && this._editorView) { + const equalContent = isEqual(this._editorView.state.doc, exportState.state.doc); + const equalTitles = dataDoc.title === exportState.title; + const unchanged = equalContent && equalTitles; + dataDoc.unchanged = unchanged; + DocumentButtonBar.Instance.setPullState(unchanged); + } + }); } clipboardTextSerializer = (slice: Slice): string => { diff --git a/src/scraping/buxton/scraper.py b/src/scraping/buxton/scraper.py index 1441a8621..ed122e544 100644 --- a/src/scraping/buxton/scraper.py +++ b/src/scraping/buxton/scraper.py @@ -16,7 +16,7 @@ filesPath = "../../server/public/files" image_dist = filesPath + "/images/buxton" db = MongoClient("localhost", 27017)["Dash"] -target_collection = db.newDocuments +target_collection = db.documents target_doc_title = "Collection 1" schema_guids = [] common_proto_id = "" diff --git a/src/server/ApiManagers/DeleteManager.ts b/src/server/ApiManagers/DeleteManager.ts index 9e70af2eb..bd80d6500 100644 --- a/src/server/ApiManagers/DeleteManager.ts +++ b/src/server/ApiManagers/DeleteManager.ts @@ -1,12 +1,12 @@ import ApiManager, { Registration } from "./ApiManager"; -import { Method, _permission_denied, PublicHandler } from "../RouteManager"; +import { Method, _permission_denied } from "../RouteManager"; import { WebSocket } from "../Websocket/Websocket"; import { Database } from "../database"; import rimraf = require("rimraf"); -import { pathToDirectory, Directory } from "./UploadManager"; import { filesDirectory } from ".."; import { DashUploadUtils } from "../DashUploadUtils"; import { mkdirSync } from "fs"; +import RouteSubscriber from "../RouteSubscriber"; export default class DeleteManager extends ApiManager { @@ -14,68 +14,39 @@ export default class DeleteManager extends ApiManager { register({ method: Method.GET, - subscription: "/delete", - secureHandler: async ({ res, isRelease }) => { + subscription: new RouteSubscriber("delete").add("target?"), + secureHandler: async ({ req, res, isRelease }) => { if (isRelease) { - return _permission_denied(res, deletionPermissionError); + return _permission_denied(res, "Cannot perform a delete operation outside of the development environment!"); } - await WebSocket.deleteFields(); - res.redirect("/home"); - } - }); - register({ - method: Method.GET, - subscription: "/deleteAll", - secureHandler: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); + const { target } = req.params; + const { doDelete } = WebSocket; + + if (!target) { + await doDelete(); + } else { + let all = false; + switch (target) { + case "all": + all = true; + case "database": + await doDelete(false); + if (!all) break; + case "files": + rimraf.sync(filesDirectory); + mkdirSync(filesDirectory); + await DashUploadUtils.buildFileDirectories(); + break; + default: + await Database.Instance.dropSchema(target); + } } - await WebSocket.deleteAll(); - res.redirect("/home"); - } - }); - register({ - method: Method.GET, - subscription: "/deleteAssets", - secureHandler: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - rimraf.sync(filesDirectory); - mkdirSync(filesDirectory); - await DashUploadUtils.buildFileDirectories(); - res.redirect("/delete"); - } - }); - - register({ - method: Method.GET, - subscription: "/deleteWithAux", - secureHandler: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.DeleteAll(); - res.redirect("/delete"); - } - }); - - register({ - method: Method.GET, - subscription: "/deleteWithGoogleCredentials", - secureHandler: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect("/delete"); + res.redirect("/home"); } }); } -} - -const deletionPermissionError = "Cannot perform a delete operation outside of the development environment!"; +} \ No newline at end of file diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts index a5240edbc..17968cc7d 100644 --- a/src/server/ApiManagers/GeneralGoogleManager.ts +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -1,10 +1,8 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method, _permission_denied } from "../RouteManager"; import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; -import { Database } from "../database"; import RouteSubscriber from "../RouteSubscriber"; - -const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; +import { Database } from "../database"; const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], @@ -20,11 +18,11 @@ export default class GeneralGoogleManager extends ApiManager { method: Method.GET, subscription: "/readGoogleAccessToken", secureHandler: async ({ user, res }) => { - const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); - if (!token) { + const { credentials } = (await GoogleApiServerUtils.retrieveCredentials(user.id)); + if (!credentials?.access_token) { return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); } - return res.send(token); + return res.send(credentials); } }); @@ -36,6 +34,15 @@ export default class GeneralGoogleManager extends ApiManager { } }); + register({ + method: Method.GET, + subscription: "/revokeGoogleAccessToken", + secureHandler: async ({ user, res }) => { + await Database.Auxiliary.GoogleAuthenticationToken.Revoke(user.id); + res.send(); + } + }); + register({ method: Method.POST, subscription: new RouteSubscriber("googleDocs").add("sector", "action"), diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index 88219423d..11841a603 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -56,7 +56,7 @@ export default class GooglePhotosManager extends ApiManager { const { media } = req.body; // first we need to ensure that we know the google account to which these photos will be uploaded - const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); + const token = (await GoogleApiServerUtils.retrieveCredentials(user.id))?.credentials?.access_token; if (!token) { return _error(res, authenticationError); } diff --git a/src/server/ApiManagers/SessionManager.ts b/src/server/ApiManagers/SessionManager.ts index bcaa6598f..fa2f6002a 100644 --- a/src/server/ApiManagers/SessionManager.ts +++ b/src/server/ApiManagers/SessionManager.ts @@ -55,7 +55,7 @@ export default class SessionManager extends ApiManager { register({ method: Method.GET, - subscription: this.secureSubscriber("delete"), + subscription: this.secureSubscriber("deleteSession"), secureHandler: this.authorizedAction(async ({ res }) => { const { error } = await sessionAgent.serverWorker.emit("delete"); res.send(error ? error.message : "Your request was successful: the server successfully deleted the database. Return to /home."); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 98f029c7d..b185d3b55 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -171,7 +171,7 @@ export default class UploadManager extends ApiManager { await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { err && console.log(err); res(); - }, true, "newDocuments")))); + }, true)))); } catch (e) { console.log(e); } unlink(path_2, () => { }); } diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index d9d346cc1..68b3107ae 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -89,8 +89,6 @@ export default class UserManager extends ApiManager { } }); - - register({ method: Method.GET, subscription: "/activity", diff --git a/src/server/DashSession/DashSessionAgent.ts b/src/server/DashSession/DashSessionAgent.ts index 5cbba13de..ef9b88541 100644 --- a/src/server/DashSession/DashSessionAgent.ts +++ b/src/server/DashSession/DashSessionAgent.ts @@ -37,7 +37,7 @@ export class DashSessionAgent extends AppliedSessionAgent { monitor.addReplCommand("debug", [/\S+\@\S+/], async ([to]) => this.dispatchZippedDebugBackup(to)); monitor.on("backup", this.backup); monitor.on("debug", async ({ to }) => this.dispatchZippedDebugBackup(to)); - monitor.on("delete", WebSocket.deleteFields); + monitor.on("delete", WebSocket.doDelete); monitor.coreHooks.onCrashDetected(this.dispatchCrashReport); return sessionKey; } diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts index 5729c3ee5..24745cbb4 100644 --- a/src/server/GarbageCollector.ts +++ b/src/server/GarbageCollector.ts @@ -76,7 +76,7 @@ async function GarbageCollect(full: boolean = true) { if (!fetchIds.length) { continue; } - const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments")); + const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res)); for (const doc of docs) { const id = doc.id; if (doc === undefined) { @@ -116,10 +116,10 @@ async function GarbageCollect(full: boolean = true) { const count = Math.min(toDelete.length, 5000); const toDeleteDocs = toDelete.slice(i, i + count); i += count; - const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments"); + const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }); deleted += result.deletedCount || 0; } - // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments"); + // const result = await Database.Instance.delete({ _id: { $in: toDelete } }); console.log(`${deleted} documents deleted`); await Search.deleteDocuments(toDelete); diff --git a/src/server/IDatabase.ts b/src/server/IDatabase.ts index 6a63df485..dd4968579 100644 --- a/src/server/IDatabase.ts +++ b/src/server/IDatabase.ts @@ -2,7 +2,6 @@ import * as mongodb from 'mongodb'; import { Transferable } from './Message'; export const DocumentsCollection = 'documents'; -export const NewDocumentsCollection = 'newDocuments'; export interface IDatabase { update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert?: boolean, collectionName?: string): Promise; updateMany(query: any, update: any, collectionName?: string): Promise; @@ -12,12 +11,13 @@ export interface IDatabase { delete(query: any, collectionName?: string): Promise; delete(id: string, collectionName?: string): Promise; - deleteAll(collectionName?: string, persist?: boolean): Promise; + dropSchema(...schemaNames: string[]): Promise; insert(value: any, collectionName?: string): Promise; getDocument(id: string, fn: (result?: Transferable) => void, collectionName?: string): void; getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName?: string): void; + getCollectionNames(): Promise; visit(ids: string[], fn: (result: any) => string[] | Promise, collectionName?: string): Promise; query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName?: string): Promise; diff --git a/src/server/MemoryDatabase.ts b/src/server/MemoryDatabase.ts index 543f96e7f..1f1d702d9 100644 --- a/src/server/MemoryDatabase.ts +++ b/src/server/MemoryDatabase.ts @@ -1,4 +1,4 @@ -import { IDatabase, DocumentsCollection, NewDocumentsCollection } from './IDatabase'; +import { IDatabase, DocumentsCollection } from './IDatabase'; import { Transferable } from './Message'; import * as mongodb from 'mongodb'; @@ -15,6 +15,10 @@ export class MemoryDatabase implements IDatabase { } } + public getCollectionNames() { + return Promise.resolve(Object.keys(this.db)); + } + public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, _upsert?: boolean, collectionName = DocumentsCollection): Promise { const collection = this.getCollection(collectionName); const set = "$set"; @@ -41,7 +45,7 @@ export class MemoryDatabase implements IDatabase { return Promise.resolve(undefined); } - public updateMany(query: any, update: any, collectionName = NewDocumentsCollection): Promise { + public updateMany(query: any, update: any, collectionName = DocumentsCollection): Promise { throw new Error("Can't updateMany a MemoryDatabase"); } @@ -58,8 +62,15 @@ export class MemoryDatabase implements IDatabase { return Promise.resolve({} as any); } - public deleteAll(collectionName = DocumentsCollection, _persist = true): Promise { - delete this.db[collectionName]; + public async dropSchema(...schemaNames: string[]): Promise { + const existing = await this.getCollectionNames(); + let valid: string[]; + if (schemaNames.length) { + valid = schemaNames.filter(collection => existing.includes(collection)); + } else { + valid = existing; + } + valid.forEach(schemaName => delete this.db[schemaName]); return Promise.resolve(); } @@ -69,14 +80,14 @@ export class MemoryDatabase implements IDatabase { return Promise.resolve(); } - public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = NewDocumentsCollection): void { + public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = DocumentsCollection): void { fn(this.getCollection(collectionName)[id]); } public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = DocumentsCollection): void { fn(ids.map(id => this.getCollection(collectionName)[id])); } - public async visit(ids: string[], fn: (result: any) => string[] | Promise, collectionName = NewDocumentsCollection): Promise { + public async visit(ids: string[], fn: (result: any) => string[] | Promise, collectionName = DocumentsCollection): Promise { const visited = new Set(); while (ids.length) { const count = Math.min(ids.length, 1000); diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index be895c4bc..844535056 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -12,6 +12,7 @@ import { timeMap } from "../ApiManagers/UserManager"; import { green } from "colors"; import { networkInterfaces } from "os"; import executeImport from "../../scraping/buxton/final/BuxtonImporter"; +import { DocumentsCollection } from "../IDatabase"; export namespace WebSocket { @@ -96,7 +97,7 @@ export namespace WebSocket { Utils.AddServerHandlerCallback(socket, MessageStore.GetField, getField); Utils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields); if (isRelease) { - Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); + Utils.AddServerHandler(socket, MessageStore.DeleteAll, () => doDelete(false)); } Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); @@ -163,24 +164,10 @@ export namespace WebSocket { } } - export async function deleteFields() { - await Database.Instance.deleteAll(); - if (process.env.DISABLE_SEARCH !== "true") { - await Search.clear(); - } - await Database.Instance.deleteAll('newDocuments'); - } - - // export async function deleteUserDocuments() { - // await Database.Instance.deleteAll(); - // await Database.Instance.deleteAll('newDocuments'); - // } - - export async function deleteAll() { - await Database.Instance.deleteAll(); - await Database.Instance.deleteAll('newDocuments'); - await Database.Instance.deleteAll('sessions'); - await Database.Instance.deleteAll('users'); + export async function doDelete(onlyFields = true) { + const target: string[] = []; + onlyFields && target.push(DocumentsCollection); + await Database.Instance.dropSchema(...target); if (process.env.DISABLE_SEARCH !== "true") { await Search.clear(); } @@ -210,11 +197,11 @@ export namespace WebSocket { } function GetRefField([id, callback]: [string, (result?: Transferable) => void]) { - Database.Instance.getDocument(id, callback, "newDocuments"); + Database.Instance.getDocument(id, callback); } function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => void]) { - Database.Instance.getDocuments(ids, callback, "newDocuments"); + Database.Instance.getDocuments(ids, callback); } const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { @@ -271,7 +258,7 @@ export namespace WebSocket { function UpdateField(socket: Socket, diff: Diff) { Database.Instance.update(diff.id, diff.diff, - () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); + () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false); const docfield = diff.diff.$set || diff.diff.$unset; if (!docfield) { return; @@ -296,7 +283,7 @@ export namespace WebSocket { } function DeleteField(socket: Socket, id: string) { - Database.Instance.delete({ _id: id }, "newDocuments").then(() => { + Database.Instance.delete({ _id: id }).then(() => { socket.broadcast.emit(MessageStore.DeleteField.Message, id); }); @@ -304,14 +291,14 @@ export namespace WebSocket { } function DeleteFields(socket: Socket, ids: string[]) { - Database.Instance.delete({ _id: { $in: ids } }, "newDocuments").then(() => { + Database.Instance.delete({ _id: { $in: ids } }).then(() => { socket.broadcast.emit(MessageStore.DeleteFields.Message, ids); }); Search.deleteDocuments(ids); } function CreateField(newValue: any) { - Database.Instance.insert(newValue, "newDocuments"); + Database.Instance.insert(newValue); } } diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 0f75833ee..48a8da89f 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -148,26 +148,6 @@ export namespace GoogleApiServerUtils { }); } - /** - * Returns the lengthy string or access token that can be passed into - * the headers of an API request or into the constructor of the Photos - * client API wrapper. - * @param userId the Dash user id of the user requesting his/her associated - * access_token - * @returns the current access_token associated with the requesting - * Dash user. The access_token is valid for only an hour, and - * is then refreshed. - */ - export async function retrieveAccessToken(userId: string): Promise { - return new Promise(async resolve => { - const { credentials } = await retrieveCredentials(userId); - if (!credentials) { - return resolve(); - } - resolve(credentials.access_token!); - }); - } - /** * Manipulates a mapping such that, in the limit, each Dash user has * an associated authenticated OAuth2 client at their disposal. This @@ -216,18 +196,6 @@ export namespace GoogleApiServerUtils { return worker.generateAuthUrl({ scope, access_type: 'offline' }); } - /** - * This is what we return to the server in processNewUser(), after the - * worker OAuth2Client has used the user-pasted authentication code - * to retrieve an access token and an info token. The avatar is the - * URL to the Google-hosted mono-color, single white letter profile 'image'. - */ - export interface GoogleAuthenticationResult { - access_token: string; - avatar: string; - name: string; - } - /** * This method receives the authentication code that the * user pasted into the overlay in the client side and uses the worker @@ -245,7 +213,7 @@ export namespace GoogleApiServerUtils { * and display basic user information in the overlay on successful authentication. * This can be expanded as needed by adding properties to the interface GoogleAuthenticationResult. */ - export async function processNewUser(userId: string, authenticationCode: string): Promise { + export async function processNewUser(userId: string, authenticationCode: string): Promise { const credentials = await new Promise((resolve, reject) => { worker.getToken(authenticationCode, async (err, credentials) => { if (err || !credentials) { @@ -257,12 +225,7 @@ export namespace GoogleApiServerUtils { }); const enriched = injectUserInfo(credentials); await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched); - const { given_name, picture } = enriched.userInfo; - return { - access_token: enriched.access_token!, - avatar: picture, - name: given_name - }; + return enriched; } /** @@ -316,15 +279,15 @@ export namespace GoogleApiServerUtils { * @returns the credentials, or undefined if the user has no stored associated credentials, * and a flag indicating whether or not they were refreshed during retrieval */ - async function retrieveCredentials(userId: string): Promise<{ credentials: Opt, refreshed: boolean }> { - let credentials: Opt = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); + export async function retrieveCredentials(userId: string): Promise<{ credentials: Opt, refreshed: boolean }> { + let credentials = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); let refreshed = false; if (!credentials) { return { credentials: undefined, refreshed }; } // check for token expiry if (credentials.expiry_date! <= new Date().getTime()) { - credentials = await refreshAccessToken(credentials, userId); + credentials = { ...credentials, ...(await refreshAccessToken(credentials, userId)) }; refreshed = true; } return { credentials, refreshed }; diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index b10c6f119..5afb90c71 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -293,7 +293,7 @@ export class CurrentUserUtils { { _width: 250, _height: 250, title: "container" }); } if (doc.emptyWebpage === undefined) { - doc.emptyWebpage = Docs.Create.WebDocument("", { title: "New Webpage", _width: 600 }) + doc.emptyWebpage = Docs.Create.WebDocument("", { title: "New Webpage", _width: 600 }); } return [ { title: "Drag a collection", label: "Col", icon: "folder", click: 'openOnRight(getCopy(this.dragFactory, true))', drag: 'getCopy(this.dragFactory, true)', dragFactory: doc.emptyCollection as Doc }, @@ -315,7 +315,8 @@ export class CurrentUserUtils { // { title: "use eraser", icon: "eraser", click: 'activateEraser(this.activePen.inkPen = sameDocs(this.activePen.inkPen, this) ? undefined : this);', ischecked: `sameDocs(this.activePen.inkPen, this)`, backgroundColor: "pink", activePen: doc }, // { title: "use drag", icon: "mouse-pointer", click: 'deactivateInk();this.activePen.inkPen = this;', ischecked: `sameDocs(this.activePen.inkPen, this)`, backgroundColor: "white", activePen: doc }, { title: "Drag a document previewer", label: "Prev", icon: "expand", click: 'openOnRight(getCopy(this.dragFactory, true))', drag: 'getCopy(this.dragFactory,true)', dragFactory: doc.emptyDocHolder as Doc }, - { title: "Drag a Calculator REPL", label: "repl", icon: "calculator", click: 'addOverlayWindow("ScriptingRepl", { x: 300, y: 100, width: 200, height: 200, title: "Scripting REPL" })' }, + { title: "Toggle a Calculator REPL", label: "repl", icon: "calculator", click: 'addOverlayWindow("ScriptingRepl", { x: 300, y: 100, width: 200, height: 200, title: "Scripting REPL" })' }, + { title: "Connect a Google Account", label: "Google Account", icon: "external-link-alt", click: 'GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(true)' }, ]; } @@ -331,21 +332,22 @@ export class CurrentUserUtils { alreadyCreatedButtons = dragDocs.map(d => StrCast(d.title)); } } - const creatorBtns = CurrentUserUtils.creatorBtnDescriptors(doc).filter(d => !alreadyCreatedButtons?.includes(d.title)). - map(data => Docs.Create.FontIconDocument({ - _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, - icon: data.icon, - title: data.title, - label: data.label, - ignoreClick: data.ignoreClick, - dropAction: "copy", - onDragStart: data.drag ? ScriptField.MakeFunction(data.drag) : undefined, - onClick: data.click ? ScriptField.MakeScript(data.click) : undefined, - ischecked: data.ischecked ? ComputedField.MakeFunction(data.ischecked) : undefined, - activePen: data.activePen, - backgroundColor: data.backgroundColor, removeDropProperties: new List(["dropAction"]), - dragFactory: data.dragFactory, - })); + const buttons = CurrentUserUtils.creatorBtnDescriptors(doc).filter(d => !alreadyCreatedButtons?.includes(d.title)); + const creatorBtns = buttons.map(({ title, label, icon, ignoreClick, drag, click, ischecked, activePen, backgroundColor, dragFactory }) => Docs.Create.FontIconDocument({ + _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, + icon, + title, + label, + ignoreClick, + dropAction: "copy", + onDragStart: drag ? ScriptField.MakeFunction(drag) : undefined, + onClick: click ? ScriptField.MakeScript(click) : undefined, + ischecked: ischecked ? ComputedField.MakeFunction(ischecked) : undefined, + activePen, + backgroundColor, + removeDropProperties: new List(["dropAction"]), + dragFactory, + })); if (dragCreatorSet === undefined) { doc.myItemCreators = new PrefetchProxy(Docs.Create.MasonryDocument(creatorBtns, { diff --git a/src/server/authentication/models/user_model.ts b/src/server/authentication/models/user_model.ts index 78e39dbc1..a0b688328 100644 --- a/src/server/authentication/models/user_model.ts +++ b/src/server/authentication/models/user_model.ts @@ -74,9 +74,9 @@ userSchema.pre("save", function save(next) { const comparePassword: comparePasswordFunction = function (this: DashUserModel, candidatePassword, cb) { // Choose one of the following bodies for authentication logic. - // secure + // secure (expected, default) bcrypt.compare(candidatePassword, this.password, cb); - // bypass password + // bypass password (debugging) // cb(undefined, true); }; diff --git a/src/server/database.ts b/src/server/database.ts index ad285765b..9ba461b65 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -4,7 +4,7 @@ import { Opt } from '../new_fields/Doc'; import { Utils, emptyFunction } from '../Utils'; import { Credentials } from 'google-auth-library'; import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; -import { IDatabase } from './IDatabase'; +import { IDatabase, DocumentsCollection } from './IDatabase'; import { MemoryDatabase } from './MemoryDatabase'; import * as mongoose from 'mongoose'; import { Upload } from './SharedMediaTypes'; @@ -14,7 +14,7 @@ export namespace Database { export let disconnect: Function; const schema = 'Dash'; const port = 27017; - export const url = `mongodb://localhost:${port}/`; + export const url = `mongodb://localhost:${port}/${schema}`; enum ConnectionStates { disconnected = 0, @@ -47,28 +47,29 @@ export namespace Database { } export class Database implements IDatabase { - public static DocumentsCollection = 'documents'; private MongoClient = mongodb.MongoClient; private currentWrites: { [id: string]: Promise } = {}; private db?: mongodb.Db; private onConnect: (() => void)[] = []; - doConnect() { + async doConnect() { console.error(`\nConnecting to Mongo with URL : ${url}\n`); - this.MongoClient.connect(url, { connectTimeoutMS: 30000, socketTimeoutMS: 30000, useUnifiedTopology: true }, (_err, client) => { - console.error("mongo connect response\n"); - if (!client) { - console.error("\nMongo connect failed with the error:\n"); - console.log(_err); - process.exit(0); - } - this.db = client.db(); - this.onConnect.forEach(fn => fn()); + return new Promise(resolve => { + this.MongoClient.connect(url, { connectTimeoutMS: 30000, socketTimeoutMS: 30000, useUnifiedTopology: true }, (_err, client) => { + console.error("mongo connect response\n"); + if (!client) { + console.error("\nMongo connect failed with the error:\n"); + console.log(_err); + process.exit(0); + } + this.db = client.db(); + this.onConnect.forEach(fn => fn()); + resolve(); + }); }); } - public async update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { - + public async update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = DocumentsCollection) { if (this.db) { const collection = this.db.collection(collectionName); const prom = this.currentWrites[id]; @@ -93,7 +94,7 @@ export namespace Database { } } - public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { + public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = DocumentsCollection) { if (this.db) { const collection = this.db.collection(collectionName); const prom = this.currentWrites[id]; @@ -117,9 +118,25 @@ export namespace Database { } } + public async getCollectionNames() { + const cursor = this.db?.listCollections(); + const collectionNames: string[] = []; + if (cursor) { + while (await cursor.hasNext()) { + const collection: any = await cursor.next(); + collection && collectionNames.push(collection.name); + } + } + return collectionNames; + } + + public async clear() { + return Promise.all((await this.getCollectionNames()).map(collection => this.dropSchema(collection))); + } + public delete(query: any, collectionName?: string): Promise; public delete(id: string, collectionName?: string): Promise; - public delete(id: any, collectionName = Database.DocumentsCollection) { + public delete(id: any, collectionName = DocumentsCollection) { if (typeof id === "string") { id = { _id: id }; } @@ -131,25 +148,26 @@ export namespace Database { } } - public async deleteAll(collectionName = Database.DocumentsCollection, persist = true): Promise { - return new Promise(resolve => { - const executor = async (database: mongodb.Db) => { - if (persist) { - await database.collection(collectionName).deleteMany({}); - } else { - await database.dropCollection(collectionName); - } - resolve(); - }; - if (this.db) { - executor(this.db); + public async dropSchema(...targetSchemas: string[]): Promise { + const executor = async (database: mongodb.Db) => { + const existing = await Instance.getCollectionNames(); + let valid: string[]; + if (targetSchemas.length) { + valid = targetSchemas.filter(collection => existing.includes(collection)); } else { - this.onConnect.push(() => this.db && executor(this.db)); + valid = existing; } - }); + const pending = Promise.all(valid.map(schemaName => database.dropCollection(schemaName))); + return (await pending).every(dropOutcome => dropOutcome); + }; + if (this.db) { + return executor(this.db); + } else { + this.onConnect.push(() => this.db && executor(this.db)); + } } - public async insert(value: any, collectionName = Database.DocumentsCollection) { + public async insert(value: any, collectionName = DocumentsCollection) { if (this.db) { if ("id" in value) { value._id = value.id; @@ -177,7 +195,7 @@ export namespace Database { } } - public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = "newDocuments") { + public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = DocumentsCollection) { if (this.db) { this.db.collection(collectionName).findOne({ _id: id }, (err, result) => { if (result) { @@ -193,7 +211,7 @@ export namespace Database { } } - public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = Database.DocumentsCollection) { + public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = DocumentsCollection) { if (this.db) { this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => { if (err) { @@ -211,7 +229,7 @@ export namespace Database { } } - public async visit(ids: string[], fn: (result: any) => string[] | Promise, collectionName = "newDocuments"): Promise { + public async visit(ids: string[], fn: (result: any) => string[] | Promise, collectionName = DocumentsCollection): Promise { if (this.db) { const visited = new Set(); while (ids.length) { @@ -238,7 +256,7 @@ export namespace Database { } } - public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise { + public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = DocumentsCollection): Promise { if (this.db) { let cursor = this.db.collection(collectionName).find(query); if (projection) { @@ -252,7 +270,7 @@ export namespace Database { } } - public updateMany(query: any, update: any, collectionName = "newDocuments") { + public updateMany(query: any, update: any, collectionName = DocumentsCollection) { if (this.db) { const db = this.db; return new Promise(res => db.collection(collectionName).update(query, update, (_, result) => res(result))); @@ -282,7 +300,8 @@ export namespace Database { export namespace Auxiliary { export enum AuxiliaryCollections { - GooglePhotosUploadHistory = "uploadedFromGooglePhotos" + GooglePhotosUploadHistory = "uploadedFromGooglePhotos", + GoogleAuthentication = "googleAuthentication" } const SanitizedCappedQuery = async (query: { [key: string]: any }, collection: string, cap: number, removeId = true) => { @@ -306,27 +325,30 @@ export namespace Database { export namespace GoogleAuthenticationToken { - const GoogleAuthentication = "googleAuthentication"; - - export type StoredCredentials = Credentials & { _id: string }; + type StoredCredentials = GoogleApiServerUtils.EnrichedCredentials & { _id: string }; export const Fetch = async (userId: string, removeId = true): Promise> => { - return SanitizedSingletonQuery({ userId }, GoogleAuthentication, removeId); + return SanitizedSingletonQuery({ userId }, AuxiliaryCollections.GoogleAuthentication, removeId); }; export const Write = async (userId: string, enrichedCredentials: GoogleApiServerUtils.EnrichedCredentials) => { - return Instance.insert({ userId, canAccess: [], ...enrichedCredentials }, GoogleAuthentication); + return Instance.insert({ userId, canAccess: [], ...enrichedCredentials }, AuxiliaryCollections.GoogleAuthentication); }; export const Update = async (userId: string, access_token: string, expiry_date: number) => { const entry = await Fetch(userId, false); if (entry) { const parameters = { $set: { access_token, expiry_date } }; - return Instance.update(entry._id, parameters, emptyFunction, true, GoogleAuthentication); + return Instance.update(entry._id, parameters, emptyFunction, true, AuxiliaryCollections.GoogleAuthentication); } }; - export const DeleteAll = () => Instance.deleteAll(GoogleAuthentication, false); + export const Revoke = async (userId: string) => { + const entry = await Fetch(userId, false); + if (entry) { + Instance.delete({ _id: entry._id }, AuxiliaryCollections.GoogleAuthentication); + } + }; } @@ -338,12 +360,6 @@ export namespace Database { return Instance.insert(bundle, AuxiliaryCollections.GooglePhotosUploadHistory); }; - export const DeleteAll = async (persist = false) => { - const collectionNames = Object.values(AuxiliaryCollections); - const pendingDeletions = collectionNames.map(name => Instance.deleteAll(name, persist)); - return Promise.all(pendingDeletions); - }; - } } diff --git a/src/server/remapUrl.ts b/src/server/remapUrl.ts index 45d2fdd33..91a3cb6bf 100644 --- a/src/server/remapUrl.ts +++ b/src/server/remapUrl.ts @@ -1,6 +1,4 @@ import { Database } from "./database"; -import { Search } from "./Search"; -import * as path from 'path'; //npx ts-node src/server/remapUrl.ts @@ -50,7 +48,7 @@ async function update() { return new Promise(res => Database.Instance.update(doc[0], doc[1], () => { console.log("wrote " + JSON.stringify(doc[1])); res(); - }, false, "newDocuments")); + }, false)); })); console.log("Done"); // await Promise.all(updates.map(update => { -- cgit v1.2.3-70-g09d2 From 7b5b04560ba24b049d77d36562fed1f7dc190d43 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 13 May 2020 01:22:16 -0700 Subject: improved buxton heuristic, but still seems intractable --- src/scraping/buxton/final/BuxtonImporter.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'src/scraping') diff --git a/src/scraping/buxton/final/BuxtonImporter.ts b/src/scraping/buxton/final/BuxtonImporter.ts index 94302c7b3..e55850b29 100644 --- a/src/scraping/buxton/final/BuxtonImporter.ts +++ b/src/scraping/buxton/final/BuxtonImporter.ts @@ -451,11 +451,23 @@ async function writeImages(zip: any): Promise { }); // if it's not an icon, by this rough heuristic, i.e. is it not square - if (Math.abs(width - height) > 10) { - valid.push({ width, height, type, mediaPath }); + const number = Number(/image(\d+)/.exec(mediaPath)![1]); + if (number > 5 || width - height > 10) { + valid.push({ width, height, type, mediaPath, number }); } } + valid.sort((a, b) => a.number - b.number); + + const [{ width: first_w, height: first_h }, { width: second_w, height: second_h }] = valid; + if (Math.abs(first_w / second_w - first_h / second_h) < 0.01) { + const first_size = first_w * first_h; + const second_size = second_w * second_h; + const target = first_size >= second_size ? 1 : 0; + valid.splice(target, 1); + console.log(`Heuristically removed image with size ${target ? second_size : first_size}`); + } + // for each valid image, output the _o, _l, _m, and _s files // THIS IS WHERE THE SCRIPT SPENDS MOST OF ITS TIME for (const { type, width, height, mediaPath } of valid) { -- cgit v1.2.3-70-g09d2