From 0538db262ff611c5273c363470886d2aa42cf760 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 1 Oct 2019 20:31:27 -0400 Subject: initial commit --- src/server/index.ts | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'src/server/index.ts') diff --git a/src/server/index.ts b/src/server/index.ts index 690836fff..1ca5d6e11 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -53,6 +53,7 @@ import { DashUploadUtils } from './DashUploadUtils'; import { BatchedArray, TimeUnit } from 'array-batcher'; import { ParsedPDF } from "./PdfTypes"; import { reject } from 'bluebird'; +import { ExifData } from 'exif'; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -590,10 +591,11 @@ const uploadDirectory = __dirname + "/public/files/"; const pdfDirectory = uploadDirectory + "text"; DashUploadUtils.createIfNotExists(pdfDirectory); -interface FileResponse { +interface ImageFileResponse { name: string; path: string; type: string; + exif: Opt; } // SETTERS @@ -604,10 +606,11 @@ app.post( form.uploadDir = uploadDirectory; form.keepExtensions = true; form.parse(req, async (_err, _fields, files) => { - let results: FileResponse[] = []; + let results: ImageFileResponse[] = []; for (const key in files) { const { type, path: location, name } = files[key]; const filename = path.basename(location); + let uploadInformation: Opt; if (filename.endsWith(".pdf")) { let dataBuffer = fs.readFileSync(uploadDirectory + filename); const result: ParsedPDF = await pdf(dataBuffer); @@ -622,9 +625,10 @@ app.post( }); }); } else { - await DashUploadUtils.UploadImage(uploadDirectory + filename, filename).catch(() => console.log(`Unable to process ${filename}`)); + uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); } - results.push({ name, type, path: `/files/${filename}` }); + const exif = uploadInformation ? uploadInformation.exifData : undefined; + results.push({ name, type, path: `/files/${filename}`, exif }); } _success(res, results); @@ -632,6 +636,15 @@ app.post( } ); +app.post(RouteStore.inspectImage, async (req, res) => { + const { source } = req.body; + if (typeof source === "string") { + const uploadInformation = await DashUploadUtils.UploadImage(source); + return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); + } + res.send({}); +}); + addSecureRoute( Method.POST, (user, res, req) => { -- cgit v1.2.3-70-g09d2 From c17fba4c07a1cc0e40d08228b4cdc4182615496b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 5 Oct 2019 15:31:25 -0400 Subject: beginning external file download --- src/client/util/Import & Export/ImageUtils.ts | 33 +++++++++++++++++++++++++-- src/server/RouteStore.ts | 1 + src/server/index.ts | 10 ++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) (limited to 'src/server/index.ts') diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index 33ca55aa9..bf482aea8 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -1,9 +1,11 @@ -import { Doc } from "../../../new_fields/Doc"; +import { Doc, DocListCast, DocListCastAsync, Opt } from "../../../new_fields/Doc"; import { ImageField } from "../../../new_fields/URLField"; -import { Cast } from "../../../new_fields/Types"; +import { Cast, StrCast } from "../../../new_fields/Types"; import { RouteStore } from "../../../server/RouteStore"; import { Docs } from "../../documents/Documents"; import { Identified } from "../../Network"; +import { Id } from "../../../new_fields/FieldSymbols"; +import { Utils } from "../../../Utils"; export namespace ImageUtils { @@ -19,4 +21,31 @@ export namespace ImageUtils { return data !== undefined; }; + export type Hierarchy = { [id: string]: string | Hierarchy }; + + export const ExportHierarchyToFileSystem = async (doc: Doc): Promise => { + const hierarchy: Hierarchy = {}; + await HierarchyTraverserRecursive(doc, hierarchy); + const a = document.createElement("a"); + a.href = Utils.prepend(`${RouteStore.imageHierarchyExport}/${JSON.stringify(hierarchy)}`); + a.download = `Full Export of ${StrCast(doc.title)}`; + a.click(); + }; + + const HierarchyTraverserRecursive = async (collection: Doc, hierarchy: Hierarchy) => { + const children = await DocListCastAsync(collection.data); + if (children) { + const local: Hierarchy = {}; + hierarchy[collection[Id]] = local; + for (const child of children) { + let imageData: Opt; + if (imageData = Cast(child.data, ImageField)) { + local[child[Id]] = imageData.url.href; + } else { + await HierarchyTraverserRecursive(child, local); + } + } + } + }; + } \ No newline at end of file diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 1e1dd6300..8a9a6baae 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -14,6 +14,7 @@ export enum RouteStore { dataUriToImage = "/uploadURI", images = "/images", inspectImage = "/inspectImage", + imageHierarchyExport = "/imageHierarchyExport", // USER AND WORKSPACES getCurrUser = "/getCurrentUser", diff --git a/src/server/index.ts b/src/server/index.ts index 1ca5d6e11..f9ca3de56 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -320,6 +320,16 @@ app.get("/serializeDoc/:docId", async (req, res) => { res.send({ docs, files: Array.from(files) }); }); +app.get(`${RouteStore.imageHierarchyExport}/:hierarchy`, async (req, res) => { + const hierarchy = JSON.parse(req.params.hierarchy); + Object.keys(hierarchy).map(key => { + let value: any; + if (value = hierarchy[key]) { + + } + }); +}); + app.get("/downloadId/:docId", async (req, res) => { res.set('Content-disposition', `attachment;`); res.set('Content-Type', "application/zip"); -- cgit v1.2.3-70-g09d2 From 6411b7fd6b782957050535850154b05c629fd95a Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 6 Oct 2019 20:53:37 -0400 Subject: implemented image hierarchy export to client file system --- src/client/util/Import & Export/ImageUtils.ts | 26 +-------- src/client/views/collections/CollectionView.tsx | 2 + src/server/database.ts | 7 +-- src/server/index.ts | 73 +++++++++++++++++++++++-- src/server/test.txt | 1 + 5 files changed, 76 insertions(+), 33 deletions(-) create mode 100644 src/server/test.txt (limited to 'src/server/index.ts') diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index bf482aea8..c9abf38fa 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -21,31 +21,11 @@ export namespace ImageUtils { return data !== undefined; }; - export type Hierarchy = { [id: string]: string | Hierarchy }; - - export const ExportHierarchyToFileSystem = async (doc: Doc): Promise => { - const hierarchy: Hierarchy = {}; - await HierarchyTraverserRecursive(doc, hierarchy); + export const ExportHierarchyToFileSystem = async (collection: Doc): Promise => { const a = document.createElement("a"); - a.href = Utils.prepend(`${RouteStore.imageHierarchyExport}/${JSON.stringify(hierarchy)}`); - a.download = `Full Export of ${StrCast(doc.title)}`; + a.href = Utils.prepend(`${RouteStore.imageHierarchyExport}/${collection[Id]}`); + a.download = `Dash Export [${StrCast(collection.title)}].zip`; a.click(); }; - const HierarchyTraverserRecursive = async (collection: Doc, hierarchy: Hierarchy) => { - const children = await DocListCastAsync(collection.data); - if (children) { - const local: Hierarchy = {}; - hierarchy[collection[Id]] = local; - for (const child of children) { - let imageData: Opt; - if (imageData = Cast(child.data, ImageField)) { - local[child[Id]] = imageData.url.href; - } else { - await HierarchyTraverserRecursive(child, local); - } - } - } - }; - } \ No newline at end of file diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index 534246326..893763840 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -18,6 +18,7 @@ import { CollectionSchemaView } from "./CollectionSchemaView"; import { CollectionStackingView } from './CollectionStackingView'; import { CollectionTreeView } from "./CollectionTreeView"; import { CollectionViewBaseChrome } from './CollectionViewChromes'; +import { ImageUtils } from '../../util/Import & Export/ImageUtils'; export const COLLECTION_BORDER_WIDTH = 2; library.add(faTh, faTree, faSquare, faProjectDiagram, faSignature, faThList, faFingerprint, faColumns, faEllipsisV, faImage, faEye as any, faCopy); @@ -123,6 +124,7 @@ export class CollectionView extends React.Component { let layoutItems: ContextMenuProps[] = existing && "subitems" in existing ? existing.subitems : []; layoutItems.push({ description: `${this.props.Document.forceActive ? "Select" : "Force"} Contents Active`, event: () => this.props.Document.forceActive = !this.props.Document.forceActive, icon: "project-diagram" }); !existing && ContextMenu.Instance.addItem({ description: "Layout...", subitems: layoutItems, icon: "hand-point-right" }); + ContextMenu.Instance.addItem({ description: "Export Image Hierarchy", icon: "columns", event: () => ImageUtils.ExportHierarchyToFileSystem(this.props.Document) }); } } diff --git a/src/server/database.ts b/src/server/database.ts index d2375ebd9..990441d5a 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -131,7 +131,7 @@ export namespace Database { } } - public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = Database.DocumentsCollection) { + public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = "newDocuments") { if (this.db) { this.db.collection(collectionName).findOne({ _id: id }, (err, result) => { if (result) { @@ -165,7 +165,7 @@ export namespace Database { } } - public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise { + public async visit(ids: string[], fn: (result: any) => string[] | Promise, collectionName = "newDocuments"): Promise { if (this.db) { const visited = new Set(); while (ids.length) { @@ -179,10 +179,9 @@ export namespace Database { for (const doc of docs) { const id = doc.id; visited.add(id); - ids.push(...fn(doc)); + ids.push(...(await fn(doc))); } } - } else { return new Promise(res => { this.onConnect.push(() => { diff --git a/src/server/index.ts b/src/server/index.ts index f9ca3de56..526cc6cce 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -54,6 +54,7 @@ import { BatchedArray, TimeUnit } from 'array-batcher'; import { ParsedPDF } from "./PdfTypes"; import { reject } from 'bluebird'; import { ExifData } from 'exif'; +import { Result } from '../client/northstar/model/idea/idea'; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -320,16 +321,76 @@ app.get("/serializeDoc/:docId", async (req, res) => { res.send({ docs, files: Array.from(files) }); }); -app.get(`${RouteStore.imageHierarchyExport}/:hierarchy`, async (req, res) => { - const hierarchy = JSON.parse(req.params.hierarchy); - Object.keys(hierarchy).map(key => { - let value: any; - if (value = hierarchy[key]) { +export type Hierarchy = { [id: string]: string | Hierarchy }; +export type ZipMutator = (file: Archiver.Archiver) => void | Promise; - } +app.get(`${RouteStore.imageHierarchyExport}/:docId`, async (req, res) => { + const id = req.params.docId; + const hierarchy: Hierarchy = {}; + await targetedVisitorRecursive(id, hierarchy); + BuildAndDispatchZip(res, async zip => { + await hierarchyTraverserRecursive(zip, hierarchy); }); }); +const BuildAndDispatchZip = async (res: Response, mutator: ZipMutator): Promise => { + const zip = Archiver('zip'); + zip.pipe(res); + await mutator(zip); + return zip.finalize(); +}; + +const targetedVisitorRecursive = async (seedId: string, hierarchy: Hierarchy): Promise => { + const local: Hierarchy = {}; + const { title, data } = await getData(seedId); + const label = `${title} (${seedId})`; + if (Array.isArray(data)) { + hierarchy[label] = local; + await Promise.all(data.map(proxy => targetedVisitorRecursive(proxy.fieldId, local))); + } else { + hierarchy[label + path.extname(data)] = data; + } +}; + +const getData = async (seedId: string): Promise<{ data: string | any[], title: string }> => { + return new Promise<{ data: string | any[], title: string }>((resolve, reject) => { + Database.Instance.getDocument(seedId, async (result: any) => { + const { data, proto, title } = result.fields; + if (data) { + if (data.url) { + resolve({ data: data.url, title }); + } else if (data.fields) { + resolve({ data: data.fields, title }); + } else { + reject(); + } + } + if (proto) { + getData(proto.fieldId).then(resolve, reject); + } + }); + }); +}; + +const hierarchyTraverserRecursive = async (file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise => { + for (const key of Object.keys(hierarchy)) { + const result = hierarchy[key]; + if (typeof result === "string") { + let path: string; + let matches: RegExpExecArray | null; + if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { + path = `${__dirname}/public/files/${matches[1]}`; + } else { + const information = await DashUploadUtils.UploadImage(result); + path = information.mediaPaths[0]; + } + file.file(path, { name: key, prefix }); + } else { + await hierarchyTraverserRecursive(file, result, `${prefix}/${key}`); + } + } +}; + app.get("/downloadId/:docId", async (req, res) => { res.set('Content-disposition', `attachment;`); res.set('Content-Type', "application/zip"); diff --git a/src/server/test.txt b/src/server/test.txt new file mode 100644 index 000000000..1c6f6d1f1 --- /dev/null +++ b/src/server/test.txt @@ -0,0 +1 @@ +"{"one":{"two":"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg","five":"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg","four":{"464d2fbb-8ad9-4bd1-833c-c04f6bb5450f":"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"},"three":"http://localhost:1050/files/upload_b94b0710bd7b72c0f201693597a2296a.jpg"}}" \ No newline at end of file -- cgit v1.2.3-70-g09d2