From 040f86c99465071301daf43481ec7c54fb593234 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 15 Feb 2020 17:19:48 -0500 Subject: commented google photos manager --- src/server/ApiManagers/GooglePhotosManager.ts | 132 +++++++++++++++++++------- 1 file changed, 99 insertions(+), 33 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index 3236d1ee2..04b724f4b 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -7,28 +7,33 @@ import { GooglePhotosUploadUtils } from "../apis/google/GooglePhotosUploadUtils" import { Opt } from "../../new_fields/Doc"; import { DashUploadUtils, InjectSize, SizeSuffix } from "../DashUploadUtils"; import { Database } from "../database"; +import { red } from "colors"; +const prefix = "google_photos_"; +const remoteUploadError = "None of the preliminary uploads to Google's servers was successful."; const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; -const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; +const localUploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; const requestError = "Unable to execute download: the body's media items were malformed."; const downloadError = "Encountered an error while executing downloads."; + interface GooglePhotosUploadFailure { batch: number; index: number; url: string; reason: string; } + interface MediaItem { baseUrl: string; } + interface NewMediaItem { description: string; simpleMediaItem: { uploadToken: string; }; } -const prefix = "google_photos_"; /** * This manager handles the creation of routes for google photos functionality. @@ -37,27 +42,47 @@ export default class GooglePhotosManager extends ApiManager { protected initialize(register: Registration): void { + /** + * This route receives a list of urls that point to images stored + * on Dash's file system, and, in a two step process, uploads them to Google's servers and + * returns the information Google generates about the associated uploaded remote images. + */ register({ method: Method.POST, - subscription: "/googlePhotosMediaUpload", + subscription: "/googlePhotosMediaPost", secureHandler: async ({ user, req, res }) => { const { media } = req.body; + + // first we need to ensure that we know the google account to which these photos will be uploaded const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); if (!token) { return _error(res, authenticationError); } + + // next, having one large list or even synchronously looping over things trips a threshold + // set on Google's servers, and would instantly return an error. So, we ease things out and send the photos to upload in + // batches of 25, where the next batch is sent 100 millieconds after we receive a response from Google's servers. const failed: GooglePhotosUploadFailure[] = []; const batched = BatchedArray.from(media, { batchSize: 25 }); + const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; const newMediaItems = await batched.batchedMapPatientInterval( - { magnitude: 100, unit: TimeUnit.Milliseconds }, + interval, async (batch: any, collector: any, { completedBatches }: any) => { for (let index = 0; index < batch.length; index++) { const { url, description } = batch[index]; + // a local function used to record failure of an upload const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, InjectSize(url, SizeSuffix.Original)).catch(fail); + // see image resizing - we store the size-agnostic url in our logic, but write out size-suffixed images to the file system + // so here, given a size agnostic url, we're just making that conversion so that the file system knows which bytes to actually upload + const imageToUpload = InjectSize(url, SizeSuffix.Original); + // STEP 1/2: send the raw bytes of the image from our server to Google's servers. We'll get back an upload token + // which acts as a pointer to those bytes that we can use to locate them later on + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, imageToUpload).catch(fail); if (!uploadToken) { fail(`${path.extname(url)} is not an accepted extension`); } else { + // gather the upload token return from Google (a pointer they give us to the raw, currently useless bytes + // we've uploaded to their servers) and put in the JSON format that the API accepts for image creation (used soon, below) collector.push({ description, simpleMediaItem: { uploadToken } @@ -66,11 +91,24 @@ export default class GooglePhotosManager extends ApiManager { } } ); - const failedCount = failed.length; - if (failedCount) { - console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + + // inform the developer / server console of any failed upload attempts + // does not abort the operation, since some subset of the uploads may have been successful + const { length } = failed; + if (length) { + console.error(`Unable to upload ${length} image${length === 1 ? "" : "s"} to Google's servers`); console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); } + + // if none of the preliminary uploads was successful, no need to try and create images + // report the failure to the client and return + if (!newMediaItems.length) { + console.error(red(`${remoteUploadError} Thus, aborting image creation. Please try again.`)); + _error(res, remoteUploadError); + return; + } + + // STEP 2/2: create the media items and return the API's response to the client, along with any failures return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( results => _success(res, { results, failed }), error => _error(res, mediaError, error) @@ -78,40 +116,68 @@ export default class GooglePhotosManager extends ApiManager { } }); + /** + * This route receives a list of urls that point to images + * stored on Google's servers and (following a *rough* heuristic) + * uploads each image to Dash's server if it hasn't already been uploaded. + * Unfortunately, since Google has so many of these images on its servers, + * these user content urls expire every 6 hours. So we can't store the url of a locally uploaded + * Google image and compare the candidate url to it to figure out if we already have it, + * since the same bytes on their server might now be associated with a new, random url. + * So, we do the next best thing and try to use an intrinsic attribute of those bytes as + * an identifier: the precise content size. This works in small cases, but has the obvious flaw of failing to upload + * an image locally if we already have uploaded another Google user content image with the exact same content size. + */ register({ method: Method.POST, - subscription: "/googlePhotosMediaDownload", + subscription: "/googlePhotosMediaGet", secureHandler: async ({ req, res }) => { const { mediaItems } = req.body as { mediaItems: MediaItem[] }; + if (!mediaItems) { + // non-starter, since the input was in an invalid format + _invalid(res, requestError); + return; + } let failed = 0; - if (mediaItems) { - const completed: Opt[] = []; - for (const { baseUrl } of mediaItems) { - const results = await DashUploadUtils.InspectImage(baseUrl); - if (results instanceof Error) { - failed++; - continue; - } - const { contentSize, ...attributes } = results; - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize); - if (!found) { - const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, undefined, prefix, false).catch(error => _error(res, downloadError, error)); - if (upload) { - completed.push(upload); - await Database.Auxiliary.LogUpload(upload); - } else { - failed++; - } + const completed: Opt[] = []; + for (const { baseUrl } of mediaItems) { + // start by getting the content size of the remote image + const results = await DashUploadUtils.InspectImage(baseUrl); + if (results instanceof Error) { + // if something went wrong here, we can't hope to upload it, so just move on to the next + failed++; + continue; + } + const { contentSize, ...attributes } = results; + // check to see if we have uploaded a Google user content image *specifically via this route* already + // that has this exact content size + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize); + if (!found) { + // if we haven't, then upload it locally to Dash's server + const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, undefined, prefix, false).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + // inform the heuristic that we've encountered an image with this content size, + // to be later checked against in future uploads + await Database.Auxiliary.LogUpload(upload); } else { - completed.push(found); + // make note of a failure to upload locallys + failed++; } + } else { + // if we have, the variable 'found' is handily the upload information of the + // existing image, so we add it to the list as if we had just uploaded it now without actually + // making a duplicate write + completed.push(found); } - if (failed) { - return _error(res, UploadError(failed)); - } - return _success(res, completed); } - _invalid(res, requestError); + // if there are any failures, report a general failure to the client + if (failed) { + return _error(res, localUploadError(failed)); + } + // otherwise, return the image upload information list corresponding to the newly (or previously) + // uploaded images + _success(res, completed); } }); -- cgit v1.2.3-70-g09d2 From f33ad290f1de3a01f2c4536f03f040e09771f82e Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 16 Feb 2020 00:50:23 -0500 Subject: improved file upload api --- src/client/Network.ts | 14 ++- .../util/Import & Export/DirectoryImportBox.tsx | 17 ++-- src/client/views/collections/CollectionSubView.tsx | 56 +++++------ src/new_fields/Doc.ts | 6 +- src/scraping/buxton/final/BuxtonImporter.ts | 7 +- src/server/ApiManagers/GooglePhotosManager.ts | 5 +- src/server/ApiManagers/UploadManager.ts | 8 +- src/server/DashUploadUtils.ts | 107 +++++++-------------- src/server/SharedMediaTypes.ts | 39 ++++++++ src/server/database.ts | 6 +- 10 files changed, 137 insertions(+), 128 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/client/Network.ts b/src/client/Network.ts index ccf60f199..6982ecf19 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -1,5 +1,6 @@ import { Utils } from "../Utils"; import requestPromise = require('request-promise'); +import { Upload } from "../server/SharedMediaTypes"; export namespace Networking { @@ -17,12 +18,21 @@ export namespace Networking { return requestPromise.post(options); } - export async function PostFormDataToServer(relativeRoute: string, formData: FormData) { + export async function UploadFilesToServer(files: File | File[]): Promise[]> { + const formData = new FormData(); + if (Array.isArray(files)) { + if (!files.length) { + return []; + } + files.forEach(file => formData.append(Utils.GenerateGuid(), file)); + } else { + formData.append(Utils.GenerateGuid(), files); + } const parameters = { method: 'POST', body: formData }; - const response = await fetch(relativeRoute, parameters); + const response = await fetch("/uploadFormData", parameters); return response.json(); } diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index d04f56e57..3d8bcbab7 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -22,7 +22,7 @@ import "./DirectoryImportBox.scss"; import { Networking } from "../../Network"; import { BatchedArray } from "array-batcher"; import * as path from 'path'; -import { AcceptibleMedia } from "../../../server/SharedMediaTypes"; +import { AcceptibleMedia, Upload } from "../../../server/SharedMediaTypes"; const unsupported = ["text/html", "text/plain"]; @@ -107,20 +107,21 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); const batched = BatchedArray.from(validated, { batchSize: 15 }); - const uploads = await batched.batchedMapAsync(async (batch, collector) => { - const formData = new FormData(); - + const uploads = await batched.batchedMapAsync>(async (batch, collector) => { batch.forEach(file => { sizes.push(file.size); modifiedDates.push(file.lastModified); - formData.append(Utils.GenerateGuid(), file); }); - - collector.push(...(await Networking.PostFormDataToServer("/uploadFormData", formData))); + collector.push(...(await Networking.UploadFilesToServer(batch))); runInAction(() => this.completed += batch.length); }); - await Promise.all(uploads.map(async ({ name, type, accessPaths, exifData }) => { + await Promise.all(uploads.map(async response => { + const { source: { type }, result } = response; + if (result instanceof Error) { + return; + } + const { accessPaths, exifData } = result; const path = Utils.prepend(accessPaths.agnostic.client); const document = await Docs.Get.DocumentFromType(type, path, { _width: 300, title: name }); const { data, error } = exifData; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 0963e1ea6..e0b2d524b 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -1,4 +1,4 @@ -import { action, computed, IReactionDisposer, reaction, trace } from "mobx"; +import { action, computed, IReactionDisposer, reaction } from "mobx"; import * as rp from 'request-promise'; import CursorField from "../../../new_fields/CursorField"; import { Doc, DocListCast, Opt, WidthSym, HeightSym } from "../../../new_fields/Doc"; @@ -25,6 +25,7 @@ import { ImageUtils } from "../../util/Import & Export/ImageUtils"; import { Networking } from "../../Network"; import { GestureUtils } from "../../../pen-gestures/GestureUtils"; import { InteractionUtils } from "../../util/InteractionUtils"; +import { Upload } from "../../../server/SharedMediaTypes"; export interface CollectionViewProps extends FieldViewProps { addDocument: (document: Doc) => boolean; @@ -288,6 +289,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { } const { items } = e.dataTransfer; const { length } = items; + const files: File[] = []; if (length) { const batch = UndoManager.StartBatch("collection view drop"); const promises: Promise[] = []; @@ -307,41 +309,31 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { }); promises.push(prom); } - const type = item.type; if (item.kind === "file") { const file = item.getAsFile(); - const formData = new FormData(); - - if (!file || !file.type) { - continue; - } - - formData.append('file', file); - const dropFileName = file ? file.name : "-empty-"; - promises.push(Networking.PostFormDataToServer("/uploadFormData", formData).then(results => { - results.map(action((result: any) => { - const { accessPaths, nativeWidth, nativeHeight, contentSize } = result; - if (Object.keys(accessPaths).length) { - const full = { ...options, _width: 300, title: dropFileName }; - const pathname = Utils.prepend(accessPaths.agnostic.client); - Docs.Get.DocumentFromType(type, pathname, full).then(doc => { - if (doc) { - const proto = Doc.GetProto(doc); - proto.fileUpload = basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, ""); - nativeWidth && (proto["data-nativeWidth"] = nativeWidth); - nativeHeight && (proto["data-nativeHeight"] = nativeHeight); - contentSize && (proto.contentSize = contentSize); - this.props?.addDocument(doc); - } - }); - } else { - alert("Upload failed..."); - } - })); - })); + file && file.type && files.push(file); } } - + (await Networking.UploadFilesToServer(files)).forEach(({ source: { name, type }, result }) => { + if (result instanceof Error) { + alert(`Upload failed: ${result.message}`); + return; + } + const full = { ...options, _width: 300, title: name }; + const pathname = Utils.prepend(result.accessPaths.agnostic.client); + Docs.Get.DocumentFromType(type, pathname, full).then(doc => { + if (doc) { + const proto = Doc.GetProto(doc); + proto.fileUpload = basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, ""); + if (Upload.isImageInformation(result)) { + proto["data-nativeWidth"] = result.nativeWidth; + proto["data-nativeHeight"] = result.nativeHeight; + proto.contentSize = result.contentSize; + } + this.props?.addDocument(doc); + } + }); + }); if (promises.length) { Promise.all(promises).finally(() => { completed && completed(); batch.end(); }); } else { diff --git a/src/new_fields/Doc.ts b/src/new_fields/Doc.ts index 55c0660c0..a722f552e 100644 --- a/src/new_fields/Doc.ts +++ b/src/new_fields/Doc.ts @@ -112,10 +112,10 @@ export class Doc extends RefField { // getPrototypeOf: (target) => Cast(target[SelfProxy].proto, Doc) || null, // TODO this might be able to replace the proto logic in getter has: (target, key) => key in target.__fields, ownKeys: target => { - let obj = {} as any; + const obj = {} as any; Object.assign(obj, target.___fields); runInAction(() => obj.__LAYOUT__ = target.__LAYOUT__); - return Object.keys(obj) + return Object.keys(obj); }, getOwnPropertyDescriptor: (target, prop) => { if (prop.toString() === "__LAYOUT__") { @@ -864,7 +864,7 @@ Scripting.addGlobal(function redo() { return UndoManager.Redo(); }); Scripting.addGlobal(function curPresentationItem() { const curPres = Doc.UserDoc().curPresentation as Doc; return curPres && DocListCast(curPres[Doc.LayoutFieldKey(curPres)])[NumCast(curPres._itemIndex)]; -}) +}); Scripting.addGlobal(function selectDoc(doc: any) { Doc.UserDoc().SelectedDocs = new List([doc]); }); Scripting.addGlobal(function selectedDocs(container: Doc, excludeCollections: boolean, prevValue: any) { const docs = DocListCast(Doc.UserDoc().SelectedDocs).filter(d => !Doc.AreProtosEqual(d, container) && !d.annotationOn && d.type !== DocumentType.DOCUMENT && d.type !== DocumentType.KVP && (!excludeCollections || !Cast(d.data, listSpec(Doc), null))); diff --git a/src/scraping/buxton/final/BuxtonImporter.ts b/src/scraping/buxton/final/BuxtonImporter.ts index 47d6bbe83..8041343fd 100644 --- a/src/scraping/buxton/final/BuxtonImporter.ts +++ b/src/scraping/buxton/final/BuxtonImporter.ts @@ -8,6 +8,7 @@ const StreamZip = require('node-stream-zip'); const createImageSizeStream = require("image-size-stream"); import { parseXml } from "libxmljs"; import { strictEqual } from "assert"; +import { Readable, PassThrough } from "stream"; interface DocumentContents { body: string; @@ -293,15 +294,15 @@ async function writeImages(zip: any): Promise { const imageUrls: ImageData[] = []; for (const mediaPath of imageEntries) { - const streamImage = () => new Promise((resolve, reject) => { + const streamImage = () => new Promise((resolve, reject) => { zip.stream(mediaPath, (error: any, stream: any) => error ? reject(error) : resolve(stream)); }); const { width, height, type } = await new Promise(async resolve => { - const sizeStream = createImageSizeStream().on('size', (dimensions: Dimensions) => { + const sizeStream = (createImageSizeStream() as PassThrough).on('size', (dimensions: Dimensions) => { readStream.destroy(); resolve(dimensions); - }); + }).on("error", () => readStream.destroy()); const readStream = await streamImage(); readStream.pipe(sizeStream); }); diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index 04b724f4b..25c54ee2e 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -8,6 +8,7 @@ import { Opt } from "../../new_fields/Doc"; import { DashUploadUtils, InjectSize, SizeSuffix } from "../DashUploadUtils"; import { Database } from "../database"; import { red } from "colors"; +import { Upload } from "../SharedMediaTypes"; const prefix = "google_photos_"; const remoteUploadError = "None of the preliminary uploads to Google's servers was successful."; @@ -139,7 +140,7 @@ export default class GooglePhotosManager extends ApiManager { return; } let failed = 0; - const completed: Opt[] = []; + const completed: Opt[] = []; for (const { baseUrl } of mediaItems) { // start by getting the content size of the remote image const results = await DashUploadUtils.InspectImage(baseUrl); @@ -151,7 +152,7 @@ export default class GooglePhotosManager extends ApiManager { const { contentSize, ...attributes } = results; // check to see if we have uploaded a Google user content image *specifically via this route* already // that has this exact content size - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize); if (!found) { // if we haven't, then upload it locally to Dash's server const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, undefined, prefix, false).catch(error => _error(res, downloadError, error)); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 4d09528f4..8f2a5ea3e 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -4,12 +4,12 @@ import * as formidable from 'formidable'; import v4 = require('uuid/v4'); const AdmZip = require('adm-zip'); import { extname, basename, dirname } from 'path'; -import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; +import { createReadStream, createWriteStream, unlink } from "fs"; import { publicDirectory, filesDirectory } from ".."; import { Database } from "../database"; -import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils"; +import { DashUploadUtils } from "../DashUploadUtils"; import * as sharp from 'sharp'; -import { AcceptibleMedia } from "../SharedMediaTypes"; +import { AcceptibleMedia, Upload } from "../SharedMediaTypes"; import { normalize } from "path"; const imageDataUri = require('image-data-uri'); @@ -47,7 +47,7 @@ export default class UploadManager extends ApiManager { form.keepExtensions = true; return new Promise(resolve => { form.parse(req, async (_err, _fields, files) => { - const results: any[] = []; + const results: Upload.FileResponse[] = []; for (const key in files) { const result = await DashUploadUtils.upload(files[key]); result && results.push(result); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 913ddc1c3..b66651ef2 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -3,9 +3,9 @@ import { Utils } from '../Utils'; import * as path from 'path'; import * as sharp from 'sharp'; import request = require('request-promise'); -import { ExifData, ExifImage } from 'exif'; +import { ExifImage } from 'exif'; import { Opt } from '../new_fields/Doc'; -import { AcceptibleMedia } from './SharedMediaTypes'; +import { AcceptibleMedia, Upload } from './SharedMediaTypes'; import { filesDirectory } from '.'; import { File } from 'formidable'; import { basename } from "path"; @@ -14,7 +14,7 @@ import { ParsedPDF } from "../server/PdfTypes"; const parse = require('pdf-parse'); import { Directory, serverPathToFile, clientPathToFile, pathToDirectory } from './ApiManagers/UploadManager'; import { red } from 'colors'; -import { Writable } from 'stream'; +import { Stream } from 'stream'; const requestImageSize = require("../client/util/request-image-size"); export enum SizeSuffix { @@ -40,13 +40,6 @@ export namespace DashUploadUtils { suffix: SizeSuffix; } - export interface ImageFileResponse { - name: string; - path: string; - type: string; - exif: Opt; - } - export const Sizes: { [size: string]: Size } = { SMALL: { width: 100, suffix: SizeSuffix.Small }, MEDIUM: { width: 400, suffix: SizeSuffix.Medium }, @@ -60,20 +53,9 @@ export namespace DashUploadUtils { const size = "content-length"; const type = "content-type"; - export interface ImageUploadInformation { - accessPaths: AccessPathInfo; - exifData: EnrichedExifData; - contentSize?: number; - contentType?: string; - } - - export interface AccessPathInfo { - [suffix: string]: { client: string, server: string }; - } - const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia; - export async function upload(file: File): Promise { + export async function upload(file: File): Promise { const { type, path, name } = file; const types = type.split("/"); @@ -83,33 +65,33 @@ export namespace DashUploadUtils { switch (category) { case "image": if (imageFormats.includes(format)) { - const results = await UploadImage(path, basename(path)); - return { ...results, name, type }; + const result = await UploadImage(path, basename(path)); + return { source: file, result }; } case "video": if (videoFormats.includes(format)) { - return MoveParsedFile(path, Directory.videos); + return MoveParsedFile(file, Directory.videos); } case "application": if (applicationFormats.includes(format)) { - return UploadPdf(path); + return UploadPdf(file); } } console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`)); - return { accessPaths: {} }; + return { source: file, result: new Error(`Could not upload unsupported file (${name}) with upload type (${type}).`) }; } - async function UploadPdf(absolutePath: string) { - const dataBuffer = readFileSync(absolutePath); + async function UploadPdf(file: File) { + const { path, name } = file; + const dataBuffer = readFileSync(path); const result: ParsedPDF = await parse(dataBuffer); - const parsedName = basename(absolutePath); await new Promise((resolve, reject) => { - const textFilename = `${parsedName.substring(0, parsedName.length - 4)}.txt`; + const textFilename = `${name.substring(0, name.length - 4)}.txt`; const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); writeStream.write(result.text, error => error ? reject(error) : resolve()); }); - return MoveParsedFile(absolutePath, Directory.pdfs); + return MoveParsedFile(file, Directory.pdfs); } /** @@ -123,13 +105,13 @@ export namespace DashUploadUtils { * @param {string} prefix is a string prepended to the generated image name in the * event that @param filename is not specified * - * @returns {ImageUploadInformation} This method returns + * @returns {ImageUploadInformation | Error} This method returns * 1) the paths to the uploaded images (plural due to resizing) - * 2) the file name of each of the resized images + * 2) the exif data embedded in the image, or the error explaining why exif couldn't be parsed * 3) the size of the image, in bytes (4432130) * 4) the content type of the image, i.e. image/(jpeg | png | ...) */ - export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise => { + export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise => { const metadata = await InspectImage(source); if (metadata instanceof Error) { return metadata; @@ -137,22 +119,6 @@ export namespace DashUploadUtils { return UploadInspectedImage(metadata, filename || metadata.filename, prefix); }; - export interface InspectionResults { - source: string; - requestable: string; - exifData: EnrichedExifData; - contentSize: number; - contentType: string; - nativeWidth: number; - nativeHeight: number; - filename?: string; - } - - export interface EnrichedExifData { - data: ExifData; - error?: string; - } - export async function buildFileDirectories() { const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`)); return Promise.all(pending); @@ -175,7 +141,7 @@ export namespace DashUploadUtils { * * @param source is the path or url to the image in question */ - export const InspectImage = async (source: string): Promise => { + export const InspectImage = async (source: string): Promise => { let rawMatches: RegExpExecArray | null; let filename: string | undefined; if ((rawMatches = /^data:image\/([a-z]+);base64,(.*)/.exec(source)) !== null) { @@ -216,14 +182,17 @@ export namespace DashUploadUtils { }; }; - export async function MoveParsedFile(absolutePath: string, destination: Directory): Promise> { + export async function MoveParsedFile(file: File, destination: Directory): Promise { + const { name, path: sourcePath } = file; return new Promise(resolve => { - const filename = basename(absolutePath); - const destinationPath = serverPathToFile(destination, filename); - rename(absolutePath, destinationPath, error => { - resolve(error ? undefined : { - accessPaths: { - agnostic: getAccessPaths(destination, filename) + const destinationPath = serverPathToFile(destination, name); + rename(sourcePath, destinationPath, error => { + resolve({ + source: file, + result: error ? error : { + accessPaths: { + agnostic: getAccessPaths(destination, name) + } } }); }); @@ -237,16 +206,16 @@ export namespace DashUploadUtils { }; } - export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = "", cleanUp = true): Promise => { + export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename?: string, prefix = "", cleanUp = true): Promise => { const { requestable, source, ...remaining } = metadata; const extension = `.${remaining.contentType.split("/")[1].toLowerCase()}`; const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}${extension}`; const { images } = Directory; - const information: ImageUploadInformation = { + const information: Upload.ImageInformation = { accessPaths: { agnostic: getAccessPaths(images, resolved) }, - ...remaining + ...metadata }; const outputPath = pathToDirectory(Directory.images); const writtenFiles = await outputResizedImages(() => request(requestable), outputPath, resolved, extension); @@ -259,9 +228,9 @@ export namespace DashUploadUtils { return information; }; - const parseExifData = async (source: string): Promise => { + const parseExifData = async (source: string): Promise => { const image = await request.get(source, { encoding: null }); - return new Promise(resolve => { + return new Promise(resolve => { new ExifImage({ image }, (error, data) => { let reason: Opt = undefined; if (error) { @@ -279,18 +248,14 @@ export namespace DashUploadUtils { force: true }; - export interface ReadStreamLike { - pipe: (dest: Writable) => Writable; - } - - export async function outputResizedImages(readStreamSource: () => ReadStreamLike | Promise, outputPath: string, fileName: string, ext: string) { + export async function outputResizedImages(readStreamSource: () => Stream | Promise, outputPath: string, fileName: string, ext: string) { const writtenFiles: { [suffix: string]: string } = {}; for (const { resizer, suffix } of resizers(ext)) { const resolved = writtenFiles[suffix] = InjectSize(fileName, suffix); await new Promise(async (resolve, reject) => { const writeStream = createWriteStream(path.resolve(outputPath, resolved)); - let readStream: ReadStreamLike; const source = readStreamSource(); + let readStream: Stream; if (source instanceof Promise) { readStream = await source; } else { @@ -320,7 +285,7 @@ export namespace DashUploadUtils { initial = initial.webp(); } else if (tiffs.includes(ext)) { initial = initial.tiff(); - } else { + } else if (ext === ".gif") { initial = undefined; } return { diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts index 274b4f01e..185e787cc 100644 --- a/src/server/SharedMediaTypes.ts +++ b/src/server/SharedMediaTypes.ts @@ -1,3 +1,6 @@ +import { ExifData } from 'exif'; +import { File } from 'formidable'; + export namespace AcceptibleMedia { export const gifs = [".gif"]; export const pngs = [".png"]; @@ -7,4 +10,40 @@ export namespace AcceptibleMedia { export const imageFormats = [...pngs, ...jpgs, ...gifs, ...webps, ...tiffs]; export const videoFormats = [".mov", ".mp4"]; export const applicationFormats = [".pdf"]; +} + +export namespace Upload { + + export function isImageInformation(uploadResponse: Upload.FileInformation): uploadResponse is Upload.ImageInformation { + return "nativeWidth" in uploadResponse; + } + + export interface FileInformation { + accessPaths: AccessPathInfo; + } + + export type FileResponse = { source: File, result: T | Error }; + + export type ImageInformation = FileInformation & InspectionResults; + + export interface AccessPathInfo { + [suffix: string]: { client: string, server: string }; + } + + export interface InspectionResults { + source: string; + requestable: string; + exifData: EnrichedExifData; + contentSize: number; + contentType: string; + nativeWidth: number; + nativeHeight: number; + filename?: string; + } + + export interface EnrichedExifData { + data: ExifData; + error?: string; + } + } \ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index 83ce865c6..055f04c49 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -2,12 +2,12 @@ import * as mongodb from 'mongodb'; import { Transferable } from './Message'; import { Opt } from '../new_fields/Doc'; import { Utils, emptyFunction } from '../Utils'; -import { DashUploadUtils } from './DashUploadUtils'; import { Credentials } from 'google-auth-library'; import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; import { IDatabase } from './IDatabase'; import { MemoryDatabase } from './MemoryDatabase'; import * as mongoose from 'mongoose'; +import { Upload } from './SharedMediaTypes'; export namespace Database { @@ -297,7 +297,7 @@ export namespace Database { }; export const QueryUploadHistory = async (contentSize: number) => { - return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); + return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); }; export namespace GoogleAuthenticationToken { @@ -326,7 +326,7 @@ export namespace Database { } - export const LogUpload = async (information: DashUploadUtils.ImageUploadInformation) => { + export const LogUpload = async (information: Upload.ImageInformation) => { const bundle = { _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)), ...information -- cgit v1.2.3-70-g09d2 From b42e1e1e2da941955d7751b6003f18fecd5f2f8d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 16 Feb 2020 12:22:10 -0500 Subject: collection sub view and google authentication manager cleanup, deleteAssets route added --- src/client/apis/GoogleAuthenticationManager.tsx | 46 ++-- .../views/collections/CollectionCarouselView.tsx | 28 +-- .../views/collections/CollectionLinearView.tsx | 2 +- .../collections/CollectionMasonryViewFieldRow.tsx | 2 +- .../views/collections/CollectionSchemaView.tsx | 7 +- .../views/collections/CollectionStackingView.tsx | 12 +- .../CollectionStackingViewFieldColumn.tsx | 2 +- src/client/views/collections/CollectionSubView.tsx | 264 +++++++++++---------- .../views/collections/CollectionTreeView.tsx | 6 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 10 +- .../CollectionMulticolumnView.tsx | 6 +- .../CollectionMultirowView.tsx | 6 +- src/server/ApiManagers/DeleteManager.ts | 31 +-- 13 files changed, 213 insertions(+), 209 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/client/apis/GoogleAuthenticationManager.tsx b/src/client/apis/GoogleAuthenticationManager.tsx index ce1277667..417dc3c3b 100644 --- a/src/client/apis/GoogleAuthenticationManager.tsx +++ b/src/client/apis/GoogleAuthenticationManager.tsx @@ -12,8 +12,8 @@ const prompt = "Paste authorization code here..."; @observer export default class GoogleAuthenticationManager extends React.Component<{}> { public static Instance: GoogleAuthenticationManager; - @observable private openState = false; private authenticationLink: Opt = undefined; + @observable private openState = false; @observable private authenticationCode: Opt = undefined; @observable private clickedState = false; @observable private success: Opt = undefined; @@ -39,24 +39,18 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { const disposer = reaction( () => this.authenticationCode, async authenticationCode => { - if (!authenticationCode) { - return; + if (authenticationCode) { + disposer(); + const { access_token, avatar, name } = await Networking.PostToServer("/writeGoogleAccessToken", { authenticationCode }); + runInAction(() => { + this.avatar = avatar; + this.username = name; + this.hasBeenClicked = false; + this.success = false; + }); + this.beginFadeout(); + resolve(access_token); } - const { access_token, avatar, name } = await Networking.PostToServer( - "/writeGoogleAccessToken", - { authenticationCode } - ); - runInAction(() => { - this.avatar = avatar; - this.username = name; - }); - this.beginFadeout(); - disposer(); - resolve(access_token); - action(() => { - this.hasBeenClicked = false; - this.success = false; - }); } ); }); @@ -86,26 +80,20 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { GoogleAuthenticationManager.Instance = this; } - private handleClick = () => { - window.open(this.authenticationLink); - setTimeout(() => this.hasBeenClicked = true, 500); - } - - private handlePaste = action((e: React.ChangeEvent) => { - this.authenticationCode = e.currentTarget.value; - }); - private get renderPrompt() { return (
{this.displayLauncher ? : (null)} {this.clickedState ? this.authenticationCode = e.currentTarget.value)} placeholder={prompt} /> : (null)} {this.avatar ? { //used for stacking and masonry view this._dropDisposer?.(); if (ele) { - this._dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this)); + this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this)); } } @@ -41,21 +41,21 @@ export class CollectionCarouselView extends CollectionSubView(CarouselDocument) } panelHeight = () => this.props.PanelHeight() - 50; - @computed get content() { + @computed get content() { const index = NumCast(this.layoutDoc._itemIndex); return !(this.childLayoutPairs?.[index]?.layout instanceof Doc) ? (null) : - <> -
- -
-
- -
- + <> +
+ +
+
+ +
+ ; } @computed get buttons() { return <> diff --git a/src/client/views/collections/CollectionLinearView.tsx b/src/client/views/collections/CollectionLinearView.tsx index 7eb316cf0..9bbc9f1b6 100644 --- a/src/client/views/collections/CollectionLinearView.tsx +++ b/src/client/views/collections/CollectionLinearView.tsx @@ -67,7 +67,7 @@ export class CollectionLinearView extends CollectionSubView(LinearDocument) { protected createDashEventsTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view this._dropDisposer && this._dropDisposer(); if (ele) { - this._dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this)); + this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this)); } } diff --git a/src/client/views/collections/CollectionMasonryViewFieldRow.tsx b/src/client/views/collections/CollectionMasonryViewFieldRow.tsx index e25a2f5eb..f3d512a97 100644 --- a/src/client/views/collections/CollectionMasonryViewFieldRow.tsx +++ b/src/client/views/collections/CollectionMasonryViewFieldRow.tsx @@ -80,7 +80,7 @@ export class CollectionMasonryViewFieldRow extends React.Component d[key] = castedValue); - this.props.parent.drop(e, de); + this.props.parent.onInternalDrop(e, de); e.stopPropagation(); } }); diff --git a/src/client/views/collections/CollectionSchemaView.tsx b/src/client/views/collections/CollectionSchemaView.tsx index fa8be5177..c422c38f1 100644 --- a/src/client/views/collections/CollectionSchemaView.tsx +++ b/src/client/views/collections/CollectionSchemaView.tsx @@ -14,7 +14,6 @@ import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import { ComputedField } from "../../../new_fields/ScriptField"; import { Cast, FieldValue, NumCast, StrCast } from "../../../new_fields/Types"; import { Docs, DocumentOptions } from "../../documents/Documents"; -import { DocumentType } from "../../documents/DocumentTypes"; import { Gateway } from "../../northstar/manager/Gateway"; import { CompileScript, Transformer, ts } from "../../util/Scripting"; import { Transform } from "../../util/Transform"; @@ -175,7 +174,7 @@ export class CollectionSchemaView extends CollectionSubView(doc => doc) { moveDocument={this.props.moveDocument} ScreenToLocalTransform={this.props.ScreenToLocalTransform} active={this.props.active} - onDrop={this.onDrop} + onDrop={this.onExternalDrop} addDocTab={this.props.addDocTab} pinToPres={this.props.pinToPres} isSelected={this.props.isSelected} @@ -199,7 +198,7 @@ export class CollectionSchemaView extends CollectionSubView(doc => doc) { render() { return
-
this.props.active(true) && e.stopPropagation()} onDrop={e => this.onDrop(e, {})} ref={this.createTarget}> +
this.props.active(true) && e.stopPropagation()} onDrop={e => this.onExternalDrop(e, {})} ref={this.createTarget}> {this.schemaTable}
{this.dividerDragger} @@ -692,7 +691,7 @@ export class SchemaTable extends React.Component { onContextMenu = (e: React.MouseEvent): void => { if (!e.isPropagationStopped() && this.props.Document[Id] !== "mainDoc") { // need to test this because GoldenLayout causes a parallel hierarchy in the React DOM for its children and the main document view7 // ContextMenu.Instance.addItem({ description: "Make DB", event: this.makeDB, icon: "table" }); - ContextMenu.Instance.addItem({ description: "Toggle text wrapping", event: this.toggleTextwrap, icon: "table" }) + ContextMenu.Instance.addItem({ description: "Toggle text wrapping", event: this.toggleTextwrap, icon: "table" }); } } diff --git a/src/client/views/collections/CollectionStackingView.tsx b/src/client/views/collections/CollectionStackingView.tsx index 055035b3e..a9cefae6a 100644 --- a/src/client/views/collections/CollectionStackingView.tsx +++ b/src/client/views/collections/CollectionStackingView.tsx @@ -189,7 +189,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { } getDocHeight(d?: Doc) { if (!d) return 0; - let layoutDoc = Doc.Layout(d, this.props.childLayoutTemplate?.()); + const layoutDoc = Doc.Layout(d, this.props.childLayoutTemplate?.()); const nw = NumCast(layoutDoc._nativeWidth); const nh = NumCast(layoutDoc._nativeHeight); let wid = this.columnWidth / (this.isStackingView ? this.numGroupColumns : 1); @@ -234,7 +234,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { @undoBatch @action - drop = (e: Event, de: DragManager.DropEvent) => { + onInternalDrop = (e: Event, de: DragManager.DropEvent) => { const where = [de.x, de.y]; let targInd = -1; let plusOne = 0; @@ -248,7 +248,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { plusOne = where[axis] > (pos[axis] + pos1[axis]) / 2 ? 1 : 0; } }); - if (super.drop(e, de)) { + if (super.onInternalDrop(e, de)) { const newDoc = de.complete.docDragData.droppedDocuments[0]; const docs = this.childDocList; if (docs) { @@ -264,7 +264,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { } @undoBatch @action - onDrop = async (e: React.DragEvent): Promise => { + onExternalDrop = async (e: React.DragEvent): Promise => { const where = [e.clientX, e.clientY]; let targInd = -1; this._docXfs.map((cd, i) => { @@ -274,7 +274,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { targInd = i; } }); - super.onDrop(e, {}, () => { + super.onExternalDrop(e, {}, () => { if (targInd !== -1) { const newDoc = this.childDocs[this.childDocs.length - 1]; const docs = this.childDocList; @@ -405,7 +405,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { transformOrigin: "top left", }} onScroll={action((e: React.UIEvent) => this._scroll = e.currentTarget.scrollTop)} - onDrop={this.onDrop.bind(this)} + onDrop={this.onExternalDrop.bind(this)} onContextMenu={this.onContextMenu} onWheel={e => this.props.active() && e.stopPropagation()} > {this.renderedSections} diff --git a/src/client/views/collections/CollectionStackingViewFieldColumn.tsx b/src/client/views/collections/CollectionStackingViewFieldColumn.tsx index 2ff477c57..87c35679f 100644 --- a/src/client/views/collections/CollectionStackingViewFieldColumn.tsx +++ b/src/client/views/collections/CollectionStackingViewFieldColumn.tsx @@ -71,7 +71,7 @@ export class CollectionStackingViewFieldColumn extends React.Component d[key] = undefined); } - this.props.parent.drop(e, de); + this.props.parent.onInternalDrop(e, de); e.stopPropagation(); } }); diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index e80e1c802..042385dcd 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -57,7 +57,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { this.gestureDisposer?.(); this.multiTouchDisposer?.(); if (ele) { - this.dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this)); + this.dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this)); this.gestureDisposer = GestureUtils.MakeGestureTarget(ele, this.onGesture.bind(this)); this.multiTouchDisposer = InteractionUtils.MakeMultiTouchTarget(ele, this.onTouchStart.bind(this)); } @@ -156,7 +156,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { @undoBatch @action - protected drop(e: Event, de: DragManager.DropEvent): boolean { + protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean { const docDragData = de.complete.docDragData; (this.props.Document.dropConverter instanceof ScriptField) && this.props.Document.dropConverter.script.run({ dragData: docDragData }); /// bcz: check this @@ -195,158 +195,172 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { @undoBatch @action - protected async onDrop(e: React.DragEvent, options: DocumentOptions, completed?: () => void) { + protected async onExternalDrop(e: React.DragEvent, options: DocumentOptions, completed?: () => void) { if (e.ctrlKey) { e.stopPropagation(); // bcz: this is a hack to stop propagation when dropping an image on a text document with shift+ctrl return; } - const html = e.dataTransfer.getData("text/html"); - const text = e.dataTransfer.getData("text/plain"); + + const { dataTransfer } = e; + const html = dataTransfer.getData("text/html"); + const text = dataTransfer.getData("text/plain"); if (text && text.startsWith(" { - if (f instanceof Doc) { - if (options.x || options.y) { f.x = options.x; f.y = options.y; } // should be in CollectionFreeFormView - (f instanceof Doc) && this.props.addDocument(f); - } - }); - } else { - this.props.addDocument && this.props.addDocument(Docs.Create.WebDocument(href, { ...options, title: href })); - } - } else if (text) { - this.props.addDocument && this.props.addDocument(Docs.Create.TextDocument(text, { ...options, _width: 100, _height: 25 })); - } + const { addDocument } = this.props; + if (!addDocument) { + alert("this.props.addDocument does not exist. Aborting drop operation."); return; } - if (html && !html.startsWith(" 1 && tags[1].startsWith("img") ? tags[1] : ""; - if (img) { - const split = img.split("src=\"")[1].split("\"")[0]; - let source = split; - if (split.startsWith("data:image") && split.includes("base64")) { - const [{ accessPaths }] = await Networking.PostToServer("/uploadRemoteImage", { sources: [split] }); - source = Utils.prepend(accessPaths.agnostic.client); + + if (html) { + if (FormattedTextBox.IsFragment(html)) { + const href = FormattedTextBox.GetHref(html); + if (href) { + const docid = FormattedTextBox.GetDocFromUrl(href); + if (docid) { // prosemirror text containing link to dash document + DocServer.GetRefField(docid).then(f => { + if (f instanceof Doc) { + if (options.x || options.y) { f.x = options.x; f.y = options.y; } // should be in CollectionFreeFormView + (f instanceof Doc) && addDocument(f); + } + }); + } else { + addDocument(Docs.Create.WebDocument(href, { ...options, title: href })); + } + } else if (text) { + addDocument(Docs.Create.TextDocument(text, { ...options, _width: 100, _height: 25 })); } - const doc = Docs.Create.ImageDocument(source, { ...options, _width: 300 }); - ImageUtils.ExtractExif(doc); - this.props.addDocument(doc); return; - } else { - const path = window.location.origin + "/doc/"; - if (text.startsWith(path)) { - const docid = text.replace(Utils.prepend("/doc/"), "").split("?")[0]; - DocServer.GetRefField(docid).then(f => { - if (f instanceof Doc) { - if (options.x || options.y) { f.x = options.x; f.y = options.y; } // should be in CollectionFreeFormView - (f instanceof Doc) && this.props.addDocument(f); - } - }); + } + if (!html.startsWith(" 1 && tags[1].startsWith("img") ? tags[1] : ""; + if (img) { + const split = img.split("src=\"")[1].split("\"")[0]; + let source = split; + if (split.startsWith("data:image") && split.includes("base64")) { + const [{ accessPaths }] = await Networking.PostToServer("/uploadRemoteImage", { sources: [split] }); + source = Utils.prepend(accessPaths.agnostic.client); + } + const doc = Docs.Create.ImageDocument(source, { ...options, _width: 300 }); + ImageUtils.ExtractExif(doc); + addDocument(doc); + return; } else { - const htmlDoc = Docs.Create.HtmlDocument(html, { ...options, title: "-web page-", _width: 300, _height: 300 }); - Doc.GetProto(htmlDoc)["data-text"] = text; - this.props.addDocument(htmlDoc); + const path = window.location.origin + "/doc/"; + if (text.startsWith(path)) { + const docid = text.replace(Utils.prepend("/doc/"), "").split("?")[0]; + DocServer.GetRefField(docid).then(f => { + if (f instanceof Doc) { + if (options.x || options.y) { f.x = options.x; f.y = options.y; } // should be in CollectionFreeFormView + (f instanceof Doc) && this.props.addDocument(f); + } + }); + } else { + const htmlDoc = Docs.Create.HtmlDocument(html, { ...options, title: "-web page-", _width: 300, _height: 300 }); + Doc.GetProto(htmlDoc)["data-text"] = text; + this.props.addDocument(htmlDoc); + } + return; } - return; } } - if (text && text.indexOf("www.youtube.com/watch") !== -1) { - const url = text.replace("youtube.com/watch?v=", "youtube.com/embed/"); - this.props.addDocument(Docs.Create.VideoDocument(url, { ...options, title: url, _width: 400, _height: 315, _nativeWidth: 600, _nativeHeight: 472.5 })); - return; - } - let matches: RegExpExecArray | null; - if ((matches = /(https:\/\/)?docs\.google\.com\/document\/d\/([^\\]+)\/edit/g.exec(text)) !== null) { - const newBox = Docs.Create.TextDocument("", { ...options, _width: 400, _height: 200, title: "Awaiting title from Google Docs..." }); - const proto = newBox.proto!; - const documentId = matches[2]; - proto[GoogleRef] = documentId; - proto.data = "Please select this document and then click on its pull button to load its contents from from Google Docs..."; - proto.backgroundColor = "#eeeeff"; - this.props.addDocument(newBox); - // const parent = Docs.Create.StackingDocument([newBox], { title: `Google Doc Import (${documentId})` }); - // CollectionDockingView.Instance.AddRightSplit(parent, undefined); - // proto.height = parent[HeightSym](); - return; - } - if ((matches = /(https:\/\/)?photos\.google\.com\/(u\/3\/)?album\/([^\\]+)/g.exec(text)) !== null) { - const albums = await GooglePhotos.Transactions.ListAlbums(); - const albumId = matches[3]; - const mediaItems = await GooglePhotos.Query.AlbumSearch(albumId); - console.log(mediaItems); - return; + + if (text) { + if (text.includes("www.youtube.com/watch")) { + const url = text.replace("youtube.com/watch?v=", "youtube.com/embed/"); + addDocument(Docs.Create.VideoDocument(url, { + ...options, + title: url, + _width: 400, + _height: 315, + _nativeWidth: 600, + _nativeHeight: 472.5 + })); + return; + } + let matches: RegExpExecArray | null; + if ((matches = /(https:\/\/)?docs\.google\.com\/document\/d\/([^\\]+)\/edit/g.exec(text)) !== null) { + const newBox = Docs.Create.TextDocument("", { ...options, _width: 400, _height: 200, title: "Awaiting title from Google Docs..." }); + const proto = newBox.proto!; + const documentId = matches[2]; + proto[GoogleRef] = documentId; + proto.data = "Please select this document and then click on its pull button to load its contents from from Google Docs..."; + proto.backgroundColor = "#eeeeff"; + addDocument(newBox); + return; + } + if ((matches = /(https:\/\/)?photos\.google\.com\/(u\/3\/)?album\/([^\\]+)/g.exec(text)) !== null) { + const albumId = matches[3]; + const mediaItems = await GooglePhotos.Query.AlbumSearch(albumId); + console.log(mediaItems); + return; + } } + const { items } = e.dataTransfer; const { length } = items; const files: File[] = []; - if (length) { - const batch = UndoManager.StartBatch("collection view drop"); - const promises: Promise[] = []; - // tslint:disable-next-line:prefer-for-of - for (let i = 0; i < length; i++) { - const item = e.dataTransfer.items[i]; - if (item.kind === "string" && item.type.indexOf("uri") !== -1) { - let str: string; - const prom = new Promise(resolve => item.getAsString(resolve)) - .then(action((s: string) => rp.head(Utils.CorsProxy(str = s)))) - .then(result => { - const type = result["content-type"]; - if (type) { - Docs.Get.DocumentFromType(type, str, options) - .then(doc => doc && this.props.addDocument(doc)); - } - }); - promises.push(prom); - } - if (item.kind === "file") { - const file = item.getAsFile(); - file && file.type && files.push(file); + const generatedDocuments: Doc[] = []; + if (!length) { + alert("No uploadable content found."); + return; + } + + const batch = UndoManager.StartBatch("collection view drop"); + for (let i = 0; i < length; i++) { + const item = e.dataTransfer.items[i]; + if (item.kind === "string" && item.type.includes("uri")) { + const stringContents = await new Promise(resolve => item.getAsString(resolve)); + const type = (await rp.head(Utils.CorsProxy(stringContents)))["content-type"]; + if (type) { + const doc = await Docs.Get.DocumentFromType(type, stringContents, options); + doc && generatedDocuments.push(doc); } } - promises.push(Networking.UploadFilesToServer(files).then(responses => responses.forEach(({ source: { name, type }, result }) => { - if (result instanceof Error) { - alert(`Upload failed: ${result.message}`); - return; - } - const full = { ...options, _width: 300, title: name }; - const pathname = Utils.prepend(result.accessPaths.agnostic.client); - Docs.Get.DocumentFromType(type, pathname, full).then(doc => { - if (doc) { - const proto = Doc.GetProto(doc); - proto.fileUpload = basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, ""); - if (Upload.isImageInformation(result)) { - proto["data-nativeWidth"] = result.nativeWidth; - proto["data-nativeHeight"] = result.nativeHeight; - proto.contentSize = result.contentSize; - } - this.props?.addDocument(doc); - } - }); - }))); - if (promises.length) { - Promise.all(promises).finally(() => { completed && completed(); batch.end(); }); - } else { - if (text && !text.includes("https://")) { - this.props.addDocument(Docs.Create.TextDocument(text, { ...options, _width: 400, _height: 315 })); - } - batch.end(); + if (item.kind === "file") { + const file = item.getAsFile(); + file && file.type && files.push(file); + } + } + for (const { source: { name, type }, result } of await Networking.UploadFilesToServer(files)) { + if (result instanceof Error) { + alert(`Upload failed: ${result.message}`); + return; } + const full = { ...options, _width: 300, title: name }; + const pathname = Utils.prepend(result.accessPaths.agnostic.client); + const doc = await Docs.Get.DocumentFromType(type, pathname, full); + if (!doc) { + continue; + } + const proto = Doc.GetProto(doc); + proto.fileUpload = basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, ""); + if (Upload.isImageInformation(result)) { + proto["data-nativeWidth"] = result.nativeWidth; + proto["data-nativeHeight"] = result.nativeHeight; + proto.contentSize = result.contentSize; + } + generatedDocuments.push(doc); + } + if (generatedDocuments.length) { + generatedDocuments.forEach(addDocument); + completed && completed(); } else { - alert("No uploadable content found."); + if (text && !text.includes("https://")) { + addDocument(Docs.Create.TextDocument(text, { ...options, _width: 400, _height: 315 })); + } } + batch.end(); } } + return CollectionSubView; } diff --git a/src/client/views/collections/CollectionTreeView.tsx b/src/client/views/collections/CollectionTreeView.tsx index 8720ce002..13ab7c1a4 100644 --- a/src/client/views/collections/CollectionTreeView.tsx +++ b/src/client/views/collections/CollectionTreeView.tsx @@ -210,7 +210,7 @@ class TreeView extends React.Component { } else { ContextMenu.Instance.addItem({ description: "Delete Workspace", event: () => this.props.deleteDoc(this.props.document), icon: "trash-alt" }); ContextMenu.Instance.addItem({ description: "Create New Workspace", event: () => MainView.Instance.createNewWorkspace(), icon: "plus" }); - } + } ContextMenu.Instance.addItem({ description: "Toggle Theme Colors", event: () => this.props.document.darkScheme = !this.props.document.darkScheme, icon: "minus" }); ContextMenu.Instance.addItem({ description: "Open Fields", event: () => { const kvp = Docs.Create.KVPDocument(this.props.document, { _width: 300, _height: 300 }); this.props.addDocTab(kvp, this.props.dataDoc ? this.props.dataDoc : kvp, "onRight"); }, icon: "layer-group" }); ContextMenu.Instance.addItem({ description: "Publish", event: () => DocUtils.Publish(this.props.document, StrCast(this.props.document.title), () => { }, () => { }), icon: "file" }); @@ -594,7 +594,7 @@ export class CollectionTreeView extends CollectionSubView(Document) { protected createTreeDropTarget = (ele: HTMLDivElement) => { this.treedropDisposer && this.treedropDisposer(); if (this._mainEle = ele) { - this.treedropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this)); + this.treedropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this)); } } @@ -702,7 +702,7 @@ export class CollectionTreeView extends CollectionSubView(Document) { !existingOnClick && ContextMenu.Instance.addItem({ description: "OnClick...", subitems: onClicks, icon: "hand-point-right" }); } outerXf = () => Utils.GetScreenTransform(this._mainEle!); - onTreeDrop = (e: React.DragEvent) => this.onDrop(e, {}); + onTreeDrop = (e: React.DragEvent) => this.onExternalDrop(e, {}); @computed get renderClearButton() { return
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 969d6b3c8..bdc5e03e3 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -117,20 +117,20 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { } @action - onDrop = (e: React.DragEvent): Promise => { + onExternalDrop = (e: React.DragEvent): Promise => { const pt = this.getTransform().transformPoint(e.pageX, e.pageY); - return super.onDrop(e, { x: pt[0], y: pt[1] }); + return super.onExternalDrop(e, { x: pt[0], y: pt[1] }); } @undoBatch @action - drop = (e: Event, de: DragManager.DropEvent) => { + onInternalDrop = (e: Event, de: DragManager.DropEvent) => { if (this.props.Document.isBackground) return false; const xf = this.getTransform(); const xfo = this.getTransformOverlay(); const [xp, yp] = xf.transformPoint(de.x, de.y); const [xpo, ypo] = xfo.transformPoint(de.x, de.y); - if (super.drop(e, de)) { + if (super.onInternalDrop(e, de)) { if (de.complete.docDragData) { if (de.complete.docDragData.droppedDocuments.length) { const firstDoc = de.complete.docDragData.droppedDocuments[0]; @@ -1079,7 +1079,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { return
{ - if (super.drop(e, de)) { + onInternalDrop = (e: Event, de: DragManager.DropEvent) => { + if (super.onInternalDrop(e, de)) { de.complete.docDragData?.droppedDocuments.forEach(action((d: Doc) => { d.dimUnit = "*"; d.dimMagnitude = 1; @@ -214,7 +214,7 @@ export class CollectionMulticolumnView extends CollectionSubView(MulticolumnDocu getTransform={dxf} onClick={this.onChildClickHandler} renderDepth={this.props.renderDepth + 1} - /> + />; } /** * @returns the resolved list of rendered child documents, displayed diff --git a/src/client/views/collections/collectionMulticolumn/CollectionMultirowView.tsx b/src/client/views/collections/collectionMulticolumn/CollectionMultirowView.tsx index 630a178cf..5e59f8237 100644 --- a/src/client/views/collections/collectionMulticolumn/CollectionMultirowView.tsx +++ b/src/client/views/collections/collectionMulticolumn/CollectionMultirowView.tsx @@ -190,8 +190,8 @@ export class CollectionMultirowView extends CollectionSubView(MultirowDocument) @undoBatch @action - drop = (e: Event, de: DragManager.DropEvent) => { - if (super.drop(e, de)) { + onInternalDrop = (e: Event, de: DragManager.DropEvent) => { + if (super.onInternalDrop(e, de)) { de.complete.docDragData?.droppedDocuments.forEach(action((d: Doc) => { d.dimUnit = "*"; d.dimMagnitude = 1; @@ -215,7 +215,7 @@ export class CollectionMultirowView extends CollectionSubView(MultirowDocument) getTransform={dxf} onClick={this.onChildClickHandler} renderDepth={this.props.renderDepth + 1} - /> + />; } /** * @returns the resolved list of rendered child documents, displayed diff --git a/src/server/ApiManagers/DeleteManager.ts b/src/server/ApiManagers/DeleteManager.ts index be452c0ff..9e70af2eb 100644 --- a/src/server/ApiManagers/DeleteManager.ts +++ b/src/server/ApiManagers/DeleteManager.ts @@ -2,6 +2,11 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method, _permission_denied, PublicHandler } from "../RouteManager"; import { WebSocket } from "../Websocket/Websocket"; import { Database } from "../database"; +import rimraf = require("rimraf"); +import { pathToDirectory, Directory } from "./UploadManager"; +import { filesDirectory } from ".."; +import { DashUploadUtils } from "../DashUploadUtils"; +import { mkdirSync } from "fs"; export default class DeleteManager extends ApiManager { @@ -31,21 +36,19 @@ export default class DeleteManager extends ApiManager { } }); - const hi: PublicHandler = async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); + register({ + method: Method.GET, + subscription: "/deleteAssets", + secureHandler: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + rimraf.sync(filesDirectory); + mkdirSync(filesDirectory); + await DashUploadUtils.buildFileDirectories(); + res.redirect("/delete"); } - await Database.Instance.deleteAll('users'); - res.redirect("/home"); - }; - - // register({ - // method: Method.GET, - // subscription: "/deleteUsers", - // onValidation: hi, - // onUnauthenticated: hi - // }); - + }); register({ method: Method.GET, -- cgit v1.2.3-70-g09d2