From 6499ac4b30c8f6ada2b0c7d74cd5f2a73f9bf180 Mon Sep 17 00:00:00 2001 From: bobzel Date: Sun, 10 Sep 2023 15:58:09 -0400 Subject: fixes for file uploads: restored progress for youtube videos, fixed info display on loadingBoxes, stop client from asking for progress on failed uploads. attempt to catch stream errors on webpages. fixed dataFieldView in text to print out layout document's field, not field of document that contains the field definition. --- src/server/ApiManagers/UploadManager.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index ebc9deab7..c264803bd 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -102,11 +102,10 @@ export default class UploadManager extends ApiManager { //req.readableBuffer.head.data return new Promise(async resolve => { req.addListener('data', async args => { - console.log(args); const payload = String.fromCharCode.apply(String, args); - const videoId = JSON.parse(payload).videoId; + const { videoId, overwriteId } = JSON.parse(payload); const results: Upload.FileResponse[] = []; - const result = await DashUploadUtils.uploadYoutube(videoId); + const result = await DashUploadUtils.uploadYoutube(videoId, overwriteId ?? videoId); result && results.push(result); _success(res, results); resolve(); @@ -123,7 +122,7 @@ export default class UploadManager extends ApiManager { req.addListener('data', args => { const payload = String.fromCharCode.apply(String, args); const videoId = JSON.parse(payload).videoId; - _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId) }); + _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId, req.user) }); resolve(); }); }); -- cgit v1.2.3-70-g09d2 From b42680588ed48d57f497ff15a18a693bb7ecb9b8 Mon Sep 17 00:00:00 2001 From: bobzel Date: Sun, 10 Sep 2023 16:54:19 -0400 Subject: instrumented pdf uploads --- src/server/ApiManagers/UploadManager.ts | 3 +++ src/server/DashUploadUtils.ts | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'src/server/ApiManagers') diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index c264803bd..854911277 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -64,11 +64,13 @@ export default class UploadManager extends ApiManager { subscription: '/uploadFormData', secureHandler: async ({ req, res }) => { const form = new formidable.IncomingForm(); + console.log('RECEIVED form: '); form.keepExtensions = true; form.uploadDir = pathToDirectory(Directory.parsed_files); return new Promise(resolve => { form.parse(req, async (_err, _fields, files) => { const results: Upload.FileResponse[] = []; + console.log('Parsed files: '); if (_err?.message) { results.push({ source: { @@ -84,6 +86,7 @@ export default class UploadManager extends ApiManager { for (const key in files) { const f = files[key]; if (!Array.isArray(f)) { + console.log('uploading file...', f.name); const result = await DashUploadUtils.upload(f, key); // key is the guid used by the client to track upload progress. result && !(result.result instanceof Error) && results.push(result); } diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 1f23ae8c1..5187d0d6f 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -282,6 +282,7 @@ export namespace DashUploadUtils { } async function UploadPdf(file: File) { + console.log('upload pdf', file.name); const fileKey = (await md5File(file.path)) + '.pdf'; const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) { @@ -290,16 +291,25 @@ export namespace DashUploadUtils { const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; const readStream = createReadStream(serverPathToFile(Directory.text, textFilename)); var rawText = ''; - readStream.on('data', chunk => (rawText += chunk.toString())).on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText))); + console.log('reading stream', file); + readStream + .on('data', chunk => { + console.log('Got chunk:' + chunk.size); + rawText += chunk.toString(); + }) + .on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText))); }); } const dataBuffer = readFileSync(file.path); const result: ParsedPDF | any = await parse(dataBuffer).catch((e: any) => e); if (!result.code) { await new Promise((resolve, reject) => { + console.log('Got text write stream', file.name); const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); writeStream.write(result?.text, error => (error ? reject(error) : resolve())); + console.log('write text stream', file.name); }); + console.log('moving', file.name); return MoveParsedFile(file, Directory.pdfs, undefined, result?.text, undefined, fileKey); } return { source: file, result: { name: 'faile pdf pupload', message: `Could not upload (${file.name}).${result.message}` } }; -- cgit v1.2.3-70-g09d2 From 2e32ffaa398f077c5ddc81e950de010199c67c8d Mon Sep 17 00:00:00 2001 From: bobzel Date: Sun, 10 Sep 2023 18:11:23 -0400 Subject: added progress for pdf uploads --- src/client/Network.ts | 5 ++++- src/server/ApiManagers/UploadManager.ts | 10 +++++++--- src/server/DashUploadUtils.ts | 15 ++------------- 3 files changed, 13 insertions(+), 17 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/client/Network.ts b/src/client/Network.ts index 631ec9122..e9f93fad3 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -58,11 +58,14 @@ export namespace Networking { ]) ); } + formData.set('fileguids', fileguidpairs.map(pair => pair.guid).join(';')); // If the fileguidpair has a guid to use (From the overwriteDoc) use that guid. Otherwise, generate a new guid. fileguidpairs.forEach(fileguidpair => formData.append(fileguidpair.guid ?? Utils.GenerateGuid(), fileguidpair.file)); } else { // Handle the case where fileguidpairs is a single file. - formData.append(fileguidpairs.guid ?? Utils.GenerateGuid(), fileguidpairs.file); + const guids = fileguidpairs.guid ?? Utils.GenerateGuid(); + formData.set('fileguids', guids); + formData.append(guids, fileguidpairs.file); } const parameters = { method: 'POST', diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 854911277..4509425bb 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -64,13 +64,18 @@ export default class UploadManager extends ApiManager { subscription: '/uploadFormData', secureHandler: async ({ req, res }) => { const form = new formidable.IncomingForm(); - console.log('RECEIVED form: '); + let fileguids = ''; + form.on('field', (e: string, value: string) => { + if (e === 'fileguids') { + (fileguids = value).split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, 'reading file')); + } + }); + form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, 'read: ' + e))); form.keepExtensions = true; form.uploadDir = pathToDirectory(Directory.parsed_files); return new Promise(resolve => { form.parse(req, async (_err, _fields, files) => { const results: Upload.FileResponse[] = []; - console.log('Parsed files: '); if (_err?.message) { results.push({ source: { @@ -86,7 +91,6 @@ export default class UploadManager extends ApiManager { for (const key in files) { const f = files[key]; if (!Array.isArray(f)) { - console.log('uploading file...', f.name); const result = await DashUploadUtils.upload(f, key); // key is the guid used by the client to track upload progress. result && !(result.result instanceof Error) && results.push(result); } diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 5187d0d6f..15678bd8d 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -140,7 +140,7 @@ export namespace DashUploadUtils { return uploadProgress.get(videoId) ?? 'pending data upload'; } - let uploadProgress = new Map(); + export let uploadProgress = new Map(); export function uploadYoutube(videoId: string, overwriteId: string): Promise { return new Promise>((res, rej) => { @@ -197,7 +197,6 @@ export namespace DashUploadUtils { const isAzureOn = usingAzure(); const { type, path, name } = file; const types = type?.split('/') ?? []; - console.log('UPLOADING:' + (overwriteGuid ?? name)); uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name. const category = types[0]; @@ -282,7 +281,6 @@ export namespace DashUploadUtils { } async function UploadPdf(file: File) { - console.log('upload pdf', file.name); const fileKey = (await md5File(file.path)) + '.pdf'; const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) { @@ -291,25 +289,16 @@ export namespace DashUploadUtils { const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; const readStream = createReadStream(serverPathToFile(Directory.text, textFilename)); var rawText = ''; - console.log('reading stream', file); - readStream - .on('data', chunk => { - console.log('Got chunk:' + chunk.size); - rawText += chunk.toString(); - }) - .on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText))); + readStream.on('data', chunk => (rawText += chunk.toString())).on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText))); }); } const dataBuffer = readFileSync(file.path); const result: ParsedPDF | any = await parse(dataBuffer).catch((e: any) => e); if (!result.code) { await new Promise((resolve, reject) => { - console.log('Got text write stream', file.name); const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); writeStream.write(result?.text, error => (error ? reject(error) : resolve())); - console.log('write text stream', file.name); }); - console.log('moving', file.name); return MoveParsedFile(file, Directory.pdfs, undefined, result?.text, undefined, fileKey); } return { source: file, result: { name: 'faile pdf pupload', message: `Could not upload (${file.name}).${result.message}` } }; -- cgit v1.2.3-70-g09d2 From e7657b4f34de651358952a15c1375a61df908f29 Mon Sep 17 00:00:00 2001 From: bobzel Date: Sun, 10 Sep 2023 19:06:49 -0400 Subject: added upload percent feedback --- src/client/Network.ts | 2 ++ src/server/ApiManagers/UploadManager.ts | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'src/server/ApiManagers') diff --git a/src/client/Network.ts b/src/client/Network.ts index e9f93fad3..89b31fdca 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -59,12 +59,14 @@ export namespace Networking { ); } formData.set('fileguids', fileguidpairs.map(pair => pair.guid).join(';')); + formData.set('filesize', fileguidpairs.reduce((sum, pair) => sum + pair.file.size, 0).toString()); // If the fileguidpair has a guid to use (From the overwriteDoc) use that guid. Otherwise, generate a new guid. fileguidpairs.forEach(fileguidpair => formData.append(fileguidpair.guid ?? Utils.GenerateGuid(), fileguidpair.file)); } else { // Handle the case where fileguidpairs is a single file. const guids = fileguidpairs.guid ?? Utils.GenerateGuid(); formData.set('fileguids', guids); + formData.set('filesize', fileguidpairs.file.size.toString()); formData.append(guids, fileguidpairs.file); } const parameters = { diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 4509425bb..826db0de8 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -65,12 +65,16 @@ export default class UploadManager extends ApiManager { secureHandler: async ({ req, res }) => { const form = new formidable.IncomingForm(); let fileguids = ''; + let filesize = ''; form.on('field', (e: string, value: string) => { if (e === 'fileguids') { (fileguids = value).split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, 'reading file')); } + if (e === 'filesize') { + filesize = value; + } }); - form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, 'read: ' + e))); + form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${+e / +filesize}%) ${e} of ' +${filesize}`))); form.keepExtensions = true; form.uploadDir = pathToDirectory(Directory.parsed_files); return new Promise(resolve => { -- cgit v1.2.3-70-g09d2 From 6e126660d3b4a68d7e5fa6cd7844821270b42a51 Mon Sep 17 00:00:00 2001 From: bobzel Date: Sun, 10 Sep 2023 19:10:09 -0400 Subject: from last --- src/server/ApiManagers/UploadManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/server/ApiManagers') diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 826db0de8..58d15e90a 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -74,7 +74,7 @@ export default class UploadManager extends ApiManager { filesize = value; } }); - form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${+e / +filesize}%) ${e} of ' +${filesize}`))); + form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${Math.round((100 * +e) / +filesize)}%) ${e} of ' +${filesize}`))); form.keepExtensions = true; form.uploadDir = pathToDirectory(Directory.parsed_files); return new Promise(resolve => { -- cgit v1.2.3-70-g09d2 From 6941f48934ae1762d349ccf1f93c107a4b60f575 Mon Sep 17 00:00:00 2001 From: bobzel Date: Sun, 10 Sep 2023 19:12:42 -0400 Subject: ugh...from last --- src/server/ApiManagers/UploadManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/server/ApiManagers') diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 58d15e90a..56a9d9b6b 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -74,7 +74,7 @@ export default class UploadManager extends ApiManager { filesize = value; } }); - form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${Math.round((100 * +e) / +filesize)}%) ${e} of ' +${filesize}`))); + form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${Math.round((100 * +e) / +filesize)}%) ${e} of ${filesize}`))); form.keepExtensions = true; form.uploadDir = pathToDirectory(Directory.parsed_files); return new Promise(resolve => { -- cgit v1.2.3-70-g09d2 From cab10c69667926b162d0613433e873f9911ddafa Mon Sep 17 00:00:00 2001 From: geireann Date: Thu, 21 Sep 2023 15:11:13 -0400 Subject: fixed image importing via zip file to do resizing. --- src/server/ApiManagers/UploadManager.ts | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 56a9d9b6b..c337144c8 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -262,9 +262,33 @@ export default class UploadManager extends ApiManager { zip.extractEntryTo(entry.entryName, publicDirectory, true, false); createReadStream(pathname).pipe(createWriteStream(targetname)); if (extension !== '.pdf') { - createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_s' + extension))); - createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_m' + extension))); - createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_l' + extension))); + const { pngs, jpgs } = AcceptableMedia; + const resizers = [ + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Small }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Large }, + ]; + let isImage = false; + if (pngs.includes(extension)) { + resizers.forEach(element => { + element.resizer = element.resizer.png(); + }); + isImage = true; + } else if (jpgs.includes(extension)) { + resizers.forEach(element => { + element.resizer = element.resizer.jpeg(); + }); + isImage = true; + } + if (isImage) { + resizers.forEach(resizer => { + createReadStream(pathname) + .on('error', e => console.log('Resizing read:' + e)) + .pipe(resizer.resizer) + .on('error', e => console.log('Resizing write: ' + e)) + .pipe(createWriteStream(targetname.replace('_o' + extension, resizer.suffix + extension)).on('error', e => console.log('Resizing write: ' + e))); + }); + } } unlink(pathname, () => {}); } catch (e) { -- cgit v1.2.3-70-g09d2 From 51d74db190ba898af9f62906ccfd1da097330fb4 Mon Sep 17 00:00:00 2001 From: Sophie Zhang Date: Fri, 22 Sep 2023 03:14:03 -0400 Subject: fix: fix report manager for videos, only add _l suffix for images --- src/client/util/reportManager/ReportManager.tsx | 28 +++++++--------------- .../util/reportManager/reportManagerUtils.ts | 3 --- src/server/ApiManagers/UploadManager.ts | 9 +++---- 3 files changed, 14 insertions(+), 26 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/client/util/reportManager/ReportManager.tsx b/src/client/util/reportManager/ReportManager.tsx index 802a2f09f..e8c69f909 100644 --- a/src/client/util/reportManager/ReportManager.tsx +++ b/src/client/util/reportManager/ReportManager.tsx @@ -333,25 +333,15 @@ export class ReportManager extends React.Component<{}> { fillWidth /> - - {({ getRootProps, getInputProps }) => ( -
- -
- -

Drop or select media that shows the bug (optional)

-
-
- )} - + { + if (!e.target.files) return; + this.setFormData({ ...this.formData, mediaFiles: [...this.formData.mediaFiles, ...Array.from(e.target.files).map(file => ({ _id: v4(), file }))] }); + }} + /> {this.formData.mediaFiles.length > 0 &&
    {this.formData.mediaFiles.map(file => this.getMediaPreview(file))}
} {this.submitting ? (