Merge branch 'master' into zach-starter

4 files changed, 70 insertions, 279 deletions

diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts
deleted file mode 100644
index 684b49eaf..000000000
--- a/src/server/ApiManagers/SearchManager.ts
+++ /dev/null
@@ -1,200 +0,0 @@
-/* eslint-disable no-use-before-define */
-import { exec } from 'child_process';
-import { cyan, green, red, yellow } from 'colors';
-import { logExecution } from '../ActionUtilities';
-import { Method } from '../RouteManager';
-import RouteSubscriber from '../RouteSubscriber';
-import { Search } from '../Search';
-import { Database } from '../database';
-import ApiManager, { Registration } from './ApiManager';
-
-export class SearchManager extends ApiManager {
-    protected initialize(register: Registration): void {
-        register({
-            method: Method.GET,
-            subscription: new RouteSubscriber('solr').add('action'),
-            secureHandler: async ({ req, res }) => {
-                const { action } = req.params;
-                switch (action) {
-                    case 'start':
-                    case 'stop':
-                        {
-                            const status = req.params.action === 'start';
-                            SolrManager.SetRunning(status);
-                        }
-                        break;
-                    case 'update':
-                        await SolrManager.update();
-                        break;
-                    default:
-                        console.log(yellow(`${action} is an unknown solr operation.`));
-                }
-                res.redirect('/home');
-            },
-        });
-
-        register({
-            method: Method.GET,
-            subscription: '/dashsearch',
-            secureHandler: async ({ req, res }) => {
-                const solrQuery: any = {};
-                ['q', 'fq', 'start', 'rows', 'sort', 'hl.maxAnalyzedChars', 'hl', 'hl.fl'].forEach(key => {
-                    solrQuery[key] = req.query[key];
-                });
-                if (solrQuery.q === undefined) {
-                    res.send([]);
-                    return;
-                }
-                const results = await Search.search(solrQuery);
-                res.send(results);
-            },
-        });
-    }
-}
-
-export namespace SolrManager {
-    export function SetRunning(status: boolean) {
-        const args = status ? 'start' : 'stop -p 8983';
-        console.log(`solr management: trying to ${args}`);
-        exec(`solr ${args}`, { cwd: './solr-8.3.1/bin' }, (error, stdout, stderr) => {
-            if (error) {
-                console.log(red(`solr management error: unable to ${args} server`));
-                console.log(red(error.message));
-            }
-            console.log(cyan(stdout));
-            console.log(yellow(stderr));
-        });
-        if (status) {
-            console.log(cyan('Start script is executing: please allow 15 seconds for solr to start on port 8983.'));
-        }
-    }
-
-    export async function update() {
-        console.log(green('Beginning update...'));
-        await logExecution<void>({
-            startMessage: 'Clearing existing Solr information...',
-            endMessage: 'Solr information successfully cleared',
-            action: Search.clear,
-            color: cyan,
-        });
-        const cursor = await logExecution({
-            startMessage: 'Connecting to and querying for all documents from database...',
-            endMessage: ({ result, error }) => {
-                const success = error === null && result !== undefined;
-                if (!success) {
-                    console.log(red('Unable to connect to the database.'));
-                    process.exit(0);
-                }
-                return 'Connection successful and query complete';
-            },
-            action: () => Database.Instance.query({}),
-            color: yellow,
-        });
-        const updates: any[] = [];
-        let numDocs = 0;
-        function updateDoc(doc: any) {
-            numDocs++;
-            if (numDocs % 50 === 0) {
-                console.log(`Batch of 50 complete, total of ${numDocs}`);
-            }
-            if (doc.__type !== 'Doc') {
-                return;
-            }
-            const { fields } = doc;
-            if (!fields) {
-                return;
-            }
-            const update2: any = { id: doc._id };
-            let dynfield = false;
-            fields.forEach((key: any) => {
-                const value = fields[key];
-                const term = ToSearchTerm(value);
-                if (term !== undefined) {
-                    const { suffix, value: tvalue } = term;
-                    if (key.endsWith('modificationDate')) {
-                        update2['modificationDate' + suffix] = tvalue;
-                    }
-                    update2[key + suffix] = value;
-                    dynfield = true;
-                }
-            });
-            if (dynfield) {
-                updates.push(update2);
-            }
-        }
-        await cursor?.forEach(updateDoc);
-        const result = await logExecution({
-            startMessage: `Dispatching updates for ${updates.length} documents`,
-            endMessage: 'Dispatched updates complete',
-            action: () => Search.updateDocuments(updates),
-            color: cyan,
-        });
-        try {
-            if (result) {
-                const { status } = JSON.parse(result).responseHeader;
-                console.log(status ? red(`Failed with status code (${status})`) : green('Success!'));
-            } else {
-                console.log(red('Solr is likely not running!'));
-            }
-        } catch (e) {
-            console.log(red('Error:'));
-            console.log(e);
-            console.log('\n');
-        }
-        await cursor?.close();
-    }
-
-    const suffixMap: { [type: string]: string | [string, string | ((json: any) => any)] } = {
-        number: '_n',
-        string: '_t',
-        boolean: '_b',
-        image: ['_t', 'url'],
-        video: ['_t', 'url'],
-        pdf: ['_t', 'url'],
-        audio: ['_t', 'url'],
-        web: ['_t', 'url'],
-        map: ['_t', 'url'],
-        date: ['_d', value => new Date(value.date).toISOString()],
-        proxy: ['_i', 'fieldId'],
-        prefetch_proxy: ['_i', 'fieldId'],
-        list: [
-            '_l',
-            list => {
-                const results = [];
-                // eslint-disable-next-line no-restricted-syntax
-                for (const value of list.fields) {
-                    const term = ToSearchTerm(value);
-                    if (term) {
-                        results.push(term.value);
-                    }
-                }
-                return results.length ? results : null;
-            },
-        ],
-    };
-
-    function ToSearchTerm(valIn: any): { suffix: string; value: any } | undefined {
-        let val = valIn;
-        if (val === null || val === undefined) {
-            return undefined;
-        }
-        const type = val.__type || typeof val;
-        let suffix = suffixMap[type];
-        if (!suffix) {
-            return undefined;
-        }
-
-        if (Array.isArray(suffix)) {
-            const accessor = suffix[1];
-            if (typeof accessor === 'function') {
-                val = accessor(val);
-            } else {
-                val = val[accessor];
-            }
-            // eslint-disable-next-line prefer-destructuring
-            suffix = suffix[0];
-        }
-
-        return { suffix, value: val };
-    }
-}
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index b2624f654..868373474 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -1,20 +1,18 @@
 import * as AdmZip from 'adm-zip';
 import * as formidable from 'formidable';
 import * as fs from 'fs';
-import { createReadStream, createWriteStream, unlink } from 'fs';
+import { unlink } from 'fs';
 import * as imageDataUri from 'image-data-uri';
-import Jimp from 'jimp';
 import * as path from 'path';
 import * as uuid from 'uuid';
 import { retrocycle } from '../../decycler/decycler';
 import { DashVersion } from '../../fields/DocSymbols';
-import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils';
+import { DashUploadUtils, InjectSize, SizeSuffix, workerResample } from '../DashUploadUtils';
 import { Method, _success } from '../RouteManager';
 import { AcceptableMedia, Upload } from '../SharedMediaTypes';
-import { clientPathToFile, Directory, pathToDirectory, publicDirectory, serverPathToFile } from '../SocketData';
+import { Directory, clientPathToFile, pathToDirectory, publicDirectory, serverPathToFile } from '../SocketData';
 import { Database } from '../database';
 import ApiManager, { Registration } from './ApiManager';
-import { SolrManager } from './SearchManager';
 
 export default class UploadManager extends ApiManager {
     protected initialize(register: Registration): void {
@@ -185,7 +183,7 @@ export default class UploadManager extends ApiManager {
                         let linkids: string[] = [];
                         try {
                             // eslint-disable-next-line no-restricted-syntax
-                            for (const name in Object.keys(files)) {
+                            for (const name in files) {
                                 if (Object.prototype.hasOwnProperty.call(files, name)) {
                                     const f = files[name];
                                     // eslint-disable-next-line no-continue
@@ -194,26 +192,17 @@ export default class UploadManager extends ApiManager {
                                     const zip = new AdmZip(path2.filepath);
                                     zip.getEntries().forEach(entry => {
                                         const entryName = entry.entryName.replace(/%%%/g, '/');
-                                        if (!entryName.startsWith('files/')) {
-                                            return;
-                                        }
-                                        const extension = path.extname(entryName);
-                                        const pathname = publicDirectory + '/' + entry.entryName;
-                                        const targetname = publicDirectory + '/' + entryName;
-                                        try {
-                                            zip.extractEntryTo(entry.entryName, publicDirectory, true, false);
-                                            createReadStream(pathname).pipe(createWriteStream(targetname));
-                                            Jimp.read(pathname).then(imgIn => {
-                                                let img = imgIn;
-                                                DashUploadUtils.imageResampleSizes(extension).forEach(({ width, suffix }) => {
-                                                    const outputPath = InjectSize(targetname, suffix);
-                                                    if (!width) createReadStream(pathname).pipe(createWriteStream(outputPath));
-                                                    else img = img.resize(width, Jimp.AUTO).write(outputPath);
-                                                });
-                                                unlink(pathname, () => {});
-                                            });
-                                        } catch (e) {
-                                            console.log(e);
+                                        if (entryName.startsWith('files/')) {
+                                            const pathname = publicDirectory + '/' + entry.entryName;
+                                            const targetname = publicDirectory + '/' + entryName;
+                                            try {
+                                                zip.extractEntryTo(entry.entryName, publicDirectory, true, false);
+                                                const extension = path.extname(targetname).toLowerCase();
+                                                const basefilename = targetname.substring(0, targetname.length - extension.length);
+                                                workerResample(pathname, basefilename.replace(/_o$/, '') + extension, SizeSuffix.Original, true);
+                                            } catch (e) {
+                                                console.log(e);
+                                            }
                                         }
                                     });
                                     const json = zip.getEntry('docs.json');
@@ -244,7 +233,6 @@ export default class UploadManager extends ApiManager {
                                     unlink(path2.filepath, () => {});
                                 }
                             }
-                            SolrManager.update();
                             res.send(JSON.stringify({ id, docids, linkids }) || 'error');
                         } catch (e) {
                             console.log(e);
@@ -287,16 +275,9 @@ export default class UploadManager extends ApiManager {
                 }
                 imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => {
                     const ext = path.extname(savedName).toLowerCase();
+                    const outputPath = serverPathToFile(Directory.images, filename + ext);
                     if (AcceptableMedia.imageFormats.includes(ext)) {
-                        Jimp.read(savedName).then(imgIn => {
-                            let img = imgIn;
-                            (!origSuffix ? [{ width: 400, suffix: SizeSuffix.Medium }] : Object.values(DashUploadUtils.Sizes)) //
-                                .forEach(({ width, suffix }) => {
-                                    const outputPath = serverPathToFile(Directory.images, InjectSize(filename, suffix) + ext);
-                                    if (!width) createReadStream(savedName).pipe(createWriteStream(outputPath));
-                                    else img = img.resize(width, Jimp.AUTO).write(outputPath);
-                                });
-                        });
+                        workerResample(savedName, outputPath, origSuffix, false);
                     }
                     res.send(clientPathToFile(Directory.images, filename + ext));
                 });
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 8f012f783..1e55a885a 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -1,5 +1,5 @@
 import axios from 'axios';
-import { spawn, exec } from 'child_process';
+import { exec, spawn } from 'child_process';
 import { green, red } from 'colors';
 import { ExifData, ExifImage } from 'exif';
 import * as exifr from 'exifr';
@@ -8,8 +8,7 @@ import * as formidable from 'formidable';
 import { File } from 'formidable';
 import * as fs from 'fs';
 import { createReadStream, createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs';
-import Jimp from 'jimp';
-import * as autorotate from 'jpeg-autorotate';
+import { Jimp } from 'jimp';
 import * as md5File from 'md5-file';
 import * as path from 'path';
 import { basename } from 'path';
@@ -23,6 +22,44 @@ import { AcceptableMedia, Upload } from './SharedMediaTypes';
 import { Directory, clientPathToFile, filesDirectory, pathToDirectory, publicDirectory, serverPathToFile } from './SocketData';
 import { resolvedServerUrl } from './server_Initialization';
 
+import { Worker, isMainThread, parentPort } from 'worker_threads';
+
+// Create an array to store worker threads
+enum workertasks {
+    JIMP = 'jimp',
+}
+const JimpWorker: Worker | undefined = isMainThread ? new Worker(__filename) : undefined;
+export const workerResample = (imgSourcePath: string, outputPath: string, origSuffix: SizeSuffix, unlinkSource: boolean) => {
+    JimpWorker?.postMessage({ task: workertasks.JIMP, imgSourcePath, outputPath, origSuffix, unlinkSource });
+};
+
+if (isMainThread) {
+    // main thread code if needed ...
+} else {
+    // Worker thread code - Listens for messages from the main thread
+    parentPort?.on('message', message => {
+        switch (message.task) {
+            case workertasks.JIMP:
+                return workerResampleImage(message);
+            default:
+        }
+    });
+
+    async function workerResampleImage(message: { imgSourcePath: string; outputPath: string; origSuffix: string; unlinkSource: boolean }) {
+        const { imgSourcePath, outputPath, origSuffix, unlinkSource } = message;
+        const sizes = !origSuffix ? [{ width: 400, suffix: SizeSuffix.Medium }] : DashUploadUtils.imageResampleSizes(path.extname(imgSourcePath));
+        // prettier-ignore
+        Jimp.read(imgSourcePath)
+            .then(img => 
+                sizes.forEach(({ width, suffix }) =>
+                    img.resize({ w: width || img.bitmap.width })
+                       .write(InjectSize(outputPath, suffix) as `${string}.${string}`)
+                ))
+            .catch(e => console.log('Error Jimp:', e))
+            .finally(() => unlinkSource && unlinkSync(imgSourcePath));
+    }
+}
+
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const requestImageSize = require('../client/util/request-image-size');
 
@@ -277,15 +314,6 @@ export namespace DashUploadUtils {
         }
     };
 
-    async function correctRotation(imgSourcePath: string) {
-        const buffer = fs.readFileSync(imgSourcePath);
-        try {
-            return (await autorotate.rotate(buffer, { quality: 30 })).buffer;
-        } catch (e) {
-            return buffer;
-        }
-    }
-
     /**
      * define the resizers to use
      * @param ext the extension
@@ -310,38 +338,23 @@ export namespace DashUploadUtils {
      * @param outputDirectory the directory to output to, usually Directory.Images
      * @returns a map with suffixes as keys and resized filenames as values.
      */
-    export async function outputResizedImages(imgSourcePath: string, outputFileName: string, outputDirectory: string) {
+    export async function outputResizedImages(imgSourcePath: string, outputFileName: string, unlinkSource: boolean) {
         const writtenFiles: { [suffix: string]: string } = {};
+        const outputPath = path.resolve(pathToDirectory(Directory.images), outputFileName);
         const sizes = imageResampleSizes(path.extname(outputFileName));
 
-        const imgBuffer = await correctRotation(imgSourcePath);
         const imgReadStream = new Duplex();
-        imgReadStream.push(imgBuffer);
+        imgReadStream.push(fs.readFileSync(imgSourcePath));
         imgReadStream.push(null);
-        const outputPath = (suffix: SizeSuffix) => {
-            writtenFiles[suffix] = InjectSize(outputFileName, suffix);
-            return path.resolve(outputDirectory, writtenFiles[suffix]);
-        };
         await Promise.all(
-            sizes.filter(({ width }) => !width).map(({ suffix }) => 
-                new Promise<void>(res => {
-                    imgReadStream.pipe(createWriteStream(outputPath(suffix))).on('close', res);
-                })
+            sizes.map(({ suffix }) => 
+                new Promise<unknown>(res =>
+                    imgReadStream.pipe(createWriteStream(writtenFiles[suffix] = InjectSize(outputPath, suffix))).on('close', res)
+                )
         )); // prettier-ignore
 
-        return Jimp.read(imgBuffer)
-            .then(async imgIn => {
-                let img = imgIn;
-                await Promise.all( sizes.filter(({ width }) => width).map(({ width, suffix }) => {
-                    img = img.resize(width, Jimp.AUTO).write(outputPath(suffix));
-                    return img;
-                } )); // prettier-ignore
-                return writtenFiles;
-            })
-            .catch(e => {
-                console.log('ERROR' + e);
-                return writtenFiles;
-            });
+        workerResample(imgSourcePath, outputPath, SizeSuffix.Original, unlinkSource);
+        return writtenFiles;
     }
 
     /**
@@ -387,8 +400,9 @@ export namespace DashUploadUtils {
                 writtenFiles = {};
             }
         } else {
+            const unlinkSrcWhenFinished = isLocal().test(source) && cleanUp;
             try {
-                writtenFiles = await outputResizedImages(metadata.source, resolved, pathToDirectory(Directory.images));
+                writtenFiles = await outputResizedImages(metadata.source, resolved, unlinkSrcWhenFinished);
             } catch (e) {
                 // input is a blob or other, try reading it to create a metadata source file.
                 const reqSource = request(metadata.source);
@@ -400,16 +414,14 @@ export namespace DashUploadUtils {
                         .on('close', () => res())
                         .on('error', () => rej());
                 });
-                writtenFiles = await outputResizedImages(readSource, resolved, pathToDirectory(Directory.images));
+                writtenFiles = await outputResizedImages(readSource, resolved, unlinkSrcWhenFinished);
                 fs.unlink(readSource, err => console.log("Couldn't unlink temporary image file:" + readSource, err));
             }
         }
         Array.from(Object.keys(writtenFiles)).forEach(suffix => {
             information.accessPaths[suffix] = getAccessPaths(images, writtenFiles[suffix]);
         });
-        if (isLocal().test(source) && cleanUp) {
-            unlinkSync(source);
-        }
+
         return information;
     };
 
diff --git a/src/server/index.ts b/src/server/index.ts
index 3e0d86814..88dbd232d 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -8,7 +8,6 @@ import DataVizManager from './ApiManagers/DataVizManager';
 import DeleteManager from './ApiManagers/DeleteManager';
 import DownloadManager from './ApiManagers/DownloadManager';
 import GeneralGoogleManager from './ApiManagers/GeneralGoogleManager';
-import { SearchManager } from './ApiManagers/SearchManager';
 import SessionManager from './ApiManagers/SessionManager';
 import UploadManager from './ApiManagers/UploadManager';
 import UserManager from './ApiManagers/UserManager';
@@ -67,7 +66,6 @@ function routeSetter({ addSupervisedRoute, logRegistrationOutcome }: RouteManage
         new UserManager(),
         new UploadManager(),
         new DownloadManager(),
-        new SearchManager(),
         new DeleteManager(),
         new UtilManager(),
         new GeneralGoogleManager(),