diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/client/views/PropertiesButtons.tsx | 20 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx | 2 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/MarqueeView.tsx | 4 | ||||
| -rw-r--r-- | src/client/views/nodes/PDFBox.tsx | 4 | ||||
| -rw-r--r-- | src/server/ApiManagers/SearchManager.ts | 200 | ||||
| -rw-r--r-- | src/server/ApiManagers/UploadManager.ts | 49 | ||||
| -rw-r--r-- | src/server/DashUploadUtils.ts | 74 | ||||
| -rw-r--r-- | src/server/index.ts | 2 |
8 files changed, 56 insertions, 299 deletions
diff --git a/src/client/views/PropertiesButtons.tsx b/src/client/views/PropertiesButtons.tsx index f346d4ba8..f96a4a255 100644 --- a/src/client/views/PropertiesButtons.tsx +++ b/src/client/views/PropertiesButtons.tsx @@ -246,26 +246,6 @@ export class PropertiesButtons extends React.Component { // ); // } - // @computed get freezeThumb() { - // return this.propertyToggleBtn( - // 'FreezeThumb', - // '_thumb-frozen', - // on => `${on ? 'Freeze' : 'Unfreeze'} thumbnail`, - // on => 'snowflake', - // (dv, doc) => { - // if (doc['thumb-frozen']) doc['thumb-frozen'] = undefined; - // else { - // document.body.focus(); // so that we can access the clipboard without an error - // setTimeout(() => - // pasteImageBitmap((data_url: any, error: any) => { - // error && console.log(error); - // data_url && Utils.convertDataUri(data_url, doc[Id] + '-thumb-frozen', true).then(returnedfilename => (doc['thumb-frozen'] = new ImageField(returnedfilename))); - // }) - // ); - // } - // } - // ); - // } @computed get snapButton() { // THESE ARE NOT COMING return this.propertyToggleBtn( diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index c4cf8dee7..dbf781e63 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1811,7 +1811,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection const contentDiv = this.DocumentView?.().ContentDiv; contentDiv && UpdateIcon( - this.layoutDoc[Id] + '-icon' + new Date().getTime(), + this.layoutDoc[Id] + '_icon_' + new Date().getTime(), contentDiv, NumCast(this.layoutDoc._width), NumCast(this.layoutDoc._height), diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index 6cc75aa4b..ccb6bc9be 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -160,8 +160,8 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps pasteImageBitmap((data: any, error: any) => { error && console.log(error); data && - ClientUtils.convertDataUri(data, this._props.Document[Id] + '-thumb-frozen').then(returnedfilename => { - this._props.Document['thumb-frozen'] = new ImageField(returnedfilename); + ClientUtils.convertDataUri(data, this._props.Document[Id] + '_icon_' + new Date().getTime()).then(returnedfilename => { + this._props.Document[DocData].icon = new ImageField(returnedfilename); }); }) ); diff --git a/src/client/views/nodes/PDFBox.tsx b/src/client/views/nodes/PDFBox.tsx index b17275a1e..cb0b0d71f 100644 --- a/src/client/views/nodes/PDFBox.tsx +++ b/src/client/views/nodes/PDFBox.tsx @@ -56,10 +56,6 @@ export class PDFBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @computed get pdfUrl() { return Cast(this.dataDoc[this._props.fieldKey], PdfField); } - @computed get pdfThumb() { - return ImageCast(this.layoutDoc['thumb-frozen'], ImageCast(this.layoutDoc.thumb))?.url; - } - constructor(props: FieldViewProps) { super(props); makeObservable(this); diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts deleted file mode 100644 index 684b49eaf..000000000 --- a/src/server/ApiManagers/SearchManager.ts +++ /dev/null @@ -1,200 +0,0 @@ -/* eslint-disable no-use-before-define */ -import { exec } from 'child_process'; -import { cyan, green, red, yellow } from 'colors'; -import { logExecution } from '../ActionUtilities'; -import { Method } from '../RouteManager'; -import RouteSubscriber from '../RouteSubscriber'; -import { Search } from '../Search'; -import { Database } from '../database'; -import ApiManager, { Registration } from './ApiManager'; - -export class SearchManager extends ApiManager { - protected initialize(register: Registration): void { - register({ - method: Method.GET, - subscription: new RouteSubscriber('solr').add('action'), - secureHandler: async ({ req, res }) => { - const { action } = req.params; - switch (action) { - case 'start': - case 'stop': - { - const status = req.params.action === 'start'; - SolrManager.SetRunning(status); - } - break; - case 'update': - await SolrManager.update(); - break; - default: - console.log(yellow(`${action} is an unknown solr operation.`)); - } - res.redirect('/home'); - }, - }); - - register({ - method: Method.GET, - subscription: '/dashsearch', - secureHandler: async ({ req, res }) => { - const solrQuery: any = {}; - ['q', 'fq', 'start', 'rows', 'sort', 'hl.maxAnalyzedChars', 'hl', 'hl.fl'].forEach(key => { - solrQuery[key] = req.query[key]; - }); - if (solrQuery.q === undefined) { - res.send([]); - return; - } - const results = await Search.search(solrQuery); - res.send(results); - }, - }); - } -} - -export namespace SolrManager { - export function SetRunning(status: boolean) { - const args = status ? 'start' : 'stop -p 8983'; - console.log(`solr management: trying to ${args}`); - exec(`solr ${args}`, { cwd: './solr-8.3.1/bin' }, (error, stdout, stderr) => { - if (error) { - console.log(red(`solr management error: unable to ${args} server`)); - console.log(red(error.message)); - } - console.log(cyan(stdout)); - console.log(yellow(stderr)); - }); - if (status) { - console.log(cyan('Start script is executing: please allow 15 seconds for solr to start on port 8983.')); - } - } - - export async function update() { - console.log(green('Beginning update...')); - await logExecution<void>({ - startMessage: 'Clearing existing Solr information...', - endMessage: 'Solr information successfully cleared', - action: Search.clear, - color: cyan, - }); - const cursor = await logExecution({ - startMessage: 'Connecting to and querying for all documents from database...', - endMessage: ({ result, error }) => { - const success = error === null && result !== undefined; - if (!success) { - console.log(red('Unable to connect to the database.')); - process.exit(0); - } - return 'Connection successful and query complete'; - }, - action: () => Database.Instance.query({}), - color: yellow, - }); - const updates: any[] = []; - let numDocs = 0; - function updateDoc(doc: any) { - numDocs++; - if (numDocs % 50 === 0) { - console.log(`Batch of 50 complete, total of ${numDocs}`); - } - if (doc.__type !== 'Doc') { - return; - } - const { fields } = doc; - if (!fields) { - return; - } - const update2: any = { id: doc._id }; - let dynfield = false; - fields.forEach((key: any) => { - const value = fields[key]; - const term = ToSearchTerm(value); - if (term !== undefined) { - const { suffix, value: tvalue } = term; - if (key.endsWith('modificationDate')) { - update2['modificationDate' + suffix] = tvalue; - } - update2[key + suffix] = value; - dynfield = true; - } - }); - if (dynfield) { - updates.push(update2); - } - } - await cursor?.forEach(updateDoc); - const result = await logExecution({ - startMessage: `Dispatching updates for ${updates.length} documents`, - endMessage: 'Dispatched updates complete', - action: () => Search.updateDocuments(updates), - color: cyan, - }); - try { - if (result) { - const { status } = JSON.parse(result).responseHeader; - console.log(status ? red(`Failed with status code (${status})`) : green('Success!')); - } else { - console.log(red('Solr is likely not running!')); - } - } catch (e) { - console.log(red('Error:')); - console.log(e); - console.log('\n'); - } - await cursor?.close(); - } - - const suffixMap: { [type: string]: string | [string, string | ((json: any) => any)] } = { - number: '_n', - string: '_t', - boolean: '_b', - image: ['_t', 'url'], - video: ['_t', 'url'], - pdf: ['_t', 'url'], - audio: ['_t', 'url'], - web: ['_t', 'url'], - map: ['_t', 'url'], - date: ['_d', value => new Date(value.date).toISOString()], - proxy: ['_i', 'fieldId'], - prefetch_proxy: ['_i', 'fieldId'], - list: [ - '_l', - list => { - const results = []; - // eslint-disable-next-line no-restricted-syntax - for (const value of list.fields) { - const term = ToSearchTerm(value); - if (term) { - results.push(term.value); - } - } - return results.length ? results : null; - }, - ], - }; - - function ToSearchTerm(valIn: any): { suffix: string; value: any } | undefined { - let val = valIn; - if (val === null || val === undefined) { - return undefined; - } - const type = val.__type || typeof val; - let suffix = suffixMap[type]; - if (!suffix) { - return undefined; - } - - if (Array.isArray(suffix)) { - const accessor = suffix[1]; - if (typeof accessor === 'function') { - val = accessor(val); - } else { - val = val[accessor]; - } - // eslint-disable-next-line prefer-destructuring - suffix = suffix[0]; - } - - return { suffix, value: val }; - } -} diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 8ab27130b..868373474 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -1,20 +1,18 @@ import * as AdmZip from 'adm-zip'; import * as formidable from 'formidable'; import * as fs from 'fs'; -import { createReadStream, createWriteStream, unlink } from 'fs'; +import { unlink } from 'fs'; import * as imageDataUri from 'image-data-uri'; -import { Jimp } from 'jimp'; import * as path from 'path'; import * as uuid from 'uuid'; import { retrocycle } from '../../decycler/decycler'; import { DashVersion } from '../../fields/DocSymbols'; -import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils'; +import { DashUploadUtils, InjectSize, SizeSuffix, workerResample } from '../DashUploadUtils'; import { Method, _success } from '../RouteManager'; import { AcceptableMedia, Upload } from '../SharedMediaTypes'; import { Directory, clientPathToFile, pathToDirectory, publicDirectory, serverPathToFile } from '../SocketData'; import { Database } from '../database'; import ApiManager, { Registration } from './ApiManager'; -import { SolrManager } from './SearchManager'; export default class UploadManager extends ApiManager { protected initialize(register: Registration): void { @@ -185,7 +183,7 @@ export default class UploadManager extends ApiManager { let linkids: string[] = []; try { // eslint-disable-next-line no-restricted-syntax - for (const name in Object.keys(files)) { + for (const name in files) { if (Object.prototype.hasOwnProperty.call(files, name)) { const f = files[name]; // eslint-disable-next-line no-continue @@ -194,25 +192,17 @@ export default class UploadManager extends ApiManager { const zip = new AdmZip(path2.filepath); zip.getEntries().forEach(entry => { const entryName = entry.entryName.replace(/%%%/g, '/'); - if (!entryName.startsWith('files/')) { - return; - } - const extension = path.extname(entryName); - const pathname = publicDirectory + '/' + entry.entryName; - const targetname = publicDirectory + '/' + entryName; - try { - zip.extractEntryTo(entry.entryName, publicDirectory, true, false); - createReadStream(pathname).pipe(createWriteStream(targetname)); - Jimp.read(pathname).then(img => { - DashUploadUtils.imageResampleSizes(extension).forEach(({ width, suffix }) => { - const outputPath = InjectSize(targetname, suffix); - if (!width) createReadStream(pathname).pipe(createWriteStream(outputPath)); - else img.resize({ w: width }).write(outputPath as `${string}.${string}`); - }); - unlink(pathname, () => {}); - }); - } catch (e) { - console.log(e); + if (entryName.startsWith('files/')) { + const pathname = publicDirectory + '/' + entry.entryName; + const targetname = publicDirectory + '/' + entryName; + try { + zip.extractEntryTo(entry.entryName, publicDirectory, true, false); + const extension = path.extname(targetname).toLowerCase(); + const basefilename = targetname.substring(0, targetname.length - extension.length); + workerResample(pathname, basefilename.replace(/_o$/, '') + extension, SizeSuffix.Original, true); + } catch (e) { + console.log(e); + } } }); const json = zip.getEntry('docs.json'); @@ -243,7 +233,6 @@ export default class UploadManager extends ApiManager { unlink(path2.filepath, () => {}); } } - SolrManager.update(); res.send(JSON.stringify({ id, docids, linkids }) || 'error'); } catch (e) { console.log(e); @@ -286,15 +275,9 @@ export default class UploadManager extends ApiManager { } imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => { const ext = path.extname(savedName).toLowerCase(); + const outputPath = serverPathToFile(Directory.images, filename + ext); if (AcceptableMedia.imageFormats.includes(ext)) { - Jimp.read(savedName).then(img => { - (!origSuffix ? [{ width: 400, suffix: SizeSuffix.Medium }] : Object.values(DashUploadUtils.Sizes)) // - .forEach(({ width, suffix }) => { - const outputPath = serverPathToFile(Directory.images, InjectSize(filename, suffix) + ext); - if (!width) createReadStream(savedName).pipe(createWriteStream(outputPath)); - else img.resize({ w: width }).write(outputPath as `${string}.${string}`); - }); - }); + workerResample(savedName, outputPath, origSuffix, false); } res.send(clientPathToFile(Directory.images, filename + ext)); }); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 5cf86b2ae..1e55a885a 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -7,7 +7,7 @@ import * as ffmpeg from 'fluent-ffmpeg'; import * as formidable from 'formidable'; import { File } from 'formidable'; import * as fs from 'fs'; -import { createReadStream, createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs'; // import { Jimp } from "@jimp/core"; +import { createReadStream, createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs'; import { Jimp } from 'jimp'; import * as md5File from 'md5-file'; import * as path from 'path'; @@ -25,32 +25,38 @@ import { resolvedServerUrl } from './server_Initialization'; import { Worker, isMainThread, parentPort } from 'worker_threads'; // Create an array to store worker threads -const workerThreads: Worker[] = []; +enum workertasks { + JIMP = 'jimp', +} +const JimpWorker: Worker | undefined = isMainThread ? new Worker(__filename) : undefined; +export const workerResample = (imgSourcePath: string, outputPath: string, origSuffix: SizeSuffix, unlinkSource: boolean) => { + JimpWorker?.postMessage({ task: workertasks.JIMP, imgSourcePath, outputPath, origSuffix, unlinkSource }); +}; + if (isMainThread) { - // Main thread code - // Create worker threads -- just one right to do image resampling - workerThreads.push(new Worker(__filename)); + // main thread code if needed ... } else { // Worker thread code - Listens for messages from the main thread - parentPort?.on('message', message => workerResampleImage(message.task)); - async function workerResampleImage(task: string) { - const [sourcePath, outputFileName, outputDirectory, unlinkSource] = task.split('::'); - const sizes = DashUploadUtils.imageResampleSizes(path.extname(sourcePath)); - const imgBuffer = await fs.readFileSync(sourcePath); - const outputPath = (suffix: SizeSuffix) => { - const writtenFile = InjectSize(outputFileName, suffix); - return path.resolve(outputDirectory, writtenFile); - }; - await Jimp.fromBuffer(imgBuffer) - .then(img => sizes.filter(({ width }) => width).map(({ width, suffix }) => img.resize({ w: width }).write(outputPath(suffix) as `${string}.${string}`))) - .catch(e => { - console.log('ERROR' + e); - }); - if (unlinkSource === 'true') { - unlinkSync(sourcePath); + parentPort?.on('message', message => { + switch (message.task) { + case workertasks.JIMP: + return workerResampleImage(message); + default: } - - // … operations to be performed to execute the task + }); + + async function workerResampleImage(message: { imgSourcePath: string; outputPath: string; origSuffix: string; unlinkSource: boolean }) { + const { imgSourcePath, outputPath, origSuffix, unlinkSource } = message; + const sizes = !origSuffix ? [{ width: 400, suffix: SizeSuffix.Medium }] : DashUploadUtils.imageResampleSizes(path.extname(imgSourcePath)); + // prettier-ignore + Jimp.read(imgSourcePath) + .then(img => + sizes.forEach(({ width, suffix }) => + img.resize({ w: width || img.bitmap.width }) + .write(InjectSize(outputPath, suffix) as `${string}.${string}`) + )) + .catch(e => console.log('Error Jimp:', e)) + .finally(() => unlinkSource && unlinkSync(imgSourcePath)); } } @@ -332,28 +338,22 @@ export namespace DashUploadUtils { * @param outputDirectory the directory to output to, usually Directory.Images * @returns a map with suffixes as keys and resized filenames as values. */ - export async function outputResizedImages(imgSourcePath: string, outputFileName: string, outputDirectory: string, unlinkSource: boolean) { + export async function outputResizedImages(imgSourcePath: string, outputFileName: string, unlinkSource: boolean) { const writtenFiles: { [suffix: string]: string } = {}; - - const outputPath = (suffix: SizeSuffix) => { - writtenFiles[suffix] = InjectSize(outputFileName, suffix); - return path.resolve(outputDirectory, writtenFiles[suffix]); - }; - + const outputPath = path.resolve(pathToDirectory(Directory.images), outputFileName); const sizes = imageResampleSizes(path.extname(outputFileName)); - const imgBuffer = fs.readFileSync(imgSourcePath); + const imgReadStream = new Duplex(); - imgReadStream.push(imgBuffer); + imgReadStream.push(fs.readFileSync(imgSourcePath)); imgReadStream.push(null); await Promise.all( sizes.map(({ suffix }) => new Promise<unknown>(res => - imgReadStream.pipe(createWriteStream(outputPath(suffix))).on('close', res) + imgReadStream.pipe(createWriteStream(writtenFiles[suffix] = InjectSize(outputPath, suffix))).on('close', res) ) )); // prettier-ignore - // Send a message to worker thread to resample image - workerThreads[0].postMessage({ task: imgSourcePath + '::' + outputFileName + '::' + outputDirectory + '::' + unlinkSource }); + workerResample(imgSourcePath, outputPath, SizeSuffix.Original, unlinkSource); return writtenFiles; } @@ -402,7 +402,7 @@ export namespace DashUploadUtils { } else { const unlinkSrcWhenFinished = isLocal().test(source) && cleanUp; try { - writtenFiles = await outputResizedImages(metadata.source, resolved, pathToDirectory(Directory.images), unlinkSrcWhenFinished); + writtenFiles = await outputResizedImages(metadata.source, resolved, unlinkSrcWhenFinished); } catch (e) { // input is a blob or other, try reading it to create a metadata source file. const reqSource = request(metadata.source); @@ -414,7 +414,7 @@ export namespace DashUploadUtils { .on('close', () => res()) .on('error', () => rej()); }); - writtenFiles = await outputResizedImages(readSource, resolved, pathToDirectory(Directory.images), unlinkSrcWhenFinished); + writtenFiles = await outputResizedImages(readSource, resolved, unlinkSrcWhenFinished); fs.unlink(readSource, err => console.log("Couldn't unlink temporary image file:" + readSource, err)); } } diff --git a/src/server/index.ts b/src/server/index.ts index 3e0d86814..88dbd232d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -8,7 +8,6 @@ import DataVizManager from './ApiManagers/DataVizManager'; import DeleteManager from './ApiManagers/DeleteManager'; import DownloadManager from './ApiManagers/DownloadManager'; import GeneralGoogleManager from './ApiManagers/GeneralGoogleManager'; -import { SearchManager } from './ApiManagers/SearchManager'; import SessionManager from './ApiManagers/SessionManager'; import UploadManager from './ApiManagers/UploadManager'; import UserManager from './ApiManagers/UserManager'; @@ -67,7 +66,6 @@ function routeSetter({ addSupervisedRoute, logRegistrationOutcome }: RouteManage new UserManager(), new UploadManager(), new DownloadManager(), - new SearchManager(), new DeleteManager(), new UtilManager(), new GeneralGoogleManager(), |