From b9316fd7280bbddbc73966ce432b5054c07c3af0 Mon Sep 17 00:00:00 2001 From: bobzel Date: Thu, 5 Dec 2024 10:48:27 -0500 Subject: playing with ink stroke editing and firefly --- src/.DS_Store | Bin 10244 -> 10244 bytes src/client/views/InkStrokeProperties.ts | 28 ++++++++++++++-- src/server/ApiManagers/FireflyManager.ts | 53 +++++++++++++++++++++++++++---- 3 files changed, 72 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/.DS_Store b/src/.DS_Store index 9b66f8d8e..1ef749033 100644 Binary files a/src/.DS_Store and b/src/.DS_Store differ diff --git a/src/client/views/InkStrokeProperties.ts b/src/client/views/InkStrokeProperties.ts index 358274f0e..8165b7f4e 100644 --- a/src/client/views/InkStrokeProperties.ts +++ b/src/client/views/InkStrokeProperties.ts @@ -74,6 +74,7 @@ export class InkStrokeProperties { doc._height = (newYrange.max - newYrange.min) * ptsYscale + NumCast(doc.stroke_width); doc.x = oldXrange.coord + (newXrange.min - oldXrange.min) * ptsXscale; doc.y = oldYrange.coord + (newYrange.min - oldYrange.min) * ptsYscale; + Doc.SetInPlace(doc, 'stroke', new InkField(newPoints), true); appliedFunc = true; } @@ -248,7 +249,7 @@ export class InkStrokeProperties { /** * Handles the movement/scaling of a control point. */ - moveControlPtHandle = undoable((inkView: DocumentView, deltaX: number, deltaY: number, controlIndex: number, origInk?: InkData) => { + moveControlPtHandle = undoable((inkView: DocumentView, deltaX: number, deltaY: number, controlIndex: number, origInk?: InkData, noPush?: boolean) => { inkView && this.applyFunction(inkView, (view: DocumentView, ink: InkData) => { const order = controlIndex % 4; @@ -288,10 +289,11 @@ export class InkStrokeProperties { const { finalCtrls: rightCtrls /* , error: errorRight */ } = FitOneCurve(samplesRight, { X: startDir.x, Y: startDir.y }, { X: endDir.x, Y: endDir.y }); finalCtrls = finalCtrls.concat(rightCtrls); newink.splice(this._currentPoint - 4, 8, ...finalCtrls); - return newink; } - return ink.map((pt, i) => { + const prevData = (inkView.ComponentView as InkingStroke).screenCtrlPts; + + const newInk = ink.map((pt, i) => { const leftHandlePoint = order === 0 && i === controlIndex + 1; const rightHandlePoint = order === 0 && controlIndex !== 0 && i === controlIndex - 2; if (controlIndex === i || (order === 0 && controlIndex !== 0 && i === controlIndex - 1) || (order === 3 && i === controlIndex - 1)) { @@ -312,6 +314,26 @@ export class InkStrokeProperties { } return pt; }); + + if (inkView.IsSelected && !noPush && !InkingStroke.IsClosed(prevData)) { + const ffview = CollectionFreeFormView.from(inkView); + ffview?.childDocs.forEach(d => { + const oinkDoc = DocumentView.getDocumentView(d); + const oinkView = oinkDoc?.ComponentView as InkingStroke; + if (oinkView instanceof InkingStroke && oinkDoc && oinkDoc.Document !== inkView.Document) { + const { nearestSeg, nearestT } = InkStrokeProperties.nearestPtToStroke(oinkView.screenCtrlPts, prevData[controlIndex]); + const locdelta = inkView.screenToContentsTransform().inverse().transformDirection(deltaX, deltaY); + const index = nearestSeg + Math.round(nearestT) * 4; + const oinkData = oinkView.inkScaledData().inkData; + const closed = InkingStroke.IsClosed(oinkData); + InkStrokeProperties.Instance.moveControlPtHandle(oinkDoc, locdelta[0], locdelta[1], index, undefined, true); + if (index === oinkData.length && closed) { + InkStrokeProperties.Instance.moveControlPtHandle(oinkDoc, locdelta[0], locdelta[1], 0, undefined, true); + } + } + }); + } + return newInk; }); }, 'move ink ctrl pt'); diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 5e3ba1f83..e10e43704 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -16,7 +16,7 @@ export default class FireflyManager extends ApiManager { }); askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { const fetched = this.getBearerToken().then(response => - response.json().then((data: { access_token: string }) => + (response as Response).json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ @@ -36,16 +36,57 @@ export default class FireflyManager extends ApiManager { ); return fetched; }; + askFireflyText = (testshotpng: Blob) => { + const fetched = this.getBearerToken().then(response => + (response as Response).json().then((data: { access_token: string }) => { + return fetch('https://sensei.adobe.io/services/v2/predict', { + method: 'POST', + headers: [ + ['Prefer', 'respond-async, wait=59'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['content-type', 'multipart/form-data'], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: ((form: FormData) => { + form.append('file', testshotpng); + form.append( + 'contentAnalyzerRequests', + JSON.stringify({ + 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + }) + ); + return form; + })(new FormData()), + }).then(response2 => + response2 + .json() + .then(json => { + console.log(json); + return ''; + }) + .catch(error => { + console.error('Error:', error); + return ''; + }) + ); + }) + ); + return fetched; + }; protected initialize(register: Registration): void { register({ method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => - this.askFirefly(req.body.prompt).then(fire => - DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { - if (info instanceof Error) _invalid(res, info.message); - else _success(res, info.accessPaths.agnostic.client); - }) + fetch('http://localhost:1050/files/images/testshot.png').then(json => + json.blob().then(file => + this.askFireflyText(file).then(fire => + DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info.accessPaths.agnostic.client); + }) + ) + ) ), }); } -- cgit v1.2.3-70-g09d2 From 629e2bef5d4dd4599de010b62e5b26304ee620dd Mon Sep 17 00:00:00 2001 From: bobzel Date: Mon, 9 Dec 2024 12:53:35 -0500 Subject: added firefly text in image request --- package-lock.json | 6 ++ package.json | 1 + src/server/ApiManagers/FireflyManager.ts | 94 ++++++++++++++++++++++++-------- src/server/ApiManagers/UploadManager.ts | 5 -- src/server/DashUploadUtils.ts | 4 +- 5 files changed, 79 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/package-lock.json b/package-lock.json index 8c2cf0694..05f6d540f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -171,6 +171,7 @@ "npm": "^10.8.1", "openai": "^4.26.0", "p-limit": "^6.1.0", + "parse-multipart-data": "^1.5.0", "passport": "^0.7.0", "passport-google-oauth20": "^2.0.0", "passport-local": "^1.0.0", @@ -30590,6 +30591,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parse-multipart-data": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/parse-multipart-data/-/parse-multipart-data-1.5.0.tgz", + "integrity": "sha512-ck5zaMF0ydjGfejNMnlo5YU2oJ+pT+80Jb1y4ybanT27j+zbVP/jkYmCrUGsEln0Ox/hZmuvgy8Ra7AxbXP2Mw==" + }, "node_modules/parse-passwd": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz", diff --git a/package.json b/package.json index 7c2730245..949f76599 100644 --- a/package.json +++ b/package.json @@ -254,6 +254,7 @@ "npm": "^10.8.1", "openai": "^4.26.0", "p-limit": "^6.1.0", + "parse-multipart-data": "^1.5.0", "passport": "^0.7.0", "passport-google-oauth20": "^2.0.0", "passport-local": "^1.0.0", diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index e10e43704..30fbdc577 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -1,6 +1,7 @@ import { DashUploadUtils } from '../DashUploadUtils'; import { _invalid, _success, Method } from '../RouteManager'; import ApiManager, { Registration } from './ApiManager'; +import * as multipart from 'parse-multipart-data'; export default class FireflyManager extends ApiManager { getBearerToken = () => @@ -27,7 +28,7 @@ export default class FireflyManager extends ApiManager { ], body: `{ "prompt": "${prompt}" }`, }) - .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) .catch(error => { console.error('Error:', error); return ''; @@ -36,7 +37,9 @@ export default class FireflyManager extends ApiManager { ); return fetched; }; - askFireflyText = (testshotpng: Blob) => { + getImageText = (testshotpng: Blob) => { + const inputFileVarName = 'infile'; + const outputVarName = 'result'; const fetched = this.getBearerToken().then(response => (response as Response).json().then((data: { access_token: string }) => { return fetch('https://sensei.adobe.io/services/v2/predict', { @@ -44,31 +47,67 @@ export default class FireflyManager extends ApiManager { headers: [ ['Prefer', 'respond-async, wait=59'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], - ['content-type', 'multipart/form-data'], + // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string ['Authorization', `Bearer ${data.access_token}`], ], body: ((form: FormData) => { - form.append('file', testshotpng); - form.append( + form.set(inputFileVarName, testshotpng); + form.set( 'contentAnalyzerRequests', JSON.stringify({ 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + 'sensei:invocation_mode': 'asynchronous', + 'sensei:invocation_batch': false, + 'sensei:engines': [ + { + 'sensei:execution_info': { + 'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + }, + 'sensei:inputs': { + documents: [ + { + 'sensei:multipart_field_name': inputFileVarName, + 'dc:format': 'image/png', + }, + ], + }, + 'sensei:params': { + correct_with_dictionary: true, + }, + 'sensei:outputs': { + result: { + 'sensei:multipart_field_name': outputVarName, + 'dc:format': 'application/json', + }, + }, + }, + ], }) ); return form; })(new FormData()), - }).then(response2 => - response2 - .json() - .then(json => { - console.log(json); - return ''; - }) - .catch(error => { - console.error('Error:', error); - return ''; - }) - ); + }).then(response2 => { + const contentType = response2.headers.get('content-type') ?? ''; + if (contentType.includes('application/json')) { + return response2.json().then((json: object) => JSON.stringify(json)); + } + if (contentType.includes('multipart')) { + return response2 + .arrayBuffer() + .then(arrayBuffer => + multipart + .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? '')) + .filter(part => part.name === outputVarName) + .map(part => JSON.parse(part.data.toString()[0])) + .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '') + ) + .catch(error => { + console.error('Error:', error); + return ''; + }); + } + return response2.text(); + }); }) ); return fetched; @@ -77,15 +116,24 @@ export default class FireflyManager extends ApiManager { register({ method: Method.POST, subscription: '/queryFireflyImage', + secureHandler: ({ req, res }) => + this.askFirefly(req.body).then(fire => { + DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info.accessPaths.agnostic.client); + }); + }), + }); + + register({ + method: Method.POST, + subscription: '/queryFireflyImageText', secureHandler: ({ req, res }) => fetch('http://localhost:1050/files/images/testshot.png').then(json => json.blob().then(file => - this.askFireflyText(file).then(fire => - DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { - if (info instanceof Error) _invalid(res, info.message); - else _success(res, info.accessPaths.agnostic.client); - }) - ) + this.getImageText(file).then(text => { + _success(res, text); + }) ) ), }); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 868373474..7bfdd5aec 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -147,13 +147,10 @@ export default class UploadManager extends ApiManager { if (doc.id) { doc.id = getId(doc.id); } - // eslint-disable-next-line no-restricted-syntax for (const key in doc.fields) { - // eslint-disable-next-line no-continue if (!Object.prototype.hasOwnProperty.call(doc.fields, key)) continue; const field = doc.fields[key]; - // eslint-disable-next-line no-continue if (field === undefined || field === null) continue; if (field.__type === 'Doc') { @@ -182,11 +179,9 @@ export default class UploadManager extends ApiManager { let docids: string[] = []; let linkids: string[] = []; try { - // eslint-disable-next-line no-restricted-syntax for (const name in files) { if (Object.prototype.hasOwnProperty.call(files, name)) { const f = files[name]; - // eslint-disable-next-line no-continue if (!f) continue; const path2 = f[0]; // what about the rest of the array? are we guaranteed only one value is set? const zip = new AdmZip(path2.filepath); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 351351ca5..028116779 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -223,7 +223,6 @@ export namespace DashUploadUtils { const parseExifData = async (source: string) => { const image = await request.get(source, { encoding: null }); const { /* data, */ error } = await new Promise<{ data: ExifData; error: string | undefined }>(resolve => { - // eslint-disable-next-line no-new new ExifImage({ image }, (exifError, data) => { resolve({ data, error: exifError?.message }); }); @@ -302,7 +301,6 @@ export namespace DashUploadUtils { // Bundle up the information into an object return { source, - // eslint-disable-next-line radix contentSize: parseInt(headers[size]), contentType: headers[type], nativeWidth, @@ -370,7 +368,7 @@ export namespace DashUploadUtils { * @returns the accessPaths for the resized files. */ export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename: string, prefix = '', cleanUp = true): Promise => { - const { requestable, source, ...remaining } = metadata; + const { requestable, ...remaining } = metadata; const dfltSuffix = remaining.contentType.split('/')[1].toLowerCase(); const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${dfltSuffix === 'xml' ? 'jpg' : dfltSuffix}`; const { images } = Directory; -- cgit v1.2.3-70-g09d2 From 48c3b802a3c8fd446ecbd33747fe702b11170dfe Mon Sep 17 00:00:00 2001 From: bobzel Date: Mon, 9 Dec 2024 12:54:49 -0500 Subject: from last --- src/server/ApiManagers/FireflyManager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 30fbdc577..cc4c218bf 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -117,12 +117,12 @@ export default class FireflyManager extends ApiManager { method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => - this.askFirefly(req.body).then(fire => { + this.askFirefly(req.body.prompt).then(fire => DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info.accessPaths.agnostic.client); - }); - }), + }) + ), }); register({ -- cgit v1.2.3-70-g09d2 From a5a7769e0c400f0a58a9b53ae13b338a26eaa919 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 11 Dec 2024 13:11:43 -0500 Subject: fixes for uploading remote images to sample and save them properly. extnesions fo firefly manager to query text in images, expand images, and tweaks to generate images. --- package-lock.json | 23 +++++-- package.json | 3 +- src/client/views/nodes/ImageBox.tsx | 31 +++++++++ src/server/ApiManagers/FireflyManager.ts | 114 +++++++++++++++++++++++++++---- src/server/ApiManagers/UploadManager.ts | 8 ++- src/server/DashUploadUtils.ts | 37 ++++++---- 6 files changed, 183 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/package-lock.json b/package-lock.json index 05f6d540f..46057f615 100644 --- a/package-lock.json +++ b/package-lock.json @@ -100,6 +100,7 @@ "depcheck": "^1.4.7", "dompurify": "^3.1.7", "dotenv": "^16.4.5", + "dropbox": "^10.34.0", "eslint-webpack-plugin": "^4.1.0", "exif": "^0.6.0", "exifr": "^7.1.3", @@ -128,7 +129,7 @@ "google-auth-library": "^9.4.1", "googleapis": "^144.0.0", "googlephotos": "^0.3.5", - "got": "^14.0.0", + "got": "^14.4.5", "howler": "^2.2.4", "html-to-image": "^1.11.11", "html-to-text": "^9.0.5", @@ -19701,6 +19702,20 @@ "resolved": "https://registry.npmjs.org/double-bits/-/double-bits-1.1.1.tgz", "integrity": "sha512-BCLEIBq0O/DWoA7BsCu/R+RP0ZXiowP8BhtJT3qeuuQEBpnS8LK/Wo6UTJQv6v8mK1fj8n90YziHLwGdM5whSg==" }, + "node_modules/dropbox": { + "version": "10.34.0", + "resolved": "https://registry.npmjs.org/dropbox/-/dropbox-10.34.0.tgz", + "integrity": "sha512-5jb5/XzU0fSnq36/hEpwT5/QIep7MgqKuxghEG44xCu7HruOAjPdOb3x0geXv5O/hd0nHpQpWO+r5MjYTpMvJg==", + "dependencies": { + "node-fetch": "^2.6.1" + }, + "engines": { + "node": ">=0.10.3" + }, + "peerDependencies": { + "@types/node-fetch": "^2.5.7" + } + }, "node_modules/dynamic-dedupe": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/dynamic-dedupe/-/dynamic-dedupe-0.3.0.tgz", @@ -22149,9 +22164,9 @@ } }, "node_modules/got": { - "version": "14.4.4", - "resolved": "https://registry.npmjs.org/got/-/got-14.4.4.tgz", - "integrity": "sha512-tqiF7eSgTBwQkxb1LxsEpva8TaMYVisbhplrFVmw9GQE3855Z+MH/mnsXLLOkDxR6hZJRFMj5VTAZ8lmTF8ZOA==", + "version": "14.4.5", + "resolved": "https://registry.npmjs.org/got/-/got-14.4.5.tgz", + "integrity": "sha512-sq+uET8TnNKRNnjEOPJzMcxeI0irT8BBNmf+GtZcJpmhYsQM1DSKmCROUjPWKsXZ5HzwD5Cf5/RV+QD9BSTxJg==", "dependencies": { "@sindresorhus/is": "^7.0.1", "@szmarczak/http-timer": "^5.0.1", diff --git a/package.json b/package.json index 949f76599..7ca0c8b38 100644 --- a/package.json +++ b/package.json @@ -183,6 +183,7 @@ "depcheck": "^1.4.7", "dompurify": "^3.1.7", "dotenv": "^16.4.5", + "dropbox": "^10.34.0", "eslint-webpack-plugin": "^4.1.0", "exif": "^0.6.0", "exifr": "^7.1.3", @@ -211,7 +212,7 @@ "google-auth-library": "^9.4.1", "googleapis": "^144.0.0", "googlephotos": "^0.3.5", - "got": "^14.0.0", + "got": "^14.4.5", "howler": "^2.2.4", "html-to-image": "^1.11.11", "html-to-text": "^9.0.5", diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index ff879a2ab..8c7ec959e 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -38,6 +38,8 @@ import { FieldView, FieldViewProps } from './FieldView'; import { FocusViewOptions } from './FocusViewOptions'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; +import { Upload } from '../../../server/SharedMediaTypes'; +import { ImageUtils } from '../../util/Import & Export/ImageUtils'; export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -309,6 +311,35 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { funcs.push({ description: 'Rotate Clockwise 90', event: this.rotate, icon: 'redo-alt' }); funcs.push({ description: `Show ${this.layoutDoc._showFullRes ? 'Dynamic Res' : 'Full Res'}`, event: this.resolution, icon: 'expand' }); funcs.push({ description: 'Set Native Pixel Size', event: this.setNativeSize, icon: 'expand-arrows-alt' }); + funcs.push({ + description: 'GetImageText', + event: () => { + Networking.PostToServer('/queryFireflyImageText', { + file: (file => { + const ext = extname(file); + return file.replace(ext, (this._error ? '_o' : this._curSuffix) + ext); + })(ImageCast(this.Document[Doc.LayoutFieldKey(this.Document)])?.url.href), + }).then(text => alert(text)); + }, + icon: 'expand-arrows-alt', + }); + funcs.push({ + description: 'Expand Image', + event: () => { + Networking.PostToServer('/expandImage', { + prompt: 'sunny skies', + file: (file => { + const ext = extname(file); + return file.replace(ext, (this._error ? '_o' : this._curSuffix) + ext); + })(ImageCast(this.Document[Doc.LayoutFieldKey(this.Document)])?.url.href), + }).then((info: Upload.ImageInformation) => { + const img = Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { title: 'expand:' + this.Document.title }); + DocUtils.assignImageInfo(info, img); + this._props.addDocTab(img, OpenWhere.addRight); + }); + }, + icon: 'expand-arrows-alt', + }); funcs.push({ description: 'Copy path', event: () => ClientUtils.CopyText(this.choosePath(field.url)), icon: 'copy' }); funcs.push({ description: 'Open Image Editor', diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index cc4c218bf..d757a23fe 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -1,7 +1,11 @@ +import { Dropbox, files } from 'dropbox'; +import * as fs from 'fs'; +import * as multipart from 'parse-multipart-data'; +import * as path from 'path'; import { DashUploadUtils } from '../DashUploadUtils'; -import { _invalid, _success, Method } from '../RouteManager'; +import { _error, _invalid, _success, Method } from '../RouteManager'; +import { Directory, filesDirectory } from '../SocketData'; import ApiManager, { Registration } from './ApiManager'; -import * as multipart from 'parse-multipart-data'; export default class FireflyManager extends ApiManager { getBearerToken = () => @@ -13,11 +17,11 @@ export default class FireflyManager extends ApiManager { body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, }).catch(error => { console.error('Error:', error); - return ''; + return undefined; }); - askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { + generateImage = (prompt: string = 'a realistic illustration of a cat coding') => { const fetched = this.getBearerToken().then(response => - (response as Response).json().then((data: { access_token: string }) => + response?.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ @@ -28,20 +32,66 @@ export default class FireflyManager extends ApiManager { ], body: `{ "prompt": "${prompt}" }`, }) - .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + .then(response2 => response2.json().then(json => (json.outputs?.[0] as { image: { url: string } })?.image.url)) .catch(error => { console.error('Error:', error); - return ''; + return undefined; }) ) ); return fetched; }; - getImageText = (testshotpng: Blob) => { + expandImage = (imgUrl: string, prompt?: string) => { + const dropboxImgUrl = imgUrl; + const fetched = this.getBearerToken().then(response => + response + ?.json() + .then((data: { access_token: string }) => { + return fetch('https://firefly-api.adobe.io/v3/images/expand', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: JSON.stringify({ + image: { + source: { + url: dropboxImgUrl, + }, + }, + numVariations: 1, + seeds: [0], + size: { + width: 3048, + height: 2048, + }, + prompt: prompt ?? 'cloudy skies', + placement: { + inset: { + left: 0, + top: 0, + right: 0, + bottom: 0, + }, + alignment: { + horizontal: 'center', + vertical: 'center', + }, + }, + }), + }); + }) + .then(resp => resp.json()) + ); + return fetched; + }; + getImageText = (imageBlob: Blob) => { const inputFileVarName = 'infile'; const outputVarName = 'result'; const fetched = this.getBearerToken().then(response => - (response as Response).json().then((data: { access_token: string }) => { + response?.json().then((data: { access_token: string }) => { return fetch('https://sensei.adobe.io/services/v2/predict', { method: 'POST', headers: [ @@ -51,7 +101,7 @@ export default class FireflyManager extends ApiManager { ['Authorization', `Bearer ${data.access_token}`], ], body: ((form: FormData) => { - form.set(inputFileVarName, testshotpng); + form.set(inputFileVarName, imageBlob); form.set( 'contentAnalyzerRequests', JSON.stringify({ @@ -98,7 +148,7 @@ export default class FireflyManager extends ApiManager { multipart .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? '')) .filter(part => part.name === outputVarName) - .map(part => JSON.parse(part.data.toString()[0])) + .map(part => JSON.parse(part.data.toString())[0]) .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '') ) .catch(error => { @@ -117,10 +167,10 @@ export default class FireflyManager extends ApiManager { method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => - this.askFirefly(req.body.prompt).then(fire => - DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { + this.generateImage(req.body.prompt).then(url => + DashUploadUtils.UploadImage(url ?? '').then(info => { if (info instanceof Error) _invalid(res, info.message); - else _success(res, info.accessPaths.agnostic.client); + else _success(res, info); }) ), }); @@ -128,6 +178,7 @@ export default class FireflyManager extends ApiManager { register({ method: Method.POST, subscription: '/queryFireflyImageText', + // eslint-disable-next-line @typescript-eslint/no-unused-vars secureHandler: ({ req, res }) => fetch('http://localhost:1050/files/images/testshot.png').then(json => json.blob().then(file => @@ -137,5 +188,40 @@ export default class FireflyManager extends ApiManager { ) ), }); + register({ + method: Method.POST, + subscription: '/expandImage', + secureHandler: ({ req, res }) => + new Promise((resolve, reject) => { + const dbx = new Dropbox({ accessToken: process.env.DROPBOX_TOKEN }); + fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(req.body.file)}`), undefined, (err, contents) => { + if (err) { + console.log('Error: ', err); + reject(); + } else { + dbx.filesUpload({ path: `/Apps/browndash/${path.basename(req.body.file)}`, contents }) + .then(response => { + dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }).then(link => { + console.log(link.result); + this.expandImage(link.result.link, req.body.prompt).then(text => { + if (text.error_code) _error(res, text.message); + else + DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + resolve(); + }); + }); + }); + }) + .catch(uploadErr => { + console.log(uploadErr); + _error(res, 'upload to dropbox failed'); + reject(); + }); + } + }); + }), + }); } } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 7bfdd5aec..5a880901b 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -70,10 +70,16 @@ export default class UploadManager extends ApiManager { ]); } else { fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `resampling images`)); + // original filenames with '.'s, such as a Macbook screenshot, can be a problem - their extension is not kept in formidable's newFilename. + // This makes sure that the extension is preserved in the newFilename. + const fixNewFilename = (f: formidable.File) => { + if (path.extname(f.originalFilename ?? '') !== path.extname(f.newFilename)) f.newFilename = f.newFilename + path.extname(f.originalFilename ?? ''); + return f; + }; const results = ( await Promise.all( Array.from(Object.keys(files)).map( - async key => (!files[key] ? undefined : DashUploadUtils.upload(files[key]![0] /* , key */)) // key is the guid used by the client to track upload progress. + async key => (!files[key] ? undefined : DashUploadUtils.upload(fixNewFilename(files[key][0]) /* , key */)) // key is the guid used by the client to track upload progress. ) ) ).filter(result => result && !(result.result instanceof Error)); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 028116779..a06fa3b6e 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -22,7 +22,6 @@ import { AzureManager } from './ApiManagers/AzureManager'; import { AcceptableMedia, Upload } from './SharedMediaTypes'; import { Directory, clientPathToFile, filesDirectory, pathToDirectory, publicDirectory, serverPathToFile } from './SocketData'; import { resolvedServerUrl } from './server_Initialization'; - import { Worker, isMainThread, parentPort } from 'worker_threads'; // Create an array to store worker threads @@ -55,7 +54,8 @@ if (isMainThread) { .then(img => sizes.forEach(({ width, suffix }) => img.resize({ w: width || img.bitmap.width }) - .write(InjectSize(outputPath, suffix) as `${string}.${string}`) + .write(InjectSize(outputPath, suffix) as `${string}.${string}`) + .catch(e => console.log("Jimp error:", e)) )) .catch(e => console.log('Error Jimp:', e)) .finally(() => unlinkSource && unlinkSync(imgSourcePath)); @@ -343,15 +343,24 @@ export namespace DashUploadUtils { const outputPath = path.resolve(pathToDirectory(Directory.images), outputFileName); const sizes = imageResampleSizes(path.extname(outputFileName)); - const imgReadStream = new Duplex(); - imgReadStream.push(fs.readFileSync(imgSourcePath)); - imgReadStream.push(null); - await Promise.all( - sizes.map(({ suffix }) => - new Promise(res => - imgReadStream.pipe(createWriteStream(writtenFiles[suffix] = InjectSize(outputPath, suffix))).on('close', res) - ) - )); // prettier-ignore + if (unlinkSource) { + const imgReadStream = new Duplex(); + imgReadStream.push(fs.readFileSync(imgSourcePath)); + imgReadStream.push(null); + await Promise.all( + sizes.map(({ suffix }) => + new Promise(res => + imgReadStream.pipe(createWriteStream(writtenFiles[suffix] = InjectSize(outputPath, suffix))).on('close', res) + ) + )); // prettier-ignore + } else { + await Promise.all( + sizes.map(({ suffix }) => + new Promise(res => + request.get(imgSourcePath).pipe(createWriteStream(writtenFiles[suffix] = InjectSize(outputPath, suffix))).on('close', res) + ) + )); // prettier-ignore + } workerResample(imgSourcePath, outputPath, SizeSuffix.Original, unlinkSource); return writtenFiles; @@ -450,7 +459,7 @@ export namespace DashUploadUtils { } const outputFile = filename || result.filename || ''; - return UploadInspectedImage(result, outputFile, prefix); + return UploadInspectedImage(result, outputFile, prefix, isLocal().exec(source) ? true : false); }; type md5 = 'md5'; @@ -568,7 +577,9 @@ export namespace DashUploadUtils { switch (category) { case 'image': if (imageFormats.includes(format)) { - const result = await UploadImage(filepath, basename(filepath)); + const outputName = basename(filepath); + const extname = path.extname(originalFilename ?? ''); + const result = await UploadImage(filepath, outputName.endsWith(extname) ? outputName : outputName + extname, undefined); return { source: file, result }; } fs.unlink(filepath, () => {}); -- cgit v1.2.3-70-g09d2 From f7c7f5f77f1170a84a22eb8b337db7c37a52a5da Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 11 Dec 2024 13:15:58 -0500 Subject: reverting changes to inkstroke properties --- src/client/views/InkStrokeProperties.ts | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) (limited to 'src') diff --git a/src/client/views/InkStrokeProperties.ts b/src/client/views/InkStrokeProperties.ts index 8165b7f4e..358274f0e 100644 --- a/src/client/views/InkStrokeProperties.ts +++ b/src/client/views/InkStrokeProperties.ts @@ -74,7 +74,6 @@ export class InkStrokeProperties { doc._height = (newYrange.max - newYrange.min) * ptsYscale + NumCast(doc.stroke_width); doc.x = oldXrange.coord + (newXrange.min - oldXrange.min) * ptsXscale; doc.y = oldYrange.coord + (newYrange.min - oldYrange.min) * ptsYscale; - Doc.SetInPlace(doc, 'stroke', new InkField(newPoints), true); appliedFunc = true; } @@ -249,7 +248,7 @@ export class InkStrokeProperties { /** * Handles the movement/scaling of a control point. */ - moveControlPtHandle = undoable((inkView: DocumentView, deltaX: number, deltaY: number, controlIndex: number, origInk?: InkData, noPush?: boolean) => { + moveControlPtHandle = undoable((inkView: DocumentView, deltaX: number, deltaY: number, controlIndex: number, origInk?: InkData) => { inkView && this.applyFunction(inkView, (view: DocumentView, ink: InkData) => { const order = controlIndex % 4; @@ -289,11 +288,10 @@ export class InkStrokeProperties { const { finalCtrls: rightCtrls /* , error: errorRight */ } = FitOneCurve(samplesRight, { X: startDir.x, Y: startDir.y }, { X: endDir.x, Y: endDir.y }); finalCtrls = finalCtrls.concat(rightCtrls); newink.splice(this._currentPoint - 4, 8, ...finalCtrls); + return newink; } - const prevData = (inkView.ComponentView as InkingStroke).screenCtrlPts; - - const newInk = ink.map((pt, i) => { + return ink.map((pt, i) => { const leftHandlePoint = order === 0 && i === controlIndex + 1; const rightHandlePoint = order === 0 && controlIndex !== 0 && i === controlIndex - 2; if (controlIndex === i || (order === 0 && controlIndex !== 0 && i === controlIndex - 1) || (order === 3 && i === controlIndex - 1)) { @@ -314,26 +312,6 @@ export class InkStrokeProperties { } return pt; }); - - if (inkView.IsSelected && !noPush && !InkingStroke.IsClosed(prevData)) { - const ffview = CollectionFreeFormView.from(inkView); - ffview?.childDocs.forEach(d => { - const oinkDoc = DocumentView.getDocumentView(d); - const oinkView = oinkDoc?.ComponentView as InkingStroke; - if (oinkView instanceof InkingStroke && oinkDoc && oinkDoc.Document !== inkView.Document) { - const { nearestSeg, nearestT } = InkStrokeProperties.nearestPtToStroke(oinkView.screenCtrlPts, prevData[controlIndex]); - const locdelta = inkView.screenToContentsTransform().inverse().transformDirection(deltaX, deltaY); - const index = nearestSeg + Math.round(nearestT) * 4; - const oinkData = oinkView.inkScaledData().inkData; - const closed = InkingStroke.IsClosed(oinkData); - InkStrokeProperties.Instance.moveControlPtHandle(oinkDoc, locdelta[0], locdelta[1], index, undefined, true); - if (index === oinkData.length && closed) { - InkStrokeProperties.Instance.moveControlPtHandle(oinkDoc, locdelta[0], locdelta[1], 0, undefined, true); - } - } - }); - } - return newInk; }); }, 'move ink ctrl pt'); -- cgit v1.2.3-70-g09d2