fixes for uploading remote images to sample and save them properly. extnesions fo firefly manager to query text in images, expand images, and tweaks to generate images.

2 files changed, 107 insertions, 15 deletions

diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts
index cc4c218bf..d757a23fe 100644
--- a/src/server/ApiManagers/FireflyManager.ts
+++ b/src/server/ApiManagers/FireflyManager.ts
@@ -1,7 +1,11 @@
+import { Dropbox, files } from 'dropbox';
+import * as fs from 'fs';
+import * as multipart from 'parse-multipart-data';
+import * as path from 'path';
 import { DashUploadUtils } from '../DashUploadUtils';
-import { _invalid, _success, Method } from '../RouteManager';
+import { _error, _invalid, _success, Method } from '../RouteManager';
+import { Directory, filesDirectory } from '../SocketData';
 import ApiManager, { Registration } from './ApiManager';
-import * as multipart from 'parse-multipart-data';
 
 export default class FireflyManager extends ApiManager {
     getBearerToken = () =>
@@ -13,11 +17,11 @@ export default class FireflyManager extends ApiManager {
             body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`,
         }).catch(error => {
             console.error('Error:', error);
-            return '';
+            return undefined;
         });
-    askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => {
+    generateImage = (prompt: string = 'a realistic illustration of a cat coding') => {
         const fetched = this.getBearerToken().then(response =>
-            (response as Response).json().then((data: { access_token: string }) =>
+            response?.json().then((data: { access_token: string }) =>
                 fetch('https://firefly-api.adobe.io/v3/images/generate', {
                     method: 'POST',
                     headers: [
@@ -28,20 +32,66 @@ export default class FireflyManager extends ApiManager {
                     ],
                     body: `{  "prompt": "${prompt}" }`,
                 })
-                    .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image)))
+                    .then(response2 => response2.json().then(json => (json.outputs?.[0] as { image: { url: string } })?.image.url))
                     .catch(error => {
                         console.error('Error:', error);
-                        return '';
+                        return undefined;
                     })
             )
         );
         return fetched;
     };
-    getImageText = (testshotpng: Blob) => {
+    expandImage = (imgUrl: string, prompt?: string) => {
+        const dropboxImgUrl = imgUrl;
+        const fetched = this.getBearerToken().then(response =>
+            response
+                ?.json()
+                .then((data: { access_token: string }) => {
+                    return fetch('https://firefly-api.adobe.io/v3/images/expand', {
+                        method: 'POST',
+                        headers: [
+                            ['Content-Type', 'application/json'],
+                            ['Accept', 'application/json'],
+                            ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+                            ['Authorization', `Bearer ${data.access_token}`],
+                        ],
+                        body: JSON.stringify({
+                            image: {
+                                source: {
+                                    url: dropboxImgUrl,
+                                },
+                            },
+                            numVariations: 1,
+                            seeds: [0],
+                            size: {
+                                width: 3048,
+                                height: 2048,
+                            },
+                            prompt: prompt ?? 'cloudy skies',
+                            placement: {
+                                inset: {
+                                    left: 0,
+                                    top: 0,
+                                    right: 0,
+                                    bottom: 0,
+                                },
+                                alignment: {
+                                    horizontal: 'center',
+                                    vertical: 'center',
+                                },
+                            },
+                        }),
+                    });
+                })
+                .then(resp => resp.json())
+        );
+        return fetched;
+    };
+    getImageText = (imageBlob: Blob) => {
         const inputFileVarName = 'infile';
         const outputVarName = 'result';
         const fetched = this.getBearerToken().then(response =>
-            (response as Response).json().then((data: { access_token: string }) => {
+            response?.json().then((data: { access_token: string }) => {
                 return fetch('https://sensei.adobe.io/services/v2/predict', {
                     method: 'POST',
                     headers: [
@@ -51,7 +101,7 @@ export default class FireflyManager extends ApiManager {
                         ['Authorization', `Bearer ${data.access_token}`],
                     ],
                     body: ((form: FormData) => {
-                        form.set(inputFileVarName, testshotpng);
+                        form.set(inputFileVarName, imageBlob);
                         form.set(
                             'contentAnalyzerRequests',
                             JSON.stringify({
@@ -98,7 +148,7 @@ export default class FireflyManager extends ApiManager {
                                 multipart
                                     .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? ''))
                                     .filter(part => part.name === outputVarName)
-                                    .map(part => JSON.parse(part.data.toString()[0]))
+                                    .map(part => JSON.parse(part.data.toString())[0])
                                     .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '')
                             )
                             .catch(error => {
@@ -117,10 +167,10 @@ export default class FireflyManager extends ApiManager {
             method: Method.POST,
             subscription: '/queryFireflyImage',
             secureHandler: ({ req, res }) =>
-                this.askFirefly(req.body.prompt).then(fire =>
-                    DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => {
+                this.generateImage(req.body.prompt).then(url =>
+                    DashUploadUtils.UploadImage(url ?? '').then(info => {
                         if (info instanceof Error) _invalid(res, info.message);
-                        else _success(res, info.accessPaths.agnostic.client);
+                        else _success(res, info);
                     })
                 ),
         });
@@ -128,6 +178,7 @@ export default class FireflyManager extends ApiManager {
         register({
             method: Method.POST,
             subscription: '/queryFireflyImageText',
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
             secureHandler: ({ req, res }) =>
                 fetch('http://localhost:1050/files/images/testshot.png').then(json =>
                     json.blob().then(file =>
@@ -137,5 +188,40 @@ export default class FireflyManager extends ApiManager {
                     )
                 ),
         });
+        register({
+            method: Method.POST,
+            subscription: '/expandImage',
+            secureHandler: ({ req, res }) =>
+                new Promise<void>((resolve, reject) => {
+                    const dbx = new Dropbox({ accessToken: process.env.DROPBOX_TOKEN });
+                    fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(req.body.file)}`), undefined, (err, contents) => {
+                        if (err) {
+                            console.log('Error: ', err);
+                            reject();
+                        } else {
+                            dbx.filesUpload({ path: `/Apps/browndash/${path.basename(req.body.file)}`, contents })
+                                .then(response => {
+                                    dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }).then(link => {
+                                        console.log(link.result);
+                                        this.expandImage(link.result.link, req.body.prompt).then(text => {
+                                            if (text.error_code) _error(res, text.message);
+                                            else
+                                                DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => {
+                                                    if (info instanceof Error) _invalid(res, info.message);
+                                                    else _success(res, info);
+                                                    resolve();
+                                                });
+                                        });
+                                    });
+                                })
+                                .catch(uploadErr => {
+                                    console.log(uploadErr);
+                                    _error(res, 'upload to dropbox failed');
+                                    reject();
+                                });
+                        }
+                    });
+                }),
+        });
     }
 }
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 7bfdd5aec..5a880901b 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -70,10 +70,16 @@ export default class UploadManager extends ApiManager {
                             ]);
                         } else {
                             fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `resampling images`));
+                            // original filenames with '.'s, such as a Macbook screenshot, can be a problem - their extension is not kept in formidable's newFilename.
+                            // This makes sure that the extension is preserved in the newFilename.
+                            const fixNewFilename = (f: formidable.File) => {
+                                if (path.extname(f.originalFilename ?? '') !== path.extname(f.newFilename)) f.newFilename = f.newFilename + path.extname(f.originalFilename ?? '');
+                                return f;
+                            };
                             const results = (
                                 await Promise.all(
                                     Array.from(Object.keys(files)).map(
-                                        async key => (!files[key] ? undefined : DashUploadUtils.upload(files[key]![0] /* , key */)) // key is the guid used by the client to track upload progress.
+                                        async key => (!files[key] ? undefined : DashUploadUtils.upload(fixNewFilename(files[key][0]) /* , key */)) // key is the guid used by the client to track upload progress.
                                     )
                                 )
                             ).filter(result => result && !(result.result instanceof Error));