added firefly text in image request

3 files changed, 72 insertions, 31 deletions

diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts
index e10e43704..30fbdc577 100644
--- a/src/server/ApiManagers/FireflyManager.ts
+++ b/src/server/ApiManagers/FireflyManager.ts
@@ -1,6 +1,7 @@
 import { DashUploadUtils } from '../DashUploadUtils';
 import { _invalid, _success, Method } from '../RouteManager';
 import ApiManager, { Registration } from './ApiManager';
+import * as multipart from 'parse-multipart-data';
 
 export default class FireflyManager extends ApiManager {
     getBearerToken = () =>
@@ -27,7 +28,7 @@ export default class FireflyManager extends ApiManager {
                     ],
                     body: `{  "prompt": "${prompt}" }`,
                 })
-                    .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image)))
+                    .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image)))
                     .catch(error => {
                         console.error('Error:', error);
                         return '';
@@ -36,7 +37,9 @@ export default class FireflyManager extends ApiManager {
         );
         return fetched;
     };
-    askFireflyText = (testshotpng: Blob) => {
+    getImageText = (testshotpng: Blob) => {
+        const inputFileVarName = 'infile';
+        const outputVarName = 'result';
         const fetched = this.getBearerToken().then(response =>
             (response as Response).json().then((data: { access_token: string }) => {
                 return fetch('https://sensei.adobe.io/services/v2/predict', {
@@ -44,31 +47,67 @@ export default class FireflyManager extends ApiManager {
                     headers: [
                         ['Prefer', 'respond-async, wait=59'],
                         ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
-                        ['content-type', 'multipart/form-data'],
+                        // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string
                         ['Authorization', `Bearer ${data.access_token}`],
                     ],
                     body: ((form: FormData) => {
-                        form.append('file', testshotpng);
-                        form.append(
+                        form.set(inputFileVarName, testshotpng);
+                        form.set(
                             'contentAnalyzerRequests',
                             JSON.stringify({
                                 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690',
+                                'sensei:invocation_mode': 'asynchronous',
+                                'sensei:invocation_batch': false,
+                                'sensei:engines': [
+                                    {
+                                        'sensei:execution_info': {
+                                            'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690',
+                                        },
+                                        'sensei:inputs': {
+                                            documents: [
+                                                {
+                                                    'sensei:multipart_field_name': inputFileVarName,
+                                                    'dc:format': 'image/png',
+                                                },
+                                            ],
+                                        },
+                                        'sensei:params': {
+                                            correct_with_dictionary: true,
+                                        },
+                                        'sensei:outputs': {
+                                            result: {
+                                                'sensei:multipart_field_name': outputVarName,
+                                                'dc:format': 'application/json',
+                                            },
+                                        },
+                                    },
+                                ],
                             })
                         );
                         return form;
                     })(new FormData()),
-                }).then(response2 =>
-                    response2
-                        .json()
-                        .then(json => {
-                            console.log(json);
-                            return '';
-                        })
-                        .catch(error => {
-                            console.error('Error:', error);
-                            return '';
-                        })
-                );
+                }).then(response2 => {
+                    const contentType = response2.headers.get('content-type') ?? '';
+                    if (contentType.includes('application/json')) {
+                        return response2.json().then((json: object) => JSON.stringify(json));
+                    }
+                    if (contentType.includes('multipart')) {
+                        return response2
+                            .arrayBuffer()
+                            .then(arrayBuffer =>
+                                multipart
+                                    .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? ''))
+                                    .filter(part => part.name === outputVarName)
+                                    .map(part => JSON.parse(part.data.toString()[0]))
+                                    .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '')
+                            )
+                            .catch(error => {
+                                console.error('Error:', error);
+                                return '';
+                            });
+                    }
+                    return response2.text();
+                });
             })
         );
         return fetched;
@@ -78,14 +117,23 @@ export default class FireflyManager extends ApiManager {
             method: Method.POST,
             subscription: '/queryFireflyImage',
             secureHandler: ({ req, res }) =>
+                this.askFirefly(req.body).then(fire => {
+                    DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => {
+                        if (info instanceof Error) _invalid(res, info.message);
+                        else _success(res, info.accessPaths.agnostic.client);
+                    });
+                }),
+        });
+
+        register({
+            method: Method.POST,
+            subscription: '/queryFireflyImageText',
+            secureHandler: ({ req, res }) =>
                 fetch('http://localhost:1050/files/images/testshot.png').then(json =>
                     json.blob().then(file =>
-                        this.askFireflyText(file).then(fire =>
-                            DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => {
-                                if (info instanceof Error) _invalid(res, info.message);
-                                else _success(res, info.accessPaths.agnostic.client);
-                            })
-                        )
+                        this.getImageText(file).then(text => {
+                            _success(res, text);
+                        })
                     )
                 ),
         });
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 868373474..7bfdd5aec 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -147,13 +147,10 @@ export default class UploadManager extends ApiManager {
                     if (doc.id) {
                         doc.id = getId(doc.id);
                     }
-                    // eslint-disable-next-line no-restricted-syntax
                     for (const key in doc.fields) {
-                        // eslint-disable-next-line no-continue
                         if (!Object.prototype.hasOwnProperty.call(doc.fields, key)) continue;
 
                         const field = doc.fields[key];
-                        // eslint-disable-next-line no-continue
                         if (field === undefined || field === null) continue;
 
                         if (field.__type === 'Doc') {
@@ -182,11 +179,9 @@ export default class UploadManager extends ApiManager {
                         let docids: string[] = [];
                         let linkids: string[] = [];
                         try {
-                            // eslint-disable-next-line no-restricted-syntax
                             for (const name in files) {
                                 if (Object.prototype.hasOwnProperty.call(files, name)) {
                                     const f = files[name];
-                                    // eslint-disable-next-line no-continue
                                     if (!f) continue;
                                     const path2 = f[0]; // what about the rest of the array? are we guaranteed only one value is set?
                                     const zip = new AdmZip(path2.filepath);
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 351351ca5..028116779 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -223,7 +223,6 @@ export namespace DashUploadUtils {
     const parseExifData = async (source: string) => {
         const image = await request.get(source, { encoding: null });
         const { /* data, */ error } = await new Promise<{ data: ExifData; error: string | undefined }>(resolve => {
-            // eslint-disable-next-line no-new
             new ExifImage({ image }, (exifError, data) => {
                 resolve({ data, error: exifError?.message });
             });
@@ -302,7 +301,6 @@ export namespace DashUploadUtils {
             // Bundle up the information into an object
             return {
                 source,
-                // eslint-disable-next-line radix
                 contentSize: parseInt(headers[size]),
                 contentType: headers[type],
                 nativeWidth,
@@ -370,7 +368,7 @@ export namespace DashUploadUtils {
      * @returns the accessPaths for the resized files.
      */
     export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename: string, prefix = '', cleanUp = true): Promise<Upload.ImageInformation> => {
-        const { requestable, source, ...remaining } = metadata;
+        const { requestable, ...remaining } = metadata;
         const dfltSuffix = remaining.contentType.split('/')[1].toLowerCase();
         const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${dfltSuffix === 'xml' ? 'jpg' : dfltSuffix}`;
         const { images } = Directory;