added firefly text in image request

1 files changed, 71 insertions, 23 deletions

diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts
index e10e43704..30fbdc577 100644
--- a/src/server/ApiManagers/FireflyManager.ts
+++ b/src/server/ApiManagers/FireflyManager.ts
@@ -1,6 +1,7 @@
 import { DashUploadUtils } from '../DashUploadUtils';
 import { _invalid, _success, Method } from '../RouteManager';
 import ApiManager, { Registration } from './ApiManager';
+import * as multipart from 'parse-multipart-data';
 
 export default class FireflyManager extends ApiManager {
     getBearerToken = () =>
@@ -27,7 +28,7 @@ export default class FireflyManager extends ApiManager {
                     ],
                     body: `{  "prompt": "${prompt}" }`,
                 })
-                    .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image)))
+                    .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image)))
                     .catch(error => {
                         console.error('Error:', error);
                         return '';
@@ -36,7 +37,9 @@ export default class FireflyManager extends ApiManager {
         );
         return fetched;
     };
-    askFireflyText = (testshotpng: Blob) => {
+    getImageText = (testshotpng: Blob) => {
+        const inputFileVarName = 'infile';
+        const outputVarName = 'result';
         const fetched = this.getBearerToken().then(response =>
             (response as Response).json().then((data: { access_token: string }) => {
                 return fetch('https://sensei.adobe.io/services/v2/predict', {
@@ -44,31 +47,67 @@ export default class FireflyManager extends ApiManager {
                     headers: [
                         ['Prefer', 'respond-async, wait=59'],
                         ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
-                        ['content-type', 'multipart/form-data'],
+                        // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string
                         ['Authorization', `Bearer ${data.access_token}`],
                     ],
                     body: ((form: FormData) => {
-                        form.append('file', testshotpng);
-                        form.append(
+                        form.set(inputFileVarName, testshotpng);
+                        form.set(
                             'contentAnalyzerRequests',
                             JSON.stringify({
                                 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690',
+                                'sensei:invocation_mode': 'asynchronous',
+                                'sensei:invocation_batch': false,
+                                'sensei:engines': [
+                                    {
+                                        'sensei:execution_info': {
+                                            'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690',
+                                        },
+                                        'sensei:inputs': {
+                                            documents: [
+                                                {
+                                                    'sensei:multipart_field_name': inputFileVarName,
+                                                    'dc:format': 'image/png',
+                                                },
+                                            ],
+                                        },
+                                        'sensei:params': {
+                                            correct_with_dictionary: true,
+                                        },
+                                        'sensei:outputs': {
+                                            result: {
+                                                'sensei:multipart_field_name': outputVarName,
+                                                'dc:format': 'application/json',
+                                            },
+                                        },
+                                    },
+                                ],
                             })
                         );
                         return form;
                     })(new FormData()),
-                }).then(response2 =>
-                    response2
-                        .json()
-                        .then(json => {
-                            console.log(json);
-                            return '';
-                        })
-                        .catch(error => {
-                            console.error('Error:', error);
-                            return '';
-                        })
-                );
+                }).then(response2 => {
+                    const contentType = response2.headers.get('content-type') ?? '';
+                    if (contentType.includes('application/json')) {
+                        return response2.json().then((json: object) => JSON.stringify(json));
+                    }
+                    if (contentType.includes('multipart')) {
+                        return response2
+                            .arrayBuffer()
+                            .then(arrayBuffer =>
+                                multipart
+                                    .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? ''))
+                                    .filter(part => part.name === outputVarName)
+                                    .map(part => JSON.parse(part.data.toString()[0]))
+                                    .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '')
+                            )
+                            .catch(error => {
+                                console.error('Error:', error);
+                                return '';
+                            });
+                    }
+                    return response2.text();
+                });
             })
         );
         return fetched;
@@ -78,14 +117,23 @@ export default class FireflyManager extends ApiManager {
             method: Method.POST,
             subscription: '/queryFireflyImage',
             secureHandler: ({ req, res }) =>
+                this.askFirefly(req.body).then(fire => {
+                    DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => {
+                        if (info instanceof Error) _invalid(res, info.message);
+                        else _success(res, info.accessPaths.agnostic.client);
+                    });
+                }),
+        });
+
+        register({
+            method: Method.POST,
+            subscription: '/queryFireflyImageText',
+            secureHandler: ({ req, res }) =>
                 fetch('http://localhost:1050/files/images/testshot.png').then(json =>
                     json.blob().then(file =>
-                        this.askFireflyText(file).then(fire =>
-                            DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => {
-                                if (info instanceof Error) _invalid(res, info.message);
-                                else _success(res, info.accessPaths.agnostic.client);
-                            })
-                        )
+                        this.getImageText(file).then(text => {
+                            _success(res, text);
+                        })
                     )
                 ),
         });