Merge branch 'master' into alyssa-starter

1 files changed, 410 insertions, 0 deletions

diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts
new file mode 100644
index 000000000..e75ede9df
--- /dev/null
+++ b/src/server/ApiManagers/FireflyManager.ts
@@ -0,0 +1,410 @@
+import axios from 'axios';
+import { Dropbox } from 'dropbox';
+import * as fs from 'fs';
+import * as multipart from 'parse-multipart-data';
+import * as path from 'path';
+import { DashUserModel } from '../authentication/DashUserModel';
+import { DashUploadUtils } from '../DashUploadUtils';
+import { _error, _invalid, _success, Method } from '../RouteManager';
+import { Directory, filesDirectory } from '../SocketData';
+import ApiManager, { Registration } from './ApiManager';
+
+export default class FireflyManager extends ApiManager {
+    getBearerToken = () =>
+        fetch('https://ims-na1.adobelogin.com/ims/token/v3', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded',
+            },
+            body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`,
+        }).catch(error => {
+            console.error('Error:', error);
+            return undefined;
+        });
+
+    generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50, styles: string[], styleUrl: string | undefined) =>
+        this.getBearerToken().then(response =>
+            response?.json().then((data: { access_token: string }) =>
+                //prettier-ignore
+                fetch('https://firefly-api.adobe.io/v3/images/generate', {
+                    method: 'POST',
+                    headers: [
+                        ['Content-Type', 'application/json'],
+                        ['Accept', 'application/json'],
+                        ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+                        ['Authorization', `Bearer ${data.access_token}`],
+                    ],
+                    body: JSON.stringify({
+                        prompt,
+                        numVariations: 4,
+                        detailLevel: 'preview', 
+                        modelVersion: 'image3_fast',
+                        size: { width, height },
+                        structure: !structureUrl
+                            ? undefined
+                            : {
+                                  strength,
+                                  imageReference: {
+                                      source: { url: structureUrl },
+                                  },
+                              },
+                        // prettier-ignore
+                        style: {
+                            presets: styles,
+                            imageReference : !styleUrl 
+                                ? undefined
+                                : {
+                                    source: { url: styleUrl },
+                                }
+                            }
+                    }),
+                })
+                .then(response2 => response2.json().then(json => 
+                {
+                    if (json.outputs?.length)
+                        return (json.outputs as {image: {url:string }}[]).map(output => output.image);
+                    throw new Error(JSON.stringify(json));
+                })
+            )
+            )
+        );
+
+    uploadImageToDropbox = (fileUrl: string, user: DashUserModel | undefined, dbx = new Dropbox({ accessToken: user?.dropboxToken || '' })) =>
+        new Promise<string | Error>((res, rej) =>
+            fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(fileUrl)}`), undefined, (err, contents) => {
+                if (err) {
+                    console.log('Error: ', err);
+                    rej();
+                } else {
+                    dbx.filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents })
+                        .then(response => {
+                            dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' })
+                                .then(link => res(link.result.link))
+                                .catch(e => res(new Error(e.toString())));
+                        })
+                        .catch(e => {
+                            if (user?.dropboxRefresh) {
+                                console.log('*********** try refresh dropbox for: ' + user.email + ' ***********');
+                                this.refreshDropboxToken(user).then(token => {
+                                    if (!token) {
+                                        console.log('Dropbox error: cannot refresh token');
+                                        res(new Error(e.toString()));
+                                    } else {
+                                        const dbxNew = new Dropbox({ accessToken: user.dropboxToken || '' });
+                                        dbxNew
+                                            .filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents })
+                                            .then(response => {
+                                                dbxNew
+                                                    .filesGetTemporaryLink({ path: response.result.path_display ?? '' })
+                                                    .then(link => res(link.result.link))
+                                                    .catch(linkErr => res(new Error(linkErr.toString())));
+                                            })
+                                            .catch(uploadErr => {
+                                                console.log('Dropbox error:', uploadErr);
+                                                res(new Error(uploadErr.toString()));
+                                            });
+                                    }
+                                });
+                            } else {
+                                console.log('Dropbox error:', e);
+                                res(new Error(e.toString()));
+                            }
+                        });
+                }
+            })
+        );
+
+    generateImage = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, seed?: number) => {
+        let body = `{  "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}} }`;
+        if (seed) {
+            console.log('RECEIVED SEED', seed);
+            body = `{  "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}}, "seeds": [${seed}]}`;
+        }
+        const fetched = this.getBearerToken().then(response =>
+            response?.json().then((data: { access_token: string }) =>
+                fetch('https://firefly-api.adobe.io/v3/images/generate', {
+                    method: 'POST',
+                    headers: [
+                        ['Content-Type', 'application/json'],
+                        ['Accept', 'application/json'],
+                        ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+                        ['Authorization', `Bearer ${data.access_token}`],
+                    ],
+                    body: body,
+                })
+                    .then(response2 => response2.json())
+                    .then(json => (json.error_code ? json : { seed: json.outputs?.[0]?.seed, url: json.outputs?.[0]?.image?.url }))
+                    .catch(error => {
+                        console.error('Error:', error);
+                        return undefined;
+                    })
+            )
+        );
+        return fetched;
+    };
+    expandImage = (imgUrl: string, prompt?: string) => {
+        const dropboxImgUrl = imgUrl;
+        const fetched = this.getBearerToken().then(response =>
+            response
+                ?.json()
+                .then((data: { access_token: string }) => {
+                    return fetch('https://firefly-api.adobe.io/v3/images/expand', {
+                        method: 'POST',
+                        headers: [
+                            ['Content-Type', 'application/json'],
+                            ['Accept', 'application/json'],
+                            ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+                            ['Authorization', `Bearer ${data.access_token}`],
+                        ],
+                        body: JSON.stringify({
+                            image: {
+                                source: {
+                                    url: dropboxImgUrl,
+                                },
+                            },
+                            numVariations: 1,
+                            seeds: [0],
+                            size: {
+                                width: 3048,
+                                height: 2048,
+                            },
+                            prompt: prompt ?? 'cloudy skies',
+                            placement: {
+                                inset: {
+                                    left: 0,
+                                    top: 0,
+                                    right: 0,
+                                    bottom: 0,
+                                },
+                                alignment: {
+                                    horizontal: 'center',
+                                    vertical: 'center',
+                                },
+                            },
+                        }),
+                    });
+                })
+                .then(resp => resp.json())
+        );
+        return fetched;
+    };
+    getImageText = (imageBlob: Blob) => {
+        const inputFileVarName = 'infile';
+        const outputVarName = 'result';
+        const fetched = this.getBearerToken().then(response =>
+            response?.json().then((data: { access_token: string }) => {
+                return fetch('https://sensei.adobe.io/services/v2/predict', {
+                    method: 'POST',
+                    headers: [
+                        ['Prefer', 'respond-async, wait=59'],
+                        ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+                        // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string
+                        ['Authorization', `Bearer ${data.access_token}`],
+                    ],
+                    body: ((form: FormData) => {
+                        form.set(inputFileVarName, imageBlob);
+                        form.set(
+                            'contentAnalyzerRequests',
+                            JSON.stringify({
+                                'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690',
+                                'sensei:invocation_mode': 'asynchronous',
+                                'sensei:invocation_batch': false,
+                                'sensei:engines': [
+                                    {
+                                        'sensei:execution_info': {
+                                            'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690',
+                                        },
+                                        'sensei:inputs': {
+                                            documents: [
+                                                {
+                                                    'sensei:multipart_field_name': inputFileVarName,
+                                                    'dc:format': 'image/png',
+                                                },
+                                            ],
+                                        },
+                                        'sensei:params': {
+                                            correct_with_dictionary: true,
+                                        },
+                                        'sensei:outputs': {
+                                            result: {
+                                                'sensei:multipart_field_name': outputVarName,
+                                                'dc:format': 'application/json',
+                                            },
+                                        },
+                                    },
+                                ],
+                            })
+                        );
+                        return form;
+                    })(new FormData()),
+                }).then(response2 => {
+                    const contentType = response2.headers.get('content-type') ?? '';
+                    if (contentType.includes('application/json')) {
+                        return response2.json().then((json: object) => JSON.stringify(json));
+                    }
+                    if (contentType.includes('multipart')) {
+                        return response2
+                            .arrayBuffer()
+                            .then(arrayBuffer =>
+                                multipart
+                                    .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? ''))
+                                    .filter(part => part.name === outputVarName)
+                                    .map(part => JSON.parse(part.data.toString())[0])
+                                    .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '')
+                            )
+                            .catch(error => {
+                                console.error('Error:', error);
+                                return '';
+                            });
+                    }
+                    return response2.text();
+                });
+            })
+        );
+        return fetched;
+    };
+
+    refreshDropboxToken = (user: DashUserModel) =>
+        axios
+            .post(
+                'https://api.dropbox.com/oauth2/token',
+                new URLSearchParams({
+                    refresh_token: user.dropboxRefresh || '',
+                    grant_type: 'refresh_token',
+                    client_id: process.env._CLIENT_DROPBOX_CLIENT_ID || '',
+                    client_secret: process.env._CLIENT_DROPBOX_SECRET || '',
+                }).toString()
+            )
+            .then(refresh => {
+                console.log('***** dropbox token refreshed for ' + user?.email + ' ******* ');
+                user.dropboxToken = refresh.data.access_token;
+                user.save();
+                return user.dropboxToken;
+            })
+            .catch(e => {
+                console.log(e);
+                return undefined;
+            });
+
+    protected initialize(register: Registration): void {
+        register({
+            method: Method.POST,
+            subscription: '/queryFireflyImageFromStructure',
+            secureHandler: ({ req, res }) =>
+                new Promise<void>(resolver => {
+                    (req.body.styleUrl ? this.uploadImageToDropbox(req.body.styleUrl, req.user as DashUserModel) : Promise.resolve(undefined))
+                        .then(styleUrl => {
+                            if (styleUrl instanceof Error) {
+                                _invalid(res, styleUrl.message);
+                                throw new Error('Error uploading images to dropbox');
+                            }
+                            this.uploadImageToDropbox(req.body.structureUrl, req.user as DashUserModel)
+                                .then(dropboxStructureUrl => {
+                                    if (dropboxStructureUrl instanceof Error) {
+                                        _invalid(res, dropboxStructureUrl.message);
+                                        throw new Error('Error uploading images to dropbox');
+                                    }
+                                    return { styleUrl, structureUrl: dropboxStructureUrl };
+                                })
+                                .then(uploads =>
+                                    this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploads.structureUrl, req.body.strength, req.body.presets, uploads.styleUrl)
+                                        .then(images => {
+                                            Promise.all((images ?? [new Error('no images were generated')]).map(fire => (fire instanceof Error ? fire : DashUploadUtils.UploadImage(fire.url))))
+                                                .then(dashImages => {
+                                                    if (dashImages.every(img => img instanceof Error)) _invalid(res, dashImages[0]!.message);
+                                                    else _success(res, JSON.stringify(dashImages.filter(img => !(img instanceof Error))));
+                                                })
+                                                .then(resolver);
+                                        })
+                                        .catch(e => {
+                                            _invalid(res, e.message);
+                                            resolver();
+                                        })
+                                );
+                        })
+                        .catch(() => {
+                            /* do nothing */
+                            resolver();
+                        });
+                }),
+        });
+        register({
+            method: Method.POST,
+            subscription: '/queryFireflyImage',
+            secureHandler: ({ req, res }) =>
+                this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img =>
+                    img.error_code
+                        ? _invalid(res, img.message)
+                        : DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => {
+                              if (info instanceof Error) _invalid(res, info.message);
+                              else _success(res, info);
+                          })
+                ),
+        });
+
+        register({
+            method: Method.POST,
+            subscription: '/queryFireflyImageText',
+            secureHandler: ({ req, res }) =>
+                fetch(req.body.file).then(json =>
+                    json.blob().then(file =>
+                        this.getImageText(file).then(text => {
+                            _success(res, text);
+                        })
+                    )
+                ),
+        });
+        register({
+            method: Method.POST,
+            subscription: '/expandImage',
+            secureHandler: ({ req, res }) =>
+                this.uploadImageToDropbox(req.body.file, req.user as DashUserModel).then(uploadUrl =>
+                    uploadUrl instanceof Error
+                        ? _invalid(res, uploadUrl.message)
+                        : this.expandImage(uploadUrl, req.body.prompt).then(text => {
+                              if (text.error_code) _error(res, text.message);
+                              else
+                                  DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => {
+                                      if (info instanceof Error) _invalid(res, info.message);
+                                      else _success(res, info);
+                                  });
+                          })
+                ),
+        });
+
+        // construct this url and send user to it.  It will allow them to authorize their dropbox account and will send the resulting token to our endpoint /refreshDropbox
+        // https://www.dropbox.com/oauth2/authorize?client_id=DROPBOX_CLIENT_ID&response_type=code&token_access_type=offline&redirect_uri=http://localhost:1050/refreshDropbox
+        // see: https://dropbox.tech/developers/using-oauth-2-0-with-offline-access
+        //
+        register({
+            method: Method.GET,
+            subscription: '/refreshDropbox',
+            secureHandler: ({ req, res }) => {
+                const user = req.user as DashUserModel;
+                console.log(`******************* dropbox authorized for ${user?.email} ******************`);
+                _success(res, 'dropbox authorized for ' + user?.email);
+
+                const data = new URLSearchParams({
+                    code: req.query.code as string,
+                    grant_type: 'authorization_code',
+                    client_id: process.env._CLIENT_DROPBOX_CLIENT_ID ?? '',
+                    client_secret: process.env._CLIENT_DROPBOX_SECRET ?? '',
+                    redirect_uri: 'http://localhost:1050/refreshDropbox',
+                });
+                axios
+                    .post('https://api.dropbox.com/oauth2/token', data.toString())
+                    .then(response => {
+                        console.log('***** dropbox token (and refresh) received for ' + user?.email + ' ******* ');
+                        user.dropboxToken = response.data.access_token;
+                        user.dropboxRefresh = response.data.refresh_token;
+                        user.save();
+
+                        setTimeout(() => this.refreshDropboxToken(user), response.data.expires_in - 600);
+                    })
+                    .catch(e => {
+                        console.log(e);
+                    });
+            },
+        });
+    }
+}