import axios from 'axios'; import { Dropbox } from 'dropbox'; import * as fs from 'fs'; import * as multipart from 'parse-multipart-data'; import * as path from 'path'; import { DashUserModel } from '../authentication/DashUserModel'; import { DashUploadUtils } from '../DashUploadUtils'; import { _error, _invalid, _success, Method } from '../RouteManager'; import { Directory, filesDirectory } from '../SocketData'; import ApiManager, { Registration } from './ApiManager'; export default class FireflyManager extends ApiManager { getBearerToken = () => fetch('https://ims-na1.adobelogin.com/ims/token/v3', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', }, body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, }).catch(error => { console.error('Error:', error); return undefined; }); generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50, styles: string[], styleUrl: string | undefined) => this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => { //prettier-ignore return fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ ['Content-Type', 'application/json'], ['Accept', 'application/json'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], body: JSON.stringify({ prompt: prompt, size: { width: width, height: height }, structure: !structureUrl ? undefined : { strength, imageReference: { source: { url: structureUrl }, }, }, // prettier-ignore style: { presets: styles, ...(styleUrl && { imageReference: { source: { uploadId: styleUrl }, }, }), } }), }) .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) .catch(error => { console.error('Error:', error); return ''; }) }) ); uploadImageToDropbox = (fileUrl: string, user: DashUserModel | undefined, dbx = new Dropbox({ accessToken: user?.dropboxToken || '' })) => new Promise((res, rej) => fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(fileUrl)}`), undefined, (err, contents) => { if (err) { console.log('Error: ', err); rej(); } else { dbx.filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }) .then(response => { dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }) .then(link => res(link.result.link)) .catch(e => res(new Error(e.toString()))); }) .catch(e => { if (user?.dropboxRefresh) { console.log('*********** try refresh dropbox for: ' + user.email + ' ***********'); this.refreshDropboxToken(user).then(token => { if (!token) { console.log('Dropbox error: cannot refresh token'); res(new Error(e.toString())); } else { const dbxNew = new Dropbox({ accessToken: user.dropboxToken || '' }); dbxNew .filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }) .then(response => { dbxNew .filesGetTemporaryLink({ path: response.result.path_display ?? '' }) .then(link => res(link.result.link)) .catch(linkErr => res(new Error(linkErr.toString()))); }) .catch(uploadErr => { console.log('Dropbox error:', uploadErr); res(new Error(uploadErr.toString())); }); } }); } else { console.log('Dropbox error:', e); res(new Error(e.toString())); } }); } }) ); generateImage = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, seed?: number) => { let body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}} }`; if (seed) { console.log('RECEIVED SEED', seed); body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}}, "seeds": [${seed}]}`; } const fetched = this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ ['Content-Type', 'application/json'], ['Accept', 'application/json'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], body: body, }) .then(response2 => response2.json().then(json => { const seed = json.outputs?.[0]?.seed; const url = json.outputs?.[0]?.image?.url; return { seed, url }; }) ) .catch(error => { console.error('Error:', error); return undefined; }) ) ); return fetched; }; expandImage = (imgUrl: string, prompt?: string) => { const dropboxImgUrl = imgUrl; const fetched = this.getBearerToken().then(response => response ?.json() .then((data: { access_token: string }) => { return fetch('https://firefly-api.adobe.io/v3/images/expand', { method: 'POST', headers: [ ['Content-Type', 'application/json'], ['Accept', 'application/json'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], body: JSON.stringify({ image: { source: { url: dropboxImgUrl, }, }, numVariations: 1, seeds: [0], size: { width: 3048, height: 2048, }, prompt: prompt ?? 'cloudy skies', placement: { inset: { left: 0, top: 0, right: 0, bottom: 0, }, alignment: { horizontal: 'center', vertical: 'center', }, }, }), }); }) .then(resp => resp.json()) ); return fetched; }; getImageText = (imageBlob: Blob) => { const inputFileVarName = 'infile'; const outputVarName = 'result'; const fetched = this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => { return fetch('https://sensei.adobe.io/services/v2/predict', { method: 'POST', headers: [ ['Prefer', 'respond-async, wait=59'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string ['Authorization', `Bearer ${data.access_token}`], ], body: ((form: FormData) => { form.set(inputFileVarName, imageBlob); form.set( 'contentAnalyzerRequests', JSON.stringify({ 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', 'sensei:invocation_mode': 'asynchronous', 'sensei:invocation_batch': false, 'sensei:engines': [ { 'sensei:execution_info': { 'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', }, 'sensei:inputs': { documents: [ { 'sensei:multipart_field_name': inputFileVarName, 'dc:format': 'image/png', }, ], }, 'sensei:params': { correct_with_dictionary: true, }, 'sensei:outputs': { result: { 'sensei:multipart_field_name': outputVarName, 'dc:format': 'application/json', }, }, }, ], }) ); return form; })(new FormData()), }).then(response2 => { const contentType = response2.headers.get('content-type') ?? ''; if (contentType.includes('application/json')) { return response2.json().then((json: object) => JSON.stringify(json)); } if (contentType.includes('multipart')) { return response2 .arrayBuffer() .then(arrayBuffer => multipart .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? '')) .filter(part => part.name === outputVarName) .map(part => JSON.parse(part.data.toString())[0]) .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '') ) .catch(error => { console.error('Error:', error); return ''; }); } return response2.text(); }); }) ); return fetched; }; refreshDropboxToken = (user: DashUserModel) => axios .post( 'https://api.dropbox.com/oauth2/token', new URLSearchParams({ refresh_token: user.dropboxRefresh || '', grant_type: 'refresh_token', client_id: process.env._CLIENT_DROPBOX_CLIENT_ID || '', client_secret: process.env._CLIENT_DROPBOX_SECRET || '', }).toString() ) .then(refresh => { console.log('***** dropbox token refreshed for ' + user?.email + ' ******* '); user.dropboxToken = refresh.data.access_token; user.save(); return user.dropboxToken; }) .catch(e => { console.log(e); return undefined; }); protected initialize(register: Registration): void { register({ method: Method.POST, subscription: '/queryFireflyImageFromStructure', secureHandler: async ({ req, res }) => { (req.body.styleRef ? this.uploadImageToDropbox(req.body.styleRef, req.user as DashUserModel) : Promise.resolve(undefined)) .then(styleUrl => { if (styleUrl instanceof Error) { _invalid(res, styleUrl.message); throw new Error('Error uploading images to dropbox'); } return this.uploadImageToDropbox(req.body.structure, req.user as DashUserModel).then(structureUrl => { if (structureUrl instanceof Error) { _invalid(res, structureUrl.message); throw new Error('Error uploading images to dropbox'); } return { styleUrl, structureUrl }; }); }) .then(uploads => this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploads.structureUrl, req.body.strength, req.body.presets, req.body.styleUrl).then(fire => { DashUploadUtils.UploadImage(JSON.parse(fire ?? '').url).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); }); }) ); }, }); register({ method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img => DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); }) ), }); register({ method: Method.POST, subscription: '/queryFireflyImageText', // eslint-disable-next-line @typescript-eslint/no-unused-vars secureHandler: ({ req, res }) => fetch(req.body.file).then(json => json.blob().then(file => this.getImageText(file).then(text => { _success(res, text); }) ) ), }); register({ method: Method.POST, subscription: '/expandImage', secureHandler: ({ req, res }) => this.uploadImageToDropbox(req.body.file, req.user as DashUserModel).then(uploadUrl => uploadUrl instanceof Error ? _invalid(res, uploadUrl.message) : this.expandImage(uploadUrl, req.body.prompt).then(text => { if (text.error_code) _error(res, text.message); else DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); }); }) ), }); // construct this url and send user to it. It will allow them to authorize their dropbox account and will send the resulting token to our endpoint /refreshDropbox // https://www.dropbox.com/oauth2/authorize?client_id=DROPBOX_CLIENT_ID&response_type=code&token_access_type=offline&redirect_uri=http://localhost:1050/refreshDropbox // see: https://dropbox.tech/developers/using-oauth-2-0-with-offline-access // register({ method: Method.GET, subscription: '/refreshDropbox', secureHandler: ({ req, res }) => { const user = req.user as DashUserModel; console.log(`******************* dropbox authorized for ${user?.email} ******************`); _success(res, 'dropbox authorized for ' + user?.email); const data = new URLSearchParams({ code: req.query.code as string, grant_type: 'authorization_code', client_id: process.env._CLIENT_DROPBOX_CLIENT_ID ?? '', client_secret: process.env._CLIENT_DROPBOX_SECRET ?? '', redirect_uri: 'http://localhost:1050/refreshDropbox', }); axios .post('https://api.dropbox.com/oauth2/token', data.toString()) .then(response => { console.log('***** dropbox token (and refresh) received for ' + user?.email + ' ******* '); user.dropboxToken = response.data.access_token; user.dropboxRefresh = response.data.refresh_token; user.save(); setTimeout(() => this.refreshDropboxToken(user), response.data.expires_in - 600); }) .catch(e => { console.log(e); }); }, }); } }