import { Dropbox } from 'dropbox'; import * as fs from 'fs'; import * as multipart from 'parse-multipart-data'; import * as path from 'path'; import { DashUploadUtils } from '../DashUploadUtils'; import { _error, _invalid, _success, Method } from '../RouteManager'; import { Directory, filesDirectory } from '../SocketData'; import ApiManager, { Registration } from './ApiManager'; export default class FireflyManager extends ApiManager { getBearerToken = () => fetch('https://ims-na1.adobelogin.com/ims/token/v3', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', }, body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, }).catch(error => { console.error('Error:', error); return undefined; }); generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50, styles: string[]) => this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => //prettier-ignore fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ ['Content-Type', 'application/json'], ['Accept', 'application/json'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], body: JSON.stringify({ prompt: prompt, size: { width: width, height: height }, structure: !structureUrl ? undefined : { strength, imageReference: { source: { url: structureUrl }, }, }, //prettier-ignore style: { presets: styles } }), }) .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) .catch(error => { console.error('Error:', error); return ''; }) ) ); uploadImageToDropbox = (fileUrl: string, dbx = new Dropbox({ accessToken: process.env.DROPBOX_TOKEN })) => new Promise((res, rej) => fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(fileUrl)}`), undefined, (err, contents) => { if (err) { console.log('Error: ', err); rej(); } else { dbx.filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }).then(response => { dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }).then(link => res(link.result.link)); }); } }) ); generateImage = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, seed?: number) => { let body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}} }`; if (seed) { console.log('RECEIVED SEED', seed); body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}}, "seeds": [${seed}]}`; } const fetched = this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ ['Content-Type', 'application/json'], ['Accept', 'application/json'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], body: body, }) .then(response2 => response2.json().then(json => { const seed = json.outputs?.[0]?.seed; const url = json.outputs?.[0]?.image?.url; return { seed, url }; }) ) .catch(error => { console.error('Error:', error); return undefined; }) ) ); return fetched; }; expandImage = (imgUrl: string, prompt?: string) => { const dropboxImgUrl = imgUrl; const fetched = this.getBearerToken().then(response => response ?.json() .then((data: { access_token: string }) => { return fetch('https://firefly-api.adobe.io/v3/images/expand', { method: 'POST', headers: [ ['Content-Type', 'application/json'], ['Accept', 'application/json'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], body: JSON.stringify({ image: { source: { url: dropboxImgUrl, }, }, numVariations: 1, seeds: [0], size: { width: 3048, height: 2048, }, prompt: prompt ?? 'cloudy skies', placement: { inset: { left: 0, top: 0, right: 0, bottom: 0, }, alignment: { horizontal: 'center', vertical: 'center', }, }, }), }); }) .then(resp => resp.json()) ); return fetched; }; getImageText = (imageBlob: Blob) => { const inputFileVarName = 'infile'; const outputVarName = 'result'; const fetched = this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => { return fetch('https://sensei.adobe.io/services/v2/predict', { method: 'POST', headers: [ ['Prefer', 'respond-async, wait=59'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string ['Authorization', `Bearer ${data.access_token}`], ], body: ((form: FormData) => { form.set(inputFileVarName, imageBlob); form.set( 'contentAnalyzerRequests', JSON.stringify({ 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', 'sensei:invocation_mode': 'asynchronous', 'sensei:invocation_batch': false, 'sensei:engines': [ { 'sensei:execution_info': { 'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', }, 'sensei:inputs': { documents: [ { 'sensei:multipart_field_name': inputFileVarName, 'dc:format': 'image/png', }, ], }, 'sensei:params': { correct_with_dictionary: true, }, 'sensei:outputs': { result: { 'sensei:multipart_field_name': outputVarName, 'dc:format': 'application/json', }, }, }, ], }) ); return form; })(new FormData()), }).then(response2 => { const contentType = response2.headers.get('content-type') ?? ''; if (contentType.includes('application/json')) { return response2.json().then((json: object) => JSON.stringify(json)); } if (contentType.includes('multipart')) { return response2 .arrayBuffer() .then(arrayBuffer => multipart .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? '')) .filter(part => part.name === outputVarName) .map(part => JSON.parse(part.data.toString())[0]) .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '') ) .catch(error => { console.error('Error:', error); return ''; }); } return response2.text(); }); }) ); return fetched; }; protected initialize(register: Registration): void { register({ method: Method.POST, subscription: '/queryFireflyImageFromStructure', secureHandler: async ({ req, res }) => this.uploadImageToDropbox(req.body.structureUrl).then(uploadUrl => this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploadUrl, req.body.strength, req.body.styles).then(fire => DashUploadUtils.UploadImage(JSON.parse(fire ?? '').url).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); }) ) ), }); register({ method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img => DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); }) ), }); register({ method: Method.POST, subscription: '/queryFireflyImageText', // eslint-disable-next-line @typescript-eslint/no-unused-vars secureHandler: ({ req, res }) => fetch(req.body.file).then(json => json.blob().then(file => this.getImageText(file).then(text => { _success(res, text); }) ) ), }); register({ method: Method.POST, subscription: '/expandImage', secureHandler: ({ req, res }) => this.uploadImageToDropbox(req.body.file).then(uploadUrl => this.expandImage(uploadUrl, req.body.prompt).then(text => { if (text.error_code) _error(res, text.message); else DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); }); }) ), }); } }