import axios from 'axios'; import { Dropbox } from 'dropbox'; import * as fs from 'fs'; import * as multipart from 'parse-multipart-data'; import * as path from 'path'; import { DashUserModel } from '../authentication/DashUserModel'; import { DashUploadUtils } from '../DashUploadUtils'; import { _error, _invalid, _success, Method } from '../RouteManager'; import { Upload } from '../SharedMediaTypes'; import { Directory, filesDirectory } from '../SocketData'; import ApiManager, { Registration } from './ApiManager'; export default class FireflyManager extends ApiManager { getBearerToken = () => fetch('https://ims-na1.adobelogin.com/ims/token/v3', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', }, body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, }).catch(error => { console.error('Error:', error); return undefined; }); generateImageFromStructure = ( prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50, styles: string[], styleUrl: string | undefined, variations: number = 4 ) => this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => //prettier-ignore fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ ['Content-Type', 'application/json'], ['Accept', 'application/json'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], body: JSON.stringify({ prompt, numVariations: variations, detailLevel: 'preview', modelVersion: 'image3_fast', size: { width, height }, structure: !structureUrl ? undefined : { strength, imageReference: { source: { url: structureUrl }, }, }, // prettier-ignore style: { presets: styles, imageReference : !styleUrl ? undefined : { source: { url: styleUrl }, } } }), }) .then(response2 => response2.json().then(json => { if (json.outputs?.length) return (json.outputs as {image: {url:string }}[]).map(output => output.image); throw new Error(JSON.stringify(json)); }) ) ) ); uploadImageToDropbox = (fileUrl: string, user: DashUserModel | undefined, dbx = new Dropbox({ accessToken: user?.dropboxToken || '' })) => new Promise((resolve, reject) => { fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(fileUrl)}`), undefined, (err, contents) => { if (err) { return reject(new Error('Error reading file:' + err.message)); } const uploadToDropbox = (dropboxClient: Dropbox) => dropboxClient .filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }) .then(response => dropboxClient .filesGetTemporaryLink({ path: response.result.path_display ?? '' }) .then(link => resolve(link.result.link)) .catch(linkErr => reject(new Error('Failed to get temporary link: ' + linkErr.message))) ) .catch(uploadErr => { if (user?.dropboxRefresh) { console.log('Attempting to refresh Dropbox token for user:', user.email); this.refreshDropboxToken(user) .then(token => { if (!token) return reject(new Error('Failed to refresh Dropbox token.' + user.email)); const dbxNew = new Dropbox({ accessToken: token }); uploadToDropbox(dbxNew).catch(finalErr => reject(new Error('Failed to refresh Dropbox token:' + finalErr.message))); }) .catch(refreshErr => reject(new Error('Failed to refresh Dropbox token: ' + refreshErr.message))); } else { reject(new Error('Dropbox error: ' + uploadErr.message)); } }); uploadToDropbox(dbx); }); }); generateImage = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, seed?: number) => { const body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}} ${seed ? ', "seeds": [' + seed + ']' : ''}}`; const fetched = this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ ['Content-Type', 'application/json'], ['Accept', 'application/json'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], body: body, }) .then(response2 => response2.json()) .then(json => (json.error_code ? json : { seed: json.outputs?.[0]?.seed, url: json.outputs?.[0]?.image?.url })) .catch(error => { console.error('Error:', error); return undefined; }) ) ); return fetched; }; expandImage = (imgUrl: string, prompt?: string) => { const dropboxImgUrl = imgUrl; const fetched = this.getBearerToken().then(response => response ?.json() .then((data: { access_token: string }) => { return fetch('https://firefly-api.adobe.io/v3/images/expand', { method: 'POST', headers: [ ['Content-Type', 'application/json'], ['Accept', 'application/json'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], body: JSON.stringify({ image: { source: { url: dropboxImgUrl, }, }, numVariations: 1, seeds: [0], size: { width: 3048, height: 2048, }, prompt: prompt ?? 'cloudy skies', placement: { inset: { left: 0, top: 0, right: 0, bottom: 0, }, alignment: { horizontal: 'center', vertical: 'center', }, }, }), }); }) .then(resp => resp.json()) ); return fetched; }; getImageText = (imageBlob: Blob) => { const inputFileVarName = 'infile'; const outputVarName = 'result'; const fetched = this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => { return fetch('https://sensei.adobe.io/services/v2/predict', { method: 'POST', headers: [ ['Prefer', 'respond-async, wait=59'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string ['Authorization', `Bearer ${data.access_token}`], ], body: ((form: FormData) => { form.set(inputFileVarName, imageBlob); form.set( 'contentAnalyzerRequests', JSON.stringify({ 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', 'sensei:invocation_mode': 'asynchronous', 'sensei:invocation_batch': false, 'sensei:engines': [ { 'sensei:execution_info': { 'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', }, 'sensei:inputs': { documents: [ { 'sensei:multipart_field_name': inputFileVarName, 'dc:format': 'image/png', }, ], }, 'sensei:params': { correct_with_dictionary: true, }, 'sensei:outputs': { result: { 'sensei:multipart_field_name': outputVarName, 'dc:format': 'application/json', }, }, }, ], }) ); return form; })(new FormData()), }).then(response2 => { const contentType = response2.headers.get('content-type') ?? ''; if (contentType.includes('application/json')) { return response2.json().then((json: object) => JSON.stringify(json)); } if (contentType.includes('multipart')) { return response2 .arrayBuffer() .then(arrayBuffer => multipart .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? '')) .filter(part => part.name === outputVarName) .map(part => JSON.parse(part.data.toString())[0]) .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '') ) .catch(error => { console.error('Error:', error); return ''; }); } return response2.text(); }); }) ); return fetched; }; refreshDropboxToken = (user: DashUserModel) => axios .post( 'https://api.dropbox.com/oauth2/token', new URLSearchParams({ refresh_token: user.dropboxRefresh || '', grant_type: 'refresh_token', client_id: process.env._CLIENT_DROPBOX_CLIENT_ID || '', client_secret: process.env._CLIENT_DROPBOX_SECRET || '', }).toString() ) .then(refresh => { console.log('***** dropbox token refreshed for ' + user?.email + ' ******* '); user.dropboxToken = refresh.data.access_token; user.save(); return user.dropboxToken; }) .catch(e => { console.log(e); return undefined; }); protected initialize(register: Registration): void { register({ method: Method.POST, subscription: '/queryFireflyImageFromStructure', secureHandler: ({ req, res }) => new Promise(resolver => (req.body.styleUrl ? this.uploadImageToDropbox(req.body.styleUrl, req.user as DashUserModel) : Promise.resolve(undefined) ) .then(styleUrl => this.uploadImageToDropbox(req.body.structureUrl, req.user as DashUserModel) .then(dropboxStructureUrl => ({ styleUrl, structureUrl: dropboxStructureUrl }) ) ) .then(uploads => this.generateImageFromStructure( req.body.prompt, req.body.width, req.body.height, uploads.structureUrl, req.body.strength, req.body.presets, uploads.styleUrl ).then(images => Promise.all((images ?? [new Error('no images were generated')]).map(fire => (fire instanceof Error ? fire : DashUploadUtils.UploadImage(fire.url)))) .then(dashImages => (dashImages.every(img => img instanceof Error)) ? _invalid(res, dashImages[0]!.message) : _success(res, JSON.stringify(dashImages.filter(img => !(img instanceof Error)))) ) ) ) .catch(e => { _invalid(res, e.message); resolver(); }) ), // prettier-ignore }); register({ method: Method.POST, subscription: '/outpaintImage', secureHandler: ({ req, res }) => new Promise(resolver => this.uploadImageToDropbox(req.body.imageUrl, req.user as DashUserModel) .then(uploadUrl => this.getBearerToken() .then(tokenResponse => tokenResponse?.json()) .then((tokenData: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/expand', { method: 'POST', headers: [ ['Content-Type', 'application/json'], ['Accept', 'application/json'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${tokenData.access_token}`], ], body: JSON.stringify({ image: { source: { url: uploadUrl }, }, size: { width: Math.round(req.body.newDimensions.width), height: Math.round(req.body.newDimensions.height), }, prompt: req.body.prompt ?? '', numVariations: 1, placement: { inset: { left: 0, // Math.round((req.body.newDimensions.width - req.body.originalDimensions.width) / 2), top: 0, // Math.round((req.body.newDimensions.height - req.body.originalDimensions.height) / 2), right: 0, // Math.round((req.body.newDimensions.width - req.body.originalDimensions.width) / 2), bottom: 0, // Math.round((req.body.newDimensions.height - req.body.originalDimensions.height) / 2), }, alignment: { horizontal: req.body.halignment || 'center', vertical: req.body.valignment || 'center', }, }, }), }) .then(expandResp => expandResp?.json()) .then(expandData => { if (expandData.error_code || !expandData.outputs?.[0]?.image?.url) { console.error('Firefly validation error:', expandData); _error(res, expandData.message ?? 'Failed to generate image'); } else { return DashUploadUtils.UploadImage(expandData.outputs[0].image.url) .then((info: Upload.ImageInformation | Error) => { if (info instanceof Error) { _invalid(res, info.message); } else { _success(res, { url: info.accessPaths.agnostic.client }); } }) .catch(uploadErr => { console.error('DashUpload Error:', uploadErr); _error(res, 'Failed to upload generated image.'); }); } }) ) ) .catch(e => { _invalid(res, e.message); resolver(); }) ), }); /* register({ method: Method.POST subscription: '/queryFireflyOutpaint', secureHandler: ({req, res}) => this.outpaintImage() })*/ register({ method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img => img.error_code ? _invalid(res, img.message) : DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); }) ), }); register({ method: Method.POST, subscription: '/queryFireflyImageText', secureHandler: ({ req, res }) => fetch(req.body.file).then(json => json.blob().then(file => this.getImageText(file).then(text => { _success(res, text); }) ) ), }); // construct this url and send user to it. It will allow them to authorize their dropbox account and will send the resulting token to our endpoint /refreshDropbox // https://www.dropbox.com/oauth2/authorize?client_id=DROPBOX_CLIENT_ID&response_type=code&token_access_type=offline&redirect_uri=http://localhost:1050/refreshDropbox // see: https://dropbox.tech/developers/using-oauth-2-0-with-offline-access // register({ method: Method.GET, subscription: '/refreshDropbox', secureHandler: ({ req, res }) => { const user = req.user as DashUserModel; console.log(`******************* dropbox authorized for ${user?.email} ******************`); _success(res, 'dropbox authorized for ' + user?.email); const data = new URLSearchParams({ code: req.query.code as string, grant_type: 'authorization_code', client_id: process.env._CLIENT_DROPBOX_CLIENT_ID ?? '', client_secret: process.env._CLIENT_DROPBOX_SECRET ?? '', redirect_uri: 'http://localhost:1050/refreshDropbox', }); axios .post('https://api.dropbox.com/oauth2/token', data.toString()) .then(response => { console.log('***** dropbox token (and refresh) received for ' + user?.email + ' ******* '); user.dropboxToken = response.data.access_token; user.dropboxRefresh = response.data.refresh_token; user.save(); setTimeout(() => this.refreshDropboxToken(user), response.data.expires_in - 600); }) .catch(e => { console.log(e); }); }, }); } }