diff options
Diffstat (limited to 'src/server/ApiManagers/FireflyManager.ts')
| -rw-r--r-- | src/server/ApiManagers/FireflyManager.ts | 407 |
1 files changed, 407 insertions, 0 deletions
diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts new file mode 100644 index 000000000..160a94d40 --- /dev/null +++ b/src/server/ApiManagers/FireflyManager.ts @@ -0,0 +1,407 @@ +import axios from 'axios'; +import { Dropbox } from 'dropbox'; +import * as fs from 'fs'; +import * as multipart from 'parse-multipart-data'; +import * as path from 'path'; +import { DashUserModel } from '../authentication/DashUserModel'; +import { DashUploadUtils } from '../DashUploadUtils'; +import { _error, _invalid, _success, Method } from '../RouteManager'; +import { Directory, filesDirectory } from '../SocketData'; +import ApiManager, { Registration } from './ApiManager'; + +export default class FireflyManager extends ApiManager { + getBearerToken = () => + fetch('https://ims-na1.adobelogin.com/ims/token/v3', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, + }).catch(error => { + console.error('Error:', error); + return undefined; + }); + + generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50, styles: string[], styleUrl: string | undefined) => + this.getBearerToken().then(response => + response?.json().then((data: { access_token: string }) => + //prettier-ignore + fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: JSON.stringify({ + prompt, + numVariations: 4, + detailLevel: 'preview', + modelVersion: 'image3_fast', + size: { width, height }, + structure: !structureUrl + ? undefined + : { + strength, + imageReference: { + source: { url: structureUrl }, + }, + }, + // prettier-ignore + style: { + presets: styles, + imageReference : !styleUrl + ? undefined + : { + source: { url: styleUrl }, + } + } + }), + }) + .then(response2 => response2.json().then(json => + { + if (json.outputs?.length) + return (json.outputs as {image: {url:string }}[]).map(output => output.image); + throw new Error(JSON.stringify(json)); + }) + ) + ) + ); + + uploadImageToDropbox = (fileUrl: string, user: DashUserModel | undefined, dbx = new Dropbox({ accessToken: user?.dropboxToken || '' })) => + new Promise<string | Error>((res, rej) => + fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(fileUrl)}`), undefined, (err, contents) => { + if (err) { + console.log('Error: ', err); + rej(); + } else { + dbx.filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }) + .then(response => { + dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }) + .then(link => res(link.result.link)) + .catch(e => res(new Error(e.toString()))); + }) + .catch(e => { + if (user?.dropboxRefresh) { + console.log('*********** try refresh dropbox for: ' + user.email + ' ***********'); + this.refreshDropboxToken(user).then(token => { + if (!token) { + console.log('Dropbox error: cannot refresh token'); + res(new Error(e.toString())); + } else { + const dbxNew = new Dropbox({ accessToken: user.dropboxToken || '' }); + dbxNew + .filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }) + .then(response => { + dbxNew + .filesGetTemporaryLink({ path: response.result.path_display ?? '' }) + .then(link => res(link.result.link)) + .catch(linkErr => res(new Error(linkErr.toString()))); + }) + .catch(uploadErr => { + console.log('Dropbox error:', uploadErr); + res(new Error(uploadErr.toString())); + }); + } + }); + } else { + console.log('Dropbox error:', e); + res(new Error(e.toString())); + } + }); + } + }) + ); + + generateImage = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, seed?: number) => { + let body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}} }`; + if (seed) { + console.log('RECEIVED SEED', seed); + body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}}, "seeds": [${seed}]}`; + } + const fetched = this.getBearerToken().then(response => + response?.json().then((data: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: body, + }) + .then(response2 => response2.json().then(json => ({ seed: json.outputs?.[0]?.seed, url: json.outputs?.[0]?.image?.url }))) + .catch(error => { + console.error('Error:', error); + return undefined; + }) + ) + ); + return fetched; + }; + expandImage = (imgUrl: string, prompt?: string) => { + const dropboxImgUrl = imgUrl; + const fetched = this.getBearerToken().then(response => + response + ?.json() + .then((data: { access_token: string }) => { + return fetch('https://firefly-api.adobe.io/v3/images/expand', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: JSON.stringify({ + image: { + source: { + url: dropboxImgUrl, + }, + }, + numVariations: 1, + seeds: [0], + size: { + width: 3048, + height: 2048, + }, + prompt: prompt ?? 'cloudy skies', + placement: { + inset: { + left: 0, + top: 0, + right: 0, + bottom: 0, + }, + alignment: { + horizontal: 'center', + vertical: 'center', + }, + }, + }), + }); + }) + .then(resp => resp.json()) + ); + return fetched; + }; + getImageText = (imageBlob: Blob) => { + const inputFileVarName = 'infile'; + const outputVarName = 'result'; + const fetched = this.getBearerToken().then(response => + response?.json().then((data: { access_token: string }) => { + return fetch('https://sensei.adobe.io/services/v2/predict', { + method: 'POST', + headers: [ + ['Prefer', 'respond-async, wait=59'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string + ['Authorization', `Bearer ${data.access_token}`], + ], + body: ((form: FormData) => { + form.set(inputFileVarName, imageBlob); + form.set( + 'contentAnalyzerRequests', + JSON.stringify({ + 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + 'sensei:invocation_mode': 'asynchronous', + 'sensei:invocation_batch': false, + 'sensei:engines': [ + { + 'sensei:execution_info': { + 'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + }, + 'sensei:inputs': { + documents: [ + { + 'sensei:multipart_field_name': inputFileVarName, + 'dc:format': 'image/png', + }, + ], + }, + 'sensei:params': { + correct_with_dictionary: true, + }, + 'sensei:outputs': { + result: { + 'sensei:multipart_field_name': outputVarName, + 'dc:format': 'application/json', + }, + }, + }, + ], + }) + ); + return form; + })(new FormData()), + }).then(response2 => { + const contentType = response2.headers.get('content-type') ?? ''; + if (contentType.includes('application/json')) { + return response2.json().then((json: object) => JSON.stringify(json)); + } + if (contentType.includes('multipart')) { + return response2 + .arrayBuffer() + .then(arrayBuffer => + multipart + .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? '')) + .filter(part => part.name === outputVarName) + .map(part => JSON.parse(part.data.toString())[0]) + .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '') + ) + .catch(error => { + console.error('Error:', error); + return ''; + }); + } + return response2.text(); + }); + }) + ); + return fetched; + }; + + refreshDropboxToken = (user: DashUserModel) => + axios + .post( + 'https://api.dropbox.com/oauth2/token', + new URLSearchParams({ + refresh_token: user.dropboxRefresh || '', + grant_type: 'refresh_token', + client_id: process.env._CLIENT_DROPBOX_CLIENT_ID || '', + client_secret: process.env._CLIENT_DROPBOX_SECRET || '', + }).toString() + ) + .then(refresh => { + console.log('***** dropbox token refreshed for ' + user?.email + ' ******* '); + user.dropboxToken = refresh.data.access_token; + user.save(); + return user.dropboxToken; + }) + .catch(e => { + console.log(e); + return undefined; + }); + + protected initialize(register: Registration): void { + register({ + method: Method.POST, + subscription: '/queryFireflyImageFromStructure', + secureHandler: ({ req, res }) => + new Promise<void>(resolver => { + (req.body.styleUrl ? this.uploadImageToDropbox(req.body.styleUrl, req.user as DashUserModel) : Promise.resolve(undefined)) + .then(styleUrl => { + if (styleUrl instanceof Error) { + _invalid(res, styleUrl.message); + throw new Error('Error uploading images to dropbox'); + } + this.uploadImageToDropbox(req.body.structure, req.user as DashUserModel) + .then(structureUrl => { + if (structureUrl instanceof Error) { + _invalid(res, structureUrl.message); + throw new Error('Error uploading images to dropbox'); + } + return { styleUrl, structureUrl }; + }) + .then(uploads => + this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploads.structureUrl, req.body.strength, req.body.presets, uploads.styleUrl) + .then(images => { + Promise.all((images ?? [new Error('no images were generated')]).map(fire => (fire instanceof Error ? fire : DashUploadUtils.UploadImage(fire.url)))) + .then(dashImages => { + if (dashImages.every(img => img instanceof Error)) _invalid(res, dashImages[0]!.message); + else _success(res, JSON.stringify(dashImages.filter(img => !(img instanceof Error)))); + }) + .then(resolver); + }) + .catch(e => { + _invalid(res, e.message); + resolver(); + }) + ); + }) + .catch(() => { + /* do nothing */ + resolver(); + }); + }), + }); + register({ + method: Method.POST, + subscription: '/queryFireflyImage', + secureHandler: ({ req, res }) => + this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img => + DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }) + ), + }); + + register({ + method: Method.POST, + subscription: '/queryFireflyImageText', + secureHandler: ({ req, res }) => + fetch(req.body.file).then(json => + json.blob().then(file => + this.getImageText(file).then(text => { + _success(res, text); + }) + ) + ), + }); + register({ + method: Method.POST, + subscription: '/expandImage', + secureHandler: ({ req, res }) => + this.uploadImageToDropbox(req.body.file, req.user as DashUserModel).then(uploadUrl => + uploadUrl instanceof Error + ? _invalid(res, uploadUrl.message) + : this.expandImage(uploadUrl, req.body.prompt).then(text => { + if (text.error_code) _error(res, text.message); + else + DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }); + }) + ), + }); + + // construct this url and send user to it. It will allow them to authorize their dropbox account and will send the resulting token to our endpoint /refreshDropbox + // https://www.dropbox.com/oauth2/authorize?client_id=DROPBOX_CLIENT_ID&response_type=code&token_access_type=offline&redirect_uri=http://localhost:1050/refreshDropbox + // see: https://dropbox.tech/developers/using-oauth-2-0-with-offline-access + // + register({ + method: Method.GET, + subscription: '/refreshDropbox', + secureHandler: ({ req, res }) => { + const user = req.user as DashUserModel; + console.log(`******************* dropbox authorized for ${user?.email} ******************`); + _success(res, 'dropbox authorized for ' + user?.email); + + const data = new URLSearchParams({ + code: req.query.code as string, + grant_type: 'authorization_code', + client_id: process.env._CLIENT_DROPBOX_CLIENT_ID ?? '', + client_secret: process.env._CLIENT_DROPBOX_SECRET ?? '', + redirect_uri: 'http://localhost:1050/refreshDropbox', + }); + axios + .post('https://api.dropbox.com/oauth2/token', data.toString()) + .then(response => { + console.log('***** dropbox token (and refresh) received for ' + user?.email + ' ******* '); + user.dropboxToken = response.data.access_token; + user.dropboxRefresh = response.data.refresh_token; + user.save(); + + setTimeout(() => this.refreshDropboxToken(user), response.data.expires_in - 600); + }) + .catch(e => { + console.log(e); + }); + }, + }); + } +} |