aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/server/ApiManagers/FireflyManager.ts217
1 files changed, 212 insertions, 5 deletions
diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts
index d9a1ef621..4425aff43 100644
--- a/src/server/ApiManagers/FireflyManager.ts
+++ b/src/server/ApiManagers/FireflyManager.ts
@@ -1,5 +1,10 @@
+import { Dropbox, files } from 'dropbox';
+import * as fs from 'fs';
+import * as multipart from 'parse-multipart-data';
+import * as path from 'path';
import { DashUploadUtils } from '../DashUploadUtils';
-import { _invalid, _success, Method } from '../RouteManager';
+import { _error, _invalid, _success, Method } from '../RouteManager';
+import { Directory, filesDirectory } from '../SocketData';
import ApiManager, { Registration } from './ApiManager';
export default class FireflyManager extends ApiManager {
@@ -12,12 +17,12 @@ export default class FireflyManager extends ApiManager {
body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`,
}).catch(error => {
console.error('Error:', error);
- return '';
+ return undefined;
});
askFireflyOld = (prompt: string = 'a realistic illustration of a cat coding', uploadId: string, strength: number) => {
const fetched = this.getBearerToken().then(response =>
- response.json().then((data: { access_token: string }) => {
+ response?.json().then((data: { access_token: string }) => {
const body: any = {
prompt: prompt,
structure: {
@@ -51,7 +56,7 @@ export default class FireflyManager extends ApiManager {
uploadImageToFirefly = (image: Blob) => {
const fetched = this.getBearerToken().then(response =>
- response.json().then((data: { access_token: string }) =>
+ response?.json().then((data: { access_token: string }) =>
fetch('https://firefly-api.adobe.io/v3/uploads', {
method: 'POST',
headers: [
@@ -71,7 +76,149 @@ export default class FireflyManager extends ApiManager {
);
return fetched;
};
-
+ generateImage = (prompt: string = 'a realistic illustration of a cat coding') => {
+ const fetched = this.getBearerToken().then(response =>
+ response?.json().then((data: { access_token: string }) =>
+ fetch('https://firefly-api.adobe.io/v3/images/generate', {
+ method: 'POST',
+ headers: [
+ ['Content-Type', 'application/json'],
+ ['Accept', 'application/json'],
+ ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+ ['Authorization', `Bearer ${data.access_token}`],
+ ],
+ body: `{ "prompt": "${prompt}" }`,
+ })
+ .then(response2 => response2.json().then(json => (json.outputs?.[0] as { image: { url: string } })?.image.url))
+ .catch(error => {
+ console.error('Error:', error);
+ return undefined;
+ })
+ )
+ );
+ return fetched;
+ };
+ expandImage = (imgUrl: string, prompt?: string) => {
+ const dropboxImgUrl = imgUrl;
+ const fetched = this.getBearerToken().then(response =>
+ response
+ ?.json()
+ .then((data: { access_token: string }) => {
+ return fetch('https://firefly-api.adobe.io/v3/images/expand', {
+ method: 'POST',
+ headers: [
+ ['Content-Type', 'application/json'],
+ ['Accept', 'application/json'],
+ ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+ ['Authorization', `Bearer ${data.access_token}`],
+ ],
+ body: JSON.stringify({
+ image: {
+ source: {
+ url: dropboxImgUrl,
+ },
+ },
+ numVariations: 1,
+ seeds: [0],
+ size: {
+ width: 3048,
+ height: 2048,
+ },
+ prompt: prompt ?? 'cloudy skies',
+ placement: {
+ inset: {
+ left: 0,
+ top: 0,
+ right: 0,
+ bottom: 0,
+ },
+ alignment: {
+ horizontal: 'center',
+ vertical: 'center',
+ },
+ },
+ }),
+ });
+ })
+ .then(resp => resp.json())
+ );
+ return fetched;
+ };
+ getImageText = (imageBlob: Blob) => {
+ const inputFileVarName = 'infile';
+ const outputVarName = 'result';
+ const fetched = this.getBearerToken().then(response =>
+ response?.json().then((data: { access_token: string }) => {
+ return fetch('https://sensei.adobe.io/services/v2/predict', {
+ method: 'POST',
+ headers: [
+ ['Prefer', 'respond-async, wait=59'],
+ ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+ // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string
+ ['Authorization', `Bearer ${data.access_token}`],
+ ],
+ body: ((form: FormData) => {
+ form.set(inputFileVarName, imageBlob);
+ form.set(
+ 'contentAnalyzerRequests',
+ JSON.stringify({
+ 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690',
+ 'sensei:invocation_mode': 'asynchronous',
+ 'sensei:invocation_batch': false,
+ 'sensei:engines': [
+ {
+ 'sensei:execution_info': {
+ 'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690',
+ },
+ 'sensei:inputs': {
+ documents: [
+ {
+ 'sensei:multipart_field_name': inputFileVarName,
+ 'dc:format': 'image/png',
+ },
+ ],
+ },
+ 'sensei:params': {
+ correct_with_dictionary: true,
+ },
+ 'sensei:outputs': {
+ result: {
+ 'sensei:multipart_field_name': outputVarName,
+ 'dc:format': 'application/json',
+ },
+ },
+ },
+ ],
+ })
+ );
+ return form;
+ })(new FormData()),
+ }).then(response2 => {
+ const contentType = response2.headers.get('content-type') ?? '';
+ if (contentType.includes('application/json')) {
+ return response2.json().then((json: object) => JSON.stringify(json));
+ }
+ if (contentType.includes('multipart')) {
+ return response2
+ .arrayBuffer()
+ .then(arrayBuffer =>
+ multipart
+ .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? ''))
+ .filter(part => part.name === outputVarName)
+ .map(part => JSON.parse(part.data.toString())[0])
+ .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '')
+ )
+ .catch(error => {
+ console.error('Error:', error);
+ return '';
+ });
+ }
+ return response2.text();
+ });
+ })
+ );
+ return fetched;
+ };
protected initialize(register: Registration): void {
register({
method: Method.POST,
@@ -87,5 +234,65 @@ export default class FireflyManager extends ApiManager {
);
},
});
+ register({
+ method: Method.POST,
+ subscription: '/queryFireflyImage',
+ secureHandler: ({ req, res }) =>
+ this.generateImage(req.body.prompt).then(url =>
+ DashUploadUtils.UploadImage(url ?? '').then(info => {
+ if (info instanceof Error) _invalid(res, info.message);
+ else _success(res, info);
+ })
+ ),
+ });
+
+ register({
+ method: Method.POST,
+ subscription: '/queryFireflyImageText',
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
+ secureHandler: ({ req, res }) =>
+ fetch('http://localhost:1050/files/images/testshot.png').then(json =>
+ json.blob().then(file =>
+ this.getImageText(file).then(text => {
+ _success(res, text);
+ })
+ )
+ ),
+ });
+ register({
+ method: Method.POST,
+ subscription: '/expandImage',
+ secureHandler: ({ req, res }) =>
+ new Promise<void>((resolve, reject) => {
+ const dbx = new Dropbox({ accessToken: process.env.DROPBOX_TOKEN });
+ fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(req.body.file)}`), undefined, (err, contents) => {
+ if (err) {
+ console.log('Error: ', err);
+ reject();
+ } else {
+ dbx.filesUpload({ path: `/Apps/browndash/${path.basename(req.body.file)}`, contents })
+ .then(response => {
+ dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }).then(link => {
+ console.log(link.result);
+ this.expandImage(link.result.link, req.body.prompt).then(text => {
+ if (text.error_code) _error(res, text.message);
+ else
+ DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => {
+ if (info instanceof Error) _invalid(res, info.message);
+ else _success(res, info);
+ resolve();
+ });
+ });
+ });
+ })
+ .catch(uploadErr => {
+ console.log(uploadErr);
+ _error(res, 'upload to dropbox failed');
+ reject();
+ });
+ }
+ });
+ }),
+ });
}
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 10:18:41 +0000