From 10a89be0b184df2750a4b3eceb48db205cb83e95 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 5 Nov 2024 10:22:31 -0500 Subject: fixed display of where control point will be added on mouse moves. fixed clickable ink line to not select fill area unless its actually painted. added end cap properties to ink stroke ui. enabled more than one ink stroke to show editable control points when multiselected. --- src/client/views/PropertiesView.scss | 1 + 1 file changed, 1 insertion(+) (limited to 'src/client/views/PropertiesView.scss') diff --git a/src/client/views/PropertiesView.scss b/src/client/views/PropertiesView.scss index a5e60b831..693c75ebf 100644 --- a/src/client/views/PropertiesView.scss +++ b/src/client/views/PropertiesView.scss @@ -508,6 +508,7 @@ display: flex; margin-bottom: 3px; margin-left: 4px; + justify-content: space-evenly; .arrows-head { display: flex; -- cgit v1.2.3-70-g09d2 From 0eff48b757ca81860a883d25e147b8a869e5fe00 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Mon, 30 Dec 2024 23:35:24 -0500 Subject: created image regeneration with dialogue --- src/client/apis/gpt/GPT.ts | 39 +++- src/client/documents/Documents.ts | 4 + src/client/views/PropertiesView.scss | 1 + src/client/views/PropertiesView.tsx | 12 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 22 +-- src/client/views/nodes/ImageBox.tsx | 10 ++ src/client/views/nodes/imageEditor/ImageEditor.tsx | 2 +- src/client/views/pdf/AnchorMenu.tsx | 1 + src/client/views/smartdraw/DrawingFillHandler.tsx | 11 +- src/client/views/smartdraw/SmartDrawHandler.scss | 8 +- src/client/views/smartdraw/SmartDrawHandler.tsx | 197 ++++++++++++++------- src/client/views/smartdraw/StickerPalette.tsx | 1 + src/server/ApiManagers/FireflyManager.ts | 20 ++- src/server/DashUploadUtils.ts | 1 - 14 files changed, 238 insertions(+), 91 deletions(-) (limited to 'src/client/views/PropertiesView.scss') diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 03380e4d6..9241eb120 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -249,6 +249,41 @@ const gptHandwriting = async (src: string): Promise => { } }; +const gptDescribeImage = async (image: string): Promise => { + try { + const response = await openai.chat.completions.create({ + model: 'gpt-4o', + temperature: 0, + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: `Identify what this drawing is, naming as many elements and their location in the drawing as possible`, + }, + { + type: 'image_url', + image_url: { + url: `${image}`, + detail: 'low', + }, + }, + ], + }, + ], + }); + if (response.choices[0].message.content) { + console.log('GPT DESCRIPTION', response.choices[0].message.content); + return response.choices[0].message.content; + } + return 'Unknown drawing'; + } catch (err) { + console.log(err); + return 'Error connecting with API'; + } +}; + const gptDrawingColor = async (image: string, coords: string[]): Promise => { try { const response = await openai.chat.completions.create({ @@ -276,11 +311,11 @@ const gptDrawingColor = async (image: string, coords: string[]): Promise if (response.choices[0].message.content) { return response.choices[0].message.content; } - return 'Missing labels'; + return 'Unknown drawing'; } catch (err) { console.log(err); return 'Error connecting with API'; } }; -export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDrawingColor }; +export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDescribeImage, gptDrawingColor }; diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index c51c1645d..785af3409 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -516,6 +516,10 @@ export class DocumentOptions { card_sort?: STRt = new StrInfo('way cards are sorted in deck view'); card_sort_isDesc?: BOOLt = new BoolInfo('whether the cards are sorted ascending or descending'); + + ai_generated?: boolean; // to mark items as ai generated + firefly_seed?: number; + firefly_prompt?: string; } export const DocOptions = new DocumentOptions(); diff --git a/src/client/views/PropertiesView.scss b/src/client/views/PropertiesView.scss index 693c75ebf..7866e67e7 100644 --- a/src/client/views/PropertiesView.scss +++ b/src/client/views/PropertiesView.scss @@ -642,6 +642,7 @@ .smooth, .color, +.strength-slider, .smooth-slider { margin-top: 7px; } diff --git a/src/client/views/PropertiesView.tsx b/src/client/views/PropertiesView.tsx index aefdeee17..5b24eb7ea 100644 --- a/src/client/views/PropertiesView.tsx +++ b/src/client/views/PropertiesView.tsx @@ -982,6 +982,9 @@ export class PropertiesView extends ObservableReactComponent { + !isNaN(val) && (this.refStrength = val); + }); return (
{!targetDoc.layout_isSvg && this.containsInkDoc && ( @@ -995,9 +998,10 @@ export class PropertiesView extends ObservableReactComponent DrawingFillHandler.drawingToImage(targetDoc, 'fill in the details of this image'), 'createImage')} + onClick={undoable(() => DrawingFillHandler.drawingToImage(targetDoc, this.refStrength, 'fill in the details of this image'), 'createImage')} />
+
{strength}
{ + doc[DocData].drawing_refStrength = Number(value); + }); + } @computed get smoothAmt() { return Number(this.getField('stroke_smoothAmount') || '5'); } // prettier-ignore set smoothAmt(value) { this.selectedStrokes.forEach(doc => { diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index acf72e5cb..4bccdd286 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1295,6 +1295,7 @@ export class CollectionFreeFormView extends CollectionSubView { - SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; - SmartDrawHandler.Instance.AddDrawing = this.addDrawing; - SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; - !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this._downX, this._downY - 10) : SmartDrawHandler.Instance.hideRegenerate(); - }), - icon: 'pen-to-square', - }); + this.layoutDoc.drawingData != undefined && + optionItems.push({ + description: 'Show Drawing Editor', + event: action(() => { + SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; + SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; + !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this._downX, this._downY - 10) : SmartDrawHandler.Instance.hideRegenerate(); + }), + icon: 'pen-to-square', + }); optionItems.push({ description: this.Document.savedAsSticker ? 'Sticker Saved!' : 'Save to Stickers', event: action(undoable(async () => await StickerPalette.addToPalette(this.Document), 'save to palette')), diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 25e7b566f..8f6a90e61 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -40,6 +40,7 @@ import { FocusViewOptions } from './FocusViewOptions'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; import { Upload } from '../../../server/SharedMediaTypes'; +import { SmartDrawHandler } from '../smartdraw/SmartDrawHandler'; export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -351,6 +352,15 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { }), icon: 'pencil-alt', }); + this.layoutDoc.ai_generated && + funcs.push({ + description: 'Regenerate AI Image', + event: action(() => { + console.log('COOOORDS', this.dataDoc.width as number, this.dataDoc.y as number); + !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this.dataDoc.x as number, (this.dataDoc.y as number) - 10) : SmartDrawHandler.Instance.hideRegenerate(); + }), + icon: 'pen-to-square', + }); funcs.push({ description: this.Document.savedAsSticker ? 'Sticker Saved!' : 'Save to Stickers', event: action(undoable(async () => await StickerPalette.addToPalette(this.Document), 'save to palette')), diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index a39878924..2a8bc034d 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -411,7 +411,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc let finalImgURL: string = url; // crop the image for these brush modes to remove excess blank space around the image contents if (currCutType == CutMode.IN || currCutType == CutMode.DRAW_IN) { - const croppedData = cropImage(image, minX, maxX, minY, maxY); + const croppedData = cropImage(image, Math.max(minX, 0), Math.min(maxX, image.width), Math.max(minY, 0), Math.min(maxY, image.height)); finalImg = croppedData; finalImgURL = croppedData.src; } diff --git a/src/client/views/pdf/AnchorMenu.tsx b/src/client/views/pdf/AnchorMenu.tsx index fe03f32a5..bb8082061 100644 --- a/src/client/views/pdf/AnchorMenu.tsx +++ b/src/client/views/pdf/AnchorMenu.tsx @@ -158,6 +158,7 @@ export class AnchorMenu extends AntimodeMenu { docData.drawingColored = opts.autoColor; docData.drawingSize = opts.size; docData.drawingData = gptRes; + docData.ai_generated = true; }); pointerDown = (e: React.PointerEvent) => { diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index 48e71bc9f..1a470f995 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -1,21 +1,26 @@ +import { imageUrlToBase64 } from '../../../ClientUtils'; import { Doc } from '../../../fields/Doc'; import { ImageCast } from '../../../fields/Types'; import { Upload } from '../../../server/SharedMediaTypes'; +import { gptDescribeImage } from '../../apis/gpt/GPT'; import { Docs } from '../../documents/Documents'; import { Networking } from '../../Network'; import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import { OpenWhere } from '../nodes/OpenWhere'; export class DrawingFillHandler { - static drawingToImage = (drawing: Doc, prompt: string) => + static drawingToImage = (drawing: Doc, strength: number, prompt: string) => DocumentView.GetDocImage(drawing)?.then(imageField => { if (imageField) { const { href } = ImageCast(imageField).url; const hrefParts = href.split('.'); const structureUrl = `${hrefParts.slice(0, -1).join('.')}_o.${hrefParts.lastElement()}`; - const strength: number = 100; - Networking.PostToServer('/queryFireflyImageFromStructure', { prompt, structureUrl, strength }).then((info: Upload.ImageInformation) => + imageUrlToBase64(structureUrl) + .then((hrefBase64: string) => gptDescribeImage(hrefBase64)) + .then((prompt: string) => { + Networking.PostToServer('/queryFireflyImageFromStructure', { prompt, structureUrl, strength }).then((info: Upload.ImageInformation) => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, {}), OpenWhere.addRight)) // prettier-ignore + }); } return false; }); diff --git a/src/client/views/smartdraw/SmartDrawHandler.scss b/src/client/views/smartdraw/SmartDrawHandler.scss index c25273876..513779512 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.scss +++ b/src/client/views/smartdraw/SmartDrawHandler.scss @@ -12,7 +12,13 @@ } } - .smartdraw-options { + .smartdraw-output-options { + display: flex; + flex-direction: row; + justify-content: center; + } + + .smartdraw-svg-options { margin-top: 5px; display: flex; flex-direction: row; diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 036ac5983..fb1a5771e 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -1,5 +1,5 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { Slider, Switch } from '@mui/material'; +import { Checkbox, Slider, Switch } from '@mui/material'; import { Button, IconButton } from 'browndash-components'; import { action, makeObservable, observable, runInAction } from 'mobx'; import { observer } from 'mobx-react'; @@ -74,6 +74,8 @@ export class SmartDrawHandler extends ObservableReactComponent { @observable private _autoColor: boolean = true; @observable private _regenInput: string = ''; @observable private _canInteract: boolean = true; + @observable private _generateDrawing: boolean = true; + @observable private _generateImage: boolean = true; @observable public ShowRegenerate: boolean = false; @@ -195,6 +197,7 @@ export class SmartDrawHandler extends ObservableReactComponent { */ @action handleSendClick = async () => { + if (!this._generateImage && !this._generateDrawing) return; this._isLoading = true; this._canInteract = false; if (this.ShowRegenerate) { @@ -212,7 +215,12 @@ export class SmartDrawHandler extends ObservableReactComponent { this._showOptions = false; }); try { - await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + if (this._generateImage) { + await this.createImageWithFirefly(this._userInput); + } + if (this._generateDrawing) { + await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + } this.hideSmartDrawHandler(); runInAction(() => { @@ -240,15 +248,12 @@ export class SmartDrawHandler extends ObservableReactComponent { drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { if (input) { this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; - - Networking.PostToServer('/queryFireflyImage', { prompt: input }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { title: input }), OpenWhere.addRight)); - const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); if (res) { const strokeData = await this.parseSvg(res, startPt, false, autoColor); const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - + this._selectedDoc = drawingDoc; this._errorOccurredOnce = false; return strokeData; } else { @@ -258,6 +263,23 @@ export class SmartDrawHandler extends ObservableReactComponent { return undefined; }; + /** + * Calls Firefly API to create an image based on user input + */ + createImageWithFirefly = (input: string, seed?: number) => { + this._lastInput.text = input; + return Networking.PostToServer('/queryFireflyImage', { prompt: input, seed: seed }).then(img => { + const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { + title: input.match(/^(.*?)~~~.*$/)?.[1] || input, + ai_generated: true, + firefly_seed: img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1], + firefly_prompt: input, + }); + DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); + this._selectedDoc = imgDoc; + }); + }; + /** * Regenerates drawings with the option to add a specific regenerate prompt/request. */ @@ -266,27 +288,39 @@ export class SmartDrawHandler extends ObservableReactComponent { if (lastInput) this._lastInput = lastInput; if (lastResponse) this._lastResponse = lastResponse; if (regenInput) this._regenInput = regenInput; - - try { - let res; + if (this._generateDrawing) { + try { + let res; + if (this._regenInput !== '') { + const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; + res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); + this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + } else { + res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + } + if (!res) { + console.error('GPT call failed'); + return; + } + const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + } catch (err) { + console.error('Error regenerating drawing', err); + } + } + if (this._generateImage) { if (this._regenInput !== '') { - const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; - res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); - this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + if (this._selectedDoc) { + const docData = this._selectedDoc[DocData]; + const newPrompt = `${docData.firefly_prompt}, ${this._regenInput}`; + const seed: number = docData?.firefly_seed as number; + await this.createImageWithFirefly(newPrompt, seed); + } } else { - res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + await this.createImageWithFirefly(this._lastInput.text); } - if (!res) { - console.error('GPT call failed'); - return; - } - const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); - this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); - drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - return strokeData; - } catch (err) { - console.error('Error regenerating drawing', err); } }; @@ -397,58 +431,87 @@ export class SmartDrawHandler extends ObservableReactComponent { {this._showOptions && (
-
-
- Auto color - this._canInteract && (this._autoColor = !this._autoColor))} - /> -
-
- Complexity - +
+ Generate Ink + this._canInteract && (this._complexity = val as number))} - valueLabelDisplay="auto" + checked={this._generateDrawing} + onChange={() => this._canInteract && (this._generateDrawing = !this._generateDrawing)} />
-
- Size (in pixels) - + Generate Image + this._canInteract && (this._size = val as number))} - valueLabelDisplay="auto" + checked={this._generateImage} + onChange={() => this._canInteract && (this._generateImage = !this._generateImage)} />
-
+ {this._generateDrawing && ( +
+
+ Auto color + this._canInteract && (this._autoColor = !this._autoColor))} + /> +
+
+ Complexity + this._canInteract && (this._complexity = val as number))} + valueLabelDisplay="auto" + /> +
+
+ Size (in pixels) + this._canInteract && (this._size = val as number))} + valueLabelDisplay="auto" + /> +
+
+ )}
)}
diff --git a/src/client/views/smartdraw/StickerPalette.tsx b/src/client/views/smartdraw/StickerPalette.tsx index d56878f10..352a02e32 100644 --- a/src/client/views/smartdraw/StickerPalette.tsx +++ b/src/client/views/smartdraw/StickerPalette.tsx @@ -186,6 +186,7 @@ export class StickerPalette extends ObservableReactComponent { + generateImage = (prompt: string = 'a realistic illustration of a cat coding', seed?: number) => { + let body = `{ "prompt": "${prompt}" }`; + if (seed) { + body = `{ "prompt": "${prompt}", "seeds": [${seed}]}`; + } const fetched = this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { @@ -76,9 +80,15 @@ export default class FireflyManager extends ApiManager { ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], - body: `{ "prompt": "${prompt}" }`, + body: body, }) - .then(response2 => response2.json().then(json => (json.outputs?.[0] as { image: { url: string } })?.image.url)) + .then(response2 => + response2.json().then(json => { + const seed = json.outputs?.[0]?.seed; + const url = json.outputs?.[0]?.image?.url; + return { seed, url }; + }) + ) .catch(error => { console.error('Error:', error); return undefined; @@ -226,8 +236,8 @@ export default class FireflyManager extends ApiManager { method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => - this.generateImage(req.body.prompt).then(url => - DashUploadUtils.UploadImage(url ?? '').then(info => { + this.generateImage(req.body.prompt, req.body.seed).then(img => + DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); }) diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 623172894..2177c5d97 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -458,7 +458,6 @@ export namespace DashUploadUtils { return { name: result.name, message: result.message }; } const outputFile = filename || result.filename || ''; - return UploadInspectedImage(result, outputFile, prefix, isLocal().exec(source) || source.startsWith('data:') ? true : false); }; -- cgit v1.2.3-70-g09d2