From 1d62d867621b293c41ff8488ca5a3bd6010723d5 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Sun, 5 Jan 2025 23:47:18 -0500 Subject: added AI image editor --- src/client/views/smartdraw/DrawingFillHandler.tsx | 2 +- src/client/views/smartdraw/SmartDrawHandler.tsx | 92 ++++++++++++----------- src/client/views/smartdraw/StickerPalette.tsx | 17 +++-- 3 files changed, 59 insertions(+), 52 deletions(-) (limited to 'src/client/views/smartdraw') diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index 52652d377..8e41ee105 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -35,7 +35,7 @@ export class DrawingFillHandler { .then((hrefBase64: string) => gptDescribeImage(hrefBase64)) .then((prompt: string) => { Networking.PostToServer('/queryFireflyImageFromStructure', { prompt: prompt, width: dims.width, height: dims.height, structureUrl: structureUrl, strength: strength, styles: styles }).then((info: Upload.ImageInformation) => - DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { ai_generated: true, nativeWidth: dims.width, nativeHeight: dims.height }), OpenWhere.addRight) + DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { ai: 'firefly', ai_firefly_prompt: prompt, nativeWidth: dims.width, nativeHeight: dims.height }), OpenWhere.addRight) ); // prettier-ignore }); } diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 6c9470480..0c67c7a13 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -58,7 +58,7 @@ export class SmartDrawHandler extends ObservableReactComponent { private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; private _lastResponse: string = ''; - private _selectedDoc: Doc | undefined = undefined; + private _selectedDocs: Doc[] = []; private _errorOccurredOnce = false; @observable private _display: boolean = false; @@ -144,14 +144,14 @@ export class SmartDrawHandler extends ObservableReactComponent { */ @action displayRegenerate = (x: number, y: number) => { - this._selectedDoc = DocumentView.SelectedDocs()?.lastElement(); + this._selectedDocs = [DocumentView.SelectedDocs()?.lastElement()]; [this._pageX, this._pageY] = [x, y]; this._display = false; this.ShowRegenerate = true; this._showEditBox = false; - const docData = this._selectedDoc[DocData]; + const docData = this._selectedDocs[0][DocData]; this._lastResponse = StrCast(docData.drawingData); - this._lastInput = { text: StrCast(docData.drawingInput), complexity: NumCast(docData.drawingComplexity), size: NumCast(docData.drawingSize), autoColor: BoolCast(docData.drawingColored), x: this._pageX, y: this._pageY }; + this._lastInput = { text: StrCast(docData.ai_drawing_input), complexity: NumCast(docData.ai_drawing_complexity), size: NumCast(docData.ai_drawing_size), autoColor: BoolCast(docData.ai_drawing_colored), x: this._pageX, y: this._pageY }; }; /** @@ -205,8 +205,9 @@ export class SmartDrawHandler extends ObservableReactComponent { this._isLoading = true; this._canInteract = false; if (this.ShowRegenerate) { - await this.regenerate(); + await this.regenerate(this._selectedDocs); runInAction(() => { + this._selectedDocs = []; this._regenInput = ''; this._showEditBox = false; }); @@ -253,7 +254,7 @@ export class SmartDrawHandler extends ObservableReactComponent { const strokeData = await this.parseSvg(res, startPt, false, autoColor); const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - this._selectedDoc = drawingDoc; + drawingDoc && this._selectedDocs.push(drawingDoc); this._errorOccurredOnce = false; return strokeData; } else { @@ -274,57 +275,62 @@ export class SmartDrawHandler extends ObservableReactComponent { title: input.match(/^(.*?)~~~.*$/)?.[1] || input, nativeWidth: dims.width, nativeHeight: dims.height, - ai_generated: true, - firefly_seed: img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1], - firefly_prompt: input, + ai: 'firefly', + ai_firefly_seed: img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1], + ai_firefly_prompt: input, }); DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); - this._selectedDoc = imgDoc; + this._selectedDocs.push(imgDoc); }); }; /** * Regenerates drawings with the option to add a specific regenerate prompt/request. + * @param doc the drawing Docs to regenerate */ @action - regenerate = async (lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string) => { + regenerate = async (drawingDocs: Doc[], lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string) => { if (lastInput) this._lastInput = lastInput; if (lastResponse) this._lastResponse = lastResponse; if (regenInput) this._regenInput = regenInput; - if (this._generateImage) { - if (this._regenInput !== '') { - if (this._selectedDoc) { - const docData = this._selectedDoc[DocData]; - const newPrompt = `${docData.firefly_prompt}, ${this._regenInput}`; - const seed: number = docData?.firefly_seed as number; - await this.createImageWithFirefly(newPrompt, seed); + await Promise.all( + drawingDocs.map(async doc => { + const docData = doc[DocData]; + if (docData.type == 'image') { + const seed: number = docData?.ai_firefly_seed as number; + if (this._regenInput !== '') { + // if (this._selectedDoc) { + const newPrompt = `${docData.ai_firefly_prompt}, ${this._regenInput}`; + await this.createImageWithFirefly(newPrompt, seed); + // } + } else { + await this.createImageWithFirefly(this._lastInput.text || StrCast(docData.ai_firefly_prompt)); + } } - } else { - await this.createImageWithFirefly(this._lastInput.text); - } - } - if (this._generateDrawing) { - try { - let res; - if (this._regenInput !== '') { - const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; - res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); - this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; - } else { - res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); - } - if (!res) { - console.error('GPT call failed'); - return; + if (docData.type == 'collection') { + try { + let res; + if (this._regenInput !== '') { + const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; + res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); + this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + } else { + res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + } + if (!res) { + console.error('GPT call failed'); + return; + } + const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, doc); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + } catch (err) { + console.error('Error regenerating drawing', err); + } } - const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); - this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); - drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - } catch (err) { - console.error('Error regenerating drawing', err); - } - } + }) + ); }; /** diff --git a/src/client/views/smartdraw/StickerPalette.tsx b/src/client/views/smartdraw/StickerPalette.tsx index 352a02e32..d23763eb9 100644 --- a/src/client/views/smartdraw/StickerPalette.tsx +++ b/src/client/views/smartdraw/StickerPalette.tsx @@ -142,19 +142,20 @@ export class StickerPalette extends ObservableReactComponent { this._isLoading = true; + const prevDrawings = DocListCast(this._props.Document[DocData].data); this._props.Document[DocData].data = undefined; SmartDrawHandler.Instance.AddDrawing = this.addDrawing; this._canInteract = false; await Promise.all( Array.from({ length: 3 }).map((_, i) => { return this._showRegenerate - ? SmartDrawHandler.Instance.regenerate(this._opts, this._gptRes[i], this._userInput) + ? SmartDrawHandler.Instance.regenerate(prevDrawings, this._opts, this._gptRes[i], this._userInput) : SmartDrawHandler.Instance.drawWithGPT({ X: 0, Y: 0 }, this._userInput, this._opts.complexity, this._opts.size, this._opts.autoColor); }) ); @@ -181,12 +182,12 @@ export class StickerPalette extends ObservableReactComponent