diff options
Diffstat (limited to 'src')
47 files changed, 2225 insertions, 945 deletions
diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 03380e4d6..4b0d58cc1 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -249,6 +249,41 @@ const gptHandwriting = async (src: string): Promise<string> => { } }; +const gptDescribeImage = async (image: string): Promise<string> => { + try { + const response = await openai.chat.completions.create({ + model: 'gpt-4o', + temperature: 0, + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: `Briefly identify what this drawing is, naming all the drawing elements and their location within the image. Do not include anything about the drawing style.`, + }, + { + type: 'image_url', + image_url: { + url: `${image}`, + detail: 'low', + }, + }, + ], + }, + ], + }); + if (response.choices[0].message.content) { + console.log('GPT DESCRIPTION', response.choices[0].message.content); + return response.choices[0].message.content; + } + return 'Unknown drawing'; + } catch (err) { + console.log(err); + return 'Error connecting with API'; + } +}; + const gptDrawingColor = async (image: string, coords: string[]): Promise<string> => { try { const response = await openai.chat.completions.create({ @@ -276,11 +311,11 @@ const gptDrawingColor = async (image: string, coords: string[]): Promise<string> if (response.choices[0].message.content) { return response.choices[0].message.content; } - return 'Missing labels'; + return 'Unknown drawing'; } catch (err) { console.log(err); return 'Error connecting with API'; } }; -export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDrawingColor }; +export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDescribeImage, gptDrawingColor }; diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 6828a1929..7f1387ff8 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -516,6 +516,10 @@ export class DocumentOptions { card_sort?: STRt = new StrInfo('way cards are sorted in deck view'); card_sort_isDesc?: BOOLt = new BoolInfo('whether the cards are sorted ascending or descending'); + + ai?: string; // to mark items as ai generated + ai_firefly_seed?: number; + ai_firefly_prompt?: string; } export const DocOptions = new DocumentOptions(); @@ -1090,7 +1094,7 @@ export namespace Docs { } export function AnnoPaletteDocument(options?: DocumentOptions) { - return InstanceFromProto(Prototypes.get(DocumentType.ANNOPALETTE), new List([Doc.MyAnnos]), { ...(options || {}) }); + return InstanceFromProto(Prototypes.get(DocumentType.ANNOPALETTE), new List([Doc.MyStickers]), { ...(options || {}) }); } export function DockDocument(documents: Array<Doc>, config: string, options: DocumentOptions, id?: string) { diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index 49a4a981a..b41fd09dc 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -161,9 +161,9 @@ export class CurrentUserUtils { return DocUtils.AssignDocField(doc, field, (opts,items) => Docs.Create.TreeDocument(items??[], opts), reqdOpts, templates, reqdScripts); } - static setupAnnoPalette(doc: Doc, field="myAnnos") { + static setupAnnoPalette(doc: Doc, field="myStickers") { const reqdOpts:DocumentOptions = { - title: "Saved Annotations", _xMargin: 0, _layout_showTitle: "title", hidden: false, _chromeHidden: true, + title: "Stickers", _xMargin: 0, _layout_showTitle: "title", hidden: false, _chromeHidden: true, _dragOnlyWithinContainer: true, layout_hideContextMenu: true, isSystem: true, _forceActive: true, _layout_autoHeight: true, _width: 500, _height: 300, _layout_fitWidth: true, _columnWidth: 35, ignoreClick: true, _lockedPosition: true, }; @@ -779,7 +779,7 @@ pie title Minerals in my tap water { title: " Size", toolTip: "Size of area pencil eraser", btnType: ButtonType.NumberSliderButton, toolType: InkProperty.EraserWidth,ignoreClick: true, scripts: {script: '{ return setInkProperty(this.toolType, value, _readOnly_);}'}, funcs: {hidden:"NotRadiusEraser()"}, numBtnMin: 1, linearBtnWidth:40}, { title: "Mask", toolTip: "Make Stroke a Stencil Mask", btnType: ButtonType.ToggleButton, icon: "user-circle", toolType: InkProperty.Mask, scripts: {onClick:'{ return setInkProperty(this.toolType, value, _readOnly_);}'}, funcs: {hidden:"IsNoviceMode()" } }, { title: "Labels", toolTip: "Show Labels Inside Shapes", btnType: ButtonType.ToggleButton, icon: "text-width", toolType: InkProperty.Labels, scripts: {onClick:'{ return setInkProperty(this.toolType, value, _readOnly_);}'}}, - { title: "Smart Draw", toolTip: "Draw with GPT", btnType: ButtonType.ToggleButton, icon: "user-pen", toolType: InkTool.SmartDraw, scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}'}, funcs: {hidden: "IsNoviceMode()"}}, + { title: "Smart Draw", toolTip: "Draw with AI", btnType: ButtonType.ToggleButton, icon: "user-pen", toolType: InkTool.SmartDraw, scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}'}, funcs: {hidden: "IsNoviceMode()"}}, ]; } diff --git a/src/client/util/bezierFit.ts b/src/client/util/bezierFit.ts index 84b27e84c..7ef370d48 100644 --- a/src/client/util/bezierFit.ts +++ b/src/client/util/bezierFit.ts @@ -703,6 +703,7 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); + coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); lastPt = { X: parseInt(match[3]), Y: parseInt(match[4]) }; } else if (match[0].startsWith('C')) { coordList.push({ X: parseInt(match[5]), Y: parseInt(match[6]) }); diff --git a/src/client/views/ContextMenuItem.tsx b/src/client/views/ContextMenuItem.tsx index 6f8f41bdd..218718b18 100644 --- a/src/client/views/ContextMenuItem.tsx +++ b/src/client/views/ContextMenuItem.tsx @@ -1,6 +1,6 @@ import { IconProp } from '@fortawesome/fontawesome-svg-core'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { action, makeObservable, observable, runInAction } from 'mobx'; +import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { SnappingManager } from '../util/SnappingManager'; @@ -26,7 +26,7 @@ export class ContextMenuItem extends ObservableReactComponent<ContextMenuProps & _hoverTimeout?: NodeJS.Timeout; _overPosY = 0; _overPosX = 0; - @observable _items: ContextMenuProps[] = []; + @observable.shallow _items: ContextMenuProps[] = []; @observable _overItem = false; constructor(props: ContextMenuProps & { selected?: boolean }) { @@ -34,8 +34,8 @@ export class ContextMenuItem extends ObservableReactComponent<ContextMenuProps & makeObservable(this); } - componentDidMount() { - runInAction(() => this._items.push(...(this._props.subitems ?? []))); + @computed get items() { + return this._items.concat(this._props.subitems ?? []); } handleEvent = async (e: React.MouseEvent<HTMLDivElement>) => { @@ -91,7 +91,7 @@ export class ContextMenuItem extends ObservableReactComponent<ContextMenuProps & }; render() { - const submenu = this._items.map(prop => <ContextMenuItem {...prop} key={prop.description} closeMenu={this._props.closeMenu} />); + const submenu = this.items.map(prop => <ContextMenuItem {...prop} key={prop.description} closeMenu={this._props.closeMenu} />); return this.props.event || this._props.noexpand ? this.renderItem(submenu) : <div className="contextMenu-inlineMenu">{submenu}</div>; } } diff --git a/src/client/views/DocumentButtonBar.tsx b/src/client/views/DocumentButtonBar.tsx index 9e010afbb..a9f03a658 100644 --- a/src/client/views/DocumentButtonBar.tsx +++ b/src/client/views/DocumentButtonBar.tsx @@ -314,6 +314,18 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( ); } + @computed + get aiEditorButton() { + const targetDoc = this.view0?.Document; + return !targetDoc ? null : ( + <Tooltip title={<div className="dash-ai-editor-button">Edit with AI</div>}> + <div className="documentButtonBar-icon" style={{ color: 'white' }} onClick={undoable(() => this.view0?.toggleAIEditor(), 'toggle AI editor')}> + <FontAwesomeIcon className="documentdecorations-icon" icon="robot" /> + </div> + </Tooltip> + ); + } + @observable _isRecording = false; _stopFunc: () => void = emptyFunction; @computed @@ -484,6 +496,7 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( <div className="documentButtonBar-button">{this.pinButton}</div> <div className="documentButtonBar-button">{this.recordButton}</div> <div className="documentButtonBar-button">{this.calendarButton}</div> + <div className="documentButtonBar-button">{this.aiEditorButton}</div> <div className="documentButtonBar-button">{this.keywordButton}</div> {!Doc.UserDoc().documentLinksButton_fullMenu ? null : <div className="documentButtonBar-button">{this.shareButton}</div>} <div className="documentButtonBar-button">{this.menuButton}</div> diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index d4300cdcb..8ab85bfaf 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -238,9 +238,9 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora if (iconViewDoc.activeFrame) { iconViewDoc.opacity = 0; // if in an animation collection, set opacity to 0 to allow inkMasks and other documents to remain in the collection and to smoothly animate when they are activated in a different animation frame } else { - // if Doc is in the annotation palette, remove the flag indicating that it's saved + // if Doc is in the sticker palette, remove the flag indicating that it's saved const dragFactory = DocCast(iconView.Document.dragFactory); - if (dragFactory && DocCast(dragFactory.cloneOf).savedAsAnno) DocCast(dragFactory.cloneOf).savedAsAnno = undefined; + if (dragFactory && DocCast(dragFactory.cloneOf).savedAsSticker) DocCast(dragFactory.cloneOf).savedAsSticker = undefined; // if this is a face Annotation doc, then just hide it. if (iconView.Document.annotationOn && iconView.Document.face) iconView.Document.hidden = true; diff --git a/src/client/views/LightboxView.tsx b/src/client/views/LightboxView.tsx index 4e72a015f..e3df01bbb 100644 --- a/src/client/views/LightboxView.tsx +++ b/src/client/views/LightboxView.tsx @@ -21,7 +21,7 @@ import { OverlayView } from './OverlayView'; import { DefaultStyleProvider, returnEmptyDocViewList, wavyBorderPath } from './StyleProvider'; import { DocumentView } from './nodes/DocumentView'; import { OpenWhere, OpenWhereMod } from './nodes/OpenWhere'; -import { AnnotationPalette } from './smartdraw/AnnotationPalette'; +import { StickerPalette } from './smartdraw/StickerPalette'; interface LightboxViewProps { PanelWidth: number; @@ -35,7 +35,7 @@ type LightboxSavedState = { [key: string]: FieldResult; }; // prettier-ignore @observer export class LightboxView extends ObservableReactComponent<LightboxViewProps> { /** - * Determines whether a DocumentView is descendant of the lightbox view (or any of its pop-ups like the annotationPalette) + * Determines whether a DocumentView is descendant of the lightbox view (or any of its pop-ups like the stickerPalette) * @param view * @returns true if a DocumentView is descendant of the lightbox view */ @@ -56,7 +56,7 @@ export class LightboxView extends ObservableReactComponent<LightboxViewProps> { }[] = []; private _savedState: LightboxSavedState = {}; private _history: { doc: Doc; target?: Doc }[] = []; - private _annoPaletteView: AnnotationPalette | null = null; + private _annoPaletteView: StickerPalette | null = null; @observable private _future: Doc[] = []; @observable private _layoutTemplate: Opt<Doc> = undefined; @observable private _layoutTemplateString: Opt<string> = undefined; @@ -211,7 +211,6 @@ export class LightboxView extends ObservableReactComponent<LightboxViewProps> { }; togglePalette = () => { this._showPalette = !this._showPalette; - // if (this._showPalette === false) AnnotationPalette.Instance.resetPalette(true); }; togglePen = () => { Doc.ActiveTool = Doc.ActiveTool === InkTool.Ink ? InkTool.None : InkTool.Ink; @@ -317,7 +316,7 @@ export class LightboxView extends ObservableReactComponent<LightboxViewProps> { </GestureOverlay> </div> - {this._showPalette && <AnnotationPalette ref={r => (this._annoPaletteView = r)} Document={DocCast(Doc.UserDoc().myLightboxDrawings)} />} + {this._showPalette && <StickerPalette ref={r => (this._annoPaletteView = r)} Document={DocCast(Doc.UserDoc().myLightboxDrawings)} />} {this.renderNavBtn(0, undefined, this._props.PanelHeight / 2 - 12.5, 'chevron-left', this._doc && this._history.length ? true : false, this.previous)} {this.renderNavBtn( this._props.PanelWidth - Math.min(this._props.PanelWidth / 4, this._props.maxBorder[0]), @@ -331,7 +330,7 @@ export class LightboxView extends ObservableReactComponent<LightboxViewProps> { <LightboxTourBtn lightboxDoc={this.lightboxDoc} navBtn={this.renderNavBtn} future={this.future} stepInto={this.stepInto} /> {toggleBtn('lightboxView-navBtn', 'toggle reading view', BoolCast(this._doc?._layout_fitWidth), 'book-open', 'book', this.toggleFitWidth)} {toggleBtn('lightboxView-tabBtn', 'open document in a tab', false, 'file-export', '', this.downloadDoc)} - {toggleBtn('lightboxView-paletteBtn', 'toggle annotation palette', this._showPalette === true, 'palette', '', this.togglePalette)} + {toggleBtn('lightboxView-paletteBtn', 'toggle sticker palette', this._showPalette === true, 'palette', '', this.togglePalette)} {toggleBtn('lightboxView-penBtn', 'toggle pen annotation', Doc.ActiveTool === InkTool.Ink, 'pen', '', this.togglePen)} {toggleBtn('lightboxView-exploreBtn', 'toggle navigate only mode', SnappingManager.ExploreMode, 'globe-americas', '', this.toggleExplore)} </div> diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 2a59f6dc3..dda543470 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -64,7 +64,7 @@ import { ImportElementBox } from './nodes/importBox/ImportElementBox'; import { PresBox, PresElementBox } from './nodes/trails'; import { FaceRecognitionHandler } from './search/FaceRecognitionHandler'; import { SearchBox } from './search/SearchBox'; -import { AnnotationPalette } from './smartdraw/AnnotationPalette'; +import { StickerPalette } from './smartdraw/StickerPalette'; dotenv.config(); @@ -117,7 +117,7 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' }; KeyValueBox.Init(); PresBox.Init(TabDocView.AllTabDocs); DocumentContentsView.Init(KeyValueBox.LayoutString(), { - AnnotationPalette, + StickerPalette: StickerPalette, FormattedTextBox, ImageBox, FontIconBox, diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 65d90a5a0..7abca5197 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -73,7 +73,7 @@ import { OpenWhere, OpenWhereMod } from './nodes/OpenWhere'; import { TaskCompletionBox } from './nodes/TaskCompletedBox'; import { DashFieldViewMenu } from './nodes/formattedText/DashFieldView'; import { RichTextMenu } from './nodes/formattedText/RichTextMenu'; -import GenerativeFill from './nodes/generativeFill/GenerativeFill'; +import ImageEditorBox from './nodes/imageEditor/ImageEditor'; import { PresBox } from './nodes/trails'; import { AnchorMenu } from './pdf/AnchorMenu'; import { GPTPopup } from './pdf/GPTPopup/GPTPopup'; @@ -446,6 +446,7 @@ export class MainView extends ObservableReactComponent<object> { fa.faAlignRight, fa.faHeading, fa.faRulerCombined, + fa.faFill, fa.faFillDrip, fa.faLink, fa.faUnlink, @@ -1172,7 +1173,7 @@ export class MainView extends ObservableReactComponent<object> { <LightboxView key="lightbox" PanelWidth={this._windowWidth} addSplit={CollectionDockingView.AddSplit} PanelHeight={this._windowHeight} maxBorder={this.lightboxMaxBorder} /> <GPTPopup key="gptpopup" /> <SchemaCSVPopUp key="schemacsvpopup" /> - <GenerativeFill imageEditorOpen={ImageEditor.Open} imageEditorSource={ImageEditor.Source} imageRootDoc={ImageEditor.RootDoc} addDoc={ImageEditor.AddDoc} /> + <ImageEditorBox imageEditorOpen={ImageEditor.Open} imageEditorSource={ImageEditor.Source} imageRootDoc={ImageEditor.RootDoc} addDoc={ImageEditor.AddDoc} /> </div> ); } diff --git a/src/client/views/PropertiesView.scss b/src/client/views/PropertiesView.scss index 693c75ebf..7866e67e7 100644 --- a/src/client/views/PropertiesView.scss +++ b/src/client/views/PropertiesView.scss @@ -642,6 +642,7 @@ .smooth, .color, +.strength-slider, .smooth-slider { margin-top: 7px; } diff --git a/src/client/views/PropertiesView.tsx b/src/client/views/PropertiesView.tsx index 8acfe6585..ae8abea2c 100644 --- a/src/client/views/PropertiesView.tsx +++ b/src/client/views/PropertiesView.tsx @@ -45,6 +45,7 @@ import { StyleProviderFuncType } from './nodes/FieldView'; import { OpenWhere } from './nodes/OpenWhere'; import { PresBox, PresEffect, PresEffectDirection } from './nodes/trails'; import { SmartDrawHandler } from './smartdraw/SmartDrawHandler'; +import { DrawingFillHandler } from './smartdraw/DrawingFillHandler'; interface PropertiesViewProps { width: number; @@ -141,7 +142,7 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps } @computed get isText() { - return [DocumentType.RTF, DocumentType.EQUATION, DocumentType.LABEL].includes(this.selectedDoc?.type as DocumentType); + return this.selectedDoc?.type === DocumentType.RTF; } @computed get isInk() { return this.selectedDoc?.type === DocumentType.INK; @@ -830,8 +831,12 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps set shapeWid(value) { this.selectedDoc && (this.selectedDoc._width = Math.round(value * 100) / 100); } // prettier-ignore @computed get shapeHgt() { return NumCast(this.selectedDoc?._height); } // prettier-ignore set shapeHgt(value) { this.selectedDoc && (this.selectedDoc._height = Math.round(value * 100) / 100); } // prettier-ignore - @computed get strokeThk(){ return NumCast(this.selectedStrokes.lastElement()?.[DocData].stroke_width, 1); } // prettier-ignore - set strokeThk(value) { this.selectedStrokes.forEach(doc => { doc[DocData].stroke_width = Math.round(value * 100) / 100; }); } // prettier-ignore + @computed get strokeThk(){ return NumCast(this.selectedStrokes.lastElement()?.[DocData].stroke_width); } // prettier-ignore + set strokeThk(value) { + this.selectedStrokes.forEach(doc => { + doc[DocData].stroke_width = Math.round(value * 100) / 100; + }); + } @computed get hgtInput() { return this.inputBoxDuo( @@ -978,22 +983,40 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps 10, 1 ); + const strength = this.getNumber('Reference Strength', '', 1, 100, this.refStrength, (val: number) => { + !isNaN(val) && (this.refStrength = val); + }); return ( <div> {!targetDoc.layout_isSvg && this.containsInkDoc && ( - <div className="color"> - <Toggle - text={'Color with GPT'} - color={SettingsManager.userColor} - icon={<FontAwesomeIcon icon="fill-drip" />} - iconPlacement="left" - align="flex-start" - fillWidth - toggleType={ToggleType.BUTTON} - onClick={undoable(() => { - SmartDrawHandler.Instance.colorWithGPT(targetDoc); - }, 'smoothStrokes')} - /> + <div> + <div className="drawing-to-image"> + <Toggle + text={'Create Image'} + color={SettingsManager.userColor} + icon={<FontAwesomeIcon icon="fill-drip" />} + iconPlacement="left" + align="flex-start" + fillWidth + toggleType={ToggleType.BUTTON} + onClick={undoable(() => DrawingFillHandler.drawingToImage(targetDoc, this.refStrength, 'fill in the details of this image'), 'createImage')} + /> + </div> + <div className="strength-slider">{strength}</div> + <div className="color"> + <Toggle + text={'Color with GPT'} + color={SettingsManager.userColor} + icon={<FontAwesomeIcon icon="fill-drip" />} + iconPlacement="left" + align="flex-start" + fillWidth + toggleType={ToggleType.BUTTON} + onClick={undoable(() => { + SmartDrawHandler.Instance.colorWithGPT(targetDoc); + }, 'colorWithGPT')} + /> + </div> </div> )} <div className="smooth"> @@ -1022,13 +1045,7 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps doc[DocData].stroke_dash = value ? this._lastDash : undefined; }); } - @computed get lineCapStk() { return (this.getField('stroke_lineCap') || 'round' ) as Property.StrokeLinecap; } // prettier-ignore - set lineCapStk(value) { - this.selectedStrokes.forEach(doc => { - doc[DocData].stroke_lineCap = value; - }); - } - @computed get widthStk() { return this.getField('stroke') || '1'; } // prettier-ignore + @computed get widthStk() { return this.getField('stroke_width') || '1'; } // prettier-ignore set widthStk(value) { this.selectedStrokes.forEach(doc => { doc[DocData].stroke_width = Number(value); @@ -1040,6 +1057,12 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps doc[DocData].stroke_markerScale = Number(value); }); } + @computed get refStrength() { return Number(this.getField('drawing_refStrength') || '50'); } // prettier-ignore + set refStrength(value) { + this.selectedStrokes.forEach(doc => { + doc[DocData].drawing_refStrength = Number(value); + }); + } @computed get smoothAmt() { return Number(this.getField('stroke_smoothAmount') || '5'); } // prettier-ignore set smoothAmt(value) { this.selectedStrokes.forEach(doc => { @@ -1130,6 +1153,7 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps <div className="arrows-head"> <div className="arrows-head-title">Arrow Head: </div> <input + key="markHead" className="arrows-head-input" type="checkbox" checked={this.markHead !== ''} @@ -1139,25 +1163,16 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps <div className="arrows-tail"> <div className="arrows-tail-title">Arrow End: </div> <input + key="markTail" className="arrows-tail-input" type="checkbox" checked={this.markTail !== ''} - onChange={undoable(action(() => { this.markTail = this.markTail ? '' : 'arrow'; }) ,"change arrow tail")} + onChange={undoable( + action(() => { this.markTail = this.markTail ? '' : 'arrow'; }) ,"change arrow tail" + )} /> </div> </div> - <div className="arrows"> - {["butt", "round", "square"].map(cap => - <div className="arrows-tail" key={cap}> - <div className="arrows-tail-title">{cap}</div> - <input - className="arrows-tail-input" - type="checkbox" - checked={this.lineCapStk === cap} - onChange={undoable(action(() => { this.lineCapStk = cap as Property.StrokeLinecap; }), `change lineCap ${cap}`)} - /> - </div>)} - </div> <div className="dashed"> <div className="dashed-title">Dashed Line:</div> <input key="markHead" className="dashed-input" type="checkbox" checked={this.dashdStk === '2'} onChange={this.changeDash} /> diff --git a/src/client/views/StyleProviderQuiz.tsx b/src/client/views/StyleProviderQuiz.tsx index f07a2c803..b3fb8c930 100644 --- a/src/client/views/StyleProviderQuiz.tsx +++ b/src/client/views/StyleProviderQuiz.tsx @@ -17,7 +17,7 @@ import { AnchorMenu } from './pdf/AnchorMenu'; import { DocumentViewProps } from './nodes/DocumentView'; import { FieldViewProps } from './nodes/FieldView'; import { ImageBox } from './nodes/ImageBox'; -import { ImageUtility } from './nodes/generativeFill/generativeFillUtils/ImageHandler'; +import { ImageUtility } from './nodes/imageEditor/imageEditorUtils/ImageHandler'; import './StyleProviderQuiz.scss'; export namespace styleProviderQuiz { diff --git a/src/client/views/ViewBoxInterface.ts b/src/client/views/ViewBoxInterface.ts index b7980d74e..a66a20cf6 100644 --- a/src/client/views/ViewBoxInterface.ts +++ b/src/client/views/ViewBoxInterface.ts @@ -60,4 +60,6 @@ export abstract class ViewBoxInterface<P> extends ObservableReactComponent<React search?: (str: string, bwd?: boolean, clear?: boolean) => boolean; dontRegisterView?: () => boolean; // KeyValueBox's don't want to register their views isUnstyledView?: () => boolean; // SchemaView and KeyValue are unstyled -- not titles, no opacity, no animations + componentAIView?: () => JSX.Element; + componentAIViewHistory?: () => JSX.Element; } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx index 79aad0ef2..272c13546 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx @@ -102,7 +102,6 @@ export function computePassLayout(poolData: Map<string, PoolData>, pivotDoc: Doc replica: '', }); }); - // eslint-disable-next-line no-use-before-define return normalizeResults(panelDim, 12, docMap, poolData, viewDefsToJSX, [], 0, []); } @@ -272,7 +271,6 @@ export function computePivotLayout(poolData: Map<string, PoolData>, pivotDoc: Do payload: pivotColumnGroups.get(key)?.filters, })); groupNames.push(...dividers); - // eslint-disable-next-line no-use-before-define return normalizeResults(panelDim, maxText, docMap, poolData, viewDefsToJSX, groupNames, 0, []); } @@ -347,7 +345,6 @@ export function computeTimelineLayout(poolData: Map<string, PoolData>, pivotDoc: if (!stack && (curTime === undefined || Math.abs(x - (curTime - minTime) * scaling) > pivotAxisWidth)) { groupNames.push({ type: 'text', text: toLabel(key), x: x, y: stack * 25, height: fontHeight, fontSize, payload: undefined }); } - // eslint-disable-next-line no-use-before-define layoutDocsAtTime(keyDocs, key); }); if (sortedKeys.length && curTime !== undefined && curTime > sortedKeys[sortedKeys.length - 1]) { diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 0f64e6e53..14b9ff4c3 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -55,7 +55,7 @@ import { FocusViewOptions } from '../../nodes/FocusViewOptions'; import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; import { OpenWhere } from '../../nodes/OpenWhere'; import { PinDocView, PinProps } from '../../PinFuncs'; -import { AnnotationPalette } from '../../smartdraw/AnnotationPalette'; +import { StickerPalette } from '../../smartdraw/StickerPalette'; import { DrawingOptions, SmartDrawHandler } from '../../smartdraw/SmartDrawHandler'; import { StyleProp } from '../../StyleProp'; import { CollectionSubView, SubCollectionViewProps } from '../CollectionSubView'; @@ -1229,7 +1229,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; SmartDrawHandler.Instance.AddDrawing = this.addDrawing; - SmartDrawHandler.Instance.displaySmartDrawHandler(x, y); + SmartDrawHandler.Instance.displaySmartDrawHandler(x, y, NumCast(this.layoutDoc[this.scaleFieldKey])); }; _drawing: Doc[] = []; @@ -1289,12 +1289,13 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection addDrawing = (doc: Doc, opts: DrawingOptions, gptRes: string) => { const docData = doc[DocData]; docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; - docData.width = opts.size; - docData.drawingInput = opts.text; - docData.drawingComplexity = opts.complexity; - docData.drawingColored = opts.autoColor; - docData.drawingSize = opts.size; - docData.drawingData = gptRes; + docData._width = opts.size; + docData.ai_drawing_input = opts.text; + docData.ai_drawing_complexity = opts.complexity; + docData.ai_drawing_colored = opts.autoColor; + docData.ai_drawing_size = opts.size; + docData.ai_drawing_data = gptRes; + docData.ai = 'gpt'; this._drawingContainer = doc; this.addDocument(doc); this._batch?.end(); @@ -1324,6 +1325,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection this.Document[this.scaleFieldKey] = Math.abs(safeScale); this.setPan(-localTransform.TranslateX / safeScale, (this._props.originTopLeft ? undefined : NumCast(this.Document.layout_scrollTop) * safeScale) || -localTransform.TranslateY / safeScale, undefined, allowScroll); } + SmartDrawHandler.Instance.hideSmartDrawHandler(); }; @action @@ -1991,20 +1993,21 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection }), icon: 'eye', }); + this.layoutDoc.drawingData != undefined && + optionItems.push({ + description: 'Regenerate AI Drawing', + event: action(() => { + SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; + SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; + !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this._downX, this._downY - 10) : SmartDrawHandler.Instance.hideRegenerate(); + }), + icon: 'pen-to-square', + }); optionItems.push({ - description: 'Show Drawing Editor', - event: action(() => { - SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; - SmartDrawHandler.Instance.AddDrawing = this.addDrawing; - SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; - !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this._downX, this._downY - 10) : SmartDrawHandler.Instance.hideRegenerate(); - }), - icon: 'pen-to-square', - }); - optionItems.push({ - description: this.Document.savedAsAnno ? 'Saved as Annotation!' : 'Save to Annotation Palette', - event: action(undoable(async () => await AnnotationPalette.addToPalette(this.Document), 'save to palette')), - icon: this.Document.savedAsAnno ? 'clipboard-check' : 'file-arrow-down', + description: this.Document.savedAsSticker ? 'Sticker Saved!' : 'Save to Stickers', + event: action(undoable(async () => await StickerPalette.addToPalette(this.Document), 'save to palette')), + icon: this.Document.savedAsSticker ? 'clipboard-check' : 'file-arrow-down', }); this._props.renderDepth && optionItems.push({ diff --git a/src/client/views/nodes/DocumentView.scss b/src/client/views/nodes/DocumentView.scss index 7568e3b57..82195b9c1 100644 --- a/src/client/views/nodes/DocumentView.scss +++ b/src/client/views/nodes/DocumentView.scss @@ -242,7 +242,7 @@ .contentFittingDocumentView * { ::-webkit-scrollbar-track { - background: none; + background: none; } } @@ -270,3 +270,19 @@ position: relative; } } + +.documentView-editorView-history { + position: absolute; + right: 0; + top: 0; + overflow-y: scroll; +} + +.documentView-editorView { + width: 100%; + overflow-y: scroll; + + .documentView-editorView-resizer { + height: 5px; + } +} diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4bfa7fc92..622eccc4f 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -52,6 +52,7 @@ import { FormattedTextBox } from './formattedText/FormattedTextBox'; import { PresEffect, PresEffectDirection } from './trails/PresEnums'; import SpringAnimation from './trails/SlideEffect'; import { SpringType, springMappings } from './trails/SpringUtils'; +import { DrawingFillHandler } from '../smartdraw/DrawingFillHandler'; export interface DocumentViewProps extends FieldViewSharedProps { hideDecorations?: boolean; // whether to suppress all DocumentDecorations when doc is selected @@ -122,6 +123,12 @@ export class DocumentViewInternal extends DocComponent<FieldViewProps & Document @observable _componentView: Opt<ViewBoxInterface<FieldViewProps>> = undefined; // needs to be accessed from DocumentView wrapper class @observable _animateScaleTime: Opt<number> = undefined; // milliseconds for animating between views. defaults to 300 if not uset @observable _animateScalingTo = 0; + @observable public _showAIEditor: boolean = false; + + @action + showAIEditor() { + this._showAIEditor = !this._showAIEditor; + } get _contentDiv() { return this._mainCont.current; } // prettier-ignore get _docView() { return this._props.DocumentView?.(); } // prettier-ignore @@ -709,32 +716,50 @@ export class DocumentViewInternal extends DocComponent<FieldViewProps & Document return this._props.styleProvider?.(doc, props, property); }; + rpw = () => this._props.PanelWidth() * 0.6; + rph = () => this.panelHeight() * 0.6; @computed get viewBoxContents() { TraceMobx(); const isInk = this.layoutDoc._layout_isSvg && !this._props.LayoutTemplateString; const noBackground = this.Document.isGroup && !this._componentView?.isUnstyledView?.() && (!this.layoutDoc.backgroundColor || this.layoutDoc.backgroundColor === 'transparent'); return ( - <div - className="documentView-contentsView" - style={{ - pointerEvents: (isInk || noBackground ? 'none' : this.contentPointerEvents()) ?? (this._mounted ? 'all' : 'none'), - height: this.headerMargin ? `calc(100% - ${this.headerMargin}px)` : undefined, - }}> - <DocumentContentsView - {...this._props} - layoutFieldKey={StrCast(this.Document.layout_fieldKey, 'layout')} - pointerEvents={this.contentPointerEvents} - setContentViewBox={this.setContentView} - childFilters={this.childFilters} - PanelHeight={this.panelHeight} - setHeight={this.setHeight} - isContentActive={this.isContentActive} - ScreenToLocalTransform={this.screenToLocalContent} - rootSelected={this.rootSelected} - onClickScript={this.onClickFunc} - setTitleFocus={this.setTitleFocus} - hideClickBehaviors={BoolCast(this.Document.hideClickBehaviors)} - /> + <div> + <div + className="documentView-contentsView" + style={{ + pointerEvents: (isInk || noBackground ? 'none' : this.contentPointerEvents()) ?? (this._mounted ? 'all' : 'none'), + width: this._showAIEditor ? this.rpw() : undefined, + height: this._showAIEditor ? this.rph() : this.headerMargin ? `calc(100% - ${this.headerMargin}px)` : undefined, + justifySelf: this._showAIEditor ? 'center' : undefined, + }}> + <DocumentContentsView + {...this._props} + layoutFieldKey={StrCast(this.Document.layout_fieldKey, 'layout')} + pointerEvents={this.contentPointerEvents} + setContentViewBox={this.setContentView} + childFilters={this.childFilters} + PanelHeight={this._showAIEditor ? this.rpw : this.panelHeight} + PanelWidth={this._showAIEditor ? this.rph : this._props.PanelWidth} + setHeight={this.setHeight} + isContentActive={this.isContentActive} + ScreenToLocalTransform={this.screenToLocalContent} + rootSelected={this.rootSelected} + onClickScript={this.onClickFunc} + setTitleFocus={this.setTitleFocus} + hideClickBehaviors={BoolCast(this.Document.hideClickBehaviors)} + /> + </div> + {this._showAIEditor && ( + <div className="documentView-editorView-history" style={{ height: this.rph(), width: (this._props.PanelWidth() - this.rpw()) / 2 }}> + {this._componentView?.componentAIViewHistory?.() ?? null} + </div> + )} + {this._showAIEditor && ( + <div className="documentView-editorView" style={{ height: this.panelHeight() - this.rph() }}> + <div className="documentView-editorView-resizer" /> + {this._componentView?.componentAIView?.() ?? null} + </div> + )} </div> ); } @@ -1072,7 +1097,7 @@ export class DocumentView extends DocComponent<DocumentViewProps>() { public static GetDocImage(doc: Doc) { return DocumentView.getDocumentView(doc) ?.ComponentView?.updateIcon?.() - .then(() => ImageCast(DocCast(doc).icon)); + .then(() => ImageCast(doc?.icon, ImageCast(doc[Doc.LayoutFieldKey(doc)]))); } public get displayName() { return 'DocumentView(' + (this.Document?.title??"") + ')'; } // prettier-ignore @@ -1284,6 +1309,11 @@ export class DocumentView extends DocComponent<DocumentViewProps>() { } }; + @action + public toggleAIEditor = () => { + this._docViewInternal && this._docViewInternal.showAIEditor(); + }; + public setTextHtmlOverlay = action((text: string | undefined, effect?: Doc) => { this._htmlOverlayText = text; this._htmlOverlayEffect = effect; @@ -1398,6 +1428,8 @@ export class DocumentView extends DocComponent<DocumentViewProps>() { NativeHeight = () => this.effectiveNativeHeight; PanelWidth = () => this.panelWidth; PanelHeight = () => this.panelHeight; + ReducedPanelWidth = () => this.panelWidth / 2; + ReducedPanelHeight = () => this.panelWidth / 2; NativeDimScaling = () => this.nativeScaling; hideLinkCount = () => !!this.hideLinkButton; isHovering = () => this._isHovering; diff --git a/src/client/views/nodes/ImageBox.scss b/src/client/views/nodes/ImageBox.scss index 3ffda5a35..e083f52b4 100644 --- a/src/client/views/nodes/ImageBox.scss +++ b/src/client/views/nodes/ImageBox.scss @@ -139,3 +139,53 @@ .imageBox-fadeBlocker-hover { opacity: 0; } + +.imageBox-aiView-history { + display: flex; + flex-direction: column; + align-items: center; + padding: 5 0 0 5; + gap: 5px; + + .imageBox-aiView-img { + width: 100%; + + &:hover { + filter: brightness(0.8); + } + } +} + +.imageBox-aiView { + text-align: center; + font-weight: bold; + align-content: center; + height: 100%; + + .imageBox-aiView-subtitle { + position: relative; + left: 5px; + align-self: start; + } + + .imageBox-aiView-regenerate-container, + .imageBox-aiView-options-container { + font-weight: normal; + text-align: start; + margin: 5px; + padding-left: 5px; + } + + .imageBox-aiView-regenerate, + .imageBox-aiView-options { + display: flex; + flex-direction: row; + justify-content: center; + flex-direction: row; + gap: 5px; + } + + .imageBox-aiView-input { + width: 50%; + } +} diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index de8a9ad21..e0ef8f423 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -1,13 +1,13 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Tooltip } from '@mui/material'; import axios from 'axios'; -import { Colors } from '@dash/components'; +import { Colors, Button, Type } from '@dash/components'; import { action, computed, IReactionDisposer, makeObservable, observable, ObservableMap, reaction } from 'mobx'; import { observer } from 'mobx-react'; import { extname } from 'path'; import * as React from 'react'; import ReactLoading from 'react-loading'; -import { ClientUtils, DashColor, returnEmptyString, returnFalse, returnOne, returnZero, setupMoveUpEvents } from '../../../ClientUtils'; +import { ClientUtils, DashColor, returnEmptyString, returnFalse, returnOne, returnZero, setupMoveUpEvents, UpdateIcon } from '../../../ClientUtils'; import { Doc, DocListCast, Opt } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; @@ -32,6 +32,7 @@ import { MarqueeAnnotator } from '../MarqueeAnnotator'; import { OverlayView } from '../OverlayView'; import { AnchorMenu } from '../pdf/AnchorMenu'; import { PinDocView, PinProps } from '../PinFuncs'; +import { StickerPalette } from '../smartdraw/StickerPalette'; import { StyleProp } from '../StyleProp'; import { DocumentView } from './DocumentView'; import { FieldView, FieldViewProps } from './FieldView'; @@ -39,6 +40,10 @@ import { FocusViewOptions } from './FocusViewOptions'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; import { Upload } from '../../../server/SharedMediaTypes'; +import { SmartDrawHandler } from '../smartdraw/SmartDrawHandler'; +import { SettingsManager } from '../../util/SettingsManager'; +import { AiOutlineSend } from 'react-icons/ai'; +import { FireflyImageData } from '../smartdraw/FireflyConstants'; export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -350,10 +355,47 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }), icon: 'pencil-alt', }); + this.layoutDoc.ai && + funcs.push({ + description: 'Regenerate AI Image', + event: action(e => { + !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(e?.x || 0, e?.y || 0) : SmartDrawHandler.Instance.hideRegenerate(); + }), + icon: 'pen-to-square', + }); + funcs.push({ + description: this.Document.savedAsSticker ? 'Sticker Saved!' : 'Save to Stickers', + event: action(undoable(async () => await StickerPalette.addToPalette(this.Document), 'save to palette')), + icon: this.Document.savedAsSticker ? 'clipboard-check' : 'file-arrow-down', + }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; + // updateIcon = () => new Promise<void>(res => res()); + updateIcon = (usePanelDimensions?: boolean) => { + const contentDiv = this._mainCont.current; + return !contentDiv + ? new Promise<void>(res => res()) + : UpdateIcon( + this.layoutDoc[Id] + '_icon_' + new Date().getTime(), + contentDiv, + usePanelDimensions ? this._props.PanelWidth() : NumCast(this.layoutDoc._width), + usePanelDimensions ? this._props.PanelHeight() : NumCast(this.layoutDoc._height), + this._props.PanelWidth(), + this._props.PanelHeight(), + 0, + 1, + false, + '', + (iconFile, nativeWidth, nativeHeight) => { + this.dataDoc.icon = new ImageField(iconFile); + this.dataDoc.icon_nativeWidth = nativeWidth; + this.dataDoc.icon_nativeHeight = nativeHeight; + } + ); + }; + choosePath = (url: URL) => { if (!url?.href) return ''; const lower = url.href.toLowerCase(); @@ -487,6 +529,145 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { ); } + @observable private _regenInput = ''; + @observable private _canInteract = true; + @observable private _regenerateLoading = false; + @observable private _prevImgs: FireflyImageData[] = []; + + componentAIViewHistory = () => { + return ( + <div className="imageBox-aiView-history"> + {this._prevImgs.map(img => ( + <img + key={img.pathname} + className="imageBox-aiView-img" + src={img.href} + onClick={() => { + this.dataDoc[this.fieldKey] = new ImageField(img.pathname); + this.dataDoc.ai_firefly_prompt = img.prompt; + this.dataDoc.ai_firefly_seed = img.seed; + }} + /> + ))} + </div> + ); + }; + + componentAIView = () => { + const field = this.dataDoc[this.fieldKey] instanceof ImageField ? Cast(this.dataDoc[this.fieldKey], ImageField, null) : new ImageField(String(this.dataDoc[this.fieldKey])); + const showRegenerate = this.Document[DocData].ai; + return ( + <div className="imageBox-aiView"> + Edit Image with AI + {showRegenerate && ( + <div className="imageBox-aiView-regenerate-container"> + <text className="imageBox-aiView-subtitle">Regenerate AI Image</text> + <div className="imageBox-aiView-regenerate"> + <input + className="imageBox-aiView-input" + aria-label="Edit instructions input" + // className="smartdraw-input" + type="text" + value={this._regenInput} + onChange={action(e => this._canInteract && (this._regenInput = e.target.value))} + // onKeyDown={this.handleKeyPress} + placeholder="Prompt (Optional)" + /> + <Button + text="Regenerate" + type={Type.SEC} + // style={{ alignSelf: 'flex-end' }} + icon={this._regenerateLoading ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} + iconPlacement="right" + onClick={undoable( + action(async () => { + this._regenerateLoading = true; + await SmartDrawHandler.Instance.regenerate([this.Document], undefined, undefined, this._regenInput, true).then(newImgs => { + if (newImgs[0]) { + const url = newImgs[0].pathname; + const imgField = new ImageField(url); + this._prevImgs.length === 0 && + this._prevImgs.push({ prompt: StrCast(this.dataDoc.ai_firefly_prompt), seed: NumCast(this.dataDoc.ai_firefly_seed), href: this.paths.lastElement(), pathname: field.url.pathname }); + this.dataDoc[this.fieldKey] = imgField; + this._prevImgs.unshift({ prompt: newImgs[0].prompt, seed: newImgs[0].seed, href: this.paths.lastElement(), pathname: url }); + this._regenerateLoading = false; + this._regenInput = ''; + } + }); + }), + 'regenerate image' + )} + /> + <Button + // style={{ alignSelf: 'flex-end' }} + text="Get Variations" + type={Type.SEC} + // icon={this._isLoading && this._regenInput !== '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} + iconPlacement="right" + // onClick={this.handleSendClick} + /> + </div> + </div> + )} + <div className="imageBox-aiView-options-container"> + {showRegenerate && <text className="imageBox-aiView-subtitle"> More Image Options </text>} + <div className="imageBox-aiView-options"> + <Button + type={Type.TERT} + text="Get Text" + icon={<FontAwesomeIcon icon="font" />} + color={SettingsManager.userBackgroundColor} + iconPlacement="right" + onClick={() => { + Networking.PostToServer('/queryFireflyImageText', { + file: (file => { + const ext = extname(file); + return file.replace(ext, (this._error ? '_o' : this._curSuffix) + ext); + })(ImageCast(this.Document[Doc.LayoutFieldKey(this.Document)])?.url.href), + }).then(text => alert(text)); + }} + /> + <Button + type={Type.TERT} + text="Generative Fill" + icon={<FontAwesomeIcon icon="fill" />} + color={SettingsManager.userBackgroundColor} + // icon={this._isLoading && this._regenInput !== '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} + iconPlacement="right" + onClick={action(() => { + ImageEditorData.Open = true; + ImageEditorData.Source = (field && this.choosePath(field.url)) || ''; + ImageEditorData.AddDoc = this._props.addDocument; + ImageEditorData.RootDoc = this.Document; + })} + /> + <Button + type={Type.TERT} + text="Expand" + icon={<FontAwesomeIcon icon="expand" />} + color={SettingsManager.userBackgroundColor} + // icon={this._isLoading && this._regenInput !== '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} + iconPlacement="right" + onClick={() => { + Networking.PostToServer('/expandImage', { + prompt: 'sunny skies', + file: (file => { + const ext = extname(file); + return file.replace(ext, (this._error ? '_o' : this._curSuffix) + ext); + })(ImageCast(this.Document[Doc.LayoutFieldKey(this.Document)])?.url.href), + }).then((info: Upload.ImageInformation) => { + const img = Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { title: 'expand:' + this.Document.title }); + DocUtils.assignImageInfo(info, img); + this._props.addDocTab(img, OpenWhere.addRight); + }); + }} + /> + </div> + </div> + </div> + ); + }; + @computed get annotationLayer() { TraceMobx(); return <div className="imageBox-annotationLayer" style={{ height: this._props.PanelHeight() }} ref={this._annotationLayer} />; diff --git a/src/client/views/nodes/formattedText/FormattedTextBox.tsx b/src/client/views/nodes/formattedText/FormattedTextBox.tsx index 35f1ec9ef..55ad543ca 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBox.tsx +++ b/src/client/views/nodes/formattedText/FormattedTextBox.tsx @@ -66,6 +66,7 @@ import { RichTextRules } from './RichTextRules'; import { schema } from './schema_rts'; import { Property } from 'csstype'; import { LabelBox } from '../LabelBox'; +import { StickerPalette } from '../../smartdraw/StickerPalette'; // import * as applyDevTools from 'prosemirror-dev-tools'; export interface FormattedTextBoxProps extends FieldViewProps { @@ -989,6 +990,11 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB }, icon: this.Document._layout_autoHeight ? 'lock' : 'unlock', }); + optionItems.push({ + description: this.Document.savedAsSticker ? 'Sticker Saved!' : 'Save to Stickers', + event: action(undoable(async () => await StickerPalette.addToPalette(this.Document), 'save to palette')), + icon: this.Document.savedAsSticker ? 'clipboard-check' : 'file-arrow-down', + }); !options && cm.addItem({ description: 'Options...', subitems: optionItems, icon: 'eye' }); const help = cm.findByDescription('Help...'); const helpItems = help?.subitems ?? []; diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts deleted file mode 100644 index 1e7801056..000000000 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts +++ /dev/null @@ -1,20 +0,0 @@ -export interface CursorData { - x: number; - y: number; - width: number; -} - -export interface Point { - x: number; - y: number; -} - -export enum BrushMode { - ADD, - SUBTRACT, -} - -export interface ImageDimensions { - width: number; - height: number; -} diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.scss b/src/client/views/nodes/imageEditor/GenerativeFillButtons.scss index 0180ef904..0180ef904 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFillButtons.scss +++ b/src/client/views/nodes/imageEditor/GenerativeFillButtons.scss diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx b/src/client/views/nodes/imageEditor/GenerativeFillButtons.tsx index d3b456273..fe9c39aad 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx +++ b/src/client/views/nodes/imageEditor/GenerativeFillButtons.tsx @@ -3,7 +3,7 @@ import * as React from 'react'; import ReactLoading from 'react-loading'; import { Button, IconButton, Type } from '@dash/components'; import { AiOutlineInfo } from 'react-icons/ai'; -import { activeColor } from './generativeFillUtils/generativeFillConstants'; +import { activeColor } from './imageEditorUtils/imageEditorConstants'; interface ButtonContainerProps { onClick: () => Promise<void>; diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.scss b/src/client/views/nodes/imageEditor/ImageEditor.scss index c2669a950..c691e6a18 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFill.scss +++ b/src/client/views/nodes/imageEditor/ImageEditor.scss @@ -2,7 +2,7 @@ $navHeight: 5rem; $canvasSize: 1024px; $scale: 0.5; -.generativeFillContainer { +.imageEditorContainer { position: absolute; top: 0; left: 0; @@ -13,7 +13,7 @@ $scale: 0.5; flex-direction: column; overflow: hidden; - .generativeFillControls { + .imageEditorTopBar { flex-shrink: 0; height: $navHeight; color: #000000; @@ -27,6 +27,12 @@ $scale: 0.5; border-bottom: 1px solid #c7cdd0; padding: 0 2rem; + .imageEditorControls { + display: flex; + align-items: center; + gap: 1.5rem; + } + h1 { font-size: 1.5rem; } @@ -69,13 +75,48 @@ $scale: 0.5; } } - .iconContainer { + .sideControlsContainer { + width: 160px; position: absolute; - top: 2rem; - left: 2rem; - display: flex; - flex-direction: column; - gap: 2rem; + left: 0; + height: 100%; + + .sideControls { + position: absolute; + width: 120px; + top: 3rem; + left: 2rem; + display: flex; + flex-direction: column; + gap: 1rem; + + .imageToolsContainer { + display: flex; + flex-direction: column; + gap: 10px; + } + + .cutToolsContainer { + display: grid; + gap: 5px; + grid-template-columns: 1fr 1fr; + } + + .undoRedoContainer { + justify-content: center; + display: flex; + flex-direction: row; + } + + .sliderContainer { + margin: 3rem 0; + height: 225px; + width: 100%; + display: flex; + justify-content: center; + cursor: pointer; + } + } } .editsBox { @@ -86,7 +127,18 @@ $scale: 0.5; flex-direction: column; gap: 1rem; + .originalImageLabel { + position: absolute; + bottom: 10; + left: 10; + color: #ffffff; + font-size: 0.8rem; + letter-spacing: 1px; + text-transform: uppercase; + } + img { + cursor: pointer; transition: all 0.2s ease-in-out; &:hover { opacity: 0.8; diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index d2910469d..809ced2f6 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -1,10 +1,7 @@ -/* eslint-disable jsx-a11y/label-has-associated-control */ -/* eslint-disable jsx-a11y/no-noninteractive-element-interactions */ -/* eslint-disable jsx-a11y/img-redundant-alt */ -/* eslint-disable jsx-a11y/click-events-have-key-events */ +/* eslint-disable no-use-before-define */ /* eslint-disable react/function-component-definition */ import { Checkbox, FormControlLabel, Slider, TextField } from '@mui/material'; -import { IconButton } from '@dash/components'; +import { Button, IconButton, Type } from '@dash/components'; import * as React from 'react'; import { useEffect, useRef, useState } from 'react'; import { CgClose } from 'react-icons/cg'; @@ -20,17 +17,16 @@ import { CollectionDockingView } from '../../collections/CollectionDockingView'; import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; import { ImageEditorData } from '../ImageBox'; import { OpenWhereMod } from '../OpenWhere'; -import './GenerativeFill.scss'; -import { EditButtons, CutButtons } from './GenerativeFillButtons'; -import { BrushHandler, BrushType } from './generativeFillUtils/BrushHandler'; -import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler'; -import { PointerHandler } from './generativeFillUtils/PointerHandler'; -import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants'; -import { CursorData, ImageDimensions, Point } from './generativeFillUtils/generativeFillInterfaces'; +import './ImageEditor.scss'; +import { ApplyFuncButtons, ImageToolButton } from './ImageEditorButtons'; +import { BrushHandler } from './imageEditorUtils/BrushHandler'; +import { APISuccess, ImageUtility } from './imageEditorUtils/ImageHandler'; +import { PointerHandler } from './imageEditorUtils/PointerHandler'; +import { activeColor, bgColor, brushWidthOffset, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './imageEditorUtils/imageEditorConstants'; +import { CutMode, CursorData, ImageDimensions, ImageEditTool, ImageToolType, Point } from './imageEditorUtils/imageEditorInterfaces'; import { DocumentView } from '../DocumentView'; -import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { ImageField } from '../../../../fields/URLField'; -import { resolve } from 'url'; +import { DocData } from '../../../../fields/DocSymbols'; +import { SettingsManager } from '../../../util/SettingsManager'; interface GenerativeFillProps { imageEditorOpen: boolean; @@ -41,7 +37,14 @@ interface GenerativeFillProps { // Added field on image doc: gen_fill_children: List of children Docs -const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => { +/** + * The image editor interface can be accessed by opening a document's context menu, then going to Options --> Open Image Editor. + * The image editor supports various operations on images. Currently, there is a Generative Fill feature that allows users to erase + * part of an image, add an optional prompt, and send this to GPT. GPT then returns a newly generated image that replaces the erased + * portion based on the optional prompt. There is also an image cutting tool that allows users to cut images in different ways to + * reshape the images, take out portions of images, and overall use them more creatively (see the header comment for cutImage() for more information). + */ +const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => { const canvasRef = useRef<HTMLCanvasElement>(null); const canvasBackgroundRef = useRef<HTMLCanvasElement>(null); const drawingAreaRef = useRef<HTMLDivElement>(null); @@ -55,13 +58,14 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // format: array of [image source, corresponding image Doc] const [edits, setEdits] = useState<{ url: string; saveRes: Doc | undefined }[]>([]); const [edited, setEdited] = useState(false); - // const [brushStyle] = useState<BrushStyle>(BrushStyle.ADD); + const [isFirstDoc, setIsFirstDoc] = useState<boolean>(true); const [input, setInput] = useState(''); const [loading, setLoading] = useState(false); const [canvasDims, setCanvasDims] = useState<ImageDimensions>({ width: canvasSize, height: canvasSize, }); + const [cutType, setCutType] = useState<CutMode>(CutMode.IN); // whether to create a new collection or not const [isNewCollection, setIsNewCollection] = useState(true); // the current image in the main canvas @@ -82,6 +86,24 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // constants for image cutting const cutPts = useRef<Point[]>([]); + /** + * + * @param type The new tool type we are changing to + */ + const changeTool = (type: ImageToolType) => { + switch (type) { + case ImageToolType.GenerativeFill: + setCurrTool(genFillTool); + setCursorData(prev => ({ ...prev, width: genFillTool.sliderDefault as number })); + break; + case ImageToolType.Cut: + setCurrTool(cutTool); + setCursorData(prev => ({ ...prev, width: cutTool.sliderDefault as number })); + break; + default: + break; + } + }; // Undo and Redo const handleUndo = () => { const ctx = ImageUtility.getCanvasContext(canvasRef); @@ -121,6 +143,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD ctx.clearRect(0, 0, canvasSize, canvasSize); undoStack.current = []; redoStack.current = []; + cutPts.current.length = 0; ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); }; @@ -161,7 +184,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD x: currPoint.x - e.movementX / canvasScale, y: currPoint.y - e.movementY / canvasScale, }; - const pts = BrushHandler.createBrushPathOverlay(lastPoint, currPoint, cursorData.width / 2 / canvasScale, ctx, eraserColor, BrushType.CUT); + const pts = BrushHandler.createBrushPathOverlay(lastPoint, currPoint, cursorData.width / 2 / canvasScale, ctx, eraserColor); cutPts.current.push(...pts); }; @@ -261,7 +284,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD })); }; - // Get AI Edit + // Get AI Edit for Generative Fill const getEdit = async () => { const img = currImg.current; if (!img) return; @@ -282,32 +305,14 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // create first image if (!newCollectionRef.current) { - if (!isNewCollection && imageRootDoc) { - // if the parent hasn't been set yet - if (!parentDoc.current) parentDoc.current = imageRootDoc; - } else { - if (!(originalImg.current && imageRootDoc)) return; - // create new collection and add it to the view - newCollectionRef.current = Docs.Create.FreeformDocument([], { - x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, - y: NumCast(imageRootDoc.y), - _width: newCollectionSize, - _height: newCollectionSize, - title: 'Image edit collection', - }); - DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); - - // opening new tab - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - - // add the doc to the main freeform - // eslint-disable-next-line no-use-before-define - await createNewImgDoc(originalImg.current, true); - } + createNewCollection(); } else { childrenDocs.current = []; } - + if (!(originalImg.current && imageRootDoc)) return; + // add the doc to the main freeform + // eslint-disable-next-line no-use-before-define + await createNewImgDoc(originalImg.current, true); originalImg.current = currImg.current; originalDoc.current = parentDoc.current; const { urls } = res as APISuccess; @@ -334,66 +339,137 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD setLoading(false); }; - const cutImage = async () => { + /** + * This function performs image cutting based on the inputted BrushMode. There are currently four ways to cut images: + * 1. By outlining the area that should be kept (BrushMode.IN) + * 2. By outlining the area that should be removed (BrushMode.OUT) + * 3. By drawing in the area that should be kept (where the image is brushed, the image will remain and everything else will be removed) (BrushMode.DRAW_IN) + * 4. By drawing the area that she be removed, so this operates as an eraser (BrushMode.ERASE) + * @param currCutType BrushMode enum that determines what kind of cutting operation to perform + * @param firstDoc boolean for whether it's the first edited image. This is for positioning of the edited images when they render on the canvas. + */ + const cutImage = async (currCutType: CutMode, brushWidth: number, prevEdits: { url: string; saveRes: Doc | undefined }[], firstDoc: boolean) => { const img = currImg.current; const canvas = canvasRef.current; if (!canvas || !img) return; - canvas.width = img.naturalWidth; - canvas.height = img.naturalHeight; const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; - ctx.globalCompositeOperation = 'source-over'; - setLoading(true); - setEdited(true); // get the original image const canvasOriginalImg = ImageUtility.getCanvasImg(img); if (!canvasOriginalImg) return; - // draw the image onto the canvas - ctx.drawImage(img, 0, 0); - // get the mask which i assume is the thing the user draws on - // const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg); - // if (!canvasMask) return; - // canvasMask.width = canvas.width; - // canvasMask.height = canvas.height; - // now put the user's path around the mask - if (cutPts.current.length) { + setLoading(true); + const currPts = [...cutPts.current]; + if (currCutType !== CutMode.ERASE) handleReset(); // gets rid of the visible brush strokes (mostly needed for line_in) unless it's erasing (which depends on the brush strokes) + let minX = img.width; + let maxX = 0; + let minY = img.height; + let maxY = 0; + // currPts is populated by the brush strokes' points, so this code is drawing a path along the points + if (currPts.length) { ctx.beginPath(); - ctx.moveTo(cutPts.current[0].x, cutPts.current[0].y); // later check edge case where cutPts is empty - for (let i = 0; i < cutPts.current.length; i++) { - ctx.lineTo(cutPts.current[i].x, cutPts.current[i].y); + ctx.moveTo(currPts[0].x, currPts[0].y); + for (let i = 0; i < currPts.length; i++) { + ctx.lineTo(currPts[i].x, currPts[i].y); + minX = Math.min(currPts[i].x, minX); + minY = Math.min(currPts[i].y, minY); + maxX = Math.max(currPts[i].x, maxX); + maxY = Math.max(currPts[i].y, maxY); + } + switch ( + currCutType // use different canvas operations depending on the type of cutting we're applying + ) { + case CutMode.IN: + ctx.closePath(); + ctx.globalCompositeOperation = 'destination-in'; + ctx.fill(); + break; + case CutMode.OUT: + ctx.closePath(); + ctx.globalCompositeOperation = 'destination-out'; + ctx.fill(); + break; + case CutMode.DRAW_IN: + ctx.globalCompositeOperation = 'destination-in'; + ctx.lineWidth = brushWidth + brushWidthOffset; // added offset because width gets cut off a little bit + ctx.stroke(); + break; } - ctx.closePath(); - ctx.stroke(); - ctx.fill(); - // ctx.clip(); } - const url = canvas.toDataURL(); // this does the same thing as convert img to canvasurl + + const url = canvas.toDataURL(); if (!newCollectionRef.current) { - if (!isNewCollection && imageRootDoc) { - // if the parent hasn't been set yet - if (!parentDoc.current) parentDoc.current = imageRootDoc; - } else { - if (!(originalImg.current && imageRootDoc)) return; - // create new collection and add it to the view - newCollectionRef.current = Docs.Create.FreeformDocument([], { - x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, - y: NumCast(imageRootDoc.y), - _width: newCollectionSize, - _height: newCollectionSize, - title: 'Image edit collection', - }); - DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); - // opening new tab - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - } + createNewCollection(); } + const image = new Image(); image.src = url; - await createNewImgDoc(image, true); - // add the doc to the main freeform - // eslint-disable-next-line no-use-before-define - setLoading(false); - cutPts.current.length = 0; + image.onload = async () => { + let finalImg: HTMLImageElement | undefined = image; + let finalImgURL: string = url; + // crop the image for these brush modes to remove excess blank space around the image contents + if (currCutType == CutMode.IN || currCutType == CutMode.DRAW_IN) { + const croppedData = cropImage(image, Math.max(minX, 0), Math.min(maxX, image.width), Math.max(minY, 0), Math.min(maxY, image.height)); + finalImg = croppedData; + finalImgURL = croppedData.src; + } + currImg.current = finalImg; + const newImgDoc = await createNewImgDoc(finalImg, firstDoc); + if (newImgDoc) { + // set the image to transparent to remove the background / brushstrokes + const docData = newImgDoc[DocData]; + docData.backgroundColor = 'transparent'; + docData.disableMixBlend = true; + if (firstDoc) setIsFirstDoc(false); + setEdits([...prevEdits, { url: finalImgURL, saveRes: undefined }]); + } + setLoading(false); + cutPts.current.length = 0; + }; + }; + + /** + * Creates a new collection to put the image edits on. Adds to a new tab on the right if "Create New Collection" is checked. + * @returns + */ + const createNewCollection = () => { + if (!isNewCollection && imageRootDoc) { + // if the parent hasn't been set yet + if (!parentDoc.current) parentDoc.current = imageRootDoc; + } else { + if (!(originalImg.current && imageRootDoc)) return; + // create new collection and add it to the view + newCollectionRef.current = Docs.Create.FreeformDocument([], { + x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, + y: NumCast(imageRootDoc.y), + _width: newCollectionSize, + _height: newCollectionSize, + title: 'Image edit collection', + }); + DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); + // opening new tab + CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + } + }; + + /** + * This function crops an image based on the inputted dimensions. This is used to automatically adjust the images that are + * edited to be smaller than the original (i.e. for cutting into a small part of the image) + */ + const cropImage = (image: HTMLImageElement, minX: number, maxX: number, minY: number, maxY: number) => { + const croppedCanvas = document.createElement('canvas'); + const croppedCtx = croppedCanvas.getContext('2d'); + if (!croppedCtx) return image; + const cropWidth = Math.abs(maxX - minX); + const cropHeight = Math.abs(maxY - minY); + croppedCanvas.width = cropWidth; + croppedCanvas.height = cropHeight; + croppedCtx.globalCompositeOperation = 'source-over'; + croppedCtx.clearRect(0, 0, cropWidth, cropHeight); + croppedCtx.drawImage(image, minX, minY, cropWidth, cropHeight, 0, 0, cropWidth, cropHeight); + const croppedURL = croppedCanvas.toDataURL(); + const croppedImage = new Image(); + croppedImage.src = croppedURL; + return croppedImage; }; // adjusts all the img positions to be aligned @@ -416,7 +492,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }; // creates a new image document and returns its reference - const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean): Promise<Doc | undefined> => { + const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean /*, parent?: Doc */): Promise<Doc | undefined> => { if (!imageRootDoc) return undefined; const { src } = img; const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] }); @@ -495,14 +571,22 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD DocumentView.addViewRenderedCb(newCollectionRef.current, dv => (dv.ComponentView as CollectionFreeFormView)?.fitContentOnce()); } setEdits([]); + setIsFirstDoc(true); }; - return ( - <div className="generativeFillContainer" style={{ display: imageEditorOpen ? 'flex' : 'none' }}> - <div className="generativeFillControls"> + // defines the tools and sets current tool + const genFillTool: ImageEditTool = { type: ImageToolType.GenerativeFill, name: 'Generative Fill', btnText: 'GET EDITS', icon: 'fill', applyFunc: getEdit, sliderMin: 25, sliderMax: 500, sliderDefault: 150 }; + const cutTool: ImageEditTool = { type: ImageToolType.Cut, name: 'Cut', btnText: 'CUT IMAGE', icon: 'scissors', applyFunc: cutImage, sliderMin: 1, sliderMax: 50, sliderDefault: 5 }; + const imageEditTools: ImageEditTool[] = [genFillTool, cutTool]; + const [currTool, setCurrTool] = useState<ImageEditTool>(genFillTool); + + // the top controls for making a new collection, resetting, and applying edits, + function renderControls() { + return ( + <div className="imageEditorTopBar"> <h1>Image Editor</h1> {/* <IconButton text="Cut out" icon={<FontAwesomeIcon icon="scissors" />} /> */} - <div style={{ display: 'flex', alignItems: 'center', gap: '1.5rem' }}> + <div className="imageEditorControls"> <FormControlLabel control={ <Checkbox @@ -518,153 +602,201 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD labelPlacement="end" sx={{ whiteSpace: 'nowrap' }} /> - <EditButtons onClick={getEdit} loading={loading} onReset={handleReset} /> - <CutButtons onClick={cutImage} loading={loading} onReset={handleReset} /> + <ApplyFuncButtons onClick={() => currTool.applyFunc(cutType, cursorData.width, edits, isFirstDoc)} loading={loading} onReset={handleReset} btnText={currTool.btnText} /> <IconButton color={activeColor} tooltip="close" icon={<CgClose size="16px" />} onClick={handleViewClose} /> </div> </div> - {/* Main canvas for editing */} - <div - className="drawingArea" // this only works if pointerevents: none is set on the custom pointer - ref={drawingAreaRef} - onPointerOver={updateCursorData} - onPointerMove={updateCursorData} - onPointerDown={handlePointerDown} - onPointerUp={handlePointerUp}> - <canvas ref={canvasRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} /> - <canvas ref={canvasBackgroundRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} /> - <div - className="pointer" - style={{ - left: cursorData.x, - top: cursorData.y, - width: cursorData.width, - height: cursorData.width, - }}> - <div className="innerPointer" /> - </div> - {/* Icons */} - <div className="iconContainer"> + ); + } + + // the side icons including tool type, the slider, and undo/redo + function renderSideIcons() { + return ( + <div className="sideControlsContainer" style={{ backgroundColor: bgColor }}> + <div className="sideControls"> + <div className="imageToolsContainer"> + {imageEditTools.map(tool => { + return ImageToolButton(tool, tool.type === currTool.type, changeTool); + })} + </div> + {currTool.type == ImageToolType.Cut && ( + <div className="cutToolsContainer"> + <Button + style={{ width: '100%' }} + text="Keep in" + type={Type.TERT} + color={cutType == CutMode.IN ? SettingsManager.userColor : bgColor} + onClick={() => { + setCutType(CutMode.IN); + }} + /> + <Button + style={{ width: '100%' }} + text="Keep out" + type={Type.TERT} + color={cutType == CutMode.OUT ? SettingsManager.userColor : bgColor} + onClick={() => { + setCutType(CutMode.OUT); + }} + /> + <Button + style={{ width: '100%' }} + text="Draw in" + type={Type.TERT} + color={cutType == CutMode.DRAW_IN ? SettingsManager.userColor : bgColor} + onClick={() => { + setCutType(CutMode.DRAW_IN); + }} + /> + <Button + style={{ width: '100%' }} + text="Erase" + type={Type.TERT} + color={cutType == CutMode.ERASE ? SettingsManager.userColor : bgColor} + onClick={() => { + setCutType(CutMode.ERASE); + }} + /> + </div> + )} + <div className="sliderContainer" onPointerDown={e => e.stopPropagation()}> + {currTool.type === ImageToolType.GenerativeFill && ( + <Slider + sx={{ + '& input[type="range"]': { + WebkitAppearance: 'slider-vertical', + }, + }} + orientation="vertical" + min={genFillTool.sliderMin} + max={genFillTool.sliderMax} + defaultValue={genFillTool.sliderDefault} + size="small" + valueLabelDisplay="auto" + onChange={(e, val) => { + setCursorData(prev => ({ ...prev, width: val as number })); + }} + /> + )} + {currTool.type === ImageToolType.Cut && ( + <Slider + sx={{ + '& input[type="range"]': { + WebkitAppearance: 'slider-vertical', + }, + }} + orientation="vertical" + min={cutTool.sliderMin} + max={cutTool.sliderMax} + defaultValue={cutTool.sliderDefault} + size="small" + valueLabelDisplay="auto" + onChange={(e, val) => { + setCursorData(prev => ({ ...prev, width: val as number })); + }} + /> + )} + </div> {/* Undo and Redo */} - <IconButton - style={{ cursor: 'pointer' }} - onPointerDown={e => { - e.stopPropagation(); - handleUndo(); - }} - onPointerUp={e => { - e.stopPropagation(); - }} - color={activeColor} - tooltip="Undo" - icon={<IoMdUndo />} - /> - <IconButton - style={{ cursor: 'pointer' }} - onPointerDown={e => { - e.stopPropagation(); - handleRedo(); - }} - onPointerUp={e => { - e.stopPropagation(); - }} - color={activeColor} - tooltip="Redo" - icon={<IoMdRedo />} - /> - <div onPointerDown={e => e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}> - <Slider - sx={{ - '& input[type="range"]': { - WebkitAppearance: 'slider-vertical', - }, + <div className="undoRedoContainer"> + <IconButton + style={{ cursor: 'pointer' }} + onPointerDown={e => { + e.stopPropagation(); + handleUndo(); }} - orientation="vertical" - min={25} - max={500} - defaultValue={150} - size="small" - valueLabelDisplay="auto" - onChange={(e: any, val: any) => { - setCursorData(prev => ({ ...prev, width: val as number })); + onPointerUp={e => { + e.stopPropagation(); }} + color={activeColor} + tooltip="Undo" + icon={<IoMdUndo />} /> - </div> - <div onPointerDown={e => e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}> - <Slider - sx={{ - '& input[type="range"]': { - WebkitAppearance: 'slider-vertical', - }, + <IconButton + style={{ cursor: 'pointer' }} + onPointerDown={e => { + e.stopPropagation(); + handleRedo(); }} - orientation="vertical" - min={1} - max={500} - defaultValue={150} - size="small" - valueLabelDisplay="auto" - onChange={(e: any, val: any) => { - setCursorData(prev => ({ ...prev, width: val as number })); + onPointerUp={e => { + e.stopPropagation(); }} + color={activeColor} + tooltip="Redo" + icon={<IoMdRedo />} /> </div> </div> - {/* Edits thumbnails */} - <div className="editsBox"> - {edits.map((edit, i) => ( + </div> + ); + } + + // circular pointer for drawing/erasing + function renderPointer() { + return ( + <div + className="pointer" + style={{ + left: cursorData.x, + top: cursorData.y, + width: cursorData.width, + height: cursorData.width, + }}> + <div className="innerPointer" /> + </div> + ); + } + + // the previews for each edit + function renderEditThumbnails() { + return ( + <div className="editsBox"> + {edits.map((edit, i) => ( + <img + // eslint-disable-next-line react/no-array-index-key + key={i} + alt="image edits" + width={75} + src={edit.url} + onClick={async () => { + const img = new Image(); + img.src = edit.url; + ImageUtility.drawImgToCanvas(img, canvasRef, img.width, img.height); + currImg.current = img; + parentDoc.current = edit.saveRes ?? null; + }} + /> + ))} + {/* Original img thumbnail */} + {edits.length > 0 && ( + <div style={{ position: 'relative' }}> + <label className="originalImageLabel">Original</label> <img - // eslint-disable-next-line react/no-array-index-key - key={i} - alt="image edits" + alt="image stuff" width={75} - src={edit.url} - style={{ cursor: 'pointer' }} - onClick={async () => { + src={originalImg.current?.src} + onClick={() => { + if (!originalImg.current) return; const img = new Image(); - img.src = edit.url; + img.src = originalImg.current.src; ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); currImg.current = img; - parentDoc.current = edit.saveRes ?? null; + if (!parentDoc.current) parentDoc.current = originalDoc.current; }} /> - ))} - {/* Original img thumbnail */} - {edits.length > 0 && ( - <div style={{ position: 'relative' }}> - <label - style={{ - position: 'absolute', - bottom: 10, - left: 10, - color: '#ffffff', - fontSize: '0.8rem', - letterSpacing: '1px', - textTransform: 'uppercase', - }}> - Original - </label> - <img - alt="image stuff" - width={75} - src={originalImg.current?.src} - style={{ cursor: 'pointer' }} - onClick={() => { - if (!originalImg.current) return; - const img = new Image(); - img.src = originalImg.current.src; - ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); - currImg.current = img; - parentDoc.current = originalDoc.current; - }} - /> - </div> - )} - </div> + </div> + )} </div> + ); + } + + // the prompt box for generative fill + function renderPromptBox() { + return ( <div> <TextField value={input} - onChange={(e: any) => setInput(e.target.value)} + onChange={e => setInput(e.target.value)} disabled={isBrushing} type="text" label="Prompt" @@ -680,8 +812,29 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }} /> </div> + ); + } + + return ( + <div className="imageEditorContainer" style={{ display: imageEditorOpen ? 'flex' : 'none' }}> + {renderControls()} + {/* Main canvas for editing */} + <div + className="drawingArea" // this only works if pointerevents: none is set on the custom pointer + ref={drawingAreaRef} + onPointerOver={updateCursorData} + onPointerMove={updateCursorData} + onPointerDown={handlePointerDown} + onPointerUp={handlePointerUp}> + <canvas ref={canvasRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} /> + <canvas ref={canvasBackgroundRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} /> + {renderPointer()} + {renderSideIcons()} + {renderEditThumbnails()} + </div> + {currTool.type === ImageToolType.GenerativeFill && renderPromptBox()} </div> ); }; -export default GenerativeFill; +export default ImageEditor; diff --git a/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx b/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx new file mode 100644 index 000000000..cb963616b --- /dev/null +++ b/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx @@ -0,0 +1,69 @@ +import './GenerativeFillButtons.scss'; +import * as React from 'react'; +import ReactLoading from 'react-loading'; +import { Button, IconButton, Type } from 'browndash-components'; +import { AiOutlineInfo } from 'react-icons/ai'; +import { bgColor } from './imageEditorUtils/imageEditorConstants'; +import { ImageEditTool, ImageToolType } from './imageEditorUtils/imageEditorInterfaces'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { SettingsManager } from '../../../util/SettingsManager'; + +interface ButtonContainerProps { + onClick: () => Promise<void>; + loading: boolean; + onReset: () => void; + btnText: string; +} + +export function ApplyFuncButtons({ loading, onClick: getEdit, onReset, btnText }: ButtonContainerProps) { + return ( + <div className="generativeFillBtnContainer"> + <Button text="RESET" type={Type.PRIM} color={SettingsManager.userVariantColor} onClick={onReset} /> + {loading ? ( + <Button + text={btnText} + type={Type.TERT} + color={SettingsManager.userVariantColor} + icon={<ReactLoading type="spin" color="#ffffff" width={20} height={20} />} + iconPlacement="right" + onClick={() => { + if (!loading) getEdit(); + }} + /> + ) : ( + <Button + text={btnText} + type={Type.TERT} + color={SettingsManager.userVariantColor} + onClick={() => { + if (!loading) getEdit(); + }} + /> + )} + <IconButton + type={Type.SEC} + color={SettingsManager.userVariantColor} + tooltip="Open Documentation" + icon={<AiOutlineInfo size="16px" />} + onClick={() => window.open('https://brown-dash.github.io/Dash-Documentation/features/generativeai/#editing', '_blank')} + /> + </div> + ); +} + +export function ImageToolButton(tool: ImageEditTool, isActive: boolean, selectTool: (type: ImageToolType) => void) { + return ( + <div className="imageEditorButtonContainer"> + <Button + style={{ width: '100%' }} + text={tool.name} + type={Type.TERT} + color={isActive ? SettingsManager.userVariantColor : bgColor} + icon={<FontAwesomeIcon icon={tool.icon} />} + onClick={() => { + selectTool(tool.type); + }} + /> + </div> + ); +} diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/BrushHandler.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/BrushHandler.ts new file mode 100644 index 000000000..ed39375e0 --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/BrushHandler.ts @@ -0,0 +1,29 @@ +import { GenerativeFillMathHelpers } from './GenerativeFillMathHelpers'; +import { eraserColor } from './imageEditorConstants'; +import { Point } from './imageEditorInterfaces'; + +export class BrushHandler { + static brushCircleOverlay = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string /* , erase: boolean */) => { + ctx.globalCompositeOperation = 'destination-out'; + ctx.fillStyle = fillColor; + ctx.shadowColor = eraserColor; + ctx.shadowBlur = 5; + ctx.beginPath(); + ctx.arc(x, y, brushRadius, 0, 2 * Math.PI); + ctx.fill(); + ctx.closePath(); + }; + + static createBrushPathOverlay = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string) => { + const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint); + const pts: Point[] = []; + for (let i = 0; i < dist; i += 5) { + const s = i / dist; + const x = startPoint.x * (1 - s) + endPoint.x * s; + const y = startPoint.y * (1 - s) + endPoint.y * s; + pts.push({ x: startPoint.x, y: startPoint.y }); + BrushHandler.brushCircleOverlay(x, y, brushRadius, ctx, fillColor); + } + return pts; + }; +} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/GenerativeFillMathHelpers.ts index 6da8c3da0..f820300f3 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/GenerativeFillMathHelpers.ts @@ -1,4 +1,4 @@ -import { Point } from './generativeFillInterfaces'; +import { Point } from './imageEditorInterfaces'; export class GenerativeFillMathHelpers { static distanceBetween = (p1: Point, p2: Point) => Math.sqrt((p2.x - p1.x) ** 2 + (p2.y - p1.y) ** 2); diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/ImageHandler.ts new file mode 100644 index 000000000..ece0f4d7f --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/ImageHandler.ts @@ -0,0 +1,312 @@ +import { RefObject } from 'react'; +import { bgColor, canvasSize } from './imageEditorConstants'; + +export interface APISuccess { + status: 'success'; + urls: string[]; +} + +export interface APIError { + status: 'error'; + message: string; +} + +export class ImageUtility { + /** + * + * @param canvas Canvas to convert + * @returns Blob of canvas + */ + static canvasToBlob = (canvas: HTMLCanvasElement): Promise<Blob> => + new Promise(resolve => { + canvas.toBlob(blob => { + if (blob) { + resolve(blob); + } + }, 'image/png'); + }); + + // given a square api image, get the cropped img + static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => { + // Create a new canvas element + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + if (ctx) { + // Clear the canvas + ctx.clearRect(0, 0, canvas.width, canvas.height); + if (width < height) { + // horizontal padding, x offset + const xOffset = (canvasSize - width) / 2; + ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - height) / 2; + ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } + return canvas; + } + return undefined; + }; + + // converts an image to a canvas data url + static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise<string> => + new Promise<string>((resolve, reject) => { + const img = new Image(); + img.onload = () => { + const canvas = this.getCroppedImg(img, width, height); + if (canvas) { + const dataUrl = canvas.toDataURL(); + resolve(dataUrl); + } + }; + img.onerror = error => { + reject(error); + }; + img.src = imageSrc; + }); + + // calls the openai api to get image edits + static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise<APISuccess | APIError> => { + const apiUrl = 'https://api.openai.com/v1/images/edits'; + const fd = new FormData(); + fd.append('image', imgBlob, 'image.png'); + fd.append('mask', maskBlob, 'mask.png'); + fd.append('prompt', prompt); + fd.append('size', '1024x1024'); + fd.append('n', n ? JSON.stringify(n) : '1'); + fd.append('response_format', 'b64_json'); + + try { + const res = await fetch(apiUrl, { + method: 'POST', + headers: { + Authorization: `Bearer ${process.env.OPENAI_KEY}`, + }, + body: fd, + }); + const data = await res.json(); + console.log(data.data); + return { + status: 'success', + urls: (data.data as { b64_json: string }[]).map(urlData => `data:image/png;base64,${urlData.b64_json}`), + }; + } catch (err) { + console.log(err); + return { status: 'error', message: 'API error.' }; + } + }; + + // mock api call + static mockGetEdit = async (mockSrc: string): Promise<APISuccess | APIError> => ({ + status: 'success', + urls: [mockSrc, mockSrc, mockSrc], + }); + + // Gets the canvas rendering context of a canvas + static getCanvasContext = (canvasRef: RefObject<HTMLCanvasElement>): CanvasRenderingContext2D | null => { + if (!canvasRef.current) return null; + const ctx = canvasRef.current.getContext('2d'); + if (!ctx) return null; + return ctx; + }; + + // Helper for downloading the canvas (for debugging) + static downloadCanvas = (canvas: HTMLCanvasElement) => { + const url = canvas.toDataURL(); + const downloadLink = document.createElement('a'); + downloadLink.href = url; + downloadLink.download = 'canvas'; + + downloadLink.click(); + downloadLink.remove(); + }; + + // Download the canvas (for debugging) + static downloadImageCanvas = (imgUrl: string) => { + const img = new Image(); + img.src = imgUrl; + img.onload = () => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); + + this.downloadCanvas(canvas); + }; + }; + + // Clears the canvas + static clearCanvas = (canvasRef: React.RefObject<HTMLCanvasElement>) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx || !canvasRef.current) return; + ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); + }; + + // Draws the image to the current canvas + static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject<HTMLCanvasElement>, width: number, height: number) => { + const drawImg = (htmlImg: HTMLImageElement) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx) return; + ctx.globalCompositeOperation = 'source-over'; + ctx.clearRect(0, 0, canvasRef.current?.width || width, canvasRef.current?.height || height); + ctx.drawImage(htmlImg, 0, 0, width, height); + }; + + if (img.complete) { + drawImg(img); + } else { + img.onload = () => { + drawImg(img); + }; + } + }; + + // Gets the image mask for the openai endpoint + static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return undefined; + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.drawImage(paddedCanvas, 0, 0); + + // extract and set padding data + if (srcCanvas.height > srcCanvas.width) { + // horizontal padding, x offset + const xOffset = (canvasSize - srcCanvas.width) / 2; + ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - srcCanvas.height) / 2; + ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height); + } + return canvas; + }; + + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) + static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const { data } = imageData; + for (let i = 0; i < canvas.height; i++) { + for (let j = 0; j < xOffset; j++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = xOffset + (xOffset - j); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let i = 0; i < canvas.height; i++) { + for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) + static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const { data } = imageData; + for (let j = 0; j < canvas.width; j++) { + for (let i = 0; i < yOffset; i++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = yOffset + (yOffset - i); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let j = 0; j < canvas.width; j++) { + for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + // Gets the unaltered (besides filling in padding) version of the image for the api call + static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return undefined; + // fix scaling + const scale = Math.min(canvasSize / img.width, canvasSize / img.height); + const width = Math.floor(img.width * scale); + const height = Math.floor(img.height * scale); + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.fillStyle = bgColor; + ctx.fillRect(0, 0, canvasSize, canvasSize); + + // extract and set padding data + if (img.naturalHeight > img.naturalWidth) { + // horizontal padding, x offset + const xOffset = Math.floor((canvasSize - width) / 2); + ctx.drawImage(img, xOffset, 0, width, height); + + // draw reflected image padding + this.drawHorizontalReflection(ctx, canvas, xOffset); + } else { + // vertical padding, y offset + const yOffset = Math.floor((canvasSize - height) / 2); + ctx.drawImage(img, 0, yOffset, width, height); + + // draw reflected image padding + this.drawVerticalReflection(ctx, canvas, yOffset); + } + return canvas; + }; + + /** + * Converts a url to base64 (tainted canvas workaround) + */ + static urlToBase64 = async (imageUrl: string): Promise<string | undefined> => { + try { + const res = await fetch(imageUrl); + const blob = await res.blob(); + + return new Promise<string>((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const base64Data = reader.result?.toString().split(',')[1]; + if (base64Data) { + resolve(base64Data); + } else { + reject(new Error('Failed to convert.')); + } + }; + reader.onerror = () => { + reject(new Error('Error reading image data')); + }; + reader.readAsDataURL(blob); + }); + } catch (err) { + console.error(err); + } + return undefined; + }; +} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/PointerHandler.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/PointerHandler.ts index 260923a64..e86f46636 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/PointerHandler.ts +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/PointerHandler.ts @@ -1,4 +1,4 @@ -import { Point } from './generativeFillInterfaces'; +import { Point } from './imageEditorInterfaces'; export class PointerHandler { static getPointRelativeToElement = (element: HTMLElement, e: React.PointerEvent | PointerEvent, scale: number): Point => { diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorConstants.ts index 4772304bc..594d6d9fc 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorConstants.ts @@ -3,6 +3,7 @@ export const freeformRenderSize = 300; export const offsetDistanceY = freeformRenderSize + 400; export const offsetX = 200; export const newCollectionSize = 500; +export const brushWidthOffset = 10; export const activeColor = '#1976d2'; export const eraserColor = '#e1e9ec'; diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts new file mode 100644 index 000000000..a14b55439 --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts @@ -0,0 +1,43 @@ +import { IconProp } from '@fortawesome/fontawesome-svg-core'; +import { Doc } from '../../../../../fields/Doc'; + +export interface CursorData { + x: number; + y: number; + width: number; +} + +export interface Point { + x: number; + y: number; +} + +export enum ImageToolType { + GenerativeFill = 'genFill', + Cut = 'cut', +} + +export enum CutMode { + IN, + OUT, + DRAW_IN, + ERASE, +} + +export interface ImageEditTool { + type: ImageToolType; + name: string; + btnText: string; + icon: IconProp; + // this is the function that the image tool applies, so it can be defined depending on the tool + applyFunc: (currCutType: CutMode, brushWidth: number, prevEdits: { url: string; saveRes: Doc | undefined }[], isFirstDoc: boolean) => Promise<void>; + // these optional parameters are here because different tools require different brush sizes and defaults + sliderMin?: number; + sliderMax?: number; + sliderDefault?: number; +} + +export interface ImageDimensions { + width: number; + height: number; +} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts b/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts index 8a66d7347..7139bebc3 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts +++ b/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts @@ -1,6 +1,6 @@ -import { GenerativeFillMathHelpers } from './GenerativeFillMathHelpers'; -import { eraserColor } from './generativeFillConstants'; -import { Point } from './generativeFillInterfaces'; +import { GenerativeFillMathHelpers } from '../imageEditorUtils/GenerativeFillMathHelpers'; +import { eraserColor } from '../imageEditorUtils/imageEditorConstants'; +import { Point } from '../imageEditorUtils/imageEditorInterfaces'; import { points } from '@turf/turf'; export enum BrushType { diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts index 24dba1778..b9723b5be 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts +++ b/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts @@ -1,5 +1,5 @@ import { RefObject } from 'react'; -import { bgColor, canvasSize } from './generativeFillConstants'; +import { bgColor, canvasSize } from '../imageEditorUtils/imageEditorConstants'; export interface APISuccess { status: 'success'; diff --git a/src/client/views/pdf/AnchorMenu.tsx b/src/client/views/pdf/AnchorMenu.tsx index 000711c19..11f2f7988 100644 --- a/src/client/views/pdf/AnchorMenu.tsx +++ b/src/client/views/pdf/AnchorMenu.tsx @@ -153,11 +153,12 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { this.AddDrawingAnnotation(drawing); const docData = drawing[DocData]; docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; - docData.drawingInput = opts.text; - docData.drawingComplexity = opts.complexity; - docData.drawingColored = opts.autoColor; - docData.drawingSize = opts.size; - docData.drawingData = gptRes; + docData.ai_drawing_input = opts.text; + docData.ai_drawing_complexity = opts.complexity; + docData.ai_drawing_colored = opts.autoColor; + docData.ai_drawing_size = opts.size; + docData.ai_drawing_data = gptRes; + docData.ai = 'gpt'; }); pointerDown = (e: React.PointerEvent) => { diff --git a/src/client/views/smartdraw/AnnotationPalette.tsx b/src/client/views/smartdraw/AnnotationPalette.tsx deleted file mode 100644 index 78833f7a2..000000000 --- a/src/client/views/smartdraw/AnnotationPalette.tsx +++ /dev/null @@ -1,361 +0,0 @@ -import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { Slider, Switch } from '@mui/material'; -import { Button } from '@dash/components'; -import { action, makeObservable, observable } from 'mobx'; -import { observer } from 'mobx-react'; -import * as React from 'react'; -import { AiOutlineSend } from 'react-icons/ai'; -import ReactLoading from 'react-loading'; -import { returnEmptyFilter, returnFalse, returnTrue } from '../../../ClientUtils'; -import { emptyFunction } from '../../../Utils'; -import { Doc, DocListCast, returnEmptyDoclist } from '../../../fields/Doc'; -import { DocData } from '../../../fields/DocSymbols'; -import { ImageCast, NumCast } from '../../../fields/Types'; -import { ImageField } from '../../../fields/URLField'; -import { DocumentType } from '../../documents/DocumentTypes'; -import { Docs } from '../../documents/Documents'; -import { makeUserTemplateButtonOrImage } from '../../util/DropConverter'; -import { SettingsManager } from '../../util/SettingsManager'; -import { Transform } from '../../util/Transform'; -import { undoBatch } from '../../util/UndoManager'; -import { ObservableReactComponent } from '../ObservableReactComponent'; -import { DefaultStyleProvider, returnEmptyDocViewList } from '../StyleProvider'; -import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; -import { FieldView } from '../nodes/FieldView'; -import './AnnotationPalette.scss'; -import { DrawingOptions, SmartDrawHandler } from './SmartDrawHandler'; - -interface AnnotationPaletteProps { - Document: Doc; -} - -/** - * The AnnotationPalette can be toggled in the lightbox view of a document. The goal of the palette - * is to offer an easy way for users to save then drag and drop repeated annotations onto a document. - * These annotations can be of any annotation type and operate similarly to user templates. - * - * On the "add" side of the palette, there is a way to create a drawing annotation with GPT. Users can - * enter the item to draw, toggle different settings, then GPT will generate three versions of the drawing - * to choose from. These drawings can then be saved to the palette as annotations. - */ -@observer -export class AnnotationPalette extends ObservableReactComponent<AnnotationPaletteProps> { - @observable private _paletteMode: 'create' | 'view' = 'view'; - @observable private _userInput: string = ''; - @observable private _isLoading: boolean = false; - @observable private _canInteract: boolean = true; - @observable private _showRegenerate: boolean = false; - @observable private _docView: DocumentView | null = null; - @observable private _docCarouselView: DocumentView | null = null; - @observable private _opts: DrawingOptions = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; - private _gptRes: string[] = []; - - constructor(props: AnnotationPaletteProps) { - super(props); - makeObservable(this); - } - - public static LayoutString(fieldKey: string) { - return FieldView.LayoutString(AnnotationPalette, fieldKey); - } - - Contains = (view: DocumentView) => { - return (this._docView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docView)) || (this._docCarouselView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docCarouselView)); - }; - - return170 = () => 170; - - @action - handleKeyPress = async (event: React.KeyboardEvent) => { - if (event.key === 'Enter') { - await this.generateDrawings(); - } - }; - - @action - setPaletteMode = (mode: 'create' | 'view') => { - this._paletteMode = mode; - }; - - @action - setUserInput = (input: string) => { - if (!this._isLoading) this._userInput = input; - }; - - @action - setDetail = (detail: number) => { - if (this._canInteract) this._opts.complexity = detail; - }; - - @action - setColor = (autoColor: boolean) => { - if (this._canInteract) this._opts.autoColor = autoColor; - }; - - @action - setSize = (size: number) => { - if (this._canInteract) this._opts.size = size; - }; - - @action - resetPalette = (changePaletteMode: boolean) => { - if (changePaletteMode) this.setPaletteMode('view'); - this.setUserInput(''); - this.setDetail(5); - this.setColor(true); - this.setSize(200); - this._showRegenerate = false; - this._canInteract = true; - this._opts = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; - this._gptRes = []; - this._props.Document[DocData].data = undefined; - }; - - /** - * Adds a doc to the annotation palette. Gets a snapshot of the document to use as a preview in the palette. When this - * preview is dragged onto a parent document, a copy of that document is added as an annotation. - */ - public static addToPalette = async (doc: Doc) => { - if (!doc.savedAsAnno) { - const docView = DocumentView.getDocumentView(doc); - await docView?.ComponentView?.updateIcon?.(true); - const { clone } = await Doc.MakeClone(doc); - clone.title = doc.title; - const image = ImageCast(doc.icon, ImageCast(clone[Doc.LayoutFieldKey(clone)]))?.url?.href; - Doc.AddDocToList(Doc.MyAnnos, 'data', makeUserTemplateButtonOrImage(clone, image)); - doc.savedAsAnno = true; - } - }; - - public static getIcon(group: Doc) { - const docView = DocumentView.getDocumentView(group); - if (docView) { - docView.ComponentView?.updateIcon?.(true); - return new Promise<ImageField | undefined>(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); - } - return undefined; - } - - /** - * Calls the draw with GPT functions in SmartDrawHandler to allow users to generate drawings straight from - * the annotation palette. - */ - @undoBatch - generateDrawings = action(async () => { - this._isLoading = true; - this._props.Document[DocData].data = undefined; - for (let i = 0; i < 3; i++) { - try { - SmartDrawHandler.Instance.AddDrawing = this.addDrawing; - this._canInteract = false; - if (this._showRegenerate) { - await SmartDrawHandler.Instance.regenerate(this._opts, this._gptRes[i], this._userInput); - } else { - await SmartDrawHandler.Instance.drawWithGPT({ X: 0, Y: 0 }, this._userInput, this._opts.complexity, this._opts.size, this._opts.autoColor); - } - } catch (e) { - console.log('Error generating drawing', e); - } - } - this._opts.text !== '' ? (this._opts.text = `${this._opts.text} ~~~ ${this._userInput}`) : (this._opts.text = this._userInput); - this._userInput = ''; - this._isLoading = false; - this._showRegenerate = true; - }); - - @action - addDrawing = (drawing: Doc, opts: DrawingOptions, gptRes: string) => { - this._gptRes.push(gptRes); - drawing[DocData].freeform_fitContentsToBox = true; - Doc.AddDocToList(this._props.Document, 'data', drawing); - }; - - /** - * Saves the currently showing, newly generated drawing to the annotation palette and sets the metadata. - * AddToPalette() is generically used to add any document to the palette, while this defines the behavior for when a user - * presses the "save drawing" button. - */ - saveDrawing = async () => { - const cIndex = NumCast(this._props.Document.carousel_index); - const focusedDrawing = DocListCast(this._props.Document.data)[cIndex]; - const docData = focusedDrawing[DocData]; - docData.title = this._opts.text.match(/^(.*?)~~~.*$/)?.[1] || this._opts.text; - docData.drawingInput = this._opts.text; - docData.drawingComplexity = this._opts.complexity; - docData.drawingColored = this._opts.autoColor; - docData.drawingSize = this._opts.size; - docData.drawingData = this._gptRes[cIndex]; - docData.width = this._opts.size; - docData.x = this._opts.x; - docData.y = this._opts.y; - await AnnotationPalette.addToPalette(focusedDrawing); - this.resetPalette(true); - }; - - render() { - return ( - <div className="annotation-palette" style={{ zIndex: 1000 }} onClick={e => e.stopPropagation()}> - {this._paletteMode === 'view' && ( - <> - <DocumentView - ref={r => (this._docView = r)} - Document={Doc.MyAnnos} - addDocument={undefined} - addDocTab={DocumentViewInternal.addDocTabFunc} - pinToPres={DocumentView.PinDoc} - containerViewPath={returnEmptyDocViewList} - styleProvider={DefaultStyleProvider} - removeDocument={returnFalse} - ScreenToLocalTransform={Transform.Identity} - PanelWidth={this.return170} - PanelHeight={this.return170} - renderDepth={0} - isContentActive={returnTrue} - focus={emptyFunction} - whenChildContentsActiveChanged={emptyFunction} - childFilters={returnEmptyFilter} - childFiltersByRanges={returnEmptyFilter} - searchFilterDocs={returnEmptyDoclist} - /> - <Button text="Add" icon={<FontAwesomeIcon icon="square-plus" />} color={SettingsManager.userColor} onClick={() => this.setPaletteMode('create')} /> - </> - )} - {this._paletteMode === 'create' && ( - <> - <div className="palette-create"> - <input - className="palette-create-input" - aria-label="label-input" - id="new-label" - type="text" - value={this._userInput} - onChange={e => { - this.setUserInput(e.target.value); - }} - placeholder={this._showRegenerate ? '(Optional) Enter edits' : 'Enter item to draw'} - onKeyDown={this.handleKeyPress} - /> - <Button - style={{ alignSelf: 'flex-end' }} - tooltip={this._showRegenerate ? 'Regenerate' : 'Send'} - icon={this._isLoading ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : this._showRegenerate ? <FontAwesomeIcon icon={'rotate'} /> : <AiOutlineSend />} - iconPlacement="right" - color={SettingsManager.userColor} - onClick={this.generateDrawings} - /> - </div> - <div className="palette-create-options"> - <div className="palette-color"> - Color - <Switch - sx={{ - '& .MuiSwitch-switchBase.Mui-checked': { - color: SettingsManager.userColor, - }, - '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { - backgroundColor: SettingsManager.userVariantColor, - }, - }} - defaultChecked={true} - value={this._opts.autoColor} - size="small" - onChange={() => this.setColor(!this._opts.autoColor)} - /> - </div> - <div className="palette-detail"> - Detail - <Slider - sx={{ - '& .MuiSlider-thumb': { - color: SettingsManager.userColor, - '&.Mui-focusVisible, &:hover, &.Mui-active': { - boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}10`, - }, - }, - '& .MuiSlider-track': { - color: SettingsManager.userVariantColor, - }, - '& .MuiSlider-rail': { - color: SettingsManager.userColor, - }, - }} - style={{ width: '80%' }} - min={1} - max={10} - step={1} - size="small" - value={this._opts.complexity} - onChange={(e, val) => { - this.setDetail(val as number); - }} - valueLabelDisplay="auto" - /> - </div> - <div className="palette-size"> - Size - <Slider - sx={{ - '& .MuiSlider-thumb': { - color: SettingsManager.userColor, - '&.Mui-focusVisible, &:hover, &.Mui-active': { - boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}20`, - }, - }, - '& .MuiSlider-track': { - color: SettingsManager.userVariantColor, - }, - '& .MuiSlider-rail': { - color: SettingsManager.userColor, - }, - }} - style={{ width: '80%' }} - min={50} - max={500} - step={10} - size="small" - value={this._opts.size} - onChange={(e, val) => { - this.setSize(val as number); - }} - valueLabelDisplay="auto" - /> - </div> - </div> - <DocumentView - ref={r => (this._docCarouselView = r)} - Document={this._props.Document} - addDocument={undefined} - addDocTab={DocumentViewInternal.addDocTabFunc} - pinToPres={DocumentView.PinDoc} - containerViewPath={returnEmptyDocViewList} - styleProvider={DefaultStyleProvider} - removeDocument={returnFalse} - ScreenToLocalTransform={Transform.Identity} - PanelWidth={this.return170} - PanelHeight={this.return170} - renderDepth={1} - isContentActive={returnTrue} - focus={emptyFunction} - whenChildContentsActiveChanged={emptyFunction} - childFilters={returnEmptyFilter} - childFiltersByRanges={returnEmptyFilter} - searchFilterDocs={returnEmptyDoclist} - /> - <div className="palette-buttons"> - <Button text="Back" tooltip="Back to All Annotations" icon={<FontAwesomeIcon icon="reply" />} color={SettingsManager.userColor} onClick={() => this.resetPalette(true)} /> - <div className="palette-save-reset"> - <Button tooltip="Save" icon={<FontAwesomeIcon icon="file-arrow-down" />} color={SettingsManager.userColor} onClick={this.saveDrawing} /> - <Button tooltip="Reset" icon={<FontAwesomeIcon icon="rotate-left" />} color={SettingsManager.userColor} onClick={() => this.resetPalette(false)} /> - </div> - </div> - </> - )} - </div> - ); - } -} - -Docs.Prototypes.TemplateMap.set(DocumentType.ANNOPALETTE, { - layout: { view: AnnotationPalette, dataField: 'data' }, - options: { acl: '' }, -}); diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx new file mode 100644 index 000000000..8e41ee105 --- /dev/null +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -0,0 +1,45 @@ +import { imageUrlToBase64 } from '../../../ClientUtils'; +import { Doc } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; +import { ImageCast } from '../../../fields/Types'; +import { Upload } from '../../../server/SharedMediaTypes'; +import { gptDescribeImage } from '../../apis/gpt/GPT'; +import { Docs } from '../../documents/Documents'; +import { Networking } from '../../Network'; +import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; +import { OpenWhere } from '../nodes/OpenWhere'; +import { AspectRatioLimits, FireflyDimensionsMap, FireflyImageDimensions, FireflyStylePresets } from './FireflyConstants'; + +export class DrawingFillHandler { + static drawingToImage = (drawing: Doc, strength: number, prompt: string) => { + const docData = drawing[DocData]; + const tags: string[] = ((docData?.tags as unknown as string[]) ?? []).map(tag => tag.slice(1)) ?? []; + const styles = tags.filter(tag => FireflyStylePresets.has(tag)); + DocumentView.GetDocImage(drawing)?.then(imageField => { + if (imageField) { + const aspectRatio = (drawing.width as number) / (drawing.height as number); + let dims: { width: number; height: number }; + if (aspectRatio > AspectRatioLimits[FireflyImageDimensions.Widescreen]) { + dims = FireflyDimensionsMap[FireflyImageDimensions.Widescreen]; + } else if (aspectRatio > AspectRatioLimits[FireflyImageDimensions.Landscape]) { + dims = FireflyDimensionsMap[FireflyImageDimensions.Landscape]; + } else if (aspectRatio < AspectRatioLimits[FireflyImageDimensions.Portrait]) { + dims = FireflyDimensionsMap[FireflyImageDimensions.Portrait]; + } else { + dims = FireflyDimensionsMap[FireflyImageDimensions.Square]; + } + const { href } = ImageCast(imageField).url; + const hrefParts = href.split('.'); + const structureUrl = `${hrefParts.slice(0, -1).join('.')}_o.${hrefParts.lastElement()}`; + imageUrlToBase64(structureUrl) + .then((hrefBase64: string) => gptDescribeImage(hrefBase64)) + .then((prompt: string) => { + Networking.PostToServer('/queryFireflyImageFromStructure', { prompt: prompt, width: dims.width, height: dims.height, structureUrl: structureUrl, strength: strength, styles: styles }).then((info: Upload.ImageInformation) => + DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { ai: 'firefly', ai_firefly_prompt: prompt, nativeWidth: dims.width, nativeHeight: dims.height }), OpenWhere.addRight) + ); // prettier-ignore + }); + } + return false; + }); + }; +} diff --git a/src/client/views/smartdraw/FireflyConstants.ts b/src/client/views/smartdraw/FireflyConstants.ts new file mode 100644 index 000000000..1f1781617 --- /dev/null +++ b/src/client/views/smartdraw/FireflyConstants.ts @@ -0,0 +1,49 @@ +export interface FireflyImageData { + prompt: string; + seed: number | undefined; + pathname: string; + href?: string; +} + +export enum FireflyImageDimensions { + Square = 'square', + Landscape = 'landscape', + Portrait = 'portrait', + Widescreen = 'widescreen', +} + +export const FireflyDimensionsMap = { + square: { width: 2048, height: 2048 }, + landscape: { width: 2304, height: 1792 }, + portrait: { width: 1792, height: 2304 }, + widescreen: { width: 2688, height: 1536 }, +}; + +export const AspectRatioLimits = { + square: 1, + landscape: 1.167, + portrait: 0.875, + widescreen: 1.472, +}; + +// prettier-ignore +export const FireflyStylePresets = + new Set<string>(['graphic', 'wireframe', + 'vector_look','bw','cool_colors','golden','monochromatic','muted_color','toned_image','vibrant_colors','warm_tone','closeup', + 'knolling','landscape_photography','macrophotography','photographed_through_window','shallow_depth_of_field','shot_from_above', + 'shot_from_below','surface_detail','wide_angle','beautiful','bohemian','chaotic','dais','divine','eclectic','futuristic','kitschy', + 'nostalgic','simple','antique_photo','bioluminescent','bokeh','color_explosion','dark','faded_image','fisheye','gomori_photography', + 'grainy_film','iridescent','isometric','misty','neon','otherworldly_depiction','ultraviolet','underwater', 'backlighting', + 'dramatic_light', 'golden_hour', 'harsh_light','long','low_lighting','multiexposure','studio_light','surreal_lighting', + '3d_patterns','charcoal','claymation','fabric','fur','guilloche_patterns','layered_paper','marble_sculpture','made_of_metal', + 'origami','paper_mache','polka','strange_patterns','wood_carving','yarn','art_deco','art_nouveau','baroque','bauhaus', + 'constructivism','cubism','cyberpunk','fantasy','fauvism', 'film_noir','glitch_art','impressionism','industrialism','maximalism', + 'minimalism','modern_art','modernism','neo','pointillism','psychedelic','science_fiction','steampunk','surrealism','synthetism', + 'synthwave','vaporwave','acrylic_paint','bold_lines','chiaroscuro','color_shift_art','daguerreotype','digital_fractal', + 'doodle_drawing','double_exposure_portrait','fresco','geometric_pen','halftone','ink','light_painting','line_drawing','linocut', + 'oil_paint','paint_spattering','painting','palette_knife','photo_manipulation','scribble_texture','sketch','splattering', + 'stippling_drawing','watercolor','3d','anime','cartoon','cinematic','comic_book','concept_art','cyber_matrix','digital_art', + 'flat_design','geometric','glassmorphism','glitch_graphic','graffiti','hyper_realistic','interior_design','line_gradient', + 'low_poly','newspaper_collage','optical_illusion','pattern_pixel','pixel_art','pop_art','product_photo','psychedelic_background', + 'psychedelic_wonderland','scandinavian','splash_images','stamp','trompe_loeil' + ]); diff --git a/src/client/views/smartdraw/SmartDrawHandler.scss b/src/client/views/smartdraw/SmartDrawHandler.scss index 0e8bd3349..cca7d77c7 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.scss +++ b/src/client/views/smartdraw/SmartDrawHandler.scss @@ -1,44 +1,77 @@ .smart-draw-handler { position: absolute; -} - -.smartdraw-input { - color: black; -} -.smartdraw-options { - display: flex; - flex-direction: row; - justify-content: space-around; - - .auto-color { + .smart-draw-main { display: flex; - flex-direction: column; - justify-content: center; - width: 30%; + flex-direction: row; + + .smartdraw-input { + color: black; + margin: auto; + } } - .complexity { + .smartdraw-output-options { display: flex; - flex-direction: column; + flex-direction: row; justify-content: center; - width: 31%; + margin-top: 3px; } - .size { + .smartdraw-options-container { + width: 265px; + padding: 5px; + font-weight: bolder; + text-align: center; display: flex; flex-direction: column; - justify-content: center; - width: 39%; - .size-slider { - width: 80%; + .smartdraw-options { + font-weight: normal; + margin-top: 5px; + display: flex; + flex-direction: row; + justify-content: space-around; + + .smartdraw-auto-color { + display: flex; + flex-direction: column; + align-items: center; + width: 30%; + margin-top: 3px; + } + + .smartdraw-complexity { + display: flex; + flex-direction: column; + align-items: center; + width: 31%; + margin-top: 3px; + } + + .smartdraw-size { + display: flex; + flex-direction: column; + align-items: center; + width: 39%; + margin-top: 3px; + } + } + + .smartdraw-dimensions { + font-weight: normal; + margin-top: 7px; + padding-left: 25px; } } -} -.regenerate-box, -.edit-box { - display: flex; - flex-direction: row; + .smartdraw-slider { + width: 65px; + } + + .regenerate-box, + .edit-box { + display: flex; + flex-direction: row; + } } diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index f6a8d7253..e9d88b315 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -1,5 +1,5 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { Slider, Switch } from '@mui/material'; +import { Checkbox, FormControlLabel, Radio, RadioGroup, Slider, Switch } from '@mui/material'; import { Button, IconButton } from '@dash/components'; import { action, makeObservable, observable, runInAction } from 'mobx'; import { observer } from 'mobx-react'; @@ -13,7 +13,6 @@ import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { InkData, InkField, InkTool } from '../../../fields/InkField'; import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; -import { Networking } from '../../Network'; import { GPTCallType, gptAPICall, gptDrawingColor } from '../../apis/gpt/GPT'; import { Docs } from '../../documents/Documents'; import { SettingsManager } from '../../util/SettingsManager'; @@ -22,9 +21,12 @@ import { SVGToBezier, SVGType } from '../../util/bezierFit'; import { InkingStroke } from '../InkingStroke'; import { ObservableReactComponent } from '../ObservableReactComponent'; import { MarqueeView } from '../collections/collectionFreeForm'; -import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveInkColor, ActiveInkDash, ActiveInkFillColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; -import { OpenWhere } from '../nodes/OpenWhere'; +import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkDash, ActiveInkFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import './SmartDrawHandler.scss'; +import { Networking } from '../../Network'; +import { OpenWhere } from '../nodes/OpenWhere'; +import { FireflyDimensionsMap, FireflyImageDimensions, FireflyImageData } from './FireflyConstants'; +import { DocumentType } from '../../documents/DocumentTypes'; export interface DrawingOptions { text: string; @@ -57,22 +59,28 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; private _lastResponse: string = ''; - private _selectedDoc: Doc | undefined = undefined; + private _selectedDocs: Doc[] = []; private _errorOccurredOnce = false; @observable private _display: boolean = false; @observable private _pageX: number = 0; @observable private _pageY: number = 0; + @observable private _scale: number = 0; @observable private _yRelativeToTop: boolean = true; @observable private _isLoading: boolean = false; + @observable private _userInput: string = ''; + @observable private _regenInput: string = ''; @observable private _showOptions: boolean = false; @observable private _showEditBox: boolean = false; @observable private _complexity: number = 5; @observable private _size: number = 200; @observable private _autoColor: boolean = true; - @observable private _regenInput: string = ''; + @observable private _imgDims: FireflyImageDimensions = FireflyImageDimensions.Square; + @observable private _canInteract: boolean = true; + @observable private _generateDrawing: boolean = true; + @observable private _generateImage: boolean = true; @observable public ShowRegenerate: boolean = false; @@ -84,8 +92,8 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { /** * AddDrawing and RemoveDrawing are defined by the other classes that call the smart draw functions (i.e. - CollectionFreeForm, FormattedTextBox, AnnotationPalette) to define how a drawing document should be added - or removed in their respective locations (to the freeform canvs, to the annotation palette's preview, etc.) + CollectionFreeForm, FormattedTextBox, StickerPalette) to define how a drawing document should be added + or removed in their respective locations (to the freeform canvas, to the sticker palette's preview, etc.) */ public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction; public RemoveDrawing: (useLastContainer: boolean, doc?: Doc) => void = unimplementedFunction; @@ -124,9 +132,10 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { }; @action - displaySmartDrawHandler = (x: number, y: number) => { + displaySmartDrawHandler = (x: number, y: number, scale: number) => { [this._pageX, this._pageY] = [x, y]; this._display = true; + this._scale = scale; }; /** @@ -136,14 +145,14 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { */ @action displayRegenerate = (x: number, y: number) => { - this._selectedDoc = DocumentView.SelectedDocs()?.lastElement(); + this._selectedDocs = [DocumentView.SelectedDocs()?.lastElement()]; [this._pageX, this._pageY] = [x, y]; this._display = false; this.ShowRegenerate = true; this._showEditBox = false; - const docData = this._selectedDoc[DocData]; + const docData = this._selectedDocs[0][DocData]; this._lastResponse = StrCast(docData.drawingData); - this._lastInput = { text: StrCast(docData.drawingInput), complexity: NumCast(docData.drawingComplexity), size: NumCast(docData.drawingSize), autoColor: BoolCast(docData.drawingColored), x: this._pageX, y: this._pageY }; + this._lastInput = { text: StrCast(docData.ai_drawing_input), complexity: NumCast(docData.ai_drawing_complexity), size: NumCast(docData.ai_drawing_size), autoColor: BoolCast(docData.ai_drawing_colored), x: this._pageX, y: this._pageY }; }; /** @@ -193,11 +202,13 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { */ @action handleSendClick = async () => { + if ((!this.ShowRegenerate && this._userInput == '') || (!this._generateImage && !this._generateDrawing)) return; this._isLoading = true; this._canInteract = false; if (this.ShowRegenerate) { - await this.regenerate(); + await this.regenerate(this._selectedDocs); runInAction(() => { + this._selectedDocs = []; this._regenInput = ''; this._showEditBox = false; }); @@ -206,7 +217,12 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { this._showOptions = false; }); try { - await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + if (this._generateImage) { + await this.createImageWithFirefly(this._userInput); + } + if (this._generateDrawing) { + await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + } this.hideSmartDrawHandler(); runInAction(() => { @@ -218,7 +234,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { this._errorOccurredOnce = false; } else { this._errorOccurredOnce = true; - await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + await this.handleSendClick(); } } } @@ -231,60 +247,99 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { /** * Calls GPT API to create a drawing based on user input. */ - @action - drawWithGPT = (startPt: { X: number; Y: number }, prompt: string, complexity: number, size: number, autoColor: boolean) => { - if (prompt === '') return; - this._lastInput = { text: prompt, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; - - Networking.PostToServer('/queryFireflyImage', { prompt }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, { title: prompt }), OpenWhere.addRight)); - - const result = gptAPICall(`"${prompt}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true).then(res => - this.parseSvg(res, startPt, false, autoColor).then(strokeData => { + drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { + if (input) { + this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; + const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); + if (res) { + const strokeData = await this.parseSvg(res, startPt, false, autoColor); const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - }) - ); + drawingDoc && this._selectedDocs.push(drawingDoc); + this._errorOccurredOnce = false; + return strokeData; + } else { + console.error('GPT call failed'); + } + } + return undefined; + }; - this._errorOccurredOnce = false; - return result; + /** + * Calls Firefly API to create an image based on user input + */ + createImageWithFirefly = (input: string, seed?: number, changeInPlace?: boolean): Promise<FireflyImageData> => { + this._lastInput.text = input; + const dims = FireflyDimensionsMap[this._imgDims]; + return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed: seed }).then(img => { + if (!changeInPlace) { + const aiseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1]; + const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { + title: input.match(/^(.*?)~~~.*$/)?.[1] || input, + nativeWidth: dims.width, + nativeHeight: dims.height, + ai: 'firefly', + ai_firefly_seed: aiseed, + ai_firefly_prompt: input, + }); + DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); + this._selectedDocs.push(imgDoc); + } + return { prompt: input, seed, pathname: img.accessPaths.agnostic.client }; + }); }; /** * Regenerates drawings with the option to add a specific regenerate prompt/request. + * @param doc the drawing Docs to regenerate */ @action - regenerate = async (lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string) => { + regenerate = (drawingDocs: Doc[], lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string, changeInPlace?: boolean) => { if (lastInput) this._lastInput = lastInput; if (lastResponse) this._lastResponse = lastResponse; if (regenInput) this._regenInput = regenInput; - - try { - let res; - if (this._regenInput !== '') { - const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; - res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); - this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; - } else { - res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); - } - if (!res) { - console.error('GPT call failed'); - return; - } - const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); - this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); - drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - return strokeData; - } catch (err) { - console.error('Error regenerating drawing', err); - } + return Promise.all( + drawingDocs.map(async doc => { + switch (doc.type) { + case DocumentType.IMG: + if (this._regenInput) { + // if (this._selectedDoc) { + const newPrompt = `${doc.ai_firefly_prompt} ~~~ ${this._regenInput}`; + return this.createImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed), changeInPlace); + // } + } + return this.createImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), undefined, changeInPlace); + case DocumentType.COL: { + try { + let res; + if (this._regenInput) { + const prompt = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; + res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); + this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + } else { + res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + } + if (res) { + const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, doc); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + } else { + console.error('GPT call failed'); + } + } catch (err) { + console.error('Error regenerating drawing', err); + } + break; + } + } + }) + ); }; /** * Parses the svg code that GPT returns into Bezier curves, with coordinates and colors. */ - @action parseSvg = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => { const svg = res.match(/<svg[^>]*>([\s\S]*?)<\/svg>/g); if (svg) { @@ -295,7 +350,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { svgStrokes.forEach(child => { const convertedBezier: InkData = SVGToBezier(child.name as SVGType, child.attributes); strokeData.push([ - convertedBezier.map(point => ({ X: point.X + startPoint.X - this._size / 1.5, Y: point.Y + startPoint.Y - this._size / 2 })), + convertedBezier.map(point => ({ X: startPoint.X + (point.X - startPoint.X) * this._scale, Y: startPoint.Y + (point.Y - startPoint.Y) * this._scale })), (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.stroke : '', (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.fill : '', ]); @@ -345,10 +400,110 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { }); }, 'color strokes'); - renderDisplay() { + renderGenerateOutputOptions = () => ( + <div className="smartdraw-output-options"> + <div className="drawing-checkbox"> + Generate Ink + <Checkbox + sx={{ + color: 'white', + '&.Mui-checked': { + color: SettingsManager.userVariantColor, + }, + }} + checked={this._generateDrawing} + onChange={() => this._canInteract && (this._generateDrawing = !this._generateDrawing)} + /> + </div> + <div className="image-checkbox"> + Generate Image + <Checkbox + sx={{ + color: 'white', + '&.Mui-checked': { + color: SettingsManager.userVariantColor, + }, + }} + checked={this._generateImage} + onChange={() => this._canInteract && (this._generateImage = !this._generateImage)} + /> + </div> + </div> + ); + + renderGenerateDrawing = () => ( + <div className="smartdraw-options-container"> + Drawing Options + <div className="smartdraw-options"> + <div className="smartdraw-auto-color"> + Auto color + <Switch + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: SettingsManager.userColor }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { backgroundColor: SettingsManager.userVariantColor }, + }} + defaultChecked={true} + value={this._autoColor} + size="small" + onChange={action(() => this._canInteract && (this._autoColor = !this._autoColor))} + /> + </div> + <div className="smartdraw-complexity"> + Complexity + <Slider + className="smartdraw-slider" + sx={{ + '& .MuiSlider-track': { color: SettingsManager.userVariantColor }, + '& .MuiSlider-rail': { color: SettingsManager.userColor }, + '& .MuiSlider-thumb': { color: SettingsManager.userColor, '&.Mui-focusVisible, &:hover, &.Mui-active': { boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}10` } }, + }} + min={1} + max={10} + step={1} + size="small" + value={this._complexity} + onChange={action((e, val) => this._canInteract && (this._complexity = val as number))} + valueLabelDisplay="auto" + /> + </div> + <div className="smartdraw-size"> + Size (in pixels) + <Slider + className="smartdraw-slider" + sx={{ + '& .MuiSlider-track': { color: SettingsManager.userVariantColor }, + '& .MuiSlider-rail': { color: SettingsManager.userColor }, + '& .MuiSlider-thumb': { color: SettingsManager.userColor, '&.Mui-focusVisible, &:hover, &.Mui-active': { boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}20` } }, + }} + min={50} + max={700} + step={10} + size="small" + value={this._size} + onChange={action((e, val) => this._canInteract && (this._size = val as number))} + valueLabelDisplay="auto" + /> + </div> + </div> + </div> + ); + + renderGenerateImage = () => ( + <div className="smartdraw-options-container"> + Image Options + <div className="smartdraw-dimensions"> + <RadioGroup row defaultValue="square" sx={{ alignItems: 'center' }}> + {Object.values(FireflyImageDimensions).map(dim => ( + <FormControlLabel sx={{ width: '40%' }} key={dim} value={dim} control={<Radio />} onChange={() => this._canInteract && (this._imgDims = dim)} label={dim} /> + ))} + </RadioGroup> + </div> + </div> + ); + + renderDisplay = () => { return ( <div - id="label-handler" className="smart-draw-handler" style={{ display: this._display ? '' : 'none', @@ -357,7 +512,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { background: SettingsManager.userBackgroundColor, color: SettingsManager.userColor, }}> - <div> + <div className="smart-draw-main"> <IconButton tooltip="Cancel" onClick={() => { @@ -388,107 +543,65 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { /> </div> {this._showOptions && ( - <div className="smartdraw-options"> - <div className="auto-color"> - Auto color - <Switch - sx={{ - '& .MuiSwitch-switchBase.Mui-checked': { color: SettingsManager.userColor }, - '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { backgroundColor: SettingsManager.userVariantColor }, - }} - defaultChecked={true} - value={this._autoColor} - size="small" - onChange={action(() => this._canInteract && (this._autoColor = !this._autoColor))} - /> - </div> - <div className="complexity"> - Complexity - <Slider - sx={{ - '& .MuiSlider-track': { color: SettingsManager.userVariantColor }, - '& .MuiSlider-rail': { color: SettingsManager.userColor }, - '& .MuiSlider-thumb': { color: SettingsManager.userColor, '&.Mui-focusVisible, &:hover, &.Mui-active': { boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}10` } }, - }} - style={{ width: '80%' }} - min={1} - max={10} - step={1} - size="small" - value={this._complexity} - onChange={action((e, val) => this._canInteract && (this._complexity = val as number))} - valueLabelDisplay="auto" - /> - </div> - <div className="size"> - Size (in pixels) - <Slider - className="size-slider" - sx={{ - '& .MuiSlider-track': { color: SettingsManager.userVariantColor }, - '& .MuiSlider-rail': { color: SettingsManager.userColor }, - '& .MuiSlider-thumb': { color: SettingsManager.userColor, '&.Mui-focusVisible, &:hover, &.Mui-active': { boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}20` } }, - }} - min={50} - max={700} - step={10} - size="small" - value={this._size} - onChange={action((e, val) => this._canInteract && (this._size = val as number))} - valueLabelDisplay="auto" - /> - </div> + <div> + {this.renderGenerateOutputOptions()} + {this._generateDrawing ? this.renderGenerateDrawing() : null} + {this._generateImage ? this.renderGenerateImage() : null} </div> )} </div> ); - } + }; - renderRegenerate() { - return ( - <div - className="smart-draw-handler" - style={{ - left: this._pageX, - ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }), - background: SettingsManager.userBackgroundColor, - color: SettingsManager.userColor, - }}> - <div className="regenerate-box"> - <IconButton - tooltip="Regenerate" - icon={this._isLoading && this._regenInput === '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <FontAwesomeIcon icon={'rotate'} />} - color={SettingsManager.userColor} - onClick={this.handleSendClick} - /> - <IconButton tooltip="Edit with GPT" icon={<FontAwesomeIcon icon="pen-to-square" />} color={SettingsManager.userColor} onClick={action(() => (this._showEditBox = !this._showEditBox))} /> - {this._showEditBox && ( - <div className="edit-box"> - <input - aria-label="Edit instructions input" - className="smartdraw-input" - type="text" - value={this._regenInput} - onChange={action(e => this._canInteract && (this._regenInput = e.target.value))} - onKeyDown={this.handleKeyPress} - placeholder="Edit instructions" - /> - <Button - style={{ alignSelf: 'flex-end' }} - text="Send" - icon={this._isLoading && this._regenInput !== '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} - iconPlacement="right" - color={SettingsManager.userColor} - onClick={this.handleSendClick} - /> - </div> - )} - </div> + renderRegenerateEditBox = () => ( + <div className="edit-box"> + <input + aria-label="Edit instructions input" + className="smartdraw-input" + type="text" + value={this._regenInput} + onChange={action(e => this._canInteract && (this._regenInput = e.target.value))} + onKeyDown={this.handleKeyPress} + placeholder="Edit instructions" + /> + <Button + style={{ alignSelf: 'flex-end' }} + text="Send" + icon={this._isLoading && this._regenInput !== '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} + iconPlacement="right" + color={SettingsManager.userColor} + onClick={this.handleSendClick} + /> + </div> + ); + + renderRegenerate = () => ( + <div + className="smart-draw-handler" + style={{ + left: this._pageX, + ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }), + background: SettingsManager.userBackgroundColor, + color: SettingsManager.userColor, + }}> + <div className="regenerate-box"> + <IconButton + tooltip="Regenerate" + icon={this._isLoading && this._regenInput === '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <FontAwesomeIcon icon={'rotate'} />} + color={SettingsManager.userColor} + onClick={this.handleSendClick} + /> + <IconButton tooltip="Edit with GPT" icon={<FontAwesomeIcon icon="pen-to-square" />} color={SettingsManager.userColor} onClick={action(() => (this._showEditBox = !this._showEditBox))} /> + {this._showEditBox ? this.renderRegenerateEditBox() : null} </div> - ); - } + </div> + ); render() { - return this._display ? this.renderDisplay() : this.ShowRegenerate ? this.renderRegenerate() : null; + return this._display + ? this.renderDisplay() // + : this.ShowRegenerate + ? this.renderRegenerate() + : null; } } diff --git a/src/client/views/smartdraw/AnnotationPalette.scss b/src/client/views/smartdraw/StickerPalette.scss index 4f11e8afc..ca99410cf 100644 --- a/src/client/views/smartdraw/AnnotationPalette.scss +++ b/src/client/views/smartdraw/StickerPalette.scss @@ -1,4 +1,4 @@ -.annotation-palette { +.sticker-palette { display: flex; flex-direction: column; align-items: center; diff --git a/src/client/views/smartdraw/StickerPalette.tsx b/src/client/views/smartdraw/StickerPalette.tsx new file mode 100644 index 000000000..e3345f547 --- /dev/null +++ b/src/client/views/smartdraw/StickerPalette.tsx @@ -0,0 +1,352 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Slider, Switch } from '@mui/material'; +import { Button } from 'browndash-components'; +import { action, makeObservable, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import * as React from 'react'; +import { AiOutlineSend } from 'react-icons/ai'; +import ReactLoading from 'react-loading'; +import { returnEmptyFilter, returnFalse, returnTrue } from '../../../ClientUtils'; +import { emptyFunction, numberRange } from '../../../Utils'; +import { Doc, DocListCast, returnEmptyDoclist } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; +import { ImageCast, NumCast } from '../../../fields/Types'; +import { ImageField } from '../../../fields/URLField'; +import { DocumentType } from '../../documents/DocumentTypes'; +import { Docs } from '../../documents/Documents'; +import { makeUserTemplateButtonOrImage } from '../../util/DropConverter'; +import { SettingsManager } from '../../util/SettingsManager'; +import { Transform } from '../../util/Transform'; +import { undoBatch } from '../../util/UndoManager'; +import { ObservableReactComponent } from '../ObservableReactComponent'; +import { DefaultStyleProvider, returnEmptyDocViewList } from '../StyleProvider'; +import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; +import { FieldView } from '../nodes/FieldView'; +import './StickerPalette.scss'; +import { DrawingOptions, SmartDrawHandler } from './SmartDrawHandler'; + +interface StickerPaletteProps { + Document: Doc; +} + +enum StickerPaletteMode { + create, + view, +} + +/** + * The StickerPalette can be toggled in the lightbox view of a document. The goal of the palette + * is to offer an easy way for users to create stickers and drag and drop them onto a document. + * These stickers can technically be of any document type and operate similarly to user templates. + * However, the palette is designed to be geared toward ink stickers and image stickers. + * + * On the "add" side of the palette, there is a way to create a drawing sticker with GPT. Users can + * enter the item to draw, toggle different settings, then GPT will generate three versions of the drawing + * to choose from. These drawings can then be saved to the palette as stickers. + */ +@observer +export class StickerPalette extends ObservableReactComponent<StickerPaletteProps> { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(StickerPalette, fieldKey); + } + /** + * Adds a doc to the sticker palette. Gets a snapshot of the document to use as a preview in the palette. When this + * preview is dragged onto a parent document, a copy of that document is added as a sticker. + */ + public static addToPalette = async (doc: Doc) => { + if (!doc.savedAsSticker) { + const docView = DocumentView.getDocumentView(doc); + await docView?.ComponentView?.updateIcon?.(true); + const { clone } = await Doc.MakeClone(doc); + clone.title = doc.title; + const image = ImageCast(doc.icon, ImageCast(clone[Doc.LayoutFieldKey(clone)]))?.url?.href; + Doc.AddDocToList(Doc.MyStickers, 'data', makeUserTemplateButtonOrImage(clone, image)); + doc.savedAsSticker = true; + } + }; + + public static getIcon(group: Doc) { + const docView = DocumentView.getDocumentView(group); + docView?.ComponentView?.updateIcon?.(true); + return !docView ? undefined : new Promise<ImageField | undefined>(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); + } + + private _gptRes: string[] = []; + + @observable private _paletteMode = StickerPaletteMode.view; + @observable private _userInput: string = ''; + @observable private _isLoading: boolean = false; + @observable private _canInteract: boolean = true; + @observable private _showRegenerate: boolean = false; + @observable private _docView: DocumentView | null = null; + @observable private _docCarouselView: DocumentView | null = null; + @observable private _opts: DrawingOptions = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; + + constructor(props: StickerPaletteProps) { + super(props); + makeObservable(this); + } + + componentWillUnmount() { + this.resetPalette(true); + } + + Contains = (view: DocumentView) => + (this._docView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docView)) || // + (this._docCarouselView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docCarouselView)); + + return170 = () => 170; + + handleKeyPress = (event: React.KeyboardEvent) => { + if (event.key === 'Enter') { + this.generateDrawings(); + } + }; + + setPaletteMode = action((mode: StickerPaletteMode) => { + this._paletteMode = mode; + }); + + setUserInput = action((input: string) => { + if (!this._isLoading) this._userInput = input; + }); + + setDetail = action((detail: number) => { + if (this._canInteract) this._opts.complexity = detail; + }); + + setColor = action((autoColor: boolean) => { + if (this._canInteract) this._opts.autoColor = autoColor; + }); + + setSize = action((size: number) => { + if (this._canInteract) this._opts.size = size; + }); + + resetPalette = action((changePaletteMode: boolean) => { + if (changePaletteMode) this.setPaletteMode(StickerPaletteMode.view); + this.setUserInput(''); + this.setDetail(5); + this.setColor(true); + this.setSize(200); + this._showRegenerate = false; + this._canInteract = true; + this._opts = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; + this._gptRes = []; + this._props.Document[DocData].data = undefined; + }); + + /** + * Calls the draw with AI functions in SmartDrawHandler to allow users to generate drawings straight from + * the sticker palette. + */ + @undoBatch + generateDrawings = action(() => { + this._isLoading = true; + const prevDrawings = DocListCast(this._props.Document[DocData].data); + this._props.Document[DocData].data = undefined; + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; + this._canInteract = false; + Promise.all( + numberRange(3).map(i => { + return this._showRegenerate + ? SmartDrawHandler.Instance.regenerate(prevDrawings, this._opts, this._gptRes[i], this._userInput) + : SmartDrawHandler.Instance.drawWithGPT({ X: 0, Y: 0 }, this._userInput, this._opts.complexity, this._opts.size, this._opts.autoColor); + }) + ).then(() => { + this._opts.text !== '' ? (this._opts.text = `${this._opts.text} ~~~ ${this._userInput}`) : (this._opts.text = this._userInput); + this._userInput = ''; + this._isLoading = false; + this._showRegenerate = true; + }); + }); + + @action + addDrawing = (drawing: Doc, opts: DrawingOptions, gptRes: string) => { + this._gptRes.push(gptRes); + drawing[DocData].freeform_fitContentsToBox = true; + Doc.AddDocToList(this._props.Document, 'data', drawing); + }; + + /** + * Saves the currently showing, newly generated drawing to the sticker palette and sets the metadata. + * AddToPalette() is generically used to add any document to the palette, while this defines the behavior for when a user + * presses the "save drawing" button. + */ + saveDrawing = () => { + const cIndex = NumCast(this._props.Document.carousel_index); + const focusedDrawing = DocListCast(this._props.Document.data)[cIndex]; + const docData = focusedDrawing[DocData]; + docData.title = this._opts.text.match(/^(.*?)~~~.*$/)?.[1] || this._opts.text; + docData.ai_drawing_input = this._opts.text; + docData.ai_drawing_complexity = this._opts.complexity; + docData.ai_drawing_colored = this._opts.autoColor; + docData.ai_drawing_size = this._opts.size; + docData.ai_drawing_data = this._gptRes[cIndex]; + docData.ai = 'gpt'; + focusedDrawing.width = this._opts.size; + docData.x = this._opts.x; + docData.y = this._opts.y; + StickerPalette.addToPalette(focusedDrawing).then(() => this.resetPalette(true)); + }; + + renderCreateInput = () => ( + <div className="palette-create"> + <input + className="palette-create-input" + aria-label="label-input" + id="new-label" + type="text" + value={this._userInput} + onChange={e => this.setUserInput(e.target.value)} + placeholder={this._showRegenerate ? '(Optional) Enter edits' : 'Enter item to draw'} + onKeyDown={this.handleKeyPress} + /> + <Button + style={{ alignSelf: 'flex-end' }} + tooltip={this._showRegenerate ? 'Regenerate' : 'Send'} + icon={this._isLoading ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : this._showRegenerate ? <FontAwesomeIcon icon={'rotate'} /> : <AiOutlineSend />} + iconPlacement="right" + color={SettingsManager.userColor} + onClick={this.generateDrawings} + /> + </div> + ); + renderCreateOptions = () => ( + <div className="palette-create-options"> + <div className="palette-color"> + Color + <Switch + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { + color: SettingsManager.userColor, + }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { + backgroundColor: SettingsManager.userVariantColor, + }, + }} + defaultChecked={true} + value={this._opts.autoColor} + size="small" + onChange={() => this.setColor(!this._opts.autoColor)} + /> + </div> + <div className="palette-detail"> + Detail + <Slider + sx={{ + '& .MuiSlider-thumb': { + color: SettingsManager.userColor, + '&.Mui-focusVisible, &:hover, &.Mui-active': { + boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}10`, + }, + }, + '& .MuiSlider-track': { + color: SettingsManager.userVariantColor, + }, + '& .MuiSlider-rail': { + color: SettingsManager.userColor, + }, + }} + style={{ width: '80%' }} + min={1} + max={10} + step={1} + size="small" + value={this._opts.complexity} + onChange={(e, val) => typeof val === 'number' && this.setDetail(val)} + valueLabelDisplay="auto" + /> + </div> + <div className="palette-size"> + Size + <Slider + sx={{ + '& .MuiSlider-thumb': { + color: SettingsManager.userColor, + '&.Mui-focusVisible, &:hover, &.Mui-active': { + boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}20`, + }, + }, + '& .MuiSlider-track': { + color: SettingsManager.userVariantColor, + }, + '& .MuiSlider-rail': { + color: SettingsManager.userColor, + }, + }} + style={{ width: '80%' }} + min={50} + max={500} + step={10} + size="small" + value={this._opts.size} + onChange={(e, val) => typeof val === 'number' && this.setSize(val)} + valueLabelDisplay="auto" + /> + </div> + </div> + ); + renderDoc = (doc: Doc, refFunc: (r: DocumentView) => void) => { + return ( + <DocumentView + ref={refFunc} + Document={doc} + addDocument={undefined} + addDocTab={DocumentViewInternal.addDocTabFunc} + pinToPres={DocumentView.PinDoc} + containerViewPath={returnEmptyDocViewList} + styleProvider={DefaultStyleProvider} + removeDocument={returnFalse} + ScreenToLocalTransform={Transform.Identity} + PanelWidth={this.return170} + PanelHeight={this.return170} + renderDepth={1} + isContentActive={returnTrue} + focus={emptyFunction} + whenChildContentsActiveChanged={emptyFunction} + childFilters={returnEmptyFilter} + childFiltersByRanges={returnEmptyFilter} + searchFilterDocs={returnEmptyDoclist} + /> + ); + }; + renderPaletteCreate = () => ( + <> + {this.renderCreateInput()} + {this.renderCreateOptions()} + {this.renderDoc(this._props.Document, (r: DocumentView) => { + this._docCarouselView = r; + })} + <div className="palette-buttons"> + <Button text="Back" tooltip="Back to All Stickers" icon={<FontAwesomeIcon icon="reply" />} color={SettingsManager.userColor} onClick={() => this.resetPalette(true)} /> + <div className="palette-save-reset"> + <Button tooltip="Save" icon={<FontAwesomeIcon icon="file-arrow-down" />} color={SettingsManager.userColor} onClick={this.saveDrawing} /> + <Button tooltip="Reset" icon={<FontAwesomeIcon icon="rotate-left" />} color={SettingsManager.userColor} onClick={() => this.resetPalette(false)} /> + </div> + </div> + </> + ); + renderPaletteView = () => ( + <> + {this.renderDoc(Doc.MyStickers, (r: DocumentView) => { + this._docView = r; + })} + <Button text="Add" icon={<FontAwesomeIcon icon="square-plus" />} color={SettingsManager.userColor} onClick={() => this.setPaletteMode(StickerPaletteMode.create)} /> + </> + ); + + render() { + return ( + <div className="sticker-palette" style={{ zIndex: 1000 }} onClick={e => e.stopPropagation()}> + {this._paletteMode === StickerPaletteMode.view ? this.renderPaletteView() : null} + {this._paletteMode === StickerPaletteMode.create ? this.renderPaletteCreate() : null} + </div> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.ANNOPALETTE, { + layout: { view: StickerPalette, dataField: 'data' }, + options: { acl: '' }, +}); diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index aef7bf330..e62ca4bb8 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -236,7 +236,7 @@ export class Doc extends RefField { public static get MyPublishedDocs() { return DocListCast(Doc.ActiveDashboard?.myPublishedDocs).concat(DocListCast(DocCast(Doc.UserDoc().myPublishedDocs)?.data)); } // prettier-ignore public static get MyDashboards() { return DocCast(Doc.UserDoc().myDashboards); } // prettier-ignore public static get MyTemplates() { return DocCast(Doc.UserDoc().myTemplates); } // prettier-ignore - public static get MyAnnos() { return DocCast(Doc.UserDoc().myAnnos); } // prettier-ignore + public static get MyStickers() { return DocCast(Doc.UserDoc().myStickers); } // prettier-ignore public static get MyLightboxDrawings() { return DocCast(Doc.UserDoc().myLightboxDrawings); } // prettier-ignore public static get MyImports() { return DocCast(Doc.UserDoc().myImports); } // prettier-ignore public static get MyFilesystem() { return DocCast(Doc.UserDoc().myFilesystem); } // prettier-ignore diff --git a/src/fields/InkField.ts b/src/fields/InkField.ts index 936e45eb2..d1dda106a 100644 --- a/src/fields/InkField.ts +++ b/src/fields/InkField.ts @@ -11,7 +11,7 @@ export enum InkTool { None = 'None', Ink = 'Ink', Eraser = 'Eraser', // not a real tool, but a class of tools - SmartDraw = 'Smartdraw', + SmartDraw = 'smartdraw', } export enum InkInkTool { diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index d757a23fe..6daa5840e 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -1,4 +1,4 @@ -import { Dropbox, files } from 'dropbox'; +import { Dropbox } from 'dropbox'; import * as fs from 'fs'; import * as multipart from 'parse-multipart-data'; import * as path from 'path'; @@ -19,7 +19,62 @@ export default class FireflyManager extends ApiManager { console.error('Error:', error); return undefined; }); - generateImage = (prompt: string = 'a realistic illustration of a cat coding') => { + + generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50, styles: string[]) => + this.getBearerToken().then(response => + response?.json().then((data: { access_token: string }) => + //prettier-ignore + fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: JSON.stringify({ + prompt: prompt, + size: { width: width, height: height }, + structure: !structureUrl + ? undefined + : { + strength, + imageReference: { + source: { url: structureUrl }, + }, + }, + //prettier-ignore + style: { presets: styles } + }), + }) + .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + .catch(error => { + console.error('Error:', error); + return ''; + }) + ) + ); + + uploadImageToDropbox = (fileUrl: string, dbx = new Dropbox({ accessToken: process.env.DROPBOX_TOKEN })) => + new Promise<string>((res, rej) => + fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(fileUrl)}`), undefined, (err, contents) => { + if (err) { + console.log('Error: ', err); + rej(); + } else { + dbx.filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }).then(response => { + dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }).then(link => res(link.result.link)); + }); + } + }) + ); + + generateImage = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, seed?: number) => { + let body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}} }`; + if (seed) { + console.log('RECEIVED SEED', seed); + body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}}, "seeds": [${seed}]}`; + } const fetched = this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { @@ -30,9 +85,15 @@ export default class FireflyManager extends ApiManager { ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], - body: `{ "prompt": "${prompt}" }`, + body: body, }) - .then(response2 => response2.json().then(json => (json.outputs?.[0] as { image: { url: string } })?.image.url)) + .then(response2 => + response2.json().then(json => { + const seed = json.outputs?.[0]?.seed; + const url = json.outputs?.[0]?.image?.url; + return { seed, url }; + }) + ) .catch(error => { console.error('Error:', error); return undefined; @@ -165,10 +226,23 @@ export default class FireflyManager extends ApiManager { protected initialize(register: Registration): void { register({ method: Method.POST, + subscription: '/queryFireflyImageFromStructure', + secureHandler: async ({ req, res }) => + this.uploadImageToDropbox(req.body.structureUrl).then(uploadUrl => + this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploadUrl, req.body.strength, req.body.styles).then(fire => + DashUploadUtils.UploadImage(JSON.parse(fire ?? '').url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }) + ) + ), + }); + register({ + method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => - this.generateImage(req.body.prompt).then(url => - DashUploadUtils.UploadImage(url ?? '').then(info => { + this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img => + DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); }) @@ -180,7 +254,7 @@ export default class FireflyManager extends ApiManager { subscription: '/queryFireflyImageText', // eslint-disable-next-line @typescript-eslint/no-unused-vars secureHandler: ({ req, res }) => - fetch('http://localhost:1050/files/images/testshot.png').then(json => + fetch(req.body.file).then(json => json.blob().then(file => this.getImageText(file).then(text => { _success(res, text); @@ -192,36 +266,16 @@ export default class FireflyManager extends ApiManager { method: Method.POST, subscription: '/expandImage', secureHandler: ({ req, res }) => - new Promise<void>((resolve, reject) => { - const dbx = new Dropbox({ accessToken: process.env.DROPBOX_TOKEN }); - fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(req.body.file)}`), undefined, (err, contents) => { - if (err) { - console.log('Error: ', err); - reject(); - } else { - dbx.filesUpload({ path: `/Apps/browndash/${path.basename(req.body.file)}`, contents }) - .then(response => { - dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }).then(link => { - console.log(link.result); - this.expandImage(link.result.link, req.body.prompt).then(text => { - if (text.error_code) _error(res, text.message); - else - DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { - if (info instanceof Error) _invalid(res, info.message); - else _success(res, info); - resolve(); - }); - }); - }); - }) - .catch(uploadErr => { - console.log(uploadErr); - _error(res, 'upload to dropbox failed'); - reject(); - }); - } - }); - }), + this.uploadImageToDropbox(req.body.file).then(uploadUrl => + this.expandImage(uploadUrl, req.body.prompt).then(text => { + if (text.error_code) _error(res, text.message); + else + DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }); + }) + ), }); } } diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 623172894..2177c5d97 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -458,7 +458,6 @@ export namespace DashUploadUtils { return { name: result.name, message: result.message }; } const outputFile = filename || result.filename || ''; - return UploadInspectedImage(result, outputFile, prefix, isLocal().exec(source) || source.startsWith('data:') ? true : false); }; |