From 2e03c9bf1af2796faef8b81b326b48f4cd136d95 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 26 Feb 2025 18:31:13 -0500 Subject: fixed toggling number dropdown. Added firefly to gpt popup menu. --- src/client/Network.ts | 17 +++-- src/client/util/DocumentManager.ts | 5 +- src/client/views/nodes/ImageBox.tsx | 2 +- src/client/views/pdf/GPTPopup/GPTPopup.tsx | 90 ++++++++++++++++++------- src/client/views/smartdraw/SmartDrawHandler.tsx | 78 ++++++++++++++------- src/server/ApiManagers/FireflyManager.ts | 13 ++-- 6 files changed, 141 insertions(+), 64 deletions(-) (limited to 'src') diff --git a/src/client/Network.ts b/src/client/Network.ts index 3b0406141..323a2648c 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -1,5 +1,4 @@ import formidable from 'formidable'; -import * as requestPromise from 'request-promise'; import { ClientUtils } from '../ClientUtils'; import { Utils } from '../Utils'; import { Upload } from '../server/SharedMediaTypes'; @@ -16,13 +15,17 @@ export namespace Networking { } export function PostToServer(relativeRoute: string, body?: unknown) { - const options = { - uri: ClientUtils.prepend(relativeRoute), + return fetch(ClientUtils.prepend(relativeRoute), { method: 'POST', - body, - json: true, - }; - return requestPromise.post(options); + headers: { + 'Content-Type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }).then(async response => { + if (response.ok) return response.json(); + + return await response.text().then(text => ({ error: '' + response.status + ':' + response.statusText + '-' + text })); + }); } /** diff --git a/src/client/util/DocumentManager.ts b/src/client/util/DocumentManager.ts index acb35f7eb..e33449782 100644 --- a/src/client/util/DocumentManager.ts +++ b/src/client/util/DocumentManager.ts @@ -354,9 +354,10 @@ export class DocumentManager { // bcz: should this delay be an options parameter? setTimeout(() => { Doc.linkFollowHighlight(viewSpec ? [docView.Document, viewSpec] : docView.Document, undefined, options.effect); - if (options.zoomTextSelections && Doc.IsUnhighlightTimerSet() && contextView && targetDoc.text_html) { + const zoomableText = StrCast(targetDoc.text_html, StrCast(targetDoc.ai_firefly_prompt)); + if (options.zoomTextSelections && Doc.IsUnhighlightTimerSet() && contextView && zoomableText) { // if the docView is a text anchor, the contextView is the PDF/Web/Text doc - contextView.setTextHtmlOverlay(StrCast(targetDoc.text_html), options.effect); + contextView.setTextHtmlOverlay(zoomableText, options.effect); DocumentManager._overlayViews.add(contextView); } Doc.AddUnHighlightWatcher(() => { diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 496c3a40a..b6b0e4a8a 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -627,7 +627,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { } else SmartDrawHandler.Instance.regenerate([this.Document], undefined, undefined, this._regenInput || StrCast(this.Document.title), true).then( action(newImgs => { - if (newImgs[0]) { + if (newImgs[0] && !(newImgs[0] instanceof Doc)) { const url = newImgs[0].pathname; const imgField = new ImageField(url); this._prevImgs.length === 0 && diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.tsx b/src/client/views/pdf/GPTPopup/GPTPopup.tsx index 2cf39bec4..efddfb841 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.tsx +++ b/src/client/views/pdf/GPTPopup/GPTPopup.tsx @@ -3,6 +3,7 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; +import { AiOutlineSend } from 'react-icons/ai'; import { CgCornerUpLeft } from 'react-icons/cg'; import ReactLoading from 'react-loading'; import { TypeAnimation } from 'react-type-animation'; @@ -16,14 +17,15 @@ import { Docs } from '../../../documents/Documents'; import { SettingsManager } from '../../../util/SettingsManager'; import { SnappingManager } from '../../../util/SnappingManager'; import { undoable } from '../../../util/UndoManager'; +import { DictationButton } from '../../DictationButton'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { TagItem } from '../../TagsView'; import { ChatSortField, docSortings } from '../../collections/CollectionSubView'; import { DocumentView } from '../../nodes/DocumentView'; +import { SmartDrawHandler } from '../../smartdraw/SmartDrawHandler'; import { AnchorMenu } from '../AnchorMenu'; import './GPTPopup.scss'; -import { DictationButton } from '../../DictationButton'; -import { AiOutlineSend } from 'react-icons/ai'; +import { FireflyImageDimensions } from '../../smartdraw/FireflyConstants'; export enum GPTPopupMode { SUMMARY, // summary of seleted document text @@ -32,6 +34,7 @@ export enum GPTPopupMode { GPT_MENU, // menu for choosing type of prompts user will provide USER_PROMPT, // user prompts for sorting,filtering and asking about docs QUIZ_RESPONSE, // user definitions or explanations to be evaluated by GPT + FIREFLY, // firefly image generation } @observer @@ -93,6 +96,7 @@ export class GPTPopup extends ObservableReactComponent { } @observable private _conversationArray: string[] = ['Hi! In this pop up, you can ask ChatGPT questions about your documents and filter / sort them. ']; + @observable private _fireflyArray: string[] = ['Hi! In this pop up, you can ask Firefly to create images. ']; @observable private _chatEnabled: boolean = false; @action private setChatEnabled = (start: boolean) => (this._chatEnabled = start); @observable private _gptProcessing: boolean = false; @@ -197,6 +201,21 @@ export class GPTPopup extends ObservableReactComponent { .catch(err => console.error('GPT call failed', err)) )) // prettier-ignore + generateFireflyImage = (imgDesc: string) => { + // if (this._fireflyRefStrength) { + // DrawingFillHandler.drawingToImage(this.props.Document, this._fireflyRefStrength, this._regenInput || StrCast(this.Document.title), this.Document)?.then( + // action(() => { + // this._regenerateLoading = false; + // }) + // ); + // } else∂ + return SmartDrawHandler.CreateWithFirefly(imgDesc, FireflyImageDimensions.Square, 0) + .then(action(() => (this._userPrompt = ''))) + .catch(e => { + alert(e); + return undefined; + }); + }; /** * Generates a response to the user's question about the docs in the collection. * The type of response depends on the chat's analysis of the type of their question @@ -338,6 +357,21 @@ export class GPTPopup extends ObservableReactComponent { gptMenu = () => (
+
); - callGpt = (isUserPrompt: boolean) => { + callGpt = action((mode: GPTPopupMode) => { this.setGptProcessing(true); - if (isUserPrompt) { - this._conversationArray.push(this._userPrompt); - return this.generateUserPromptResponse(this._userPrompt).then(action(() => (this._userPrompt = ''))); + switch (mode) { + case GPTPopupMode.FIREFLY: + this._fireflyArray.push(this._userPrompt); + return this.generateFireflyImage(this._userPrompt).then(action(() => (this._userPrompt = ''))); + case GPTPopupMode.USER_PROMPT: + this._conversationArray.push(this._userPrompt); + return this.generateUserPromptResponse(this._userPrompt).then(action(() => (this._userPrompt = ''))); + case GPTPopupMode.QUIZ_RESPONSE: + this._conversationArray.push(this._quizAnswer); + return this.generateQuizAnswerAnalysis(DocumentView.SelectedDocs().lastElement(), this._quizAnswer).then(action(() => (this._quizAnswer = ''))); } - this._conversationArray.push(this._quizAnswer); - return this.generateQuizAnswerAnalysis(DocumentView.SelectedDocs().lastElement(), this._quizAnswer).then(action(() => (this._quizAnswer = ''))); - }; + }); @action - handleKeyPress = async (e: React.KeyboardEvent, isUserPrompt: boolean) => { + handleKeyPress = async (e: React.KeyboardEvent, mode: GPTPopupMode) => { this._askDictation?.stopDictation(); if (e.key === 'Enter') { e.stopPropagation(); - this.callGpt(isUserPrompt).then(() => { + this.callGpt(mode)?.then(() => { this.setGptProcessing(false); this.scrollToBottom(); }); @@ -401,7 +440,7 @@ export class GPTPopup extends ObservableReactComponent {
- {this._conversationArray.map((message, index) => ( + {(this._mode === GPTPopupMode.FIREFLY ? this._fireflyArray : this._conversationArray).map((message, index) => (
{message}
@@ -414,21 +453,21 @@ export class GPTPopup extends ObservableReactComponent { ); - promptBox = (isUserPrompt: boolean) => ( + promptBox = (heading: string, value: string, onChange: (e: string) => string, placeholder: string) => ( <>
- {this.heading(isUserPrompt ? 'ASK' : 'QUIZ')} + {this.heading(heading)} {this.gptUserInput()}
(isUserPrompt ? this.setUserPrompt : this.setQuizAnswer)(e.target.value)} - onKeyDown={e => this.handleKeyPress(e, isUserPrompt)} + onChange={e => onChange(e.target.value)} + onKeyDown={e => this.handleKeyPress(e, this._mode)} type="text" - placeholder={`${isUserPrompt ? 'Have ChatGPT sort, tag, define, or filter your documents for you!' : 'Describe/answer the selected document!'}`} + placeholder={placeholder} />
); @@ -461,7 +500,7 @@ export class GPTPopup extends ObservableReactComponent { dalle generation -
+
@@ -599,7 +638,7 @@ export class GPTPopup extends ObservableReactComponent { color={Doc.hasDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag) ? 'red' : 'transparent'} onClick={() => this._collectionContext && Doc.setDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag, 'remove')} /> - {(this._mode === GPTPopupMode.USER_PROMPT || this._mode === GPTPopupMode.QUIZ_RESPONSE) && ( + {[GPTPopupMode.USER_PROMPT, GPTPopupMode.QUIZ_RESPONSE, GPTPopupMode.FIREFLY].includes(this._mode) && ( } onClick={() => (this._mode = GPTPopupMode.GPT_MENU)} /> )} @@ -613,8 +652,9 @@ export class GPTPopup extends ObservableReactComponent { {(() => { //prettier-ignore switch (this._mode) { - case GPTPopupMode.USER_PROMPT: - case GPTPopupMode.QUIZ_RESPONSE: return this.promptBox(this._mode === GPTPopupMode.USER_PROMPT); + case GPTPopupMode.USER_PROMPT: return this.promptBox("ASK", this._userPrompt, this.setUserPrompt, 'Ask GPT to sort, tag, define, or filter your documents for you!'); + case GPTPopupMode.FIREFLY: return this.promptBox("CREATE", this._userPrompt, this.setUserPrompt, 'Ask Firefly to generate images'); + case GPTPopupMode.QUIZ_RESPONSE: return this.promptBox("QUIZ", this._quizAnswer, this.setQuizAnswer, 'Describe/answer the selected document!'); case GPTPopupMode.GPT_MENU: return this.menuBox(); case GPTPopupMode.SUMMARY: return this.summaryBox(); case GPTPopupMode.DATA: return this.dataAnalysisBox(); diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 7db9ef133..9d67d111b 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -1,6 +1,6 @@ +import { Button, IconButton } from '@dash/components'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Checkbox, FormControlLabel, Radio, RadioGroup, Slider, Switch } from '@mui/material'; -import { Button, IconButton } from '@dash/components'; import { action, makeObservable, observable, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import React from 'react'; @@ -13,7 +13,9 @@ import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { InkData, InkField, InkTool } from '../../../fields/InkField'; import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; +import { Networking } from '../../Network'; import { GPTCallType, gptAPICall, gptDrawingColor } from '../../apis/gpt/GPT'; +import { DocumentType } from '../../documents/DocumentTypes'; import { Docs } from '../../documents/Documents'; import { SettingsManager } from '../../util/SettingsManager'; import { undoable } from '../../util/UndoManager'; @@ -21,12 +23,10 @@ import { SVGToBezier, SVGType } from '../../util/bezierFit'; import { InkingStroke } from '../InkingStroke'; import { ObservableReactComponent } from '../ObservableReactComponent'; import { MarqueeView } from '../collections/collectionFreeForm'; -import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkDash, ActiveInkFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; -import './SmartDrawHandler.scss'; -import { Networking } from '../../Network'; +import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveInkColor, ActiveInkDash, ActiveInkFillColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import { OpenWhere } from '../nodes/OpenWhere'; -import { FireflyDimensionsMap, FireflyImageDimensions, FireflyImageData } from './FireflyConstants'; -import { DocumentType } from '../../documents/DocumentTypes'; +import { FireflyDimensionsMap, FireflyImageData, FireflyImageDimensions } from './FireflyConstants'; +import './SmartDrawHandler.scss'; export interface DrawingOptions { text: string; @@ -268,28 +268,56 @@ export class SmartDrawHandler extends ObservableReactComponent { /** * Calls Firefly API to create an image based on user input */ - createImageWithFirefly = (input: string, seed?: number, changeInPlace?: boolean): Promise => { + createImageWithFirefly = (input: string, seed?: number): Promise => { + this._lastInput.text = input; + return SmartDrawHandler.CreateWithFirefly(input, this._imgDims, seed); + }; /** + * Calls Firefly API to create an image based on user input + */ + recreateImageWithFirefly = (input: string, seed?: number): Promise => { this._lastInput.text = input; - const dims = FireflyDimensionsMap[this._imgDims]; + return SmartDrawHandler.ReCreateWithFirefly(input, this._imgDims, seed); + }; + public static ReCreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise { + const dims = FireflyDimensionsMap[imgDims]; return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed }) .then(img => { - const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1]; - if (!changeInPlace) { - const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { - title: input.match(/^(.*?)~~~.*$/)?.[1] || input, - nativeWidth: dims.width, - nativeHeight: dims.height, - ai: 'firefly', - ai_firefly_seed: newseed, - ai_firefly_prompt: input, - }); - DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); - this._selectedDocs.push(imgDoc); + if (img.error) { + alert('recreate image failed: ' + img.error); + return undefined; } return { prompt: input, seed, pathname: img.accessPaths.agnostic.client }; }) - .catch(e => alert('create image failed: ' + e.toString())); - }; + .catch(e => { + alert('recreate image failed: ' + e.toString()); + return undefined; + }); + } + public static CreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise { + const dims = FireflyDimensionsMap[imgDims]; + return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed }) + .then(img => { + if (img.error) { + alert('create image failed: ' + img.error); + return undefined; + } + const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1]; + const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { + title: input.match(/^(.*?)~~~.*$/)?.[1] || input, + nativeWidth: dims.width, + nativeHeight: dims.height, + ai: 'firefly', + ai_firefly_seed: newseed, + ai_firefly_prompt: input, + }); + DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); + return imgDoc; + }) + .catch(e => { + alert('create image failed: ' + e.toString()); + return undefined; + }); + } /** * Regenerates drawings with the option to add a specific regenerate prompt/request. @@ -307,10 +335,12 @@ export class SmartDrawHandler extends ObservableReactComponent { if (this._regenInput) { // if (this._selectedDoc) { const newPrompt = doc.ai_firefly_prompt ? `${doc.ai_firefly_prompt} ~~~ ${this._regenInput}` : this._regenInput; - return this.createImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed), changeInPlace); + return changeInPlace ? this.recreateImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed)) : this.createImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed)); // } } - return this.createImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), undefined, changeInPlace); + return changeInPlace + ? this.recreateImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), NumCast(doc?.ai_firefly_seed)) + : this.createImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), NumCast(doc?.ai_firefly_seed)); case DocumentType.COL: { try { let res; diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 160a94d40..8a310aed8 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -132,7 +132,8 @@ export default class FireflyManager extends ApiManager { ], body: body, }) - .then(response2 => response2.json().then(json => ({ seed: json.outputs?.[0]?.seed, url: json.outputs?.[0]?.image?.url }))) + .then(response2 => response2.json()) + .then(json => (json.error_code ? json : { seed: json.outputs?.[0]?.seed, url: json.outputs?.[0]?.image?.url })) .catch(error => { console.error('Error:', error); return undefined; @@ -332,10 +333,12 @@ export default class FireflyManager extends ApiManager { subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img => - DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { - if (info instanceof Error) _invalid(res, info.message); - else _success(res, info); - }) + img.error_code + ? _invalid(res, img.message) + : DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }) ), }); -- cgit v1.2.3-70-g09d2