fixed toggling number dropdown. Added firefly to gpt popup menu.

6 files changed, 141 insertions, 64 deletions

diff --git a/src/client/Network.ts b/src/client/Network.ts
index 3b0406141..323a2648c 100644
--- a/src/client/Network.ts
+++ b/src/client/Network.ts
@@ -1,5 +1,4 @@
 import formidable from 'formidable';
-import * as requestPromise from 'request-promise';
 import { ClientUtils } from '../ClientUtils';
 import { Utils } from '../Utils';
 import { Upload } from '../server/SharedMediaTypes';
@@ -16,13 +15,17 @@ export namespace Networking {
     }
 
     export function PostToServer(relativeRoute: string, body?: unknown) {
-        const options = {
-            uri: ClientUtils.prepend(relativeRoute),
+        return fetch(ClientUtils.prepend(relativeRoute), {
             method: 'POST',
-            body,
-            json: true,
-        };
-        return requestPromise.post(options);
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: body ? JSON.stringify(body) : undefined,
+        }).then(async response => {
+            if (response.ok) return response.json();
+
+            return await response.text().then(text => ({ error: '' + response.status + ':' + response.statusText + '-' + text }));
+        });
     }
 
     /**
diff --git a/src/client/util/DocumentManager.ts b/src/client/util/DocumentManager.ts
index acb35f7eb..e33449782 100644
--- a/src/client/util/DocumentManager.ts
+++ b/src/client/util/DocumentManager.ts
@@ -354,9 +354,10 @@ export class DocumentManager {
             // bcz: should this delay be an options parameter?
             setTimeout(() => {
                 Doc.linkFollowHighlight(viewSpec ? [docView.Document, viewSpec] : docView.Document, undefined, options.effect);
-                if (options.zoomTextSelections && Doc.IsUnhighlightTimerSet() && contextView && targetDoc.text_html) {
+                const zoomableText = StrCast(targetDoc.text_html, StrCast(targetDoc.ai_firefly_prompt));
+                if (options.zoomTextSelections && Doc.IsUnhighlightTimerSet() && contextView && zoomableText) {
                     // if the docView is a text anchor, the contextView is the PDF/Web/Text doc
-                    contextView.setTextHtmlOverlay(StrCast(targetDoc.text_html), options.effect);
+                    contextView.setTextHtmlOverlay(zoomableText, options.effect);
                     DocumentManager._overlayViews.add(contextView);
                 }
                 Doc.AddUnHighlightWatcher(() => {
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index 496c3a40a..b6b0e4a8a 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -627,7 +627,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                                 } else
                                     SmartDrawHandler.Instance.regenerate([this.Document], undefined, undefined, this._regenInput || StrCast(this.Document.title), true).then(
                                         action(newImgs => {
-                                            if (newImgs[0]) {
+                                            if (newImgs[0] && !(newImgs[0] instanceof Doc)) {
                                                 const url = newImgs[0].pathname;
                                                 const imgField = new ImageField(url);
                                                 this._prevImgs.length === 0 &&
diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.tsx b/src/client/views/pdf/GPTPopup/GPTPopup.tsx
index 2cf39bec4..efddfb841 100644
--- a/src/client/views/pdf/GPTPopup/GPTPopup.tsx
+++ b/src/client/views/pdf/GPTPopup/GPTPopup.tsx
@@ -3,6 +3,7 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
 import { action, makeObservable, observable, reaction } from 'mobx';
 import { observer } from 'mobx-react';
 import * as React from 'react';
+import { AiOutlineSend } from 'react-icons/ai';
 import { CgCornerUpLeft } from 'react-icons/cg';
 import ReactLoading from 'react-loading';
 import { TypeAnimation } from 'react-type-animation';
@@ -16,14 +17,15 @@ import { Docs } from '../../../documents/Documents';
 import { SettingsManager } from '../../../util/SettingsManager';
 import { SnappingManager } from '../../../util/SnappingManager';
 import { undoable } from '../../../util/UndoManager';
+import { DictationButton } from '../../DictationButton';
 import { ObservableReactComponent } from '../../ObservableReactComponent';
 import { TagItem } from '../../TagsView';
 import { ChatSortField, docSortings } from '../../collections/CollectionSubView';
 import { DocumentView } from '../../nodes/DocumentView';
+import { SmartDrawHandler } from '../../smartdraw/SmartDrawHandler';
 import { AnchorMenu } from '../AnchorMenu';
 import './GPTPopup.scss';
-import { DictationButton } from '../../DictationButton';
-import { AiOutlineSend } from 'react-icons/ai';
+import { FireflyImageDimensions } from '../../smartdraw/FireflyConstants';
 
 export enum GPTPopupMode {
     SUMMARY, // summary of seleted document text
@@ -32,6 +34,7 @@ export enum GPTPopupMode {
     GPT_MENU, // menu for choosing type of prompts user will provide
     USER_PROMPT, // user prompts for sorting,filtering and asking about docs
     QUIZ_RESPONSE, // user definitions or explanations to be evaluated by GPT
+    FIREFLY, // firefly image generation
 }
 
 @observer
@@ -93,6 +96,7 @@ export class GPTPopup extends ObservableReactComponent<object> {
     }
 
     @observable private _conversationArray: string[] = ['Hi! In this pop up, you can ask ChatGPT questions about your documents and filter / sort them. '];
+    @observable private _fireflyArray: string[] = ['Hi! In this pop up, you can ask Firefly to create images. '];
     @observable private _chatEnabled: boolean = false;
     @action private setChatEnabled = (start: boolean) => (this._chatEnabled = start);
     @observable private _gptProcessing: boolean = false;
@@ -197,6 +201,21 @@ export class GPTPopup extends ObservableReactComponent<object> {
                 .catch(err => console.error('GPT call failed', err))
         )) // prettier-ignore
 
+    generateFireflyImage = (imgDesc: string) => {
+        // if (this._fireflyRefStrength) {
+        //     DrawingFillHandler.drawingToImage(this.props.Document, this._fireflyRefStrength, this._regenInput || StrCast(this.Document.title), this.Document)?.then(
+        //         action(() => {
+        //             this._regenerateLoading = false;
+        //         })
+        //     );
+        // }  else∂
+        return SmartDrawHandler.CreateWithFirefly(imgDesc, FireflyImageDimensions.Square, 0)
+            .then(action(() => (this._userPrompt = '')))
+            .catch(e => {
+                alert(e);
+                return undefined;
+            });
+    };
     /**
      * Generates a response to the user's question about the docs in the collection.
      * The type of response depends on the chat's analysis of the type of their question
@@ -339,6 +358,21 @@ export class GPTPopup extends ObservableReactComponent<object> {
     gptMenu = () => (
         <div className="btns-wrapper-gpt">
             <Button
+                tooltip="Ask Firefly to create images"
+                text="Ask Firefly"
+                onClick={() => this.setMode(GPTPopupMode.FIREFLY)}
+                color={StrCast(Doc.UserDoc().userVariantColor)}
+                type={Type.TERT}
+                style={{
+                    width: '100%',
+                    height: '40%',
+                    textAlign: 'center',
+                    color: '#ffffff',
+                    fontSize: '16px',
+                    marginBottom: '10px',
+                }}
+            />
+            <Button
                 tooltip="Ask GPT to sort, tag, define, or filter your Docs!"
                 text="Ask GPT"
                 onClick={() => this.setMode(GPTPopupMode.USER_PROMPT)}
@@ -365,32 +399,37 @@ export class GPTPopup extends ObservableReactComponent<object> {
                 type={Type.TERT}
                 style={{
                     width: '100%',
+                    height: '40%',
                     textAlign: 'center',
                     color: '#ffffff',
                     fontSize: '16px',
-                    height: '40%',
                 }}
             />
         </div>
     );
 
-    callGpt = (isUserPrompt: boolean) => {
+    callGpt = action((mode: GPTPopupMode) => {
         this.setGptProcessing(true);
-        if (isUserPrompt) {
-            this._conversationArray.push(this._userPrompt);
-            return this.generateUserPromptResponse(this._userPrompt).then(action(() => (this._userPrompt = '')));
+        switch (mode) {
+            case GPTPopupMode.FIREFLY:
+                this._fireflyArray.push(this._userPrompt);
+                return this.generateFireflyImage(this._userPrompt).then(action(() => (this._userPrompt = '')));
+            case GPTPopupMode.USER_PROMPT:
+                this._conversationArray.push(this._userPrompt);
+                return this.generateUserPromptResponse(this._userPrompt).then(action(() => (this._userPrompt = '')));
+            case GPTPopupMode.QUIZ_RESPONSE:
+                this._conversationArray.push(this._quizAnswer);
+                return this.generateQuizAnswerAnalysis(DocumentView.SelectedDocs().lastElement(), this._quizAnswer).then(action(() => (this._quizAnswer = '')));
         }
-        this._conversationArray.push(this._quizAnswer);
-        return this.generateQuizAnswerAnalysis(DocumentView.SelectedDocs().lastElement(), this._quizAnswer).then(action(() => (this._quizAnswer = '')));
-    };
+    });
 
     @action
-    handleKeyPress = async (e: React.KeyboardEvent, isUserPrompt: boolean) => {
+    handleKeyPress = async (e: React.KeyboardEvent, mode: GPTPopupMode) => {
         this._askDictation?.stopDictation();
         if (e.key === 'Enter') {
             e.stopPropagation();
 
-            this.callGpt(isUserPrompt).then(() => {
+            this.callGpt(mode)?.then(() => {
                 this.setGptProcessing(false);
                 this.scrollToBottom();
             });
@@ -401,7 +440,7 @@ export class GPTPopup extends ObservableReactComponent<object> {
         <div className="btns-wrapper-gpt">
             <div className="chat-wrapper">
                 <div className="chat-bubbles">
-                    {this._conversationArray.map((message, index) => (
+                    {(this._mode === GPTPopupMode.FIREFLY ? this._fireflyArray : this._conversationArray).map((message, index) => (
                         <div key={index} className={`chat-bubble ${index % 2 === 1 ? 'user-message' : 'chat-message'}`}>
                             {message}
                         </div>
@@ -414,21 +453,21 @@ export class GPTPopup extends ObservableReactComponent<object> {
         </div>
     );
 
-    promptBox = (isUserPrompt: boolean) => (
+    promptBox = (heading: string, value: string, onChange: (e: string) => string, placeholder: string) => (
         <>
             <div className="gptPopup-sortBox">
-                {this.heading(isUserPrompt ? 'ASK' : 'QUIZ')}
+                {this.heading(heading)}
                 {this.gptUserInput()}
             </div>
             <div className="inputWrapper">
                 <input
                     className="searchBox-input"
-                    value={isUserPrompt ? this._userPrompt : this._quizAnswer} // Controlled input
+                    value={value} // Controlled input
                     autoComplete="off"
-                    onChange={e => (isUserPrompt ? this.setUserPrompt : this.setQuizAnswer)(e.target.value)}
-                    onKeyDown={e => this.handleKeyPress(e, isUserPrompt)}
+                    onChange={e => onChange(e.target.value)}
+                    onKeyDown={e => this.handleKeyPress(e, this._mode)}
                     type="text"
-                    placeholder={`${isUserPrompt ? 'Have ChatGPT sort, tag, define, or filter your documents for you!' : 'Describe/answer the selected document!'}`}
+                    placeholder={placeholder}
                 />
                 <Button //
                     text="Send"
@@ -436,9 +475,9 @@ export class GPTPopup extends ObservableReactComponent<object> {
                     icon={<AiOutlineSend />}
                     iconPlacement="right"
                     color={SnappingManager.userVariantColor}
-                    onClick={() => this.callGpt(isUserPrompt)}
+                    onClick={() => this.callGpt(this._mode)}
                 />
-                <DictationButton ref={r => (this._askDictation = r)} setInput={isUserPrompt ? this.setUserPrompt : this.setQuizAnswer} />
+                <DictationButton ref={r => (this._askDictation = r)} setInput={onChange} />
             </div>
         </>
     );
@@ -461,7 +500,7 @@ export class GPTPopup extends ObservableReactComponent<object> {
                                 <img key={rawSrc[0]} src={rawSrc[0]} width={150} height={150} alt="dalle generation" />
                             </div>
                         </div>
-                        <div className="btn-container">
+                        <div key={rawSrc[0] + i + 'btn'} className="btn-container">
                             <Button text="Save Image" onClick={() => this.transferToImage(rawSrc[1])} color={StrCast(Doc.UserDoc().userColor)} type={Type.TERT} />
                         </div>
                     </>
@@ -599,7 +638,7 @@ export class GPTPopup extends ObservableReactComponent<object> {
                         color={Doc.hasDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag) ? 'red' : 'transparent'}
                         onClick={() => this._collectionContext && Doc.setDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag, 'remove')}
                     />
-                    {(this._mode === GPTPopupMode.USER_PROMPT || this._mode === GPTPopupMode.QUIZ_RESPONSE) && (
+                    {[GPTPopupMode.USER_PROMPT, GPTPopupMode.QUIZ_RESPONSE, GPTPopupMode.FIREFLY].includes(this._mode) && (
                         <IconButton color={StrCast(SettingsManager.userVariantColor)} tooltip="back" icon={<CgCornerUpLeft size="16px" />} onClick={() => (this._mode = GPTPopupMode.GPT_MENU)} />
                     )}
                 </>
@@ -613,8 +652,9 @@ export class GPTPopup extends ObservableReactComponent<object> {
                 {(() => {
                     //prettier-ignore
                     switch (this._mode) {
-                        case GPTPopupMode.USER_PROMPT: 
-                        case GPTPopupMode.QUIZ_RESPONSE: return this.promptBox(this._mode === GPTPopupMode.USER_PROMPT);
+                        case GPTPopupMode.USER_PROMPT:   return this.promptBox("ASK", this._userPrompt, this.setUserPrompt, 'Ask GPT to sort, tag, define, or filter your documents for you!');
+                        case GPTPopupMode.FIREFLY:       return this.promptBox("CREATE", this._userPrompt, this.setUserPrompt, 'Ask Firefly to generate images');
+                        case GPTPopupMode.QUIZ_RESPONSE: return this.promptBox("QUIZ", this._quizAnswer, this.setQuizAnswer, 'Describe/answer the selected document!');
                         case GPTPopupMode.GPT_MENU:      return this.menuBox();
                         case GPTPopupMode.SUMMARY:       return this.summaryBox();
                         case GPTPopupMode.DATA:          return this.dataAnalysisBox();
diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx
index 7db9ef133..9d67d111b 100644
--- a/src/client/views/smartdraw/SmartDrawHandler.tsx
+++ b/src/client/views/smartdraw/SmartDrawHandler.tsx
@@ -1,6 +1,6 @@
+import { Button, IconButton } from '@dash/components';
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
 import { Checkbox, FormControlLabel, Radio, RadioGroup, Slider, Switch } from '@mui/material';
-import { Button, IconButton } from '@dash/components';
 import { action, makeObservable, observable, runInAction } from 'mobx';
 import { observer } from 'mobx-react';
 import React from 'react';
@@ -13,7 +13,9 @@ import { Doc, DocListCast } from '../../../fields/Doc';
 import { DocData } from '../../../fields/DocSymbols';
 import { InkData, InkField, InkTool } from '../../../fields/InkField';
 import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types';
+import { Networking } from '../../Network';
 import { GPTCallType, gptAPICall, gptDrawingColor } from '../../apis/gpt/GPT';
+import { DocumentType } from '../../documents/DocumentTypes';
 import { Docs } from '../../documents/Documents';
 import { SettingsManager } from '../../util/SettingsManager';
 import { undoable } from '../../util/UndoManager';
@@ -21,12 +23,10 @@ import { SVGToBezier, SVGType } from '../../util/bezierFit';
 import { InkingStroke } from '../InkingStroke';
 import { ObservableReactComponent } from '../ObservableReactComponent';
 import { MarqueeView } from '../collections/collectionFreeForm';
-import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkDash, ActiveInkFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView';
-import './SmartDrawHandler.scss';
-import { Networking } from '../../Network';
+import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveInkColor, ActiveInkDash, ActiveInkFillColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView';
 import { OpenWhere } from '../nodes/OpenWhere';
-import { FireflyDimensionsMap, FireflyImageDimensions, FireflyImageData } from './FireflyConstants';
-import { DocumentType } from '../../documents/DocumentTypes';
+import { FireflyDimensionsMap, FireflyImageData, FireflyImageDimensions } from './FireflyConstants';
+import './SmartDrawHandler.scss';
 
 export interface DrawingOptions {
     text: string;
@@ -268,28 +268,56 @@ export class SmartDrawHandler extends ObservableReactComponent<object> {
     /**
      * Calls Firefly API to create an image based on user input
      */
-    createImageWithFirefly = (input: string, seed?: number, changeInPlace?: boolean): Promise<FireflyImageData> => {
+    createImageWithFirefly = (input: string, seed?: number): Promise<FireflyImageData | Doc | undefined> => {
+        this._lastInput.text = input;
+        return SmartDrawHandler.CreateWithFirefly(input, this._imgDims, seed);
+    }; /**
+     * Calls Firefly API to create an image based on user input
+     */
+    recreateImageWithFirefly = (input: string, seed?: number): Promise<FireflyImageData | Doc | undefined> => {
         this._lastInput.text = input;
-        const dims = FireflyDimensionsMap[this._imgDims];
+        return SmartDrawHandler.ReCreateWithFirefly(input, this._imgDims, seed);
+    };
+    public static ReCreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise<FireflyImageData | Doc | undefined> {
+        const dims = FireflyDimensionsMap[imgDims];
         return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed })
             .then(img => {
-                const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1];
-                if (!changeInPlace) {
-                    const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, {
-                        title: input.match(/^(.*?)~~~.*$/)?.[1] || input,
-                        nativeWidth: dims.width,
-                        nativeHeight: dims.height,
-                        ai: 'firefly',
-                        ai_firefly_seed: newseed,
-                        ai_firefly_prompt: input,
-                    });
-                    DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight);
-                    this._selectedDocs.push(imgDoc);
+                if (img.error) {
+                    alert('recreate image failed: ' + img.error);
+                    return undefined;
                 }
                 return { prompt: input, seed, pathname: img.accessPaths.agnostic.client };
             })
-            .catch(e => alert('create image failed: ' + e.toString()));
-    };
+            .catch(e => {
+                alert('recreate image failed: ' + e.toString());
+                return undefined;
+            });
+    }
+    public static CreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise<FireflyImageData | Doc | undefined> {
+        const dims = FireflyDimensionsMap[imgDims];
+        return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed })
+            .then(img => {
+                if (img.error) {
+                    alert('create image failed: ' + img.error);
+                    return undefined;
+                }
+                const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1];
+                const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, {
+                    title: input.match(/^(.*?)~~~.*$/)?.[1] || input,
+                    nativeWidth: dims.width,
+                    nativeHeight: dims.height,
+                    ai: 'firefly',
+                    ai_firefly_seed: newseed,
+                    ai_firefly_prompt: input,
+                });
+                DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight);
+                return imgDoc;
+            })
+            .catch(e => {
+                alert('create image failed: ' + e.toString());
+                return undefined;
+            });
+    }
 
     /**
      * Regenerates drawings with the option to add a specific regenerate prompt/request.
@@ -307,10 +335,12 @@ export class SmartDrawHandler extends ObservableReactComponent<object> {
                         if (this._regenInput) {
                             // if (this._selectedDoc) {
                             const newPrompt = doc.ai_firefly_prompt ? `${doc.ai_firefly_prompt} ~~~ ${this._regenInput}` : this._regenInput;
-                            return this.createImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed), changeInPlace);
+                            return changeInPlace ? this.recreateImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed)) : this.createImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed));
                             // }
                         }
-                        return this.createImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), undefined, changeInPlace);
+                        return changeInPlace
+                            ? this.recreateImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), NumCast(doc?.ai_firefly_seed))
+                            : this.createImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), NumCast(doc?.ai_firefly_seed));
                     case DocumentType.COL: {
                         try {
                             let res;
diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts
index 160a94d40..8a310aed8 100644
--- a/src/server/ApiManagers/FireflyManager.ts
+++ b/src/server/ApiManagers/FireflyManager.ts
@@ -132,7 +132,8 @@ export default class FireflyManager extends ApiManager {
                     ],
                     body: body,
                 })
-                    .then(response2 => response2.json().then(json => ({ seed: json.outputs?.[0]?.seed, url: json.outputs?.[0]?.image?.url })))
+                    .then(response2 => response2.json())
+                    .then(json => (json.error_code ? json : { seed: json.outputs?.[0]?.seed, url: json.outputs?.[0]?.image?.url }))
                     .catch(error => {
                         console.error('Error:', error);
                         return undefined;
@@ -332,10 +333,12 @@ export default class FireflyManager extends ApiManager {
             subscription: '/queryFireflyImage',
             secureHandler: ({ req, res }) =>
                 this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img =>
-                    DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => {
-                        if (info instanceof Error) _invalid(res, info.message);
-                        else _success(res, info);
-                    })
+                    img.error_code
+                        ? _invalid(res, img.message)
+                        : DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => {
+                              if (info instanceof Error) _invalid(res, info.message);
+                              else _success(res, info);
+                          })
                 ),
         });