image generation working!

1 files changed, 133 insertions, 60 deletions

diff --git a/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx b/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx
index 8f7bf8713..dcdd52c73 100644
--- a/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx
+++ b/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx
@@ -2,7 +2,7 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
 import { IReactionDisposer, ObservableMap, action, computed, makeObservable, observable, reaction, runInAction } from 'mobx';
 import { observer } from 'mobx-react';
 import * as React from 'react';
-import { returnAll, returnFalse, returnNone, returnOne, returnZero, setupMoveUpEvents } from '../../../../ClientUtils';
+import { ClientUtils, returnAll, returnFalse, returnNone, returnOne, returnZero, setupMoveUpEvents } from '../../../../ClientUtils';
 import { Doc, NumListCast, StrListCast } from '../../../../fields/Doc';
 import { DocCast, ImageCast, ScriptCast, StrCast } from '../../../../fields/Types';
 import { ImageField } from '../../../../fields/URLField';
@@ -17,7 +17,7 @@ import { Id } from '../../../../fields/FieldSymbols';
 import { Colors, IconButton, Size } from 'browndash-components';
 import { MakeTemplate } from '../../../util/DropConverter';
 import { DragManager } from '../../../util/DragManager';
-import { GPTCallType, gptAPICall } from '../../../apis/gpt/GPT';
+import { GPTCallType, gptAPICall, gptImageCall } from '../../../apis/gpt/GPT';
 import { CollectionFreeFormView } from '../../collections/collectionFreeForm/CollectionFreeFormView';
 import { Docs } from '../../../documents/Documents';
 import { OpenWhere } from '../OpenWhere';
@@ -32,6 +32,7 @@ import { ImageBox } from '../ImageBox';
 import { a } from '@react-spring/web';
 import { RichTextMenu } from '../formattedText/RichTextMenu';
 import e from 'cors';
+import { Networking } from '../../../Network';
 
 export enum LayoutType {
     Stacked = 'stacked', 
@@ -423,7 +424,7 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
 
 
 
-    testTemplate = () => { 
+    testTemplate = async() => { 
         // const temp = TemplateLayouts.FourField001;
         // const title: Doc = FieldFuncs.TextField({tl: temp.fields[0].tl, br: temp.fields[0].br}, temp.height, temp.width, 'title', 'Title', {backgroundColor: 'transparent'});
         // const image: Doc = FieldFuncs.ImageField({tl: temp.fields[1].tl, br: temp.fields[1].br}, temp.height, temp.width, 'title', '', {borderColor: '#159fe4', borderWidth: '10', cornerRounding: 10, rotation: 40});
@@ -450,11 +451,22 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
         // console.log(this._dataViz?.colsInfo.get("IMG")?.size, this._dataViz?.colsInfo.get("IMG")?.type)
         // console.log(this.fieldsInfos)
 
+        try {
+            const res = await gptImageCall('Image of panda eating a cookie');
+
+            if (res){
+                const result = await Networking.PostToServer('/uploadRemoteImage', { sources: res });
+
+                console.log(result);
+            }
+        } catch (e) {
+            console.log(e);
+        }
         
     };
 
     @action addField = () => {
-        const newFields: Col[] = this._columns.concat([{title: '', type: TemplateFieldType.UNSET, desc: '', sizes: [TemplateFieldSize.MEDIUM]}])
+        const newFields: Col[] = this._columns.concat([{title: '', type: TemplateFieldType.UNSET, desc: '', sizes: []}])
         this._columns = newFields;
     };
 
@@ -479,7 +491,7 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
         }   
     };
 
-    setColTitle = (column: Col, title: string) => {
+    @action setColTitle = (column: Col, title: string) => {
         if (this.selectedFields.includes(column.title)) {
             this._dataViz?.setColumnTitle(column.title, title);
         } else {
@@ -488,7 +500,7 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
         this.forceUpdate();
     };
 
-    setColType = (column: Col, type: TemplateFieldType) => {
+    @action setColType = (column: Col, type: TemplateFieldType) => {
         if (this.selectedFields.includes(column.title)) {
             this._dataViz?.setColumnType(column.title, type);
         } else {
@@ -507,7 +519,6 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
                 column.sizes.push(size);
             }
         }
-        console.log(column.sizes)
         this.forceUpdate();
     };
 
@@ -520,6 +531,22 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
         this.forceUpdate();
     };
 
+    generateGPTImage = async(prompt: string): Promise<string | undefined> => {
+        console.log(prompt)
+
+        try {
+            const res = await gptImageCall(prompt);
+
+            if (res){
+                const result = await Networking.PostToServer('/uploadRemoteImage', { sources: res });
+                const source = ClientUtils.prepend(result[0].accessPaths.agnostic.client);
+                return source;
+            }
+        } catch (e) {
+            console.log(e);
+        }
+    }
+
     matchesForTemplate = (template: TemplateDocInfos, cols: Col[]): number[][] => {
         const colMatchesField = (col: Col, field : Field) => { return field.sizes?.some(size => col.sizes?.includes(size)) && field.types?.includes(col.type) };
 
@@ -539,13 +566,11 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
     maxMatches = (fieldsCt: number, matches: number[][]) => {
         const used: boolean[] = Array(fieldsCt).fill(false);
         const mt: number[] = Array(fieldsCt).fill(-1);
-        console.log(fieldsCt, matches)
 
         const augmentingPath = (v: number): boolean => {
             if (used[v]) return false;
             used[v] = true;
             for (const to of matches[v]) {
-                console.log(mt[to]);
                 if (mt[to] === -1 || augmentingPath(mt[to])) {
                     mt[to] = v;
                     return true;
@@ -582,7 +607,6 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
 
         validTemplates = validTemplates.map(title => TemplateLayouts.fieldByTitle(title));
 
-        console.log(validTemplates);
         return validTemplates;
     };
 
@@ -626,20 +650,105 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
             }
         }
 
+        const renderTextCalls = async(): Promise<Doc[]> => {
+            const rendered: Doc[] = [];
+
+            if (GPTTextCalls.length) {
+
+                try {
+                    const prompt = fieldContent + GPTTextAssignment;
+    
+                    const res = await gptAPICall(prompt, GPTCallType.FILL);
+    
+                    if (res){
+    
+                        const assignments: {[title: string]: {number: string, content: string}} = JSON.parse(res);
+                        //console.log('assignments', GPTAssignments, 'assignment string', GPTAssignmentString, 'field content', fieldContent, 'response', res, 'assignments', assignments);
+                        Object.entries(assignments).forEach(([title, info]) => {
+                            const field: Field = template.fields[Number(info.number)];
+                            const col = this.getColByTitle(title);
+    
+                            const doc = FieldFuncs.TextField({
+                                tl: field.tl, 
+                                br: field.br }, 
+                                template.height, 
+                                template.width, 
+                                col.title, 
+                                info.content ?? '', 
+                                field.opts
+                            );
+    
+                            rendered.push(doc);
+                        });
+    
+                    }
+                } catch(err) {
+                    console.log(err);
+                }
+            }
+
+            return rendered;
+        };
+
+        const createGeneratedImage = async(fieldNum: string, col: Col, prompt: string) => {
+            const url = await this.generateGPTImage(prompt);
+            const field: Field = template.fields[Number(fieldNum)];
+            const doc = FieldFuncs.ImageField({
+                tl: field.tl, 
+                br: field.br }, 
+                template.height, 
+                template.width, 
+                col.title, 
+                url ?? '', 
+                field.opts
+            );
+
+            return doc;
+        }
+
+        const renderImageCalls = async(): Promise<Doc[]> => {
+            const rendered: Doc[] = [];
+            const calls = GPTIMGCalls;
+
+            if (calls.length) {
+                try {
+                    const renderedImages: Doc[] = await Promise.all(
+                        calls.map(async ([fieldNum, col]) => {
+                            const sysPrompt = 'Your job is to create a prompt for an AI image generator to help it generate an image based on existing content in a template and a user prompt. ONLY INCLUDE THE PROMPT, NO OTHER TEXT OR EXPLANATION. The existing content is as follows: ' + fieldContent + ' **** The user prompt is: ' + col.desc;
+                            
+                            const prompt = await gptAPICall(sysPrompt, GPTCallType.COMPLETEPROMPT);
+                            console.log(sysPrompt, prompt);
+                            
+                            return createGeneratedImage(fieldNum, col, prompt);
+                        })
+                    );
+                
+                    const renderedTemplates: Doc[] = await Promise.all(renderedImages);
+                    renderedTemplates.forEach(doc => rendered.push(doc));
+                } catch (e){
+                    console.log(e);
+                }
+            }
+
+            return rendered;
+        }
+
         const fields: Doc[] = [];
 
         const GPTAssignments = Object.entries(assignments).filter(([f, col]) => this._columns.includes(col));
         const nonGPTAssignments: [string, Col][] = Object.entries(assignments).filter(a => !GPTAssignments.includes(a));
+        const GPTTextCalls = GPTAssignments.filter(([str, col]) => col.type === TemplateFieldType.TEXT);
+        const GPTIMGCalls = GPTAssignments.filter(([str, col]) => col.type === TemplateFieldType.VISUAL);
 
-        const stringifyGPTInfo = (): string => {
+        const stringifyGPTInfo = (calls: [string, Col][]): string => {
             let string: string = '*** COLUMN INFO:';
-            GPTAssignments.forEach(([fieldNum, col]) => {
+            calls.forEach(([fieldNum, col]) => {
                 string += `--- title: ${col.title}, prompt: ${col.desc}, word limit: ${wordLimit(col.sizes[0])} words, assigned field: ${fieldNum} ---`
             });
             return string += ' ***';
         };
 
-        const GPTAssignmentString = stringifyGPTInfo();
+        const GPTTextAssignment = stringifyGPTInfo(GPTTextCalls);
 
         let fieldContent: string = '';
 
@@ -695,47 +804,14 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
             return main;
         }
 
-        if (GPTAssignments.length) {
-            
-            try {
-                const prompt = fieldContent + GPTAssignmentString;
-
-                const res = await gptAPICall(prompt, GPTCallType.FILL);
-
-                if (res){
-                    console.log('response', res);
-
-                    const assignments: {[title: string]: {number: string, content: string}} = JSON.parse(res);
-                    console.log('assignments', GPTAssignments, 'assignment string', GPTAssignmentString, 'field content', fieldContent, 'response', res, 'assignments', assignments);
-                    Object.entries(assignments).forEach(([title, info]) => {
-                        const field: Field = template.fields[Number(info.number)];
-                        const col = this.getColByTitle(title);
-
-                        const doc = (col.type === TemplateFieldType.VISUAL ? FieldFuncs.ImageField : FieldFuncs.TextField)({
-                            tl: field.tl, 
-                            br: field.br }, 
-                            template.height, 
-                            template.width, 
-                            col.title, 
-                            info.content ?? '', 
-                            field.opts
-                        );
-
-                        fields.push(doc);
-                    });
-
-                    return createMainDoc();
-                }
-            } catch(err) {
-                console.log(err);
-            }
-            
-        } else {
-            return createMainDoc();
-        }
+        const textCalls = await renderTextCalls();
+        const imageCalls = await renderImageCalls();
 
-        return new Doc;
-    };
+        textCalls.forEach(doc => {fields.push(doc)});
+        imageCalls.forEach(doc => {fields.push(doc)});
+
+        return createMainDoc();
+    }
 
     compileFieldDescriptions = (templates: TemplateDocInfos[]): string => {
         let descriptions: string = '';
@@ -765,7 +841,6 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
 
         const inputText = fieldDescriptions.concat(colDescriptions);
 
-        console.log(inputText);
         ++this._callCount;
         const origCount = this._callCount;
 
@@ -779,14 +854,12 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
             if (res && this._callCount === origCount) {
                 this._GPTLoading = false;
 
-                console.log(res);
-
                 const assignments: {[templateTitle: string]: {[field: string]: string}} = JSON.parse(res);
                 const brokenDownAssignments: [TemplateDocInfos, {[field: number]: Col}][] = [];
+
                 Object.entries(assignments).forEach(([tempTitle, assignment]) => {
                     const template = TemplateLayouts.fieldByTitle(tempTitle);
                     if (!template) return;
-                    console.log(assignments)
                     const toObj = Object.entries(assignment).reduce((a, [fieldNum, colTitle]) => {
                         a[Number(fieldNum)] = this.getColByTitle(colTitle);
                         return a;
@@ -849,7 +922,7 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
                     <div className='docCreatorMenu-section' style={{height: this._GPTOpt ? 200 : 200}}>
                         <div className='docCreatorMenu-section-topbar'>
                             <div className='docCreatorMenu-section-title'>Suggested Templates</div>
-                            <button className='docCreatorMenu-menu-button section-reveal-options' onPointerDown={e => this.setUpButtonClick(e, () => this._menuContent = 'dashboard')}>
+                            <button className='docCreatorMenu-menu-button section-reveal-options' onPointerDown={e => this.setUpButtonClick(e, () => runInAction(() => this._menuContent = 'dashboard'))}>
                                 <FontAwesomeIcon icon='gear'/>
                             </button>
                         </div>
@@ -1197,7 +1270,7 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
                     <div className='opts-bar'> 
                         <div className='opt-box'>
                             <div className='top-bar'> Title </div>
-                            <textarea className='content' style={{width: '100%', height: 'calc(100% - 20px)'}} defaultValue={field.title} placeholder={'Enter title'}  onChange={(e) => this.setColTitle(field, e.target.value)}/>
+                            <textarea className='content' style={{width: '100%', height: 'calc(100% - 20px)'}} defaultValue={field.title} placeholder={'Enter title'} onChange={(e) => this.setColTitle(field, e.target.value)}/>
                         </div>
                         <div className='opt-box'>
                             <div className='top-bar'> Type </div>
@@ -1237,7 +1310,7 @@ export class DocCreatorMenu extends ObservableReactComponent<FieldViewProps> {
                 <button className='docCreatorMenu-menu-button section-reveal-options' onPointerDown={e => this.setUpButtonClick(e, this.addField)}>
                     <FontAwesomeIcon icon='plus'/>
                 </button>
-                <button className='docCreatorMenu-menu-button section-reveal-options float-right' onPointerDown={e => this.setUpButtonClick(e, () => this._menuContent = 'templates')}>
+                <button className='docCreatorMenu-menu-button section-reveal-options float-right' onPointerDown={e => this.setUpButtonClick(e, () => runInAction(() => this._menuContent = 'templates'))}>
                     <FontAwesomeIcon icon='arrow-left'/>
                 </button>
             </div>