From 984a470094399e4bbd74ddb7daa3e6f08a0a4793 Mon Sep 17 00:00:00 2001 From: Nathan-SR <144961007+Nathan-SR@users.noreply.github.com> Date: Sun, 8 Sep 2024 17:44:11 -0400 Subject: image generation working! --- src/client/views/nodes/DataVizBox/DataVizBox.tsx | 15 +- .../views/nodes/DataVizBox/DocCreatorMenu.tsx | 193 ++++++++++++++------- 2 files changed, 138 insertions(+), 70 deletions(-) (limited to 'src/client/views') diff --git a/src/client/views/nodes/DataVizBox/DataVizBox.tsx b/src/client/views/nodes/DataVizBox/DataVizBox.tsx index 0efe8ead0..0e3b602d6 100644 --- a/src/client/views/nodes/DataVizBox/DataVizBox.tsx +++ b/src/client/views/nodes/DataVizBox/DataVizBox.tsx @@ -157,13 +157,11 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { @action setColumnType = (colTitle: string, type: TemplateFieldType) => { const colInfo = this.colsInfo.get(colTitle); - console.log(colInfo) if (colInfo) { colInfo.type = type; } else { this.colsInfo.set(colTitle, {title: colTitle, desc: '', type: type, sizes: [TemplateFieldSize.MEDIUM]}) } - console.log(colInfo?.title, colInfo?.type) } @action modifyColumnSizes = (colTitle: string, size: TemplateFieldSize, valid: boolean) => { @@ -183,8 +181,9 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { const colInfo = this.colsInfo.get(colTitle); if (colInfo) { colInfo.title = newTitle; + console.log(colInfo.title) } else { - this.colsInfo.set(colTitle, {title: newTitle, desc: '', type: TemplateFieldType.UNSET, sizes: [TemplateFieldSize.MEDIUM]}) + this.colsInfo.set(colTitle, {title: newTitle, desc: '', type: TemplateFieldType.UNSET, sizes: []}) } } @@ -194,7 +193,7 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { if (!desc) { colInfo.desc = this.GPTSummary?.get(colTitle)?.desc ?? ''; } else { colInfo.desc = desc; } } else { - this.colsInfo.set(colTitle, {title: colTitle, desc: desc, type: TemplateFieldType.UNSET, sizes: [TemplateFieldSize.MEDIUM]}) + this.colsInfo.set(colTitle, {title: colTitle, desc: desc, type: TemplateFieldType.UNSET, sizes: []}) } } @@ -203,7 +202,7 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { if (colInfo) { colInfo.defaultContent = cont; } else { - this.colsInfo.set(colTitle, {title: colTitle, desc: '', type: TemplateFieldType.UNSET, sizes: [TemplateFieldSize.MEDIUM], defaultContent: cont}) + this.colsInfo.set(colTitle, {title: colTitle, desc: '', type: TemplateFieldType.UNSET, sizes: [], defaultContent: cont}) } } @@ -593,11 +592,9 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { const prompt = this.getColSummary(); - console.log(prompt) - const cols = Array.from(Object.keys(this.records[0])).filter(header => header !== '' && header !== undefined); cols.forEach(col => { - if (!this.colsInfo.get(col)) this.colsInfo.set(col, {title: col, desc: '', sizes: [TemplateFieldSize.MEDIUM], type: TemplateFieldType.UNSET}); + if (!this.colsInfo.get(col)) this.colsInfo.set(col, {title: col, desc: '', sizes: [], type: TemplateFieldType.UNSET}); }); try { @@ -607,7 +604,6 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { ]); if (res1) { - console.log(res1); this.GPTSummary = new ObservableMap(); const descs: { [col: string]: string } = JSON.parse(res1); for (const [key, val] of Object.entries(descs)) { @@ -619,7 +615,6 @@ export class DataVizBox extends ViewBoxAnnotatableComponent() { if (res2) { !this.GPTSummary && (this.GPTSummary = new ObservableMap()); const info: { [col: string]: { type: TemplateFieldType, size: TemplateFieldSize } } = JSON.parse(res2); - console.log(info); for (const [key, val] of Object.entries(info)) { const colSummary = this.GPTSummary.get(key); if (colSummary) { diff --git a/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx b/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx index 8f7bf8713..dcdd52c73 100644 --- a/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx +++ b/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx @@ -2,7 +2,7 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { IReactionDisposer, ObservableMap, action, computed, makeObservable, observable, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; -import { returnAll, returnFalse, returnNone, returnOne, returnZero, setupMoveUpEvents } from '../../../../ClientUtils'; +import { ClientUtils, returnAll, returnFalse, returnNone, returnOne, returnZero, setupMoveUpEvents } from '../../../../ClientUtils'; import { Doc, NumListCast, StrListCast } from '../../../../fields/Doc'; import { DocCast, ImageCast, ScriptCast, StrCast } from '../../../../fields/Types'; import { ImageField } from '../../../../fields/URLField'; @@ -17,7 +17,7 @@ import { Id } from '../../../../fields/FieldSymbols'; import { Colors, IconButton, Size } from 'browndash-components'; import { MakeTemplate } from '../../../util/DropConverter'; import { DragManager } from '../../../util/DragManager'; -import { GPTCallType, gptAPICall } from '../../../apis/gpt/GPT'; +import { GPTCallType, gptAPICall, gptImageCall } from '../../../apis/gpt/GPT'; import { CollectionFreeFormView } from '../../collections/collectionFreeForm/CollectionFreeFormView'; import { Docs } from '../../../documents/Documents'; import { OpenWhere } from '../OpenWhere'; @@ -32,6 +32,7 @@ import { ImageBox } from '../ImageBox'; import { a } from '@react-spring/web'; import { RichTextMenu } from '../formattedText/RichTextMenu'; import e from 'cors'; +import { Networking } from '../../../Network'; export enum LayoutType { Stacked = 'stacked', @@ -423,7 +424,7 @@ export class DocCreatorMenu extends ObservableReactComponent { - testTemplate = () => { + testTemplate = async() => { // const temp = TemplateLayouts.FourField001; // const title: Doc = FieldFuncs.TextField({tl: temp.fields[0].tl, br: temp.fields[0].br}, temp.height, temp.width, 'title', 'Title', {backgroundColor: 'transparent'}); // const image: Doc = FieldFuncs.ImageField({tl: temp.fields[1].tl, br: temp.fields[1].br}, temp.height, temp.width, 'title', '', {borderColor: '#159fe4', borderWidth: '10', cornerRounding: 10, rotation: 40}); @@ -450,11 +451,22 @@ export class DocCreatorMenu extends ObservableReactComponent { // console.log(this._dataViz?.colsInfo.get("IMG")?.size, this._dataViz?.colsInfo.get("IMG")?.type) // console.log(this.fieldsInfos) + try { + const res = await gptImageCall('Image of panda eating a cookie'); + + if (res){ + const result = await Networking.PostToServer('/uploadRemoteImage', { sources: res }); + + console.log(result); + } + } catch (e) { + console.log(e); + } }; @action addField = () => { - const newFields: Col[] = this._columns.concat([{title: '', type: TemplateFieldType.UNSET, desc: '', sizes: [TemplateFieldSize.MEDIUM]}]) + const newFields: Col[] = this._columns.concat([{title: '', type: TemplateFieldType.UNSET, desc: '', sizes: []}]) this._columns = newFields; }; @@ -479,7 +491,7 @@ export class DocCreatorMenu extends ObservableReactComponent { } }; - setColTitle = (column: Col, title: string) => { + @action setColTitle = (column: Col, title: string) => { if (this.selectedFields.includes(column.title)) { this._dataViz?.setColumnTitle(column.title, title); } else { @@ -488,7 +500,7 @@ export class DocCreatorMenu extends ObservableReactComponent { this.forceUpdate(); }; - setColType = (column: Col, type: TemplateFieldType) => { + @action setColType = (column: Col, type: TemplateFieldType) => { if (this.selectedFields.includes(column.title)) { this._dataViz?.setColumnType(column.title, type); } else { @@ -507,7 +519,6 @@ export class DocCreatorMenu extends ObservableReactComponent { column.sizes.push(size); } } - console.log(column.sizes) this.forceUpdate(); }; @@ -520,6 +531,22 @@ export class DocCreatorMenu extends ObservableReactComponent { this.forceUpdate(); }; + generateGPTImage = async(prompt: string): Promise => { + console.log(prompt) + + try { + const res = await gptImageCall(prompt); + + if (res){ + const result = await Networking.PostToServer('/uploadRemoteImage', { sources: res }); + const source = ClientUtils.prepend(result[0].accessPaths.agnostic.client); + return source; + } + } catch (e) { + console.log(e); + } + } + matchesForTemplate = (template: TemplateDocInfos, cols: Col[]): number[][] => { const colMatchesField = (col: Col, field : Field) => { return field.sizes?.some(size => col.sizes?.includes(size)) && field.types?.includes(col.type) }; @@ -539,13 +566,11 @@ export class DocCreatorMenu extends ObservableReactComponent { maxMatches = (fieldsCt: number, matches: number[][]) => { const used: boolean[] = Array(fieldsCt).fill(false); const mt: number[] = Array(fieldsCt).fill(-1); - console.log(fieldsCt, matches) const augmentingPath = (v: number): boolean => { if (used[v]) return false; used[v] = true; for (const to of matches[v]) { - console.log(mt[to]); if (mt[to] === -1 || augmentingPath(mt[to])) { mt[to] = v; return true; @@ -582,7 +607,6 @@ export class DocCreatorMenu extends ObservableReactComponent { validTemplates = validTemplates.map(title => TemplateLayouts.fieldByTitle(title)); - console.log(validTemplates); return validTemplates; }; @@ -626,20 +650,105 @@ export class DocCreatorMenu extends ObservableReactComponent { } } + const renderTextCalls = async(): Promise => { + const rendered: Doc[] = []; + + if (GPTTextCalls.length) { + + try { + const prompt = fieldContent + GPTTextAssignment; + + const res = await gptAPICall(prompt, GPTCallType.FILL); + + if (res){ + + const assignments: {[title: string]: {number: string, content: string}} = JSON.parse(res); + //console.log('assignments', GPTAssignments, 'assignment string', GPTAssignmentString, 'field content', fieldContent, 'response', res, 'assignments', assignments); + Object.entries(assignments).forEach(([title, info]) => { + const field: Field = template.fields[Number(info.number)]; + const col = this.getColByTitle(title); + + const doc = FieldFuncs.TextField({ + tl: field.tl, + br: field.br }, + template.height, + template.width, + col.title, + info.content ?? '', + field.opts + ); + + rendered.push(doc); + }); + + } + } catch(err) { + console.log(err); + } + } + + return rendered; + }; + + const createGeneratedImage = async(fieldNum: string, col: Col, prompt: string) => { + const url = await this.generateGPTImage(prompt); + const field: Field = template.fields[Number(fieldNum)]; + const doc = FieldFuncs.ImageField({ + tl: field.tl, + br: field.br }, + template.height, + template.width, + col.title, + url ?? '', + field.opts + ); + + return doc; + } + + const renderImageCalls = async(): Promise => { + const rendered: Doc[] = []; + const calls = GPTIMGCalls; + + if (calls.length) { + try { + const renderedImages: Doc[] = await Promise.all( + calls.map(async ([fieldNum, col]) => { + const sysPrompt = 'Your job is to create a prompt for an AI image generator to help it generate an image based on existing content in a template and a user prompt. ONLY INCLUDE THE PROMPT, NO OTHER TEXT OR EXPLANATION. The existing content is as follows: ' + fieldContent + ' **** The user prompt is: ' + col.desc; + + const prompt = await gptAPICall(sysPrompt, GPTCallType.COMPLETEPROMPT); + console.log(sysPrompt, prompt); + + return createGeneratedImage(fieldNum, col, prompt); + }) + ); + + const renderedTemplates: Doc[] = await Promise.all(renderedImages); + renderedTemplates.forEach(doc => rendered.push(doc)); + } catch (e){ + console.log(e); + } + } + + return rendered; + } + const fields: Doc[] = []; const GPTAssignments = Object.entries(assignments).filter(([f, col]) => this._columns.includes(col)); const nonGPTAssignments: [string, Col][] = Object.entries(assignments).filter(a => !GPTAssignments.includes(a)); + const GPTTextCalls = GPTAssignments.filter(([str, col]) => col.type === TemplateFieldType.TEXT); + const GPTIMGCalls = GPTAssignments.filter(([str, col]) => col.type === TemplateFieldType.VISUAL); - const stringifyGPTInfo = (): string => { + const stringifyGPTInfo = (calls: [string, Col][]): string => { let string: string = '*** COLUMN INFO:'; - GPTAssignments.forEach(([fieldNum, col]) => { + calls.forEach(([fieldNum, col]) => { string += `--- title: ${col.title}, prompt: ${col.desc}, word limit: ${wordLimit(col.sizes[0])} words, assigned field: ${fieldNum} ---` }); return string += ' ***'; }; - const GPTAssignmentString = stringifyGPTInfo(); + const GPTTextAssignment = stringifyGPTInfo(GPTTextCalls); let fieldContent: string = ''; @@ -695,47 +804,14 @@ export class DocCreatorMenu extends ObservableReactComponent { return main; } - if (GPTAssignments.length) { - - try { - const prompt = fieldContent + GPTAssignmentString; - - const res = await gptAPICall(prompt, GPTCallType.FILL); - - if (res){ - console.log('response', res); - - const assignments: {[title: string]: {number: string, content: string}} = JSON.parse(res); - console.log('assignments', GPTAssignments, 'assignment string', GPTAssignmentString, 'field content', fieldContent, 'response', res, 'assignments', assignments); - Object.entries(assignments).forEach(([title, info]) => { - const field: Field = template.fields[Number(info.number)]; - const col = this.getColByTitle(title); - - const doc = (col.type === TemplateFieldType.VISUAL ? FieldFuncs.ImageField : FieldFuncs.TextField)({ - tl: field.tl, - br: field.br }, - template.height, - template.width, - col.title, - info.content ?? '', - field.opts - ); - - fields.push(doc); - }); - - return createMainDoc(); - } - } catch(err) { - console.log(err); - } - - } else { - return createMainDoc(); - } + const textCalls = await renderTextCalls(); + const imageCalls = await renderImageCalls(); - return new Doc; - }; + textCalls.forEach(doc => {fields.push(doc)}); + imageCalls.forEach(doc => {fields.push(doc)}); + + return createMainDoc(); + } compileFieldDescriptions = (templates: TemplateDocInfos[]): string => { let descriptions: string = ''; @@ -765,7 +841,6 @@ export class DocCreatorMenu extends ObservableReactComponent { const inputText = fieldDescriptions.concat(colDescriptions); - console.log(inputText); ++this._callCount; const origCount = this._callCount; @@ -779,14 +854,12 @@ export class DocCreatorMenu extends ObservableReactComponent { if (res && this._callCount === origCount) { this._GPTLoading = false; - console.log(res); - const assignments: {[templateTitle: string]: {[field: string]: string}} = JSON.parse(res); const brokenDownAssignments: [TemplateDocInfos, {[field: number]: Col}][] = []; + Object.entries(assignments).forEach(([tempTitle, assignment]) => { const template = TemplateLayouts.fieldByTitle(tempTitle); if (!template) return; - console.log(assignments) const toObj = Object.entries(assignment).reduce((a, [fieldNum, colTitle]) => { a[Number(fieldNum)] = this.getColByTitle(colTitle); return a; @@ -849,7 +922,7 @@ export class DocCreatorMenu extends ObservableReactComponent {
Suggested Templates
-
@@ -1197,7 +1270,7 @@ export class DocCreatorMenu extends ObservableReactComponent {
Title
-