From 57e3c9b9977228a561e8972a469a67f17f4bcd9c Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Wed, 18 Dec 2024 20:34:33 -0500 Subject: trying new image generation plus new implementaion of video and audio --- .../views/nodes/chatbot/tools/ImageCreationTool.ts | 74 ++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 src/client/views/nodes/chatbot/tools/ImageCreationTool.ts (limited to 'src/client/views/nodes/chatbot/tools/ImageCreationTool.ts') diff --git a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts new file mode 100644 index 000000000..cf9e8cfc8 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts @@ -0,0 +1,74 @@ +import { v4 as uuidv4 } from 'uuid'; +import { Networking } from '../../../../Network'; +import { BaseTool } from './BaseTool'; +import { Observation } from '../types/types'; +import { ParametersType, ToolInfo } from '../types/tool_types'; +import { DocumentOptions } from '../../../../documents/Documents'; + +const imageCreationToolParams = [ + { + name: 'image_prompt', + type: 'string', + description: 'The prompt for the image to be created. This should be a string that describes the image to be created in extreme detail for an AI image generator.', + required: true, + }, +] as const; + +type ImageCreationToolParamsType = typeof imageCreationToolParams; + +const imageCreationToolInfo: ToolInfo = { + name: 'imageCreationTool', + citationRules: 'No citation needed. Cannot cite image generation for a response.', + parameterRules: imageCreationToolParams, + description: 'Create an image of any style, content, or design, based on a prompt. The prompt should be a detailed description of the image to be created.', +}; + +export class ImageCreationTool extends BaseTool { + private _addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void; + constructor(addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void) { + super(imageCreationToolInfo); + this._addLinkedDoc = addLinkedDoc; + } + + async execute(args: ParametersType): Promise { + const image_prompt = args.image_prompt; + + console.log(`Generating image for prompt: ${image_prompt}`); + // Create an array of promises, each one handling a search for a query + try { + try { + const { image_url } = await Networking.PostToServer('/generateImage', { + image_prompt, + }); + if (res) { + const result = await Networking.PostToServer('/uploadRemoteImage', { sources: res }); + const source = ClientUtils.prepend(result[0].accessPaths.agnostic.client); + return source; + } + } catch (e) { + console.log(e); + } + + const { base64_data, image_path } = await Networking.PostToServer('/generateImage', { + image_prompt, + }); + const id = uuidv4(); + + this._addLinkedDoc('image', image_path, {}, id); + return [ + { + type: 'image_url', + image_url: { url: `data:image/jpeg;base64,${base64_data}` }, + }, + ]; + } catch (error) { + console.log(error); + return [ + { + type: 'text', + text: `An error occurred while generating image.`, + }, + ]; + } + } +} -- cgit v1.2.3-70-g09d2 From 9b4c554cca11f5c3105085b54646e684dd235f1d Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Wed, 18 Dec 2024 20:46:27 -0500 Subject: image creation works but is weird --- .../views/nodes/DataVizBox/DocCreatorMenu.tsx | 29 ++++++++------ .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 21 ++++++++++- .../views/nodes/chatbot/tools/ImageCreationTool.ts | 44 +++++++++++----------- src/server/ApiManagers/AssistantManager.ts | 24 ++---------- 4 files changed, 61 insertions(+), 57 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/ImageCreationTool.ts') diff --git a/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx b/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx index 16c016d6c..94a37a19f 100644 --- a/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx +++ b/src/client/views/nodes/DataVizBox/DocCreatorMenu.tsx @@ -245,7 +245,7 @@ export class DocCreatorMenu extends ObservableReactComponent { } updateIcons = (docs: Doc[]) => { - console.log('called') + console.log('called'); docs.map(this.getIcon); }; @@ -919,17 +919,17 @@ export class DocCreatorMenu extends ObservableReactComponent { @action setExpandedView = (info: { icon: ImageField; doc: Doc } | undefined) => { if (info) { const doc = info.doc; - const wrapper: Doc = Docs.Create.FreeformDocument([info.doc], { _height: NumListCast(doc._height)[0], _width: NumListCast(doc._width)[0], title: ''}); - const newInfo = {icon: new ImageField(''), doc: wrapper} + const wrapper: Doc = Docs.Create.FreeformDocument([info.doc], { _height: NumListCast(doc._height)[0], _width: NumListCast(doc._width)[0], title: '' }); + const newInfo = { icon: new ImageField(''), doc: wrapper }; this._expandedPreview = newInfo; } else { this._expandedPreview = info; } }; - get editingWindow(){ + get editingWindow() { const doc = this._expandedPreview?.doc ?? new Doc(); - const rendered = + const rendered = (
{ removeDocument={returnFalse} PanelWidth={() => this._menuDimensions.width - 10} PanelHeight={() => this._menuDimensions.height - 60} - ScreenToLocalTransform={() => new Transform(-this._pageX,-this._pageY, 1)} + ScreenToLocalTransform={() => new Transform(-this._pageX, -this._pageY, 1)} renderDepth={5} whenChildContentsActiveChanged={emptyFunction} focus={emptyFunction} @@ -961,14 +961,21 @@ export class DocCreatorMenu extends ObservableReactComponent { yPadding={0} />
- + ); return (
-
+
{rendered}
-
- ); } @@ -1639,7 +1645,7 @@ export interface FieldOpts { fieldViewType?: 'freeform' | 'stacked'; } -type Field = { +export type Field = { tl: [number, number]; br: [number, number]; opts: FieldOpts; @@ -2359,4 +2365,3 @@ export class FieldUtils { // }] // }; // } - diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index baa4ad521..e5a90ab4a 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -39,6 +39,7 @@ import { AudioBox } from '../../AudioBox'; import { DiagramBox } from '../../DiagramBox'; import { ImageField } from '../../../../../fields/URLField'; import { DashUploadUtils } from '../../../../../server/DashUploadUtils'; +import { DocCreatorMenu, Field, FieldUtils } from '../../DataVizBox/DocCreatorMenu'; dotenv.config(); @@ -398,6 +399,23 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.addCSVForAnalysis(doc, id); }; + @action + createImageInDash = async (url: string, title: string, id: string, data: string) => { + const doc = FieldUtils.ImageField( + { + tl: [0, 0], + br: [300, 300], + }, + 300, + 300, + title, + url ?? '', + {} + ); + + return doc; + }; + /** * Creates a text document in the dashboard and adds it for analysis. * @param title The title of the doc. @@ -415,8 +433,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { break; case 'image': console.log('imageURL: ' + data); - //DashUploadUtils.UploadImage(data!); - doc = Docs.Create.ImageDocument(data || '', options); + doc = await this.createImageInDash(data || '', options.title as string, '', data || ''); break; case 'pdf': doc = Docs.Create.PdfDocument(data || '', options); diff --git a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts index cf9e8cfc8..3db401b14 100644 --- a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts +++ b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts @@ -4,6 +4,7 @@ import { BaseTool } from './BaseTool'; import { Observation } from '../types/types'; import { ParametersType, ToolInfo } from '../types/tool_types'; import { DocumentOptions } from '../../../../documents/Documents'; +import { ClientUtils } from '../../../../../ClientUtils'; const imageCreationToolParams = [ { @@ -36,31 +37,30 @@ export class ImageCreationTool extends BaseTool { console.log(`Generating image for prompt: ${image_prompt}`); // Create an array of promises, each one handling a search for a query try { - try { - const { image_url } = await Networking.PostToServer('/generateImage', { - image_prompt, - }); - if (res) { - const result = await Networking.PostToServer('/uploadRemoteImage', { sources: res }); - const source = ClientUtils.prepend(result[0].accessPaths.agnostic.client); - return source; - } - } catch (e) { - console.log(e); - } - - const { base64_data, image_path } = await Networking.PostToServer('/generateImage', { + const { url } = await Networking.PostToServer('/generateImage', { image_prompt, }); - const id = uuidv4(); + if (url) { + const result = await Networking.PostToServer('/uploadRemoteImage', { sources: [url] }); + const source = ClientUtils.prepend(result[0].accessPaths.agnostic.client); - this._addLinkedDoc('image', image_path, {}, id); - return [ - { - type: 'image_url', - image_url: { url: `data:image/jpeg;base64,${base64_data}` }, - }, - ]; + const id = uuidv4(); + + this._addLinkedDoc('image', source, {}, id); + return [ + { + type: 'image_url', + image_url: { url }, + }, + ]; + } else { + return [ + { + type: 'text', + text: `An error occurred while generating image.`, + }, + ]; + } } catch (error) { console.log(error); return [ diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index 83bb1b228..425365348 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -426,30 +426,12 @@ export default class AssistantManager extends ApiManager { } try { - const image = await openai.images.generate({ model: 'dall-e-3', prompt: image_prompt, response_format: 'b64_json' }); + const image = await openai.images.generate({ model: 'dall-e-3', prompt: image_prompt, response_format: 'url' }); console.log(image); - const base64String = image.data[0].b64_json; - if (!base64String) { - throw new Error('No base64 data received from image generation'); - } - // Generate a UUID for the file to ensure unique naming - const uuidv4 = uuid.v4(); - const fullFilename = `${uuidv4}.jpg`; // Prefix the file name with the UUID - - // Get the full server path where the file will be saved - const serverFilePath = serverPathToFile(Directory.images, fullFilename); - - const binaryData = Buffer.from(base64String, 'base64'); + const url = image.data[0].url; - // Write the CSV data (which is a raw string) to the file - await writeFileAsync(serverFilePath, binaryData); - - // Construct the client-accessible URL for the file - const fileUrl = clientPathToFile(Directory.images, fullFilename); - - // Send the file URL and UUID back to the client - res.send({ base64_data: base64String, image_path: fileUrl }); + res.send({ url }); } catch (error) { console.error('Error fetching the URL:', error); res.status(500).send({ -- cgit v1.2.3-70-g09d2 From f915013d2ccfaeb7f04bf8bfea57e6d7d1f66b81 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Thu, 19 Dec 2024 11:45:00 -0500 Subject: image generation works better --- .../views/nodes/chatbot/agentsystem/Agent.ts | 11 ++-- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 54 +++++++++++------- .../views/nodes/chatbot/tools/CreateAnyDocTool.ts | 8 ++- .../views/nodes/chatbot/tools/DictionaryTool.ts | 64 ++++++++++++++++++++++ .../views/nodes/chatbot/tools/ImageCreationTool.ts | 16 +++--- src/client/views/nodes/chatbot/tools/RAGTool.ts | 6 +- src/client/views/nodes/chatbot/types/types.ts | 1 + .../views/nodes/chatbot/vectorstore/Vectorstore.ts | 4 +- src/server/ApiManagers/AssistantManager.ts | 7 +-- 9 files changed, 130 insertions(+), 41 deletions(-) create mode 100644 src/client/views/nodes/chatbot/tools/DictionaryTool.ts (limited to 'src/client/views/nodes/chatbot/tools/ImageCreationTool.ts') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 1eb5e3963..1cf6ca030 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -21,6 +21,7 @@ import { CreateTextDocTool } from '../tools/CreateTextDocumentTool'; import { DocumentOptions } from '../../../../documents/Documents'; import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; import { ImageCreationTool } from '../tools/ImageCreationTool'; +import { DictionaryTool } from '../tools/DictionaryTool'; dotenv.config(); @@ -60,7 +61,8 @@ export class Agent { csvData: () => { filename: string; id: string; text: string }[], addLinkedUrlDoc: (url: string, id: string) => void, addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void, - createCSVInDash: (url: string, title: string, id: string, data: string) => void + createCSVInDash: (url: string, title: string, id: string, data: string) => void, + createImage: (result: any, options: DocumentOptions) => void ) { // Initialize OpenAI client with API key from environment this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); @@ -74,13 +76,14 @@ export class Agent { calculate: new CalculateTool(), rag: new RAGTool(this.vectorstore), dataAnalysis: new DataAnalysisTool(csvData), - //websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), - //searchTool: new SearchTool(addLinkedUrlDoc), + websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), + searchTool: new SearchTool(addLinkedUrlDoc), createCSV: new CreateCSVTool(createCSVInDash), noTool: new NoTool(), - imageCreationTool: new ImageCreationTool(addLinkedDoc), + imageCreationTool: new ImageCreationTool(createImage), //createTextDoc: new CreateTextDocTool(addLinkedDoc), createAnyDocument: new CreateAnyDocumentTool(addLinkedDoc), + dictionary: new DictionaryTool(), }; } diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index e5a90ab4a..d2931106a 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -40,6 +40,11 @@ import { DiagramBox } from '../../DiagramBox'; import { ImageField } from '../../../../../fields/URLField'; import { DashUploadUtils } from '../../../../../server/DashUploadUtils'; import { DocCreatorMenu, Field, FieldUtils } from '../../DataVizBox/DocCreatorMenu'; +import { ImageUtils } from '../../../../util/Import & Export/ImageUtils'; +import { ScriptManager } from '../../../../util/ScriptManager'; +import { CompileError, CompileScript } from '../../../../util/Scripting'; +import { ScriptField } from '../../../../../fields/ScriptField'; +import { ScriptingBox } from '../../ScriptingBox'; dotenv.config(); @@ -96,7 +101,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); } this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); - this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createDocInDash, this.createCSVInDash); + this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createDocInDash, this.createCSVInDash, this.createImageInDash); this.messagesRef = React.createRef(); // Reaction to update dataDoc when chat history changes @@ -400,20 +405,17 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }; @action - createImageInDash = async (url: string, title: string, id: string, data: string) => { - const doc = FieldUtils.ImageField( - { - tl: [0, 0], - br: [300, 300], - }, - 300, - 300, - title, - url ?? '', - {} - ); - - return doc; + createImageInDash = async (result: any, options: DocumentOptions) => { + const newImgSrc = + result.accessPaths.agnostic.client.indexOf('dashblobstore') === -1 // + ? ClientUtils.prepend(result.accessPaths.agnostic.client) + : result.accessPaths.agnostic.client; + const doc = Docs.Create.ImageDocument(newImgSrc, options); + this.addDocument(ImageUtils.AssignImgInfo(doc, result)); + const linkDoc = Docs.Create.LinkDocument(this.Document, doc); + LinkManager.Instance.addLink(linkDoc); + doc && this._props.addDocument?.(doc); + await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); }; /** @@ -431,10 +433,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { case 'text': doc = Docs.Create.TextDocument(data || '', options); break; - case 'image': - console.log('imageURL: ' + data); - doc = await this.createImageInDash(data || '', options.title as string, '', data || ''); - break; case 'pdf': doc = Docs.Create.PdfDocument(data || '', options); break; @@ -471,6 +469,24 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { case 'chat': doc = Docs.Create.ChatDocument(options); break; + case 'script': + const result = !data!.trim() ? ({ compiled: false, errors: [] } as CompileError) : CompileScript(data!, {}); + const script_field = result.compiled ? new ScriptField(result, undefined, data!) : undefined; + doc = Docs.Create.ScriptingDocument(script_field, options); + await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + const firstView = Array.from(doc[DocViews])[0] as DocumentView; + (firstView.ComponentView as ScriptingBox)?.onApply?.(); + (firstView.ComponentView as ScriptingBox)?.onRun?.(); + }); + + break; + // this.dataDoc.script = this.rawScript; + + // ScriptManager.Instance.addScript(this.dataDoc); + + // this._scriptKeys = ScriptingGlobals.getGlobals(); + // this._scriptingDescriptions = ScriptingGlobals.getDescriptions(); + // this._scriptingParams = ScriptingGlobals.getParameters(); // Add more cases for other document types default: console.error('Unknown or unsupported document type:', doc_type); diff --git a/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts b/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts index 4c059177b..36f133503 100644 --- a/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts @@ -7,8 +7,8 @@ import { DocumentOptions, Docs } from '../../../../documents/Documents'; /** * List of supported document types that can be created via text LLM. */ -type supportedDocumentTypesType = 'text' | 'html' | 'equation' | 'function_plot' | 'dataviz' | 'note_taking' | 'rtf' | 'message' | 'mermaid_diagram'; -const supportedDocumentTypes: supportedDocumentTypesType[] = ['text', 'html', 'equation', 'function_plot', 'dataviz', 'note_taking', 'rtf', 'message', 'mermaid_diagram']; +type supportedDocumentTypesType = 'text' | 'html' | 'equation' | 'function_plot' | 'dataviz' | 'note_taking' | 'rtf' | 'message' | 'mermaid_diagram' | 'script'; +const supportedDocumentTypes: supportedDocumentTypesType[] = ['text', 'html', 'equation', 'function_plot', 'dataviz', 'note_taking', 'rtf', 'message', 'mermaid_diagram', 'script']; /** * Description of document options and data field for each type. @@ -50,6 +50,10 @@ const documentTypesInfo = { options: ['title', 'backgroundColor', 'layout'], dataDescription: 'The Mermaid diagram content.', }, + script: { + options: ['title', 'backgroundColor', 'layout'], + dataDescription: 'The compilable JavaScript code. Use this for creating scripts.', + }, }; const createAnyDocumentToolParams = [ diff --git a/src/client/views/nodes/chatbot/tools/DictionaryTool.ts b/src/client/views/nodes/chatbot/tools/DictionaryTool.ts new file mode 100644 index 000000000..fa554e7b3 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/DictionaryTool.ts @@ -0,0 +1,64 @@ +import { Observation } from '../types/types'; +import { ParametersType, ToolInfo } from '../types/tool_types'; +import { BaseTool } from './BaseTool'; + +// Define the tool's parameters +const dictionaryToolParams = [ + { + name: 'word', + type: 'string', + description: 'The word to look up in the dictionary.', + required: true, + }, +] as const; + +type DictionaryToolParamsType = typeof dictionaryToolParams; + +// Define the tool's metadata and rules +const dictionaryToolInfo: ToolInfo = { + name: 'dictionary', + citationRules: 'No citation needed.', + parameterRules: dictionaryToolParams, + description: 'Fetches the definition of a given word using an open dictionary API.', +}; + +export class DictionaryTool extends BaseTool { + constructor() { + super(dictionaryToolInfo); + } + + async execute(args: ParametersType): Promise { + const url = `https://api.dictionaryapi.dev/api/v2/entries/en/${args.word}`; + + try { + const response = await fetch(url); + const data = await response.json(); + + // Handle cases where the word is not found + if (data.title === 'No Definitions Found') { + return [ + { + type: 'text', + text: `Sorry, I couldn't find a definition for the word "${args.word}".`, + }, + ]; + } + + // Extract the first definition + const definition = data[0]?.meanings[0]?.definitions[0]?.definition; + return [ + { + type: 'text', + text: `The definition of "${args.word}" is: ${definition}`, + }, + ]; + } catch (error) { + return [ + { + type: 'text', + text: `An error occurred while fetching the definition: ${error}`, + }, + ]; + } + } +} diff --git a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts index 3db401b14..ba1aa987a 100644 --- a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts +++ b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts @@ -5,6 +5,8 @@ import { Observation } from '../types/types'; import { ParametersType, ToolInfo } from '../types/tool_types'; import { DocumentOptions } from '../../../../documents/Documents'; import { ClientUtils } from '../../../../../ClientUtils'; +import { DashUploadUtils } from '../../../../../server/DashUploadUtils'; +import { RTFCast, StrCast } from '../../../../../fields/Types'; const imageCreationToolParams = [ { @@ -25,10 +27,10 @@ const imageCreationToolInfo: ToolInfo = { }; export class ImageCreationTool extends BaseTool { - private _addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void; - constructor(addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void) { + private _createImage: (result: any, options: DocumentOptions) => void; + constructor(createImage: (result: any, options: DocumentOptions) => void) { super(imageCreationToolInfo); - this._addLinkedDoc = addLinkedDoc; + this._createImage = createImage; } async execute(args: ParametersType): Promise { @@ -37,16 +39,14 @@ export class ImageCreationTool extends BaseTool { console.log(`Generating image for prompt: ${image_prompt}`); // Create an array of promises, each one handling a search for a query try { - const { url } = await Networking.PostToServer('/generateImage', { + const { result, url } = await Networking.PostToServer('/generateImage', { image_prompt, }); + console.log('Image generation result:', result); + this._createImage(result, { text: RTFCast(image_prompt) }); if (url) { - const result = await Networking.PostToServer('/uploadRemoteImage', { sources: [url] }); - const source = ClientUtils.prepend(result[0].accessPaths.agnostic.client); - const id = uuidv4(); - this._addLinkedDoc('image', source, {}, id); return [ { type: 'image_url', diff --git a/src/client/views/nodes/chatbot/tools/RAGTool.ts b/src/client/views/nodes/chatbot/tools/RAGTool.ts index 1f73986a7..2db61c768 100644 --- a/src/client/views/nodes/chatbot/tools/RAGTool.ts +++ b/src/client/views/nodes/chatbot/tools/RAGTool.ts @@ -23,8 +23,9 @@ const ragToolInfo: ToolInfo = { 1. **Grounded Text Guidelines**: - Each tag must correspond to exactly one citation, ensuring a one-to-one relationship. - Always cite a **subset** of the chunk, never the full text. The citation should be as short as possible while providing the relevant information (typically one to two sentences). - - Do not paraphrase the chunk text in the citation; use the original subset directly from the chunk. + - Do not paraphrase the chunk text in the citation; use the original subset directly from the chunk. IT MUST BE EXACT AND WORD FOR WORD FROM THE ORIGINAL CHUNK! - If multiple citations are needed for different sections of the response, create new tags for each. + - !!!IMPORTANT: For video transcript citations, use a subset of the exact text from the transcript as the citation content. It should be just before the start of the section of the transcript that is relevant to the grounded_text tag. 2. **Citation Guidelines**: - The citation must include only the relevant excerpt from the chunk being referenced. @@ -56,7 +57,8 @@ const ragToolInfo: ToolInfo = { ***NOTE***: - Prefer to cite visual elements (i.e. chart, image, table, etc.) over text, if they both can be used. Only if a visual element is not going to be helpful, then use text. Otherwise, use both! - Use as many citations as possible (even when one would be sufficient), thus keeping text as grounded as possible. - - Cite from as many documents as possible and always use MORE, and as granular, citations as possible.`, + - Cite from as many documents as possible and always use MORE, and as granular, citations as possible. + - CITATION TEXT MUST BE EXACTLY AS IT APPEARS IN THE CHUNK. DO NOT PARAPHRASE!`, parameterRules: ragToolParams, }; diff --git a/src/client/views/nodes/chatbot/types/types.ts b/src/client/views/nodes/chatbot/types/types.ts index 54fd7c979..995ac531d 100644 --- a/src/client/views/nodes/chatbot/types/types.ts +++ b/src/client/views/nodes/chatbot/types/types.ts @@ -19,6 +19,7 @@ export enum CHUNK_TYPE { URL = 'url', CSV = 'CSV', MEDIA = 'media', + VIDEO = 'video', } export enum PROCESSING_TYPE { diff --git a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts index 3ed433778..d962b887f 100644 --- a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts +++ b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts @@ -159,7 +159,7 @@ export class Vectorstore { start_time: chunk.start, end_time: chunk.end, text: chunk.text, - chunkType: 'text', + type: CHUNK_TYPE.VIDEO, }, })), type: 'media', @@ -176,7 +176,7 @@ export class Vectorstore { start_time: chunk.metadata.start_time, end_time: chunk.metadata.end_time, indexes: chunk.metadata.indexes, - chunkType: CHUNK_TYPE.TEXT, + chunkType: CHUNK_TYPE.VIDEO, text: chunk.metadata.text, })); doc.chunk_simpl = JSON.stringify({ chunks: simplifiedChunks }); diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index 425365348..fbda74194 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -30,6 +30,7 @@ import ffmpeg from 'fluent-ffmpeg'; import OpenAI from 'openai'; import * as xmlbuilder from 'xmlbuilder'; import { last } from 'lodash'; +import { DashUploadUtils } from '../DashUploadUtils'; // Enumeration of directories where different file types are stored export enum Directory { @@ -370,9 +371,6 @@ export default class AssistantManager extends ApiManager { } // Step 5: Return the JSON result res.send({ full: originalSegments, condensed: combinedSegments, summary }); - - // Step 5: Return the JSON result - res.send({ full: originalSegments, condensed: combinedSegments, summary: summary }); } catch (error) { console.error('Error processing media file:', error); res.status(500).send({ error: 'Failed to process media file' }); @@ -428,10 +426,11 @@ export default class AssistantManager extends ApiManager { try { const image = await openai.images.generate({ model: 'dall-e-3', prompt: image_prompt, response_format: 'url' }); console.log(image); + const result = await DashUploadUtils.UploadImage(image.data[0].url!); const url = image.data[0].url; - res.send({ url }); + res.send({ result, url }); } catch (error) { console.error('Error fetching the URL:', error); res.status(500).send({ -- cgit v1.2.3-70-g09d2 From f6581aebd603dcdd84e033acffe46352ce5a4485 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 19 Feb 2025 15:05:47 -0500 Subject: more gptpopup cleanup. --- src/client/apis/gpt/GPT.ts | 59 +++++++------- src/client/util/CurrentUserUtils.ts | 6 +- src/client/util/Import & Export/ImageUtils.ts | 12 +-- src/client/views/StyleProviderQuiz.tsx | 2 +- src/client/views/collections/CollectionSubView.tsx | 33 ++++---- src/client/views/global/globalScripts.ts | 14 ++-- src/client/views/nodes/ComparisonBox.tsx | 6 +- .../views/nodes/chatbot/agentsystem/Agent.ts | 3 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 3 +- .../views/nodes/chatbot/tools/ImageCreationTool.ts | 36 ++++---- src/client/views/pdf/GPTPopup/GPTPopup.tsx | 95 ++++++++++------------ src/fields/Doc.ts | 2 +- 12 files changed, 126 insertions(+), 145 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/ImageCreationTool.ts') diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index e1ae99d97..6d9bc1d06 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -1,12 +1,11 @@ import { ChatCompletionMessageParam, Image } from 'openai/resources'; import { openai } from './setup'; -export enum GPTTypeStyle { +export enum GPTDocCommand { AssignTags = 1, Filter = 2, - DocInfo = 3, - GeneralInfo = 4, - SortDocs = 5, + GetInfo = 3, + Sort = 4, } export const DescriptionSeperator = '======'; @@ -18,8 +17,6 @@ enum GPTCallType { EDIT = 'edit', CHATCARD = 'chatcard', // a single flashcard style response to a question FLASHCARD = 'flashcard', // a set of flashcard qustion/answer responses to a topic - QUIZ = 'quiz', - SORT = 'sort', DESCRIBE = 'describe', MERMAID = 'mermaid', DATA = 'data', @@ -27,15 +24,17 @@ enum GPTCallType { PRONUNCIATION = 'pronunciation', DRAW = 'draw', COLOR = 'color', - RUBRIC = 'rubric', // needs to be filled in below - TYPE = 'type', // needs to be filled in below - SUBSET = 'subset', // needs to be filled in below - INFO = 'info', // needs to be filled in below TEMPLATE = 'template', VIZSUM = 'vizsum', VIZSUM2 = 'vizsum2', FILL = 'fill', COMPLETEPROMPT = 'completeprompt', + QUIZDOC = 'quiz_doc', + MAKERUBRIC = 'make_rubric', // create a definition rubric for a document to be used when quizzing the user + COMMANDTYPE = 'command_type', // Determine the type of command being made (GPTQueryType - eg., AssignTags, Sort, Filter, DocInfo, GenInfo) and possibly some parameters (eg, Tag type for Tags) + SUBSETDOCS = 'subset_docs', // select a subset of documents based on their descriptions + DOCINFO = 'doc_info', // provide information about a document + SORTDOCS = 'sort_docs', } type GPTCallOpts = { @@ -69,7 +68,7 @@ const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { temp: 0.5, prompt: "You are a helpful resarch assistant. Analyze the user's data to find meaningful patterns and/or correlation. Please only return a JSON with a correlation column 1 propert, a correlation column 2 property, and an analysis property. ", }, - sort: { + sort_docs: { model: 'gpt-4o', maxTokens: 2048, temp: 0.25, @@ -89,7 +88,7 @@ const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { prompt: 'Make flashcards out of this text with each question and answer labeled as question and answer. Create a title for each question and asnwer that is labeled as "title". Do not label each flashcard and do not include asterisks: ', }, chatcard: { model: 'gpt-4-turbo', maxTokens: 512, temp: 0.5, prompt: 'Answer the following question as a short flashcard response. Do not include a label.' }, - quiz: { + quiz_doc: { model: 'gpt-4-turbo', maxTokens: 1024, temp: 0, @@ -138,20 +137,20 @@ const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { temp: 0.5, prompt: 'You will be coloring drawings. You will be given what the drawing is, then a list of descriptions for parts of the drawing. Based on each description, respond with the stroke and fill color that it should be. Follow the rules: 1. Avoid using black for stroke color 2. Make the stroke color 1-3 shades darker than the fill color 3. Use the same colors when possible. Format as {#abcdef #abcdef}, making sure theres a color for each description, and do not include any additional text.', }, - type: { + command_type: { model: 'gpt-4-turbo', maxTokens: 1024, temp: 0, prompt: `I'm going to provide you with a question. Based on the question, is the user asking you to - ${GPTTypeStyle.AssignTags}. Assigns docs with tags(like star / heart etc)/labels, - ${GPTTypeStyle.DocInfo}. Provide information about a specific doc - ${GPTTypeStyle.Filter}. Filter docs based on a question/information - ${GPTTypeStyle.GeneralInfo}. Provide general information - ${GPTTypeStyle.SortDocs}. Put cards in a specific order. - Answer with only the number for 2-5. For number one, provide the number (1) and the appropriate tag`, + ${GPTDocCommand.AssignTags}. Assigns docs with tags(like star / heart etc)/labels. + ${GPTDocCommand.GetInfo}. Provide information about a specific doc. + ${GPTDocCommand.Filter}. Filter docs based on a question/information. + ${GPTDocCommand.Sort}. Put docs in a specific order. + Answer with only the number for ${GPTDocCommand.GetInfo}-${GPTDocCommand.Sort}. + For number one, provide the number (${GPTDocCommand.AssignTags}) and the appropriate tag`, }, - subset: { + subset_docs: { model: 'gpt-4-turbo', maxTokens: 1024, temp: 0, @@ -164,14 +163,14 @@ const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { It is VERY important that you format it exactly as described, ensuring the proper number of '${DescriptionSeperator[0]}' and '${DocSeperator[0]}' (${DescriptionSeperator.length} of each) and NO commas`, }, - info: { + doc_info: { model: 'gpt-4-turbo', maxTokens: 1024, temp: 0, prompt: `Answer the user's question with a short (<100 word) response. If a particular document is selected I will provide that information (which may help with your response)`, }, - rubric: { + make_rubric: { model: 'gpt-4-turbo', maxTokens: 1024, temp: 0, @@ -188,17 +187,15 @@ let lastResp = ''; * @returns AI Output */ const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: string, dontCache?: boolean) => { - const inputText = inputTextIn + ([GPTCallType.SUMMARY, GPTCallType.FLASHCARD, GPTCallType.QUIZ, GPTCallType.STACK].includes(callType) ? '.' : ''); + const inputText = inputTextIn + ([GPTCallType.SUMMARY, GPTCallType.FLASHCARD, GPTCallType.QUIZDOC, GPTCallType.STACK].includes(callType) ? '.' : ''); const opts = callTypeMap[callType]; if (!opts) { console.log('The query type:' + callType + ' requires a configuration.'); return 'Error connecting with API.'; } - if (lastCall === inputText && dontCache !== true) return lastResp; + if (lastCall === inputText && dontCache !== true && lastResp) return lastResp; try { - lastCall = inputText; - - const usePrompt = prompt ? prompt + opts.prompt : opts.prompt; + const usePrompt = prompt ? prompt + '.' + opts.prompt : opts.prompt; const messages: ChatCompletionMessageParam[] = [ { role: 'system', content: usePrompt }, { role: 'user', content: inputText }, @@ -210,8 +207,12 @@ const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: s temperature: opts.temp, max_tokens: opts.maxTokens, }); - lastResp = response.choices[0].message.content ?? ''; - return lastResp; + const result = response.choices[0].message.content ?? ''; + if (!dontCache) { + lastResp = result; + lastCall = inputText; + } + return result; } catch (err) { console.log(err); return 'Error connecting with API.'; diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index 6ca181d92..fb349abd9 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -710,11 +710,7 @@ pie title Minerals in my tap water { title: "Type", icon:"eye", toolTip:"Sort by document type", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"docType", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, { title: "Color", icon:"palette", toolTip:"Sort by document color", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"color", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, { title: "Tags", icon:"bolt", toolTip:"Sort by document's tags", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"tag", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, - { title: "Sort", icon: "sort" , toolTip: "Manage sort order / lock status", btnType: ButtonType.MultiToggleButton, toolType:"alignment", ignoreClick: true, - subMenu: [ - { title: "Ascending", toolTip: "Sort the cards in ascending order", btnType: ButtonType.ToggleButton, icon: "sort-up", toolType:"up", ignoreClick: true, scripts: {onClick: '{ return showFreeform(this.toolType, _readOnly_);}'} }, - { title: "Descending",toolTip: "Sort the cards in descending order",btnType: ButtonType.ToggleButton, icon: "sort-down",toolType:"down",ignoreClick: true, scripts: {onClick: '{ return showFreeform(this.toolType, _readOnly_);}'} }, - ]}, + { title: "Reverse", icon: "sort-up", toolTip: "Sort the cards in reverse order", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"reverse", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'} }, ] } diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index 8d4eefa7e..f73149fdc 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -4,22 +4,16 @@ import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; import { Cast, NumCast, StrCast } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; +import { Upload } from '../../../server/SharedMediaTypes'; import { Networking } from '../../Network'; export namespace ImageUtils { - export type imgInfo = { - contentSize: number; - nativeWidth: number; - nativeHeight: number; - source: string; - exifData: { error: string | undefined; data: string }; - }; - export const ExtractImgInfo = async (document: Doc): Promise => { + export const ExtractImgInfo = async (document: Doc): Promise => { const field = Cast(document.data, ImageField); return field ? Networking.PostToServer('/inspectImage', { source: field.url.href }) : undefined; }; - export const AssignImgInfo = (document: Doc, data?: imgInfo) => { + export const AssignImgInfo = (document: Doc, data?: Upload.InspectionResults) => { if (data) { data.nativeWidth && (document._height = (NumCast(document._width) * data.nativeHeight) / data.nativeWidth); const proto = document[DocData]; diff --git a/src/client/views/StyleProviderQuiz.tsx b/src/client/views/StyleProviderQuiz.tsx index b3fb8c930..d8eeb3490 100644 --- a/src/client/views/StyleProviderQuiz.tsx +++ b/src/client/views/StyleProviderQuiz.tsx @@ -265,7 +265,7 @@ export namespace styleProviderQuiz { '. ' + rubricText + '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."'; - const response = await gptAPICall(queryText, GPTCallType.QUIZ); + const response = await gptAPICall(queryText, GPTCallType.QUIZDOC); const hexSent = extractHexAndSentences(response); doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); doc.backgroundColor = '#' + hexSent.hexNumber; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 954f9aa4c..b40cd2761 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -37,6 +37,9 @@ export enum docSortings { Tag = 'tag', None = '', } + +export const ChatSortField = 'chat_sortIndex'; + export interface CollectionViewProps extends React.PropsWithChildren { isAnnotationOverlay?: boolean; // is the collection an annotation overlay (eg an overlay on an image/video/etc) isAnnotationOverlayScrollable?: boolean; // whether the annotation overlay can be vertically scrolled (just for tree views, currently) @@ -229,23 +232,21 @@ export function CollectionSubView() { childSortedDocs = (docsIn: Doc[], dragIndex: number) => { const sortType = StrCast(this.Document[this._props.fieldKey + '_sort']) as docSortings; - const isDesc = BoolCast(this.Document[this._props.fieldKey + '_sort_desc']); + const isDesc = BoolCast(this.Document[this._props.fieldKey + '_sort_reverse']); const docs = docsIn.slice(); - if (sortType) { - docs.sort((docA, docB) => { - const [typeA, typeB] = (() => { - switch (sortType) { - default: - case docSortings.Type: return [StrCast(docA.type), StrCast(docB.type)]; - case docSortings.Chat: return [NumCast(docA.chatIndex, 9999), NumCast(docB.chatIndex,9999)]; - case docSortings.Time: return [DateCast(docA.author_date)?.date ?? Date.now(), DateCast(docB.author_date)?.date ?? Date.now()]; - case docSortings.Color:return [DashColor(StrCast(docA.backgroundColor)).hsv().hue(), DashColor(StrCast(docB.backgroundColor)).hsv().hue()]; - case docSortings.Tag: return [StrListCast(docA.tags).join(""), StrListCast(docB.tags).join("")]; - } - })(); //prettier-ignore - return (typeA < typeB ? -1 : typeA > typeB ? 1 : 0) * (isDesc ? 1 : -1); - }); - } + sortType && docs.sort((docA, docB) => { + const [typeA, typeB] = (() => { + switch (sortType) { + default: + case docSortings.Type: return [StrCast(docA.type), StrCast(docB.type)]; + case docSortings.Chat: return [NumCast(docA[ChatSortField], 9999), NumCast(docB[ChatSortField], 9999)]; + case docSortings.Time: return [DateCast(docA.author_date)?.date ?? Date.now(), DateCast(docB.author_date)?.date ?? Date.now()]; + case docSortings.Color:return [DashColor(StrCast(docA.backgroundColor)).hsv().hue(), DashColor(StrCast(docB.backgroundColor)).hsv().hue()]; + case docSortings.Tag: return [StrListCast(docA.tags).join(""), StrListCast(docB.tags).join("")]; + } + })(); + return (typeA < typeB ? -1 : typeA > typeB ? 1 : 0) * (isDesc ? -1 : 1); + }); //prettier-ignore if (dragIndex !== -1) { const draggedDoc = DragManager.docsBeingDragged[0]; const originalIndex = docs.findIndex(doc => doc === draggedDoc); diff --git a/src/client/views/global/globalScripts.ts b/src/client/views/global/globalScripts.ts index 2bc0e3338..79873ed8f 100644 --- a/src/client/views/global/globalScripts.ts +++ b/src/client/views/global/globalScripts.ts @@ -152,7 +152,7 @@ ScriptingGlobals.add(function toggleOverlay(checkResult?: boolean) { // eslint-disable-next-line prefer-arrow-callback ScriptingGlobals.add(function showFreeform( - attr: 'flashcards' | 'hcenter' | 'vcenter' | 'grid' | 'snaplines' | 'clusters' | 'viewAll' | 'fitOnce' | 'time' | 'docType' | 'color' | 'chat' | 'up' | 'down' | 'toggle-chat' | 'toggle-tags' | 'tag', + attr: 'flashcards' | 'hcenter' | 'vcenter' | 'grid' | 'snaplines' | 'clusters' | 'viewAll' | 'fitOnce' | 'time' | 'docType' | 'color' | 'chat' | 'reverse' | 'toggle-chat' | 'toggle-tags' | 'tag', checkResult?: boolean, persist?: boolean ) { @@ -163,7 +163,7 @@ ScriptingGlobals.add(function showFreeform( } // prettier-ignore - const map: Map<'flashcards' | 'hcenter' | 'vcenter' | 'grid' | 'snaplines' | 'clusters' | 'viewAll' | 'fitOnce' | 'time' | 'docType' | 'color' | 'chat' | 'up' | 'down'| 'toggle-chat' | 'toggle-tags' | 'tag', + const map: Map<'flashcards' | 'hcenter' | 'vcenter' | 'grid' | 'snaplines' | 'clusters' | 'viewAll' | 'fitOnce' | 'time' | 'docType' | 'color' | 'chat' | 'reverse'| 'toggle-chat' | 'toggle-tags' | 'tag', { waitForRender?: boolean; checkResult: (doc: Doc) => boolean; @@ -214,13 +214,9 @@ ScriptingGlobals.add(function showFreeform( checkResult: (doc: Doc) => StrCast(doc?.[Doc.LayoutFieldKey(doc)+"_sort"]) === "tag", setDoc: (doc: Doc, dv: DocumentView) => { doc[Doc.LayoutFieldKey(doc)+"_sort"] === "tag" ? doc[Doc.LayoutFieldKey(doc)+"_sort"] = '' : doc[Doc.LayoutFieldKey(doc)+"_sort"] = docSortings.Tag}, // prettier-ignore }], - ['up', { - checkResult: (doc: Doc) => BoolCast(!doc?.[Doc.LayoutFieldKey(doc)+"_sort_desc"]), - setDoc: (doc: Doc, dv: DocumentView) => { doc[Doc.LayoutFieldKey(doc)+"_sort_desc"] = undefined; }, - }], - ['down', { - checkResult: (doc: Doc) => BoolCast(doc?.[Doc.LayoutFieldKey(doc)+"_sort_desc"]), - setDoc: (doc: Doc, dv: DocumentView) => { doc[Doc.LayoutFieldKey(doc)+"_sort_desc"] = true; }, + ['reverse', { + checkResult: (doc: Doc) => BoolCast(doc?.[Doc.LayoutFieldKey(doc)+"_sort_reverse"]), + setDoc: (doc: Doc, dv: DocumentView) => { doc[Doc.LayoutFieldKey(doc)+"_sort_reverse"] = !doc[Doc.LayoutFieldKey(doc)+"_sort_reverse"]; }, }], ['toggle-chat', { checkResult: (doc: Doc) => SnappingManager.ChatVisible, diff --git a/src/client/views/nodes/ComparisonBox.tsx b/src/client/views/nodes/ComparisonBox.tsx index cb0831d3c..5315612e1 100644 --- a/src/client/views/nodes/ComparisonBox.tsx +++ b/src/client/views/nodes/ComparisonBox.tsx @@ -291,7 +291,7 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent() this.askGPTPhonemes(this._inputValue); this._renderSide = this.backKey; this._outputValue = ''; - } else if (this._inputValue) this.askGPT(GPTCallType.QUIZ); + } else if (this._inputValue) this.askGPT(GPTCallType.QUIZDOC); }; onPointerMove = ({ movementX }: PointerEvent) => { @@ -511,7 +511,7 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent() */ askGPT = async (callType: GPTCallType) => { const questionText = this.frontText; - const queryText = questionText + (callType == GPTCallType.QUIZ ? ' UserAnswer: ' + this._inputValue + '. ' + ' Rubric: ' + this.backText : ''); + const queryText = questionText + (callType == GPTCallType.QUIZDOC ? ' UserAnswer: ' + this._inputValue + '. ' + ' Rubric: ' + this.backText : ''); this.loading = true; const res = !this.frontText @@ -522,7 +522,7 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent() case GPTCallType.CHATCARD: DocCast(this.dataDoc[this.backKey])[DocData].text = resp; break; - case GPTCallType.QUIZ: + case GPTCallType.QUIZDOC: this._renderSide = this.backKey; this._outputValue = resp.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); break; diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index b2b0c9aea..2ed808622 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -22,6 +22,7 @@ import { ChatCompletionMessageParam } from 'openai/resources'; import { Doc } from '../../../../../fields/Doc'; import { parsedDoc } from '../chatboxcomponents/ChatBox'; import { WebsiteInfoScraperTool } from '../tools/WebsiteInfoScraperTool'; +import { Upload } from '../../../../../server/SharedMediaTypes'; //import { CreateTextDocTool } from '../tools/CreateTextDocumentTool'; dotenv.config(); @@ -61,7 +62,7 @@ export class Agent { history: () => string, csvData: () => { filename: string; id: string; text: string }[], addLinkedUrlDoc: (url: string, id: string) => void, - createImage: (result: any, options: DocumentOptions) => void, + createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void, addLinkedDoc: (doc: parsedDoc) => Doc | undefined, // eslint-disable-next-line @typescript-eslint/no-unused-vars createCSVInDash: (url: string, title: string, id: string, data: string) => void diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 16da360fc..6e9307d37 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -42,6 +42,7 @@ import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; import { ProgressBar } from './ProgressBar'; import { OpenWhere } from '../../OpenWhere'; +import { Upload } from '../../../../../server/SharedMediaTypes'; dotenv.config(); @@ -412,7 +413,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }); @action - createImageInDash = async (result: any, options: DocumentOptions) => { + createImageInDash = async (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => { const newImgSrc = result.accessPaths.agnostic.client.indexOf('dashblobstore') === -1 // ? ClientUtils.prepend(result.accessPaths.agnostic.client) diff --git a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts index 177552c5c..dc6140871 100644 --- a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts +++ b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts @@ -1,10 +1,10 @@ -import { v4 as uuidv4 } from 'uuid'; import { RTFCast } from '../../../../../fields/Types'; import { DocumentOptions } from '../../../../documents/Documents'; import { Networking } from '../../../../Network'; import { ParametersType, ToolInfo } from '../types/tool_types'; import { Observation } from '../types/types'; import { BaseTool } from './BaseTool'; +import { Upload } from '../../../../../server/SharedMediaTypes'; const imageCreationToolParams = [ { @@ -25,8 +25,8 @@ const imageCreationToolInfo: ToolInfo = { }; export class ImageCreationTool extends BaseTool { - private _createImage: (result: any, options: DocumentOptions) => void; - constructor(createImage: (result: any, options: DocumentOptions) => void) { + private _createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void; + constructor(createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void) { super(imageCreationToolInfo); this._createImage = createImage; } @@ -42,23 +42,19 @@ export class ImageCreationTool extends BaseTool { }); console.log('Image generation result:', result); this._createImage(result, { text: RTFCast(image_prompt) }); - if (url) { - const id = uuidv4(); - - return [ - { - type: 'image_url', - image_url: { url }, - }, - ]; - } else { - return [ - { - type: 'text', - text: `An error occurred while generating image.`, - }, - ]; - } + return url + ? [ + { + type: 'image_url', + image_url: { url }, + }, + ] + : [ + { + type: 'text', + text: `An error occurred while generating image.`, + }, + ]; } catch (error) { console.log(error); return [ diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.tsx b/src/client/views/pdf/GPTPopup/GPTPopup.tsx index 72381cfad..cb3e9b2d7 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.tsx +++ b/src/client/views/pdf/GPTPopup/GPTPopup.tsx @@ -10,7 +10,7 @@ import { ClientUtils } from '../../../../ClientUtils'; import { Doc } from '../../../../fields/Doc'; import { NumCast, StrCast } from '../../../../fields/Types'; import { Networking } from '../../../Network'; -import { DescriptionSeperator, DocSeperator, GPTCallType, GPTTypeStyle, gptAPICall, gptImageCall } from '../../../apis/gpt/GPT'; +import { DescriptionSeperator, DocSeperator, GPTCallType, GPTDocCommand, gptAPICall, gptImageCall } from '../../../apis/gpt/GPT'; import { DocUtils } from '../../../documents/DocUtils'; import { Docs } from '../../../documents/Documents'; import { SettingsManager } from '../../../util/SettingsManager'; @@ -18,7 +18,7 @@ import { SnappingManager } from '../../../util/SnappingManager'; import { undoable } from '../../../util/UndoManager'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { TagItem } from '../../TagsView'; -import { docSortings } from '../../collections/CollectionSubView'; +import { ChatSortField, docSortings } from '../../collections/CollectionSubView'; import { DocumentView } from '../../nodes/DocumentView'; import { AnchorMenu } from '../AnchorMenu'; import './GPTPopup.scss'; @@ -36,6 +36,7 @@ export enum GPTPopupMode { export class GPTPopup extends ObservableReactComponent { // eslint-disable-next-line no-use-before-define static Instance: GPTPopup; + static ChatTag = '#chat'; // tag used by GPT popup to filter docs private _messagesEndRef: React.RefObject; private _correlatedColumns: string[] = []; private _dataChatPrompt: string | undefined = undefined; @@ -77,10 +78,10 @@ export class GPTPopup extends ObservableReactComponent { if (hasChildDocs) { this._textToDocMap.clear(); this.setCollectionContext(selDoc.Document); - this.onGptResponse = (sortResult: string, questionType: GPTTypeStyle, tag?: string) => this.processGptResponse(selDoc, this._textToDocMap, sortResult, questionType, tag); + this.onGptResponse = (sortResult: string, questionType: GPTDocCommand, args?: string) => this.processGptResponse(selDoc, this._textToDocMap, sortResult, questionType, args); this.onQuizRandom = () => this.randomlyChooseDoc(selDoc.Document, hasChildDocs()); this._documentDescriptions = Promise.all(hasChildDocs().map(doc => - Doc.getDescription(doc).then(text => this._textToDocMap.set(text, doc) && `${DescriptionSeperator}${text}${DescriptionSeperator}`) + Doc.getDescription(doc).then(text => this._textToDocMap.set(text.trim(), doc) && `${DescriptionSeperator}${text}${DescriptionSeperator}`) )).then(docDescriptions => docDescriptions.join()); // prettier-ignore } }, @@ -110,8 +111,8 @@ export class GPTPopup extends ObservableReactComponent { @action public setMode = (mode: GPTPopupMode) => (this._mode = mode); onQuizRandom?: () => void; - onGptResponse?: (sortResult: string, questionType: GPTTypeStyle, tag?: string) => void; - questionTypeNumberToStyle = (questionType: string) => +questionType.split(' ')[0][0]; + onGptResponse?: (sortResult: string, questionType: GPTDocCommand, args?: string) => void; + NumberToCommandType = (questionType: string) => +questionType.split(' ')[0][0]; /** * Processes gpt's output depending on the type of question the user asked. Converts gpt's string output to @@ -120,44 +121,38 @@ export class GPTPopup extends ObservableReactComponent { * @param questionType * @param tag */ - processGptResponse = (docView: DocumentView, textToDocMap: Map, gptOutput: string, questionType: GPTTypeStyle, tag?: string) => + processGptResponse = (docView: DocumentView, textToDocMap: Map, gptOutput: string, questionType: GPTDocCommand, args?: string) => undoable(() => { - // Split the string into individual list items - const listItems = gptOutput.split('======').filter(item => item.trim() !== ''); - - if (questionType === GPTTypeStyle.Filter) { - docView.ComponentView?.hasChildDocs?.().forEach(d => TagItem.removeTagFromDoc(d, '#chat')); - } - - if (questionType === GPTTypeStyle.SortDocs) { - docView.Document[docView.ComponentView?.fieldKey + '_sort'] = docSortings.Chat; - } - - listItems.forEach((item, index) => { - const normalizedItem = item.replace(/\n/g, ' ').trim(); - // find the corresponding Doc in the textToDoc map - const doc = textToDocMap.get(normalizedItem); - if (doc) { + switch (questionType) { // reset collection based on question typefc + case GPTDocCommand.Sort: + docView.Document[docView.ComponentView?.fieldKey + '_sort'] = docSortings.Chat; + break; + case GPTDocCommand.Filter: + docView.ComponentView?.hasChildDocs?.().forEach(d => TagItem.removeTagFromDoc(d, GPTPopup.ChatTag)); + break; + } // prettier-ignore + + gptOutput.split('======').filter(item => item.trim() !== '') // Split output into individual document contents + .map(docContentRaw => textToDocMap.get(docContentRaw.replace(/\n/g, ' ').trim())) // the find the corresponding Doc using textToDoc map + .filter(doc => doc).map(doc => doc!) // filter out undefined values + .forEach((doc, index) => { switch (questionType) { - case GPTTypeStyle.SortDocs: - doc.chatIndex = index; + case GPTDocCommand.Sort: + doc[ChatSortField] = index; break; - case GPTTypeStyle.AssignTags: - if (tag) { - const hashTag = tag.startsWith('#') ? tag : '#' + tag[0].toLowerCase() + tag.slice(1); - const filterTag = Doc.MyFilterHotKeys.map(key => StrCast(key.toolType)).find(key => key.includes(tag)) ?? hashTag; + case GPTDocCommand.AssignTags: + if (args) { + const hashTag = args.startsWith('#') ? args : '#' + args[0].toLowerCase() + args.slice(1); + const filterTag = Doc.MyFilterHotKeys.map(key => StrCast(key.toolType)).find(key => key.includes(args)) ?? hashTag; TagItem.addTagToDoc(doc, filterTag); } break; - case GPTTypeStyle.Filter: - TagItem.addTagToDoc(doc, '#chat'); - Doc.setDocFilter(docView.Document, 'tags', '#chat', 'check'); + case GPTDocCommand.Filter: + TagItem.addTagToDoc(doc, GPTPopup.ChatTag); + Doc.setDocFilter(docView.Document, 'tags', GPTPopup.ChatTag, 'check'); break; } - } else { - console.warn(`No matching document found for item: ${normalizedItem}`); - } - }); + }); // prettier-ignore }, '')(); /** @@ -173,7 +168,7 @@ export class GPTPopup extends ObservableReactComponent { StrCast(doc.gptRubric) ? Promise.resolve(StrCast(doc.gptRubric)) : Doc.getDescription(doc).then(desc => - gptAPICall(desc, GPTCallType.RUBRIC) + gptAPICall(desc, GPTCallType.MAKERUBRIC) .then(res => (doc.gptRubric = res)) .catch(err => console.error('GPT call failed', err)) ); @@ -191,7 +186,7 @@ export class GPTPopup extends ObservableReactComponent { `Question: ${desc}; UserAnswer: ${quizAnswer}; Rubric: ${StrCast(doc.gptRubric)}`, - GPTCallType.QUIZ + GPTCallType.QUIZDOC ).then(res => { this._conversationArray.push(res || 'GPT provided no answer'); this.onQuizRandom?.(); @@ -205,20 +200,20 @@ export class GPTPopup extends ObservableReactComponent { * @param userPrompt the user's input that chat will respond to */ generateUserPromptResponse = (userPrompt: string) => - gptAPICall(userPrompt, GPTCallType.TYPE).then(questionType => + gptAPICall(userPrompt, GPTCallType.COMMANDTYPE, undefined, true).then((commandType, args = commandType.split(' ').slice(1).join(' ')) => (async () => { - switch (this.questionTypeNumberToStyle(questionType)) { - case GPTTypeStyle.AssignTags: - case GPTTypeStyle.Filter: return this._documentDescriptions?.then(descs => gptAPICall(descs, GPTCallType.SUBSET, userPrompt)) ?? ""; - case GPTTypeStyle.SortDocs: return this._documentDescriptions?.then(descs => gptAPICall(descs, GPTCallType.SORT, userPrompt)) ?? ""; - default: return Doc.getDescription(DocumentView.SelectedDocs().lastElement()).then(desc => gptAPICall(desc, GPTCallType.INFO, userPrompt)); + switch (this.NumberToCommandType(commandType)) { + case GPTDocCommand.AssignTags: + case GPTDocCommand.Filter: return this._documentDescriptions?.then(descs => gptAPICall(userPrompt, GPTCallType.SUBSETDOCS, descs)) ?? ""; + case GPTDocCommand.Sort: return this._documentDescriptions?.then(descs => gptAPICall(userPrompt, GPTCallType.SORTDOCS, descs)) ?? ""; + default: return Doc.getDescription(DocumentView.SelectedDocs().lastElement()).then(desc => gptAPICall(userPrompt, GPTCallType.DOCINFO, desc)); } // prettier-ignore })().then( action(res => { // Trigger the callback with the result - this.onGptResponse?.(res || 'Something went wrong :(', this.questionTypeNumberToStyle(questionType), questionType.split(' ').slice(1).join(' ')); + this.onGptResponse?.(res || 'Something went wrong :(', this.NumberToCommandType(commandType), args); this._conversationArray.push( - [GPTTypeStyle.GeneralInfo, GPTTypeStyle.DocInfo].includes(this.questionTypeNumberToStyle(questionType)) ? res: + this.NumberToCommandType(commandType) === GPTDocCommand.GetInfo ? res: // Extract explanation surrounded by the DocSeperator string (defined in GPT.ts) at the top or both at the top and bottom (res.match(new RegExp(`${DocSeperator}\\s*([\\s\\S]*?)\\s*(?:${DocSeperator}|$)`)) ?? [])[1]?.trim() ?? 'No explanation found' ); @@ -583,10 +578,10 @@ export class GPTPopup extends ObservableReactComponent { tooltip="Clear Chat filter" toggleType={ToggleType.BUTTON} type={Type.PRIM} - toggleStatus={Doc.hasDocFilter(this._collectionContext, 'tags', '#chat')} - text={Doc.hasDocFilter(this._collectionContext, 'tags', '#chat') ? 'filtered' : ''} - color={Doc.hasDocFilter(this._collectionContext, 'tags', '#chat') ? 'red' : 'transparent'} - onClick={() => this._collectionContext && Doc.setDocFilter(this._collectionContext, 'tags', '#chat', 'remove')} + toggleStatus={Doc.hasDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag)} + text={Doc.hasDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag) ? 'filtered' : ''} + color={Doc.hasDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag) ? 'red' : 'transparent'} + onClick={() => this._collectionContext && Doc.setDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag, 'remove')} /> {(this._mode === GPTPopupMode.USER_PROMPT || this._mode === GPTPopupMode.QUIZ_RESPONSE) && ( } onClick={() => (this._mode = GPTPopupMode.GPT_MENU)} /> diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 950d9047c..bdd41b0bb 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -1482,7 +1482,7 @@ export namespace Doc { case DocumentType.IMG: return curDescription || imageUrlToBase64(ImageCastWithSuffix(Doc.LayoutField(tdoc), '_o') ?? '') .then(hrefBase64 => gptImageLabel(hrefBase64, 'Give three to five labels to describe this image.')); case DocumentType.RTF: return RTFCast(tdoc[Doc.LayoutFieldKey(tdoc)]).Text; - default: return StrCast(tdoc.title); + default: return StrCast(tdoc.title).startsWith("Untitled") ? "" : StrCast(tdoc.title); }}); // prettier-ignore return docText(doc).then(text => (doc[DocData][Doc.LayoutFieldKey(doc) + '_description'] = text)); } -- cgit v1.2.3-70-g09d2 From a9a1a6a507616a77f70d6525dab5027f5b7a60e6 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 26 Feb 2025 20:48:51 -0500 Subject: added typing to PostToServer calls. made smartDraw popup create images locally. --- src/client/Network.ts | 2 +- src/client/apis/GoogleAuthenticationManager.tsx | 27 ++++--- .../apis/google_docs/GoogleApiClientUtils.ts | 19 ++--- .../apis/google_docs/GooglePhotosClientUtils.ts | 18 ++--- src/client/util/Import & Export/ImageUtils.ts | 4 +- src/client/util/PingManager.ts | 2 +- src/client/util/SettingsManager.tsx | 3 +- src/client/views/collections/CollectionSubView.tsx | 3 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 5 +- .../views/nodes/DataVizBox/DocCreatorMenu.tsx | 3 +- src/client/views/nodes/ImageBox.tsx | 20 +++-- .../views/nodes/RecordingBox/RecordingView.tsx | 3 +- .../views/nodes/chatbot/tools/CreateCSVTool.ts | 4 +- .../views/nodes/chatbot/tools/ImageCreationTool.ts | 4 +- src/client/views/nodes/chatbot/tools/RAGTool.ts | 2 +- src/client/views/nodes/chatbot/tools/SearchTool.ts | 8 +- src/client/views/pdf/GPTPopup/GPTPopup.tsx | 13 ++- src/client/views/smartdraw/FireflyConstants.ts | 5 ++ src/client/views/smartdraw/SmartDrawHandler.tsx | 94 ++++++++++------------ 19 files changed, 123 insertions(+), 116 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/ImageCreationTool.ts') diff --git a/src/client/Network.ts b/src/client/Network.ts index 323a2648c..a2ecf1bea 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -22,7 +22,7 @@ export namespace Networking { }, body: body ? JSON.stringify(body) : undefined, }).then(async response => { - if (response.ok) return response.json(); + if (response.ok) return response.json() as object; return await response.text().then(text => ({ error: '' + response.status + ':' + response.statusText + '-' + text })); }); diff --git a/src/client/apis/GoogleAuthenticationManager.tsx b/src/client/apis/GoogleAuthenticationManager.tsx index 5269f763b..94ce42d8d 100644 --- a/src/client/apis/GoogleAuthenticationManager.tsx +++ b/src/client/apis/GoogleAuthenticationManager.tsx @@ -11,7 +11,8 @@ const AuthenticationUrl = 'https://accounts.google.com/o/oauth2/v2/auth'; const prompt = 'Paste authorization code here...'; @observer -export class GoogleAuthenticationManager extends React.Component<{}> { +export class GoogleAuthenticationManager extends React.Component { + // eslint-disable-next-line no-use-before-define public static Instance: GoogleAuthenticationManager; private authenticationLink: Opt = undefined; @observable private openState = false; @@ -19,7 +20,7 @@ export class GoogleAuthenticationManager extends React.Component<{}> { @observable private showPasteTargetState = false; @observable private success: Opt = undefined; @observable private displayLauncher = true; - @observable private credentials: any; + @observable private credentials: { user_info: { name: string; picture: string }; access_token: string } | undefined = undefined; private disposer: Opt; private set isOpen(value: boolean) { @@ -35,25 +36,25 @@ export class GoogleAuthenticationManager extends React.Component<{}> { } public fetchOrGenerateAccessToken = async (displayIfFound = false) => { - let response: any = await Networking.FetchFromServer('/readGoogleAccessToken'); + const response = await Networking.FetchFromServer('/readGoogleAccessToken'); // if this is an authentication url, activate the UI to register the new access token if (new RegExp(AuthenticationUrl).test(response)) { this.isOpen = true; this.authenticationLink = response; - return new Promise(async resolve => { + return new Promise(resolve => { this.disposer?.(); this.disposer = reaction( () => this.authenticationCode, async authenticationCode => { if (authenticationCode && /\d{1}\/[\w-]{55}/.test(authenticationCode)) { this.disposer?.(); - const response = await Networking.PostToServer('/writeGoogleAccessToken', { authenticationCode }); + const response2 = await Networking.PostToServer('/writeGoogleAccessToken', { authenticationCode }); runInAction(() => { this.success = true; - this.credentials = response; + this.credentials = response2 as { user_info: { name: string; picture: string }; access_token: string }; }); this.resetState(); - resolve(response.access_token); + resolve((response2 as { access_token: string }).access_token); } } ); @@ -61,16 +62,16 @@ export class GoogleAuthenticationManager extends React.Component<{}> { } // otherwise, we already have a valid, stored access token and user info - response = JSON.parse(response); + const response2 = JSON.parse(response) as { user_info: { name: string; picture: string }; access_token: string }; if (displayIfFound) { runInAction(() => { this.success = true; - this.credentials = response; + this.credentials = response2; }); this.resetState(-1, -1); this.isOpen = true; } - return response.access_token; + return (response2 as { access_token: string }).access_token; }; resetState = action((visibleForMS: number = 3000, fadesOutInMS: number = 500) => { @@ -106,7 +107,7 @@ export class GoogleAuthenticationManager extends React.Component<{}> { } }); - constructor(props: {}) { + constructor(props: object) { super(props); GoogleAuthenticationManager.Instance = this; } @@ -128,8 +129,8 @@ export class GoogleAuthenticationManager extends React.Component<{}> { {this.showPasteTargetState ? (this.authenticationCode = e.currentTarget.value))} placeholder={prompt} /> : null} {this.credentials ? ( <> - - Welcome to Dash, {this.credentials.userInfo.name} + + Welcome to Dash, {this.credentials.user_info.name}
{ diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index 0b303eacf..c1ac352b1 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -1,7 +1,5 @@ -/* eslint-disable no-restricted-syntax */ /* eslint-disable no-use-before-define */ import { docs_v1 as docsV1 } from 'googleapis'; -// eslint-disable-next-line node/no-deprecated-api import { isArray } from 'util'; import { EditorState } from 'prosemirror-state'; import { Opt } from '../../../fields/Doc'; @@ -37,7 +35,7 @@ export namespace GoogleApiClientUtils { text: string | string[]; requests: docsV1.Schema$Request[]; } - export type IdHandler = (id: DocumentId) => any; + export type IdHandler = (id: DocumentId) => unknown; export type CreationResult = Opt; export type ReadLinesResult = Opt<{ title?: string; bodyLines?: string[] }>; export type ReadResult = { title: string; body: string }; @@ -145,7 +143,7 @@ export namespace GoogleApiClientUtils { if (paragraphs.length) { const target = paragraphs[paragraphs.length - 1]; if (target.paragraph && target.paragraph.elements) { - length = target.paragraph.elements.length; + const length = target.paragraph.elements.length; if (length) { const final = target.paragraph.elements[length - 1]; return final.endIndex ? final.endIndex - 1 : undefined; @@ -208,13 +206,13 @@ export namespace GoogleApiClientUtils { }); export const setStyle = async (options: UpdateOptions) => { - const replies: any = await update({ + const replies = await update({ documentId: options.documentId, requests: options.requests, }); - if ('errors' in replies) { + if (replies && 'errors' in replies) { console.log('Write operation failed:'); - console.log(replies.errors.map((error: any) => error.message)); + console.log(replies); //.errors.map((error: any) => error.message)); } return replies; }; @@ -229,7 +227,6 @@ export namespace GoogleApiClientUtils { const { mode } = options; if (!(index && mode === WriteMode.Insert)) { const schema = await retrieve({ documentId }); - // eslint-disable-next-line no-cond-assign if (!schema || !(index = Utils.endOf(schema))) { return undefined; } @@ -258,10 +255,10 @@ export namespace GoogleApiClientUtils { return undefined; } requests.push(...options.content.requests); - const replies: any = await update({ documentId, requests }); - if ('errors' in replies) { + const replies = await update({ documentId, requests }); + if (replies && 'errors' in replies) { console.log('Write operation failed:'); - console.log(replies.errors.map((error: any) => error.message)); + console.log(replies); // .errors.map((error: any) => error.message)); } return replies; }; diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index b238f07e9..4b86a8341 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,5 +1,5 @@ /* eslint-disable no-use-before-define */ -import Photos = require('googlephotos'); +import Photos from 'googlephotos'; import { AssertionError } from 'assert'; import { EditorState } from 'prosemirror-state'; import { ClientUtils } from '../../../ClientUtils'; @@ -118,7 +118,7 @@ export namespace GooglePhotos { } export namespace Import { - export type CollectionConstructor = (data: Array, options: DocumentOptions, ...args: any) => Doc; + export type CollectionConstructor = (data: Array, options: DocumentOptions, ...args: unknown[]) => Doc; export const CollectionFromSearch = async (constructor: CollectionConstructor, requested: Opt>): Promise => { await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); @@ -147,7 +147,7 @@ export namespace GooglePhotos { values.forEach(async value => { const searched = (await ContentSearch({ included: [value] }))?.mediaItems?.map(({ id }) => id); searched?.forEach(async id => { - const image = await Cast(idMapping[id], Doc); + const image = await Cast(idMapping[id as string], Doc); if (image) { const key = image[Id]; const tags = tagMapping.get(key); @@ -193,7 +193,7 @@ export namespace GooglePhotos { } export interface SearchResponse { - mediaItems: any[]; + mediaItems: MediaItem[]; nextPageToken: string; } @@ -204,7 +204,7 @@ export namespace GooglePhotos { const found = 0; do { // eslint-disable-next-line no-await-in-loop - const response: any = await photos.mediaItems.search(albumId, pageSize, nextPageTokenStored); + const response = await photos.mediaItems.search(albumId, pageSize, nextPageTokenStored); mediaItems.push(...response.mediaItems); nextPageTokenStored = response.nextPageToken; } while (found); @@ -278,9 +278,9 @@ export namespace GooglePhotos { return undefined; }; - export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise => { + export const WriteMediaItemsToServer = async (body: { mediaItems: MediaItem[] }): Promise => { const uploads = await Networking.PostToServer('/googlePhotosMediaGet', body); - return uploads; + return uploads as UploadInformation[]; }; export const UploadThenFetch = async (sources: Doc[], album?: AlbumReference, descriptionKey = 'caption') => { @@ -320,7 +320,7 @@ export namespace GooglePhotos { }); if (media.length) { const results = await Networking.PostToServer('/googlePhotosMediaPost', { media, album }); - return results; + return results as Opt; } return undefined; }; @@ -331,7 +331,7 @@ export namespace GooglePhotos { if (typeof target === 'string') { description = target; } else if (target instanceof RichTextField) { - description = RichTextUtils.ToPlainText(EditorState.fromJSON(new FormattedTextBox({} as any).config, JSON.parse(target.Data))); + description = RichTextUtils.ToPlainText(EditorState.fromJSON(FormattedTextBox.MakeConfig(undefined, undefined), JSON.parse(target.Data))); } return description; }; diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index f73149fdc..43807397f 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -8,9 +8,9 @@ import { Upload } from '../../../server/SharedMediaTypes'; import { Networking } from '../../Network'; export namespace ImageUtils { - export const ExtractImgInfo = async (document: Doc): Promise => { + export const ExtractImgInfo = async (document: Doc) => { const field = Cast(document.data, ImageField); - return field ? Networking.PostToServer('/inspectImage', { source: field.url.href }) : undefined; + return field ? (Networking.PostToServer('/inspectImage', { source: field.url.href }) as Promise) : undefined; }; export const AssignImgInfo = (document: Doc, data?: Upload.InspectionResults) => { diff --git a/src/client/util/PingManager.ts b/src/client/util/PingManager.ts index 170632836..0e4f8cab0 100644 --- a/src/client/util/PingManager.ts +++ b/src/client/util/PingManager.ts @@ -29,7 +29,7 @@ export class PingManager { }); Networking.PostToServer('/ping', { date: new Date() }) .then(res => { - SnappingManager.SetServerVersion(res.message); + SnappingManager.SetServerVersion((res as { message: string }).message); !this.IsBeating && setIsBeating(true); }) .catch(() => this.IsBeating && setIsBeating(false)); diff --git a/src/client/util/SettingsManager.tsx b/src/client/util/SettingsManager.tsx index 5d041f7b4..6ea242fc3 100644 --- a/src/client/util/SettingsManager.tsx +++ b/src/client/util/SettingsManager.tsx @@ -598,7 +598,8 @@ export class SettingsManager extends React.Component { }); } else { const passwordBundle = { curr_pass: this._curr_password, new_pass: this._new_password, new_confirm: this._new_confirm }; - const { error } = await Networking.PostToServer('/internalResetPassword', passwordBundle); + const reset = await Networking.PostToServer('/internalResetPassword', passwordBundle); + const { error } = reset as { error: { msg: string }[] }; runInAction(() => { this._passwordResultText = error ? 'Error: ' + error[0].msg + '...' : 'Password successfully updated!'; }); diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index b40cd2761..ca830aa6f 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -28,6 +28,7 @@ import { FieldViewProps } from '../nodes/FieldView'; import { DocumentView, DocumentViewProps } from '../nodes/DocumentView'; import { FlashcardPracticeUI } from './FlashcardPracticeUI'; import { OpenWhere, OpenWhereMod } from '../nodes/OpenWhere'; +import { Upload } from '../../../server/SharedMediaTypes'; export enum docSortings { Time = 'time', @@ -407,7 +408,7 @@ export function CollectionSubView() { const imgSrc = img.split('src="')[1].split('"')[0]; const imgOpts = { ...options, _width: 300 }; if (imgSrc.startsWith('data:image') && imgSrc.includes('base64')) { - const result = (await Networking.PostToServer('/uploadRemoteImage', { sources: [imgSrc] })).lastElement(); + const result = ((await Networking.PostToServer('/uploadRemoteImage', { sources: [imgSrc] })) as Upload.ImageInformation[]).lastElement(); const newImgSrc = result.accessPaths.agnostic.client.indexOf('dashblobstore') === -1 // ? ClientUtils.prepend(result.accessPaths.agnostic.client) diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 3c31b584e..b33c267ee 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1291,7 +1291,7 @@ export class CollectionFreeFormView extends CollectionSubView { + addDrawing = (doc: Doc, opts: DrawingOptions, gptRes: string, x?: number, y?: number) => { const docData = doc[DocData]; docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; docData._width = opts.size; @@ -1302,6 +1302,9 @@ export class CollectionFreeFormView extends CollectionSubView { const res = await gptImageCall(prompt); if (res) { - const result = await Networking.PostToServer('/uploadRemoteImage', { sources: res }); + const result = (await Networking.PostToServer('/uploadRemoteImage', { sources: res })) as Upload.FileInformation[]; const source = ClientUtils.prepend(result[0].accessPaths.agnostic.client); return source; } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index b6b0e4a8a..9395863d8 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -36,7 +36,7 @@ import { OverlayView } from '../OverlayView'; import { AnchorMenu } from '../pdf/AnchorMenu'; import { PinDocView, PinProps } from '../PinFuncs'; import { DrawingFillHandler } from '../smartdraw/DrawingFillHandler'; -import { FireflyImageData } from '../smartdraw/FireflyConstants'; +import { FireflyImageData, isFireflyImageData } from '../smartdraw/FireflyConstants'; import { SmartDrawHandler } from '../smartdraw/SmartDrawHandler'; import { StickerPalette } from '../smartdraw/StickerPalette'; import { StyleProp } from '../StyleProp'; @@ -354,7 +354,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { const ext = extname(file); return file.replace(ext, (this._error ? '_o' : this._curSuffix) + ext); })(ImageCast(this.Document[Doc.LayoutFieldKey(this.Document)])?.url.href), - }).then((info: Upload.ImageInformation) => { + }).then(res => { + const info = res as Upload.ImageInformation; const img = Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { title: 'expand:' + this.Document.title }); DocUtils.assignImageInfo(info, img); this._props.addDocTab(img, OpenWhere.addRight); @@ -627,14 +628,15 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { } else SmartDrawHandler.Instance.regenerate([this.Document], undefined, undefined, this._regenInput || StrCast(this.Document.title), true).then( action(newImgs => { - if (newImgs[0] && !(newImgs[0] instanceof Doc)) { - const url = newImgs[0].pathname; + const firstImg = newImgs[0]; + if (isFireflyImageData(firstImg)) { + const url = firstImg.pathname; const imgField = new ImageField(url); this._prevImgs.length === 0 && this._prevImgs.push({ prompt: StrCast(this.dataDoc.ai_firefly_prompt), seed: this.dataDoc.ai_firefly_seed as number, href: this.paths.lastElement(), pathname: field.url.pathname }); - this._prevImgs.unshift({ prompt: newImgs[0].prompt, seed: newImgs[0].seed, pathname: url }); + this._prevImgs.unshift({ prompt: firstImg.prompt, seed: firstImg.seed, pathname: url }); this.dataDoc.ai_firefly_history = JSON.stringify(this._prevImgs); - this.dataDoc.ai_firefly_prompt = newImgs[0].prompt; + this.dataDoc.ai_firefly_prompt = firstImg.prompt; this.dataDoc[this.fieldKey] = imgField; this._regenerateLoading = false; this._regenInput = ''; @@ -688,7 +690,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { const ext = extname(file); return file.replace(ext, (this._error ? '_o' : this._curSuffix) + ext); })(ImageCast(this.Document[Doc.LayoutFieldKey(this.Document)])?.url.href), - }).then((info: Upload.ImageInformation) => { + }).then(res => { + const info = res as Upload.ImageInformation; const img = Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { title: 'expand:' + this.Document.title }); DocUtils.assignImageInfo(info, img); const genratedDocs = this.Document.generatedDocs @@ -741,7 +744,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { focus = (anchor: Doc, options: FocusViewOptions) => (anchor.type === DocumentType.CONFIG ? undefined : this._ffref.current?.focus(anchor, options)); renderedPixelDimensions = async () => { - const { nativeWidth: width, nativeHeight: height } = await Networking.PostToServer('/inspectImage', { source: this.paths[0] }); + const res = await Networking.PostToServer('/inspectImage', { source: this.paths[0] }); + const { nativeWidth: width, nativeHeight: height } = res as { nativeWidth: number; nativeHeight: number }; return { width, height }; }; savedAnnotations = () => this._savedAnnotations; diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx index 37ffca2d6..e7a6193d4 100644 --- a/src/client/views/nodes/RecordingBox/RecordingView.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx @@ -1,4 +1,3 @@ -/* eslint-disable react/button-has-type */ import * as React from 'react'; import { useEffect, useRef, useState } from 'react'; import { IconContext } from 'react-icons'; @@ -72,7 +71,7 @@ export function RecordingView(props: IRecordingViewProps) { const serverPaths: string[] = (await Networking.UploadFilesToServer(videoFiles.map(file => ({ file })))).map(res => (res.result instanceof Error ? '' : res.result.accessPaths.agnostic.server)); // concat the segments together using post call - const result: Upload.AccessPathInfo | Error = await Networking.PostToServer('/concatVideos', serverPaths); + const result = (await Networking.PostToServer('/concatVideos', serverPaths)) as Upload.AccessPathInfo | Error; !(result instanceof Error) ? props.setResult(result, concatPres || undefined) : console.error('video conversion failed'); })(); } diff --git a/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts b/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts index e8ef3fbfe..290c48d6c 100644 --- a/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts @@ -38,10 +38,10 @@ export class CreateCSVTool extends BaseTool { async execute(args: ParametersType): Promise { try { console.log('Creating CSV file:', args.filename, ' with data:', args.csvData); - const { fileUrl, id } = await Networking.PostToServer('/createCSV', { + const { fileUrl, id } = (await Networking.PostToServer('/createCSV', { filename: args.filename, data: args.csvData, - }); + })) as { fileUrl: string; id: string }; this._handleCSVResult(fileUrl, args.filename, id, args.csvData); diff --git a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts index dc6140871..e92d87dfd 100644 --- a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts +++ b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts @@ -37,9 +37,9 @@ export class ImageCreationTool extends BaseTool { console.log(`Generating image for prompt: ${image_prompt}`); // Create an array of promises, each one handling a search for a query try { - const { result, url } = await Networking.PostToServer('/generateImage', { + const { result, url } = (await Networking.PostToServer('/generateImage', { image_prompt, - }); + })) as { result: Upload.FileInformation & Upload.InspectionResults; url: string }; console.log('Image generation result:', result); this._createImage(result, { text: RTFCast(image_prompt) }); return url diff --git a/src/client/views/nodes/chatbot/tools/RAGTool.ts b/src/client/views/nodes/chatbot/tools/RAGTool.ts index 2db61c768..ef374ed22 100644 --- a/src/client/views/nodes/chatbot/tools/RAGTool.ts +++ b/src/client/views/nodes/chatbot/tools/RAGTool.ts @@ -75,7 +75,7 @@ export class RAGTool extends BaseTool { async getFormattedChunks(relevantChunks: RAGChunk[]): Promise { try { - const { formattedChunks } = await Networking.PostToServer('/formatChunks', { relevantChunks }); + const { formattedChunks } = await Networking.PostToServer('/formatChunks', { relevantChunks }) as { formattedChunks: Observation[]} if (!formattedChunks) { throw new Error('Failed to format chunks'); diff --git a/src/client/views/nodes/chatbot/tools/SearchTool.ts b/src/client/views/nodes/chatbot/tools/SearchTool.ts index 5fc6ab768..6a11407a5 100644 --- a/src/client/views/nodes/chatbot/tools/SearchTool.ts +++ b/src/client/views/nodes/chatbot/tools/SearchTool.ts @@ -41,15 +41,15 @@ export class SearchTool extends BaseTool { // Create an array of promises, each one handling a search for a query const searchPromises = queries.map(async query => { try { - const { results } = await Networking.PostToServer('/getWebSearchResults', { + const { results } = (await Networking.PostToServer('/getWebSearchResults', { query, max_results: this._max_results, - }); + })) as { results: { url: string; snippet: string }[] }; const data = results.map((result: { url: string; snippet: string }) => { const id = uuidv4(); this._addLinkedUrlDoc(result.url, id); return { - type: 'text', + type: 'text' as const, text: `${result.url}${result.snippet}`, }; }); @@ -58,7 +58,7 @@ export class SearchTool extends BaseTool { console.log(error); return [ { - type: 'text', + type: 'text' as const, text: `An error occurred while performing the web search for query: ${query}`, }, ]; diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.tsx b/src/client/views/pdf/GPTPopup/GPTPopup.tsx index efddfb841..f4ab2f41c 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.tsx +++ b/src/client/views/pdf/GPTPopup/GPTPopup.tsx @@ -21,11 +21,13 @@ import { DictationButton } from '../../DictationButton'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { TagItem } from '../../TagsView'; import { ChatSortField, docSortings } from '../../collections/CollectionSubView'; -import { DocumentView } from '../../nodes/DocumentView'; +import { DocumentView, DocumentViewInternal } from '../../nodes/DocumentView'; import { SmartDrawHandler } from '../../smartdraw/SmartDrawHandler'; import { AnchorMenu } from '../AnchorMenu'; import './GPTPopup.scss'; import { FireflyImageDimensions } from '../../smartdraw/FireflyConstants'; +import { Upload } from '../../../../server/SharedMediaTypes'; +import { OpenWhere } from '../../nodes/OpenWhere'; export enum GPTPopupMode { SUMMARY, // summary of seleted document text @@ -210,7 +212,12 @@ export class GPTPopup extends ObservableReactComponent { // ); // } else∂ return SmartDrawHandler.CreateWithFirefly(imgDesc, FireflyImageDimensions.Square, 0) - .then(action(() => (this._userPrompt = ''))) + .then( + action(doc => { + doc instanceof Doc && DocumentViewInternal.addDocTabFunc(doc, OpenWhere.addRight); + this._userPrompt = ''; + }) + ) .catch(e => { alert(e); return undefined; @@ -259,7 +266,7 @@ export class GPTPopup extends ObservableReactComponent { .then(imageUrls => imageUrls?.[0] ? Networking.PostToServer('/uploadRemoteImage', { sources: [imageUrls[0]] }).then(res => { - const source = ClientUtils.prepend(res[0].accessPaths.agnostic.client); + const source = ClientUtils.prepend((res as Upload.FileInformation[])[0].accessPaths.agnostic.client); return this.setImgUrls([[imageUrls[0]!, source]]); }) : undefined diff --git a/src/client/views/smartdraw/FireflyConstants.ts b/src/client/views/smartdraw/FireflyConstants.ts index 1f1781617..8cc9e36a5 100644 --- a/src/client/views/smartdraw/FireflyConstants.ts +++ b/src/client/views/smartdraw/FireflyConstants.ts @@ -5,6 +5,11 @@ export interface FireflyImageData { href?: string; } +export function isFireflyImageData(obj: unknown): obj is FireflyImageData { + const tobj = obj as FireflyImageData; + return typeof obj === 'object' && obj !== null && typeof tobj.pathname === 'string' && typeof tobj.prompt === 'string' && typeof tobj.seed === 'number'; +} + export enum FireflyImageDimensions { Square = 'square', Landscape = 'landscape', diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 9d67d111b..532391ac6 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -23,10 +23,10 @@ import { SVGToBezier, SVGType } from '../../util/bezierFit'; import { InkingStroke } from '../InkingStroke'; import { ObservableReactComponent } from '../ObservableReactComponent'; import { MarqueeView } from '../collections/collectionFreeForm'; -import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveInkColor, ActiveInkDash, ActiveInkFillColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; -import { OpenWhere } from '../nodes/OpenWhere'; +import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveInkColor, ActiveInkDash, ActiveInkFillColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../nodes/DocumentView'; import { FireflyDimensionsMap, FireflyImageData, FireflyImageDimensions } from './FireflyConstants'; import './SmartDrawHandler.scss'; +import { Upload } from '../../../server/SharedMediaTypes'; export interface DrawingOptions { text: string; @@ -60,7 +60,6 @@ export class SmartDrawHandler extends ObservableReactComponent { private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; private _lastResponse: string = ''; private _selectedDocs: Doc[] = []; - private _errorOccurredOnce = false; @observable private _display: boolean = false; @observable private _pageX: number = 0; @@ -95,7 +94,7 @@ export class SmartDrawHandler extends ObservableReactComponent { CollectionFreeForm, FormattedTextBox, StickerPalette) to define how a drawing document should be added or removed in their respective locations (to the freeform canvas, to the sticker palette's preview, etc.) */ - public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction; + public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string, x?: number, y?: number) => void = unimplementedFunction; public RemoveDrawing: (useLastContainer: boolean, doc?: Doc) => void = unimplementedFunction; /** * This creates the ink document that represents a drawing, so it goes through the strokes that make up the drawing, @@ -206,16 +205,15 @@ export class SmartDrawHandler extends ObservableReactComponent { this._isLoading = true; this._canInteract = false; if (this.ShowRegenerate) { - await this.regenerate(this._selectedDocs); - runInAction(() => { - this._selectedDocs = []; - this._regenInput = ''; - this._showEditBox = false; - }); + await this.regenerate(this._selectedDocs).then( + action(() => { + this._selectedDocs = []; + this._regenInput = ''; + this._showEditBox = false; + }) + ); } else { - runInAction(() => { - this._showOptions = false; - }); + this._showOptions = false; try { if (this._generateImage) { await this.createImageWithFirefly(this._userInput); @@ -224,18 +222,8 @@ export class SmartDrawHandler extends ObservableReactComponent { await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); } this.hideSmartDrawHandler(); - - runInAction(() => { - this.ShowRegenerate = true; - }); } catch (err) { - if (this._errorOccurredOnce) { - console.error('GPT call failed', err); - this._errorOccurredOnce = false; - } else { - this._errorOccurredOnce = true; - await this.handleSendClick(); - } + console.error('GPT call failed', err); } } runInAction(() => { @@ -256,7 +244,6 @@ export class SmartDrawHandler extends ObservableReactComponent { const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); drawingDoc && this._selectedDocs.push(drawingDoc); - this._errorOccurredOnce = false; return strokeData; } else { console.error('GPT call failed'); @@ -270,7 +257,10 @@ export class SmartDrawHandler extends ObservableReactComponent { */ createImageWithFirefly = (input: string, seed?: number): Promise => { this._lastInput.text = input; - return SmartDrawHandler.CreateWithFirefly(input, this._imgDims, seed); + return SmartDrawHandler.CreateWithFirefly(input, this._imgDims, seed).then(doc => { + doc instanceof Doc && this.AddDrawing(doc, this._lastInput, input, this._pageX, this._pageY); + return doc; + }); }; /** * Calls Firefly API to create an image based on user input */ @@ -281,9 +271,11 @@ export class SmartDrawHandler extends ObservableReactComponent { public static ReCreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise { const dims = FireflyDimensionsMap[imgDims]; return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed }) - .then(img => { - if (img.error) { - alert('recreate image failed: ' + img.error); + .then(res => { + const img = res as Upload.FileInformation; + const error = res as { error: string }; + if ('error' in error) { + alert('recreate image failed: ' + error.error); return undefined; } return { prompt: input, seed, pathname: img.accessPaths.agnostic.client }; @@ -296,21 +288,22 @@ export class SmartDrawHandler extends ObservableReactComponent { public static CreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise { const dims = FireflyDimensionsMap[imgDims]; return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed }) - .then(img => { - if (img.error) { - alert('create image failed: ' + img.error); + .then(res => { + const img = res as Upload.FileInformation; + const error = res as { error: string }; + if ('error' in error) { + alert('create image failed: ' + error.error); return undefined; } - const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1]; + const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)?.[1]; const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { title: input.match(/^(.*?)~~~.*$/)?.[1] || input, nativeWidth: dims.width, nativeHeight: dims.height, ai: 'firefly', - ai_firefly_seed: newseed, + ai_firefly_seed: +(newseed ?? 0), ai_firefly_prompt: input, }); - DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); return imgDoc; }) .catch(e => { @@ -331,26 +324,21 @@ export class SmartDrawHandler extends ObservableReactComponent { return Promise.all( drawingDocs.map(async doc => { switch (doc.type) { - case DocumentType.IMG: - if (this._regenInput) { - // if (this._selectedDoc) { - const newPrompt = doc.ai_firefly_prompt ? `${doc.ai_firefly_prompt} ~~~ ${this._regenInput}` : this._regenInput; - return changeInPlace ? this.recreateImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed)) : this.createImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed)); - // } - } - return changeInPlace - ? this.recreateImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), NumCast(doc?.ai_firefly_seed)) - : this.createImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), NumCast(doc?.ai_firefly_seed)); + case DocumentType.IMG: { + const func = changeInPlace ? this.recreateImageWithFirefly : this.createImageWithFirefly; + const newPrompt = doc.ai_firefly_prompt ? `${doc.ai_firefly_prompt} ~~~ ${this._regenInput}` : this._regenInput; + return this._regenInput ? func(newPrompt, NumCast(doc?.ai_firefly_seed)) : func(this._lastInput.text || StrCast(doc.ai_firefly_prompt)); + } case DocumentType.COL: { try { - let res; - if (this._regenInput) { - const prompt = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; - res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); - this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; - } else { - res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); - } + const res = await (async () => { + if (this._regenInput) { + const prompt = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; + this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + return gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); + } + return gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + })(); if (res) { const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, doc); -- cgit v1.2.3-70-g09d2 From db3bdb19ff7bc1c69c544797c05a6db3b72b1464 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 5 Mar 2025 14:54:18 -0500 Subject: added:hover mode for images to make primary image fade in instead of alternate. --- src/client/documents/Documents.ts | 3 ++ src/client/views/MainView.tsx | 2 +- src/client/views/TagsView.tsx | 3 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 2 +- src/client/views/nodes/DocumentView.scss | 1 + src/client/views/nodes/DocumentView.tsx | 16 +++++---- src/client/views/nodes/ImageBox.scss | 6 ++++ src/client/views/nodes/ImageBox.tsx | 40 +++++++++++++++------- .../views/nodes/chatbot/tools/ImageCreationTool.ts | 3 +- src/client/views/pdf/GPTPopup/GPTPopup.tsx | 3 ++ src/client/views/smartdraw/DrawingFillHandler.tsx | 4 +++ src/client/views/smartdraw/SmartDrawHandler.tsx | 8 +++-- 12 files changed, 65 insertions(+), 26 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/ImageCreationTool.ts') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 1ce25165c..21d3c978b 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -384,6 +384,9 @@ export class DocumentOptions { presentation_duration?: NUMt = new NumInfo('the duration of the slide in presentation view', false); presentation_zoomText?: BOOLt = new BoolInfo('whether text anchors should shown in a larger box when following links to make them stand out', false); + data_annotations?: List; + _data_usePath?: STRt = new StrInfo("description of field key to display in image box ('alternate','alternate:hover', 'data:hover'). defaults to primary", false); + data_alternates?: List; data?: FieldType; data_useCors?: BOOLt = new BoolInfo('whether CORS protocol should be used for web page'); _face_showImages?: BOOLt = new BoolInfo('whether to show images in uniqe face Doc'); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index afefe3f03..cc7c1a42b 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -686,7 +686,7 @@ export class MainView extends ObservableReactComponent { ); } @computed get mainDocView() { - const headerBar = null; // this._hideUI || !this.headerBarDocHeight?.() ? null : this.headerBarDocView; + const headerBar = this._hideUI || !this.headerBarDocHeight?.() ? null : this.headerBarDocView; return ( <> {headerBar} diff --git a/src/client/views/TagsView.tsx b/src/client/views/TagsView.tsx index b70e21918..93d6fb684 100644 --- a/src/client/views/TagsView.tsx +++ b/src/client/views/TagsView.tsx @@ -398,8 +398,7 @@ export class TagsView extends ObservableReactComponent { e.stopPropagation(); }} type="text" - placeholder="Input tags for document..." - aria-label="tagsView-input" + placeholder="Enter #tags or @metadata" className="tagsView-input" style={{ width: '100%', borderRadius: '5px' }} /> diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index b3d908da4..43addfc29 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -344,7 +344,7 @@ export class CollectionFreeFormView extends CollectionSubView { const { pointFocus, zoomTime, didMove } = options; if (!this.Document.isGroup && pointFocus && !didMove) { - const dfltScale = this.isAnnotationOverlay ? 1 : 0.5; + const dfltScale = this.isAnnotationOverlay ? 1 : 0.25; if (this.layoutDoc[this.scaleFieldKey] !== dfltScale) { this.zoomSmoothlyAboutPt(this.screenToFreeformContentsXf.transformPoint(pointFocus.X, pointFocus.Y), dfltScale, zoomTime); options.didMove = true; diff --git a/src/client/views/nodes/DocumentView.scss b/src/client/views/nodes/DocumentView.scss index 294af4d96..dd5fd0d0c 100644 --- a/src/client/views/nodes/DocumentView.scss +++ b/src/client/views/nodes/DocumentView.scss @@ -291,6 +291,7 @@ justify-items: center; background-color: rgb(223, 223, 223); transform-origin: top left; + background: transparent; .documentView-editorView-resizer { height: 5px; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 5f5dd1210..595abc7f8 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -279,16 +279,17 @@ export class DocumentViewInternal extends DocComponent this._titleRef.current?.setIsFocused(true)); // use timeout in case title wasn't shown to allow re-render so that titleref will be defined }; onBrowseClick = (e: React.MouseEvent) => { - const browseTransitionTime = 500; + //const browseTransitionTime = 500; DocumentView.DeselectAll(); DocumentView.showDocument(this.Document, { zoomScale: 0.8, willZoomCentered: true }, (focused: boolean) => { - const options: FocusViewOptions = { pointFocus: { X: e.clientX, Y: e.clientY }, zoomTime: browseTransitionTime }; + // const options: FocusViewOptions = { pointFocus: { X: e.clientX, Y: e.clientY }, zoomTime: browseTransitionTime }; if (!focused && this._docView) { - this._docView - .docViewPath() - .reverse() - .forEach(cont => cont.ComponentView?.focus?.(cont.Document, options)); - Doc.linkFollowHighlight(this.Document, false); + DocumentView.showDocument(this.Document, { zoomScale: 0.3, willZoomCentered: true }); + // this._docView + // .docViewPath() + // .reverse() + // .forEach(cont => cont.ComponentView?.focus?.(cont.Document, options)); + // Doc.linkFollowHighlight(this.Document, false); } }); e.stopPropagation(); @@ -797,6 +798,7 @@ export class DocumentViewInternal extends DocComponent() { }; getScrollHeight = () => (this._props.fitWidth?.(this.Document) !== false && NumCast(this.layoutDoc._freeform_scale, 1) === NumCast(this.dataDoc._freeform_scaleMin, 1) ? this.nativeSize.nativeHeight : undefined); + @computed get usingAlternate() { + const usePath = StrCast(this.Document[this.fieldKey + '_usePath']); + return 'alternate' === usePath || ('alternate:hover' === usePath && this._isHovering) || (':hover' === usePath && !this._isHovering); + } + @computed get nativeSize() { TraceMobx(); if (this.paths.length && this.paths[0].includes('icon-hi')) return { nativeWidth: NumCast(this.layoutDoc._width), nativeHeight: NumCast(this.layoutDoc._height), nativeOrientation: 0 }; @@ -471,10 +476,13 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { alternate, - and show alternate on hover + and show + + primary on hover + }>
() { ref={this._overlayIconRef} onPointerDown={e => setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => { - this.layoutDoc[`_${this.fieldKey}_usePath`] = usePath === undefined ? 'alternate' : usePath === 'alternate' ? 'alternate:hover' : undefined; + this.layoutDoc[`_${this.fieldKey}_usePath`] = usePath === undefined ? 'alternate' : usePath === 'alternate' ? 'alternate:hover' : usePath === 'alternate:hover' ? ':hover' : undefined; }) } style={{ @@ -527,7 +535,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { .filter(url => url) .map(url => this.choosePath(url)) ?? []; // acc ess the primary layout data of the alternate documents const paths = field ? [this.choosePath(field.url), ...altpaths] : altpaths; - return paths.length ? paths : [defaultUrl.href]; + return paths.length ? paths.reverse() : [defaultUrl.href]; } @computed get content() { @@ -552,7 +560,6 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { transformOrigin = 'right top'; transform = `translate(-100%, 0%) rotate(${rotation}deg) scale(${aspect})`; } - const usePath = this.layoutDoc[`_${this.fieldKey}_usePath`]; return (
() { ref={action((r: HTMLImageElement | null) => (this.imageRef = r))} key="paths" src={srcpath} - style={{ transform, transformOrigin, objectFit: 'fill', height: '100%' }} + style={{ transform, transformOrigin }} onError={action(e => { this._error = e.toString(); })} @@ -579,7 +586,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { width={nativeWidth} /> {fadepath === srcpath ? null : ( -
+
)} @@ -619,8 +626,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { return (
- Firefly: + + Firefly: + () {
- Similarity + + Similarity + () { return { width, height }; }; savedAnnotations = () => this._savedAnnotations; - render() { TraceMobx(); const borderRad = this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.BorderRounding) as string; const borderRadius = borderRad?.includes('px') ? `${Number(borderRad.split('px')[0]) / (this._props.NativeDimScaling?.() || 1)}px` : borderRad; + const alts = DocListCast(this.dataDoc[this.fieldKey + '_alternates']); + const doc = this.usingAlternate ? (alts.lastElement() ?? this.Document) : this.Document; return (
() { { image_prompt, })) as { result: Upload.FileInformation & Upload.InspectionResults; url: string }; console.log('Image generation result:', result); - this._createImage(result, { text: RTFCast(image_prompt) }); + this._createImage(result, { text: RTFCast(image_prompt), ai: 'dall-e-3', tags: new List(['@ai']) }); return url ? [ { diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.tsx b/src/client/views/pdf/GPTPopup/GPTPopup.tsx index 79f5121ed..4dc45e6a0 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.tsx +++ b/src/client/views/pdf/GPTPopup/GPTPopup.tsx @@ -30,6 +30,7 @@ import { Upload } from '../../../../server/SharedMediaTypes'; import { OpenWhere } from '../../nodes/OpenWhere'; import { DrawingFillHandler } from '../../smartdraw/DrawingFillHandler'; import { ImageField } from '../../../../fields/URLField'; +import { List } from '../../../../fields/List'; export enum GPTPopupMode { SUMMARY, // summary of seleted document text @@ -352,6 +353,8 @@ export class GPTPopup extends ObservableReactComponent { y: NumCast(textAnchor.y), _height: 200, _width: 200, + ai: 'dall-e', + tags: new List(['@ai']), data_nativeWidth: 1024, data_nativeHeight: 1024, }); diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index 0a30b14b8..c672bc718 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -1,6 +1,7 @@ import { imageUrlToBase64 } from '../../../ClientUtils'; import { Doc, StrListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; +import { List } from '../../../fields/List'; import { DocCast, ImageCast } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; import { Upload } from '../../../server/SharedMediaTypes'; @@ -53,7 +54,10 @@ export class DrawingFillHandler { undefined, Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { ai: 'firefly', + tags: new List(['@ai']), title: newPrompt, + _data_usePath: 'alternate:hover', + data_alternates: new List([drawing]), ai_firefly_prompt: newPrompt, _width: 500, data_nativeWidth: info.nativeWidth, diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index ca308015d..1cceabed3 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -28,6 +28,7 @@ import { FireflyDimensionsMap, FireflyImageData, FireflyImageDimensions } from ' import './SmartDrawHandler.scss'; import { Upload } from '../../../server/SharedMediaTypes'; import { PointData } from '../../../pen-gestures/GestureTypes'; +import { List } from '../../../fields/List'; export interface DrawingOptions { text?: string; @@ -293,15 +294,17 @@ export class SmartDrawHandler extends ObservableReactComponent { return undefined; } const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)?.[1]; - const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { + return Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { title: input, nativeWidth: dims.width, nativeHeight: dims.height, + tags: new List(['@ai']), + _width: Math.min(400, dims.width), + _height: (Math.min(400, dims.width) * dims.height) / dims.width, ai: 'firefly', ai_firefly_seed: +(newseed ?? 0), ai_firefly_prompt: input, }); - return imgDoc; }) .catch(e => { alert('create image failed: ' + e.toString()); @@ -568,6 +571,7 @@ export class SmartDrawHandler extends ObservableReactComponent { color={SettingsManager.userColor} />