From c079b7aa916d7eb35ce186806ca312c84e910cb2 Mon Sep 17 00:00:00 2001 From: bobzel Date: Fri, 14 Feb 2025 14:07:18 -0500 Subject: major restructure of GPTPopup so that it works with all collections. --- src/client/views/collections/CollectionSubView.tsx | 1 + 1 file changed, 1 insertion(+) (limited to 'src/client/views/collections/CollectionSubView.tsx') diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 5e99bec39..954f9aa4c 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -121,6 +121,7 @@ export function CollectionSubView() { return this.dataDoc[this._props.fieldKey]; // this used to be 'layoutDoc', but then template fields will get ignored since the template is not a proto of the layout. hopefully nothing depending on the previous code. } + hasChildDocs = () => this.childLayoutPairs.map(pair => pair.layout); @computed get childLayoutPairs(): { layout: Doc; data: Doc }[] { const { Document, TemplateDataDocument } = this._props; const validPairs = this.childDocs -- cgit v1.2.3-70-g09d2 From f6581aebd603dcdd84e033acffe46352ce5a4485 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 19 Feb 2025 15:05:47 -0500 Subject: more gptpopup cleanup. --- src/client/apis/gpt/GPT.ts | 59 +++++++------- src/client/util/CurrentUserUtils.ts | 6 +- src/client/util/Import & Export/ImageUtils.ts | 12 +-- src/client/views/StyleProviderQuiz.tsx | 2 +- src/client/views/collections/CollectionSubView.tsx | 33 ++++---- src/client/views/global/globalScripts.ts | 14 ++-- src/client/views/nodes/ComparisonBox.tsx | 6 +- .../views/nodes/chatbot/agentsystem/Agent.ts | 3 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 3 +- .../views/nodes/chatbot/tools/ImageCreationTool.ts | 36 ++++---- src/client/views/pdf/GPTPopup/GPTPopup.tsx | 95 ++++++++++------------ src/fields/Doc.ts | 2 +- 12 files changed, 126 insertions(+), 145 deletions(-) (limited to 'src/client/views/collections/CollectionSubView.tsx') diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index e1ae99d97..6d9bc1d06 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -1,12 +1,11 @@ import { ChatCompletionMessageParam, Image } from 'openai/resources'; import { openai } from './setup'; -export enum GPTTypeStyle { +export enum GPTDocCommand { AssignTags = 1, Filter = 2, - DocInfo = 3, - GeneralInfo = 4, - SortDocs = 5, + GetInfo = 3, + Sort = 4, } export const DescriptionSeperator = '======'; @@ -18,8 +17,6 @@ enum GPTCallType { EDIT = 'edit', CHATCARD = 'chatcard', // a single flashcard style response to a question FLASHCARD = 'flashcard', // a set of flashcard qustion/answer responses to a topic - QUIZ = 'quiz', - SORT = 'sort', DESCRIBE = 'describe', MERMAID = 'mermaid', DATA = 'data', @@ -27,15 +24,17 @@ enum GPTCallType { PRONUNCIATION = 'pronunciation', DRAW = 'draw', COLOR = 'color', - RUBRIC = 'rubric', // needs to be filled in below - TYPE = 'type', // needs to be filled in below - SUBSET = 'subset', // needs to be filled in below - INFO = 'info', // needs to be filled in below TEMPLATE = 'template', VIZSUM = 'vizsum', VIZSUM2 = 'vizsum2', FILL = 'fill', COMPLETEPROMPT = 'completeprompt', + QUIZDOC = 'quiz_doc', + MAKERUBRIC = 'make_rubric', // create a definition rubric for a document to be used when quizzing the user + COMMANDTYPE = 'command_type', // Determine the type of command being made (GPTQueryType - eg., AssignTags, Sort, Filter, DocInfo, GenInfo) and possibly some parameters (eg, Tag type for Tags) + SUBSETDOCS = 'subset_docs', // select a subset of documents based on their descriptions + DOCINFO = 'doc_info', // provide information about a document + SORTDOCS = 'sort_docs', } type GPTCallOpts = { @@ -69,7 +68,7 @@ const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { temp: 0.5, prompt: "You are a helpful resarch assistant. Analyze the user's data to find meaningful patterns and/or correlation. Please only return a JSON with a correlation column 1 propert, a correlation column 2 property, and an analysis property. ", }, - sort: { + sort_docs: { model: 'gpt-4o', maxTokens: 2048, temp: 0.25, @@ -89,7 +88,7 @@ const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { prompt: 'Make flashcards out of this text with each question and answer labeled as question and answer. Create a title for each question and asnwer that is labeled as "title". Do not label each flashcard and do not include asterisks: ', }, chatcard: { model: 'gpt-4-turbo', maxTokens: 512, temp: 0.5, prompt: 'Answer the following question as a short flashcard response. Do not include a label.' }, - quiz: { + quiz_doc: { model: 'gpt-4-turbo', maxTokens: 1024, temp: 0, @@ -138,20 +137,20 @@ const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { temp: 0.5, prompt: 'You will be coloring drawings. You will be given what the drawing is, then a list of descriptions for parts of the drawing. Based on each description, respond with the stroke and fill color that it should be. Follow the rules: 1. Avoid using black for stroke color 2. Make the stroke color 1-3 shades darker than the fill color 3. Use the same colors when possible. Format as {#abcdef #abcdef}, making sure theres a color for each description, and do not include any additional text.', }, - type: { + command_type: { model: 'gpt-4-turbo', maxTokens: 1024, temp: 0, prompt: `I'm going to provide you with a question. Based on the question, is the user asking you to - ${GPTTypeStyle.AssignTags}. Assigns docs with tags(like star / heart etc)/labels, - ${GPTTypeStyle.DocInfo}. Provide information about a specific doc - ${GPTTypeStyle.Filter}. Filter docs based on a question/information - ${GPTTypeStyle.GeneralInfo}. Provide general information - ${GPTTypeStyle.SortDocs}. Put cards in a specific order. - Answer with only the number for 2-5. For number one, provide the number (1) and the appropriate tag`, + ${GPTDocCommand.AssignTags}. Assigns docs with tags(like star / heart etc)/labels. + ${GPTDocCommand.GetInfo}. Provide information about a specific doc. + ${GPTDocCommand.Filter}. Filter docs based on a question/information. + ${GPTDocCommand.Sort}. Put docs in a specific order. + Answer with only the number for ${GPTDocCommand.GetInfo}-${GPTDocCommand.Sort}. + For number one, provide the number (${GPTDocCommand.AssignTags}) and the appropriate tag`, }, - subset: { + subset_docs: { model: 'gpt-4-turbo', maxTokens: 1024, temp: 0, @@ -164,14 +163,14 @@ const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { It is VERY important that you format it exactly as described, ensuring the proper number of '${DescriptionSeperator[0]}' and '${DocSeperator[0]}' (${DescriptionSeperator.length} of each) and NO commas`, }, - info: { + doc_info: { model: 'gpt-4-turbo', maxTokens: 1024, temp: 0, prompt: `Answer the user's question with a short (<100 word) response. If a particular document is selected I will provide that information (which may help with your response)`, }, - rubric: { + make_rubric: { model: 'gpt-4-turbo', maxTokens: 1024, temp: 0, @@ -188,17 +187,15 @@ let lastResp = ''; * @returns AI Output */ const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: string, dontCache?: boolean) => { - const inputText = inputTextIn + ([GPTCallType.SUMMARY, GPTCallType.FLASHCARD, GPTCallType.QUIZ, GPTCallType.STACK].includes(callType) ? '.' : ''); + const inputText = inputTextIn + ([GPTCallType.SUMMARY, GPTCallType.FLASHCARD, GPTCallType.QUIZDOC, GPTCallType.STACK].includes(callType) ? '.' : ''); const opts = callTypeMap[callType]; if (!opts) { console.log('The query type:' + callType + ' requires a configuration.'); return 'Error connecting with API.'; } - if (lastCall === inputText && dontCache !== true) return lastResp; + if (lastCall === inputText && dontCache !== true && lastResp) return lastResp; try { - lastCall = inputText; - - const usePrompt = prompt ? prompt + opts.prompt : opts.prompt; + const usePrompt = prompt ? prompt + '.' + opts.prompt : opts.prompt; const messages: ChatCompletionMessageParam[] = [ { role: 'system', content: usePrompt }, { role: 'user', content: inputText }, @@ -210,8 +207,12 @@ const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: s temperature: opts.temp, max_tokens: opts.maxTokens, }); - lastResp = response.choices[0].message.content ?? ''; - return lastResp; + const result = response.choices[0].message.content ?? ''; + if (!dontCache) { + lastResp = result; + lastCall = inputText; + } + return result; } catch (err) { console.log(err); return 'Error connecting with API.'; diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index 6ca181d92..fb349abd9 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -710,11 +710,7 @@ pie title Minerals in my tap water { title: "Type", icon:"eye", toolTip:"Sort by document type", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"docType", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, { title: "Color", icon:"palette", toolTip:"Sort by document color", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"color", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, { title: "Tags", icon:"bolt", toolTip:"Sort by document's tags", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"tag", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, - { title: "Sort", icon: "sort" , toolTip: "Manage sort order / lock status", btnType: ButtonType.MultiToggleButton, toolType:"alignment", ignoreClick: true, - subMenu: [ - { title: "Ascending", toolTip: "Sort the cards in ascending order", btnType: ButtonType.ToggleButton, icon: "sort-up", toolType:"up", ignoreClick: true, scripts: {onClick: '{ return showFreeform(this.toolType, _readOnly_);}'} }, - { title: "Descending",toolTip: "Sort the cards in descending order",btnType: ButtonType.ToggleButton, icon: "sort-down",toolType:"down",ignoreClick: true, scripts: {onClick: '{ return showFreeform(this.toolType, _readOnly_);}'} }, - ]}, + { title: "Reverse", icon: "sort-up", toolTip: "Sort the cards in reverse order", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"reverse", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'} }, ] } diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index 8d4eefa7e..f73149fdc 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -4,22 +4,16 @@ import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; import { Cast, NumCast, StrCast } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; +import { Upload } from '../../../server/SharedMediaTypes'; import { Networking } from '../../Network'; export namespace ImageUtils { - export type imgInfo = { - contentSize: number; - nativeWidth: number; - nativeHeight: number; - source: string; - exifData: { error: string | undefined; data: string }; - }; - export const ExtractImgInfo = async (document: Doc): Promise => { + export const ExtractImgInfo = async (document: Doc): Promise => { const field = Cast(document.data, ImageField); return field ? Networking.PostToServer('/inspectImage', { source: field.url.href }) : undefined; }; - export const AssignImgInfo = (document: Doc, data?: imgInfo) => { + export const AssignImgInfo = (document: Doc, data?: Upload.InspectionResults) => { if (data) { data.nativeWidth && (document._height = (NumCast(document._width) * data.nativeHeight) / data.nativeWidth); const proto = document[DocData]; diff --git a/src/client/views/StyleProviderQuiz.tsx b/src/client/views/StyleProviderQuiz.tsx index b3fb8c930..d8eeb3490 100644 --- a/src/client/views/StyleProviderQuiz.tsx +++ b/src/client/views/StyleProviderQuiz.tsx @@ -265,7 +265,7 @@ export namespace styleProviderQuiz { '. ' + rubricText + '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."'; - const response = await gptAPICall(queryText, GPTCallType.QUIZ); + const response = await gptAPICall(queryText, GPTCallType.QUIZDOC); const hexSent = extractHexAndSentences(response); doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); doc.backgroundColor = '#' + hexSent.hexNumber; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 954f9aa4c..b40cd2761 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -37,6 +37,9 @@ export enum docSortings { Tag = 'tag', None = '', } + +export const ChatSortField = 'chat_sortIndex'; + export interface CollectionViewProps extends React.PropsWithChildren { isAnnotationOverlay?: boolean; // is the collection an annotation overlay (eg an overlay on an image/video/etc) isAnnotationOverlayScrollable?: boolean; // whether the annotation overlay can be vertically scrolled (just for tree views, currently) @@ -229,23 +232,21 @@ export function CollectionSubView() { childSortedDocs = (docsIn: Doc[], dragIndex: number) => { const sortType = StrCast(this.Document[this._props.fieldKey + '_sort']) as docSortings; - const isDesc = BoolCast(this.Document[this._props.fieldKey + '_sort_desc']); + const isDesc = BoolCast(this.Document[this._props.fieldKey + '_sort_reverse']); const docs = docsIn.slice(); - if (sortType) { - docs.sort((docA, docB) => { - const [typeA, typeB] = (() => { - switch (sortType) { - default: - case docSortings.Type: return [StrCast(docA.type), StrCast(docB.type)]; - case docSortings.Chat: return [NumCast(docA.chatIndex, 9999), NumCast(docB.chatIndex,9999)]; - case docSortings.Time: return [DateCast(docA.author_date)?.date ?? Date.now(), DateCast(docB.author_date)?.date ?? Date.now()]; - case docSortings.Color:return [DashColor(StrCast(docA.backgroundColor)).hsv().hue(), DashColor(StrCast(docB.backgroundColor)).hsv().hue()]; - case docSortings.Tag: return [StrListCast(docA.tags).join(""), StrListCast(docB.tags).join("")]; - } - })(); //prettier-ignore - return (typeA < typeB ? -1 : typeA > typeB ? 1 : 0) * (isDesc ? 1 : -1); - }); - } + sortType && docs.sort((docA, docB) => { + const [typeA, typeB] = (() => { + switch (sortType) { + default: + case docSortings.Type: return [StrCast(docA.type), StrCast(docB.type)]; + case docSortings.Chat: return [NumCast(docA[ChatSortField], 9999), NumCast(docB[ChatSortField], 9999)]; + case docSortings.Time: return [DateCast(docA.author_date)?.date ?? Date.now(), DateCast(docB.author_date)?.date ?? Date.now()]; + case docSortings.Color:return [DashColor(StrCast(docA.backgroundColor)).hsv().hue(), DashColor(StrCast(docB.backgroundColor)).hsv().hue()]; + case docSortings.Tag: return [StrListCast(docA.tags).join(""), StrListCast(docB.tags).join("")]; + } + })(); + return (typeA < typeB ? -1 : typeA > typeB ? 1 : 0) * (isDesc ? -1 : 1); + }); //prettier-ignore if (dragIndex !== -1) { const draggedDoc = DragManager.docsBeingDragged[0]; const originalIndex = docs.findIndex(doc => doc === draggedDoc); diff --git a/src/client/views/global/globalScripts.ts b/src/client/views/global/globalScripts.ts index 2bc0e3338..79873ed8f 100644 --- a/src/client/views/global/globalScripts.ts +++ b/src/client/views/global/globalScripts.ts @@ -152,7 +152,7 @@ ScriptingGlobals.add(function toggleOverlay(checkResult?: boolean) { // eslint-disable-next-line prefer-arrow-callback ScriptingGlobals.add(function showFreeform( - attr: 'flashcards' | 'hcenter' | 'vcenter' | 'grid' | 'snaplines' | 'clusters' | 'viewAll' | 'fitOnce' | 'time' | 'docType' | 'color' | 'chat' | 'up' | 'down' | 'toggle-chat' | 'toggle-tags' | 'tag', + attr: 'flashcards' | 'hcenter' | 'vcenter' | 'grid' | 'snaplines' | 'clusters' | 'viewAll' | 'fitOnce' | 'time' | 'docType' | 'color' | 'chat' | 'reverse' | 'toggle-chat' | 'toggle-tags' | 'tag', checkResult?: boolean, persist?: boolean ) { @@ -163,7 +163,7 @@ ScriptingGlobals.add(function showFreeform( } // prettier-ignore - const map: Map<'flashcards' | 'hcenter' | 'vcenter' | 'grid' | 'snaplines' | 'clusters' | 'viewAll' | 'fitOnce' | 'time' | 'docType' | 'color' | 'chat' | 'up' | 'down'| 'toggle-chat' | 'toggle-tags' | 'tag', + const map: Map<'flashcards' | 'hcenter' | 'vcenter' | 'grid' | 'snaplines' | 'clusters' | 'viewAll' | 'fitOnce' | 'time' | 'docType' | 'color' | 'chat' | 'reverse'| 'toggle-chat' | 'toggle-tags' | 'tag', { waitForRender?: boolean; checkResult: (doc: Doc) => boolean; @@ -214,13 +214,9 @@ ScriptingGlobals.add(function showFreeform( checkResult: (doc: Doc) => StrCast(doc?.[Doc.LayoutFieldKey(doc)+"_sort"]) === "tag", setDoc: (doc: Doc, dv: DocumentView) => { doc[Doc.LayoutFieldKey(doc)+"_sort"] === "tag" ? doc[Doc.LayoutFieldKey(doc)+"_sort"] = '' : doc[Doc.LayoutFieldKey(doc)+"_sort"] = docSortings.Tag}, // prettier-ignore }], - ['up', { - checkResult: (doc: Doc) => BoolCast(!doc?.[Doc.LayoutFieldKey(doc)+"_sort_desc"]), - setDoc: (doc: Doc, dv: DocumentView) => { doc[Doc.LayoutFieldKey(doc)+"_sort_desc"] = undefined; }, - }], - ['down', { - checkResult: (doc: Doc) => BoolCast(doc?.[Doc.LayoutFieldKey(doc)+"_sort_desc"]), - setDoc: (doc: Doc, dv: DocumentView) => { doc[Doc.LayoutFieldKey(doc)+"_sort_desc"] = true; }, + ['reverse', { + checkResult: (doc: Doc) => BoolCast(doc?.[Doc.LayoutFieldKey(doc)+"_sort_reverse"]), + setDoc: (doc: Doc, dv: DocumentView) => { doc[Doc.LayoutFieldKey(doc)+"_sort_reverse"] = !doc[Doc.LayoutFieldKey(doc)+"_sort_reverse"]; }, }], ['toggle-chat', { checkResult: (doc: Doc) => SnappingManager.ChatVisible, diff --git a/src/client/views/nodes/ComparisonBox.tsx b/src/client/views/nodes/ComparisonBox.tsx index cb0831d3c..5315612e1 100644 --- a/src/client/views/nodes/ComparisonBox.tsx +++ b/src/client/views/nodes/ComparisonBox.tsx @@ -291,7 +291,7 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent() this.askGPTPhonemes(this._inputValue); this._renderSide = this.backKey; this._outputValue = ''; - } else if (this._inputValue) this.askGPT(GPTCallType.QUIZ); + } else if (this._inputValue) this.askGPT(GPTCallType.QUIZDOC); }; onPointerMove = ({ movementX }: PointerEvent) => { @@ -511,7 +511,7 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent() */ askGPT = async (callType: GPTCallType) => { const questionText = this.frontText; - const queryText = questionText + (callType == GPTCallType.QUIZ ? ' UserAnswer: ' + this._inputValue + '. ' + ' Rubric: ' + this.backText : ''); + const queryText = questionText + (callType == GPTCallType.QUIZDOC ? ' UserAnswer: ' + this._inputValue + '. ' + ' Rubric: ' + this.backText : ''); this.loading = true; const res = !this.frontText @@ -522,7 +522,7 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent() case GPTCallType.CHATCARD: DocCast(this.dataDoc[this.backKey])[DocData].text = resp; break; - case GPTCallType.QUIZ: + case GPTCallType.QUIZDOC: this._renderSide = this.backKey; this._outputValue = resp.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); break; diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index b2b0c9aea..2ed808622 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -22,6 +22,7 @@ import { ChatCompletionMessageParam } from 'openai/resources'; import { Doc } from '../../../../../fields/Doc'; import { parsedDoc } from '../chatboxcomponents/ChatBox'; import { WebsiteInfoScraperTool } from '../tools/WebsiteInfoScraperTool'; +import { Upload } from '../../../../../server/SharedMediaTypes'; //import { CreateTextDocTool } from '../tools/CreateTextDocumentTool'; dotenv.config(); @@ -61,7 +62,7 @@ export class Agent { history: () => string, csvData: () => { filename: string; id: string; text: string }[], addLinkedUrlDoc: (url: string, id: string) => void, - createImage: (result: any, options: DocumentOptions) => void, + createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void, addLinkedDoc: (doc: parsedDoc) => Doc | undefined, // eslint-disable-next-line @typescript-eslint/no-unused-vars createCSVInDash: (url: string, title: string, id: string, data: string) => void diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 16da360fc..6e9307d37 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -42,6 +42,7 @@ import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; import { ProgressBar } from './ProgressBar'; import { OpenWhere } from '../../OpenWhere'; +import { Upload } from '../../../../../server/SharedMediaTypes'; dotenv.config(); @@ -412,7 +413,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { }); @action - createImageInDash = async (result: any, options: DocumentOptions) => { + createImageInDash = async (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => { const newImgSrc = result.accessPaths.agnostic.client.indexOf('dashblobstore') === -1 // ? ClientUtils.prepend(result.accessPaths.agnostic.client) diff --git a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts index 177552c5c..dc6140871 100644 --- a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts +++ b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts @@ -1,10 +1,10 @@ -import { v4 as uuidv4 } from 'uuid'; import { RTFCast } from '../../../../../fields/Types'; import { DocumentOptions } from '../../../../documents/Documents'; import { Networking } from '../../../../Network'; import { ParametersType, ToolInfo } from '../types/tool_types'; import { Observation } from '../types/types'; import { BaseTool } from './BaseTool'; +import { Upload } from '../../../../../server/SharedMediaTypes'; const imageCreationToolParams = [ { @@ -25,8 +25,8 @@ const imageCreationToolInfo: ToolInfo = { }; export class ImageCreationTool extends BaseTool { - private _createImage: (result: any, options: DocumentOptions) => void; - constructor(createImage: (result: any, options: DocumentOptions) => void) { + private _createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void; + constructor(createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void) { super(imageCreationToolInfo); this._createImage = createImage; } @@ -42,23 +42,19 @@ export class ImageCreationTool extends BaseTool { }); console.log('Image generation result:', result); this._createImage(result, { text: RTFCast(image_prompt) }); - if (url) { - const id = uuidv4(); - - return [ - { - type: 'image_url', - image_url: { url }, - }, - ]; - } else { - return [ - { - type: 'text', - text: `An error occurred while generating image.`, - }, - ]; - } + return url + ? [ + { + type: 'image_url', + image_url: { url }, + }, + ] + : [ + { + type: 'text', + text: `An error occurred while generating image.`, + }, + ]; } catch (error) { console.log(error); return [ diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.tsx b/src/client/views/pdf/GPTPopup/GPTPopup.tsx index 72381cfad..cb3e9b2d7 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.tsx +++ b/src/client/views/pdf/GPTPopup/GPTPopup.tsx @@ -10,7 +10,7 @@ import { ClientUtils } from '../../../../ClientUtils'; import { Doc } from '../../../../fields/Doc'; import { NumCast, StrCast } from '../../../../fields/Types'; import { Networking } from '../../../Network'; -import { DescriptionSeperator, DocSeperator, GPTCallType, GPTTypeStyle, gptAPICall, gptImageCall } from '../../../apis/gpt/GPT'; +import { DescriptionSeperator, DocSeperator, GPTCallType, GPTDocCommand, gptAPICall, gptImageCall } from '../../../apis/gpt/GPT'; import { DocUtils } from '../../../documents/DocUtils'; import { Docs } from '../../../documents/Documents'; import { SettingsManager } from '../../../util/SettingsManager'; @@ -18,7 +18,7 @@ import { SnappingManager } from '../../../util/SnappingManager'; import { undoable } from '../../../util/UndoManager'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { TagItem } from '../../TagsView'; -import { docSortings } from '../../collections/CollectionSubView'; +import { ChatSortField, docSortings } from '../../collections/CollectionSubView'; import { DocumentView } from '../../nodes/DocumentView'; import { AnchorMenu } from '../AnchorMenu'; import './GPTPopup.scss'; @@ -36,6 +36,7 @@ export enum GPTPopupMode { export class GPTPopup extends ObservableReactComponent { // eslint-disable-next-line no-use-before-define static Instance: GPTPopup; + static ChatTag = '#chat'; // tag used by GPT popup to filter docs private _messagesEndRef: React.RefObject; private _correlatedColumns: string[] = []; private _dataChatPrompt: string | undefined = undefined; @@ -77,10 +78,10 @@ export class GPTPopup extends ObservableReactComponent { if (hasChildDocs) { this._textToDocMap.clear(); this.setCollectionContext(selDoc.Document); - this.onGptResponse = (sortResult: string, questionType: GPTTypeStyle, tag?: string) => this.processGptResponse(selDoc, this._textToDocMap, sortResult, questionType, tag); + this.onGptResponse = (sortResult: string, questionType: GPTDocCommand, args?: string) => this.processGptResponse(selDoc, this._textToDocMap, sortResult, questionType, args); this.onQuizRandom = () => this.randomlyChooseDoc(selDoc.Document, hasChildDocs()); this._documentDescriptions = Promise.all(hasChildDocs().map(doc => - Doc.getDescription(doc).then(text => this._textToDocMap.set(text, doc) && `${DescriptionSeperator}${text}${DescriptionSeperator}`) + Doc.getDescription(doc).then(text => this._textToDocMap.set(text.trim(), doc) && `${DescriptionSeperator}${text}${DescriptionSeperator}`) )).then(docDescriptions => docDescriptions.join()); // prettier-ignore } }, @@ -110,8 +111,8 @@ export class GPTPopup extends ObservableReactComponent { @action public setMode = (mode: GPTPopupMode) => (this._mode = mode); onQuizRandom?: () => void; - onGptResponse?: (sortResult: string, questionType: GPTTypeStyle, tag?: string) => void; - questionTypeNumberToStyle = (questionType: string) => +questionType.split(' ')[0][0]; + onGptResponse?: (sortResult: string, questionType: GPTDocCommand, args?: string) => void; + NumberToCommandType = (questionType: string) => +questionType.split(' ')[0][0]; /** * Processes gpt's output depending on the type of question the user asked. Converts gpt's string output to @@ -120,44 +121,38 @@ export class GPTPopup extends ObservableReactComponent { * @param questionType * @param tag */ - processGptResponse = (docView: DocumentView, textToDocMap: Map, gptOutput: string, questionType: GPTTypeStyle, tag?: string) => + processGptResponse = (docView: DocumentView, textToDocMap: Map, gptOutput: string, questionType: GPTDocCommand, args?: string) => undoable(() => { - // Split the string into individual list items - const listItems = gptOutput.split('======').filter(item => item.trim() !== ''); - - if (questionType === GPTTypeStyle.Filter) { - docView.ComponentView?.hasChildDocs?.().forEach(d => TagItem.removeTagFromDoc(d, '#chat')); - } - - if (questionType === GPTTypeStyle.SortDocs) { - docView.Document[docView.ComponentView?.fieldKey + '_sort'] = docSortings.Chat; - } - - listItems.forEach((item, index) => { - const normalizedItem = item.replace(/\n/g, ' ').trim(); - // find the corresponding Doc in the textToDoc map - const doc = textToDocMap.get(normalizedItem); - if (doc) { + switch (questionType) { // reset collection based on question typefc + case GPTDocCommand.Sort: + docView.Document[docView.ComponentView?.fieldKey + '_sort'] = docSortings.Chat; + break; + case GPTDocCommand.Filter: + docView.ComponentView?.hasChildDocs?.().forEach(d => TagItem.removeTagFromDoc(d, GPTPopup.ChatTag)); + break; + } // prettier-ignore + + gptOutput.split('======').filter(item => item.trim() !== '') // Split output into individual document contents + .map(docContentRaw => textToDocMap.get(docContentRaw.replace(/\n/g, ' ').trim())) // the find the corresponding Doc using textToDoc map + .filter(doc => doc).map(doc => doc!) // filter out undefined values + .forEach((doc, index) => { switch (questionType) { - case GPTTypeStyle.SortDocs: - doc.chatIndex = index; + case GPTDocCommand.Sort: + doc[ChatSortField] = index; break; - case GPTTypeStyle.AssignTags: - if (tag) { - const hashTag = tag.startsWith('#') ? tag : '#' + tag[0].toLowerCase() + tag.slice(1); - const filterTag = Doc.MyFilterHotKeys.map(key => StrCast(key.toolType)).find(key => key.includes(tag)) ?? hashTag; + case GPTDocCommand.AssignTags: + if (args) { + const hashTag = args.startsWith('#') ? args : '#' + args[0].toLowerCase() + args.slice(1); + const filterTag = Doc.MyFilterHotKeys.map(key => StrCast(key.toolType)).find(key => key.includes(args)) ?? hashTag; TagItem.addTagToDoc(doc, filterTag); } break; - case GPTTypeStyle.Filter: - TagItem.addTagToDoc(doc, '#chat'); - Doc.setDocFilter(docView.Document, 'tags', '#chat', 'check'); + case GPTDocCommand.Filter: + TagItem.addTagToDoc(doc, GPTPopup.ChatTag); + Doc.setDocFilter(docView.Document, 'tags', GPTPopup.ChatTag, 'check'); break; } - } else { - console.warn(`No matching document found for item: ${normalizedItem}`); - } - }); + }); // prettier-ignore }, '')(); /** @@ -173,7 +168,7 @@ export class GPTPopup extends ObservableReactComponent { StrCast(doc.gptRubric) ? Promise.resolve(StrCast(doc.gptRubric)) : Doc.getDescription(doc).then(desc => - gptAPICall(desc, GPTCallType.RUBRIC) + gptAPICall(desc, GPTCallType.MAKERUBRIC) .then(res => (doc.gptRubric = res)) .catch(err => console.error('GPT call failed', err)) ); @@ -191,7 +186,7 @@ export class GPTPopup extends ObservableReactComponent { `Question: ${desc}; UserAnswer: ${quizAnswer}; Rubric: ${StrCast(doc.gptRubric)}`, - GPTCallType.QUIZ + GPTCallType.QUIZDOC ).then(res => { this._conversationArray.push(res || 'GPT provided no answer'); this.onQuizRandom?.(); @@ -205,20 +200,20 @@ export class GPTPopup extends ObservableReactComponent { * @param userPrompt the user's input that chat will respond to */ generateUserPromptResponse = (userPrompt: string) => - gptAPICall(userPrompt, GPTCallType.TYPE).then(questionType => + gptAPICall(userPrompt, GPTCallType.COMMANDTYPE, undefined, true).then((commandType, args = commandType.split(' ').slice(1).join(' ')) => (async () => { - switch (this.questionTypeNumberToStyle(questionType)) { - case GPTTypeStyle.AssignTags: - case GPTTypeStyle.Filter: return this._documentDescriptions?.then(descs => gptAPICall(descs, GPTCallType.SUBSET, userPrompt)) ?? ""; - case GPTTypeStyle.SortDocs: return this._documentDescriptions?.then(descs => gptAPICall(descs, GPTCallType.SORT, userPrompt)) ?? ""; - default: return Doc.getDescription(DocumentView.SelectedDocs().lastElement()).then(desc => gptAPICall(desc, GPTCallType.INFO, userPrompt)); + switch (this.NumberToCommandType(commandType)) { + case GPTDocCommand.AssignTags: + case GPTDocCommand.Filter: return this._documentDescriptions?.then(descs => gptAPICall(userPrompt, GPTCallType.SUBSETDOCS, descs)) ?? ""; + case GPTDocCommand.Sort: return this._documentDescriptions?.then(descs => gptAPICall(userPrompt, GPTCallType.SORTDOCS, descs)) ?? ""; + default: return Doc.getDescription(DocumentView.SelectedDocs().lastElement()).then(desc => gptAPICall(userPrompt, GPTCallType.DOCINFO, desc)); } // prettier-ignore })().then( action(res => { // Trigger the callback with the result - this.onGptResponse?.(res || 'Something went wrong :(', this.questionTypeNumberToStyle(questionType), questionType.split(' ').slice(1).join(' ')); + this.onGptResponse?.(res || 'Something went wrong :(', this.NumberToCommandType(commandType), args); this._conversationArray.push( - [GPTTypeStyle.GeneralInfo, GPTTypeStyle.DocInfo].includes(this.questionTypeNumberToStyle(questionType)) ? res: + this.NumberToCommandType(commandType) === GPTDocCommand.GetInfo ? res: // Extract explanation surrounded by the DocSeperator string (defined in GPT.ts) at the top or both at the top and bottom (res.match(new RegExp(`${DocSeperator}\\s*([\\s\\S]*?)\\s*(?:${DocSeperator}|$)`)) ?? [])[1]?.trim() ?? 'No explanation found' ); @@ -583,10 +578,10 @@ export class GPTPopup extends ObservableReactComponent { tooltip="Clear Chat filter" toggleType={ToggleType.BUTTON} type={Type.PRIM} - toggleStatus={Doc.hasDocFilter(this._collectionContext, 'tags', '#chat')} - text={Doc.hasDocFilter(this._collectionContext, 'tags', '#chat') ? 'filtered' : ''} - color={Doc.hasDocFilter(this._collectionContext, 'tags', '#chat') ? 'red' : 'transparent'} - onClick={() => this._collectionContext && Doc.setDocFilter(this._collectionContext, 'tags', '#chat', 'remove')} + toggleStatus={Doc.hasDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag)} + text={Doc.hasDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag) ? 'filtered' : ''} + color={Doc.hasDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag) ? 'red' : 'transparent'} + onClick={() => this._collectionContext && Doc.setDocFilter(this._collectionContext, 'tags', GPTPopup.ChatTag, 'remove')} /> {(this._mode === GPTPopupMode.USER_PROMPT || this._mode === GPTPopupMode.QUIZ_RESPONSE) && ( } onClick={() => (this._mode = GPTPopupMode.GPT_MENU)} /> diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 950d9047c..bdd41b0bb 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -1482,7 +1482,7 @@ export namespace Doc { case DocumentType.IMG: return curDescription || imageUrlToBase64(ImageCastWithSuffix(Doc.LayoutField(tdoc), '_o') ?? '') .then(hrefBase64 => gptImageLabel(hrefBase64, 'Give three to five labels to describe this image.')); case DocumentType.RTF: return RTFCast(tdoc[Doc.LayoutFieldKey(tdoc)]).Text; - default: return StrCast(tdoc.title); + default: return StrCast(tdoc.title).startsWith("Untitled") ? "" : StrCast(tdoc.title); }}); // prettier-ignore return docText(doc).then(text => (doc[DocData][Doc.LayoutFieldKey(doc) + '_description'] = text)); } -- cgit v1.2.3-70-g09d2 From a9a1a6a507616a77f70d6525dab5027f5b7a60e6 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 26 Feb 2025 20:48:51 -0500 Subject: added typing to PostToServer calls. made smartDraw popup create images locally. --- src/client/Network.ts | 2 +- src/client/apis/GoogleAuthenticationManager.tsx | 27 ++++--- .../apis/google_docs/GoogleApiClientUtils.ts | 19 ++--- .../apis/google_docs/GooglePhotosClientUtils.ts | 18 ++--- src/client/util/Import & Export/ImageUtils.ts | 4 +- src/client/util/PingManager.ts | 2 +- src/client/util/SettingsManager.tsx | 3 +- src/client/views/collections/CollectionSubView.tsx | 3 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 5 +- .../views/nodes/DataVizBox/DocCreatorMenu.tsx | 3 +- src/client/views/nodes/ImageBox.tsx | 20 +++-- .../views/nodes/RecordingBox/RecordingView.tsx | 3 +- .../views/nodes/chatbot/tools/CreateCSVTool.ts | 4 +- .../views/nodes/chatbot/tools/ImageCreationTool.ts | 4 +- src/client/views/nodes/chatbot/tools/RAGTool.ts | 2 +- src/client/views/nodes/chatbot/tools/SearchTool.ts | 8 +- src/client/views/pdf/GPTPopup/GPTPopup.tsx | 13 ++- src/client/views/smartdraw/FireflyConstants.ts | 5 ++ src/client/views/smartdraw/SmartDrawHandler.tsx | 94 ++++++++++------------ 19 files changed, 123 insertions(+), 116 deletions(-) (limited to 'src/client/views/collections/CollectionSubView.tsx') diff --git a/src/client/Network.ts b/src/client/Network.ts index 323a2648c..a2ecf1bea 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -22,7 +22,7 @@ export namespace Networking { }, body: body ? JSON.stringify(body) : undefined, }).then(async response => { - if (response.ok) return response.json(); + if (response.ok) return response.json() as object; return await response.text().then(text => ({ error: '' + response.status + ':' + response.statusText + '-' + text })); }); diff --git a/src/client/apis/GoogleAuthenticationManager.tsx b/src/client/apis/GoogleAuthenticationManager.tsx index 5269f763b..94ce42d8d 100644 --- a/src/client/apis/GoogleAuthenticationManager.tsx +++ b/src/client/apis/GoogleAuthenticationManager.tsx @@ -11,7 +11,8 @@ const AuthenticationUrl = 'https://accounts.google.com/o/oauth2/v2/auth'; const prompt = 'Paste authorization code here...'; @observer -export class GoogleAuthenticationManager extends React.Component<{}> { +export class GoogleAuthenticationManager extends React.Component { + // eslint-disable-next-line no-use-before-define public static Instance: GoogleAuthenticationManager; private authenticationLink: Opt = undefined; @observable private openState = false; @@ -19,7 +20,7 @@ export class GoogleAuthenticationManager extends React.Component<{}> { @observable private showPasteTargetState = false; @observable private success: Opt = undefined; @observable private displayLauncher = true; - @observable private credentials: any; + @observable private credentials: { user_info: { name: string; picture: string }; access_token: string } | undefined = undefined; private disposer: Opt; private set isOpen(value: boolean) { @@ -35,25 +36,25 @@ export class GoogleAuthenticationManager extends React.Component<{}> { } public fetchOrGenerateAccessToken = async (displayIfFound = false) => { - let response: any = await Networking.FetchFromServer('/readGoogleAccessToken'); + const response = await Networking.FetchFromServer('/readGoogleAccessToken'); // if this is an authentication url, activate the UI to register the new access token if (new RegExp(AuthenticationUrl).test(response)) { this.isOpen = true; this.authenticationLink = response; - return new Promise(async resolve => { + return new Promise(resolve => { this.disposer?.(); this.disposer = reaction( () => this.authenticationCode, async authenticationCode => { if (authenticationCode && /\d{1}\/[\w-]{55}/.test(authenticationCode)) { this.disposer?.(); - const response = await Networking.PostToServer('/writeGoogleAccessToken', { authenticationCode }); + const response2 = await Networking.PostToServer('/writeGoogleAccessToken', { authenticationCode }); runInAction(() => { this.success = true; - this.credentials = response; + this.credentials = response2 as { user_info: { name: string; picture: string }; access_token: string }; }); this.resetState(); - resolve(response.access_token); + resolve((response2 as { access_token: string }).access_token); } } ); @@ -61,16 +62,16 @@ export class GoogleAuthenticationManager extends React.Component<{}> { } // otherwise, we already have a valid, stored access token and user info - response = JSON.parse(response); + const response2 = JSON.parse(response) as { user_info: { name: string; picture: string }; access_token: string }; if (displayIfFound) { runInAction(() => { this.success = true; - this.credentials = response; + this.credentials = response2; }); this.resetState(-1, -1); this.isOpen = true; } - return response.access_token; + return (response2 as { access_token: string }).access_token; }; resetState = action((visibleForMS: number = 3000, fadesOutInMS: number = 500) => { @@ -106,7 +107,7 @@ export class GoogleAuthenticationManager extends React.Component<{}> { } }); - constructor(props: {}) { + constructor(props: object) { super(props); GoogleAuthenticationManager.Instance = this; } @@ -128,8 +129,8 @@ export class GoogleAuthenticationManager extends React.Component<{}> { {this.showPasteTargetState ? (this.authenticationCode = e.currentTarget.value))} placeholder={prompt} /> : null} {this.credentials ? ( <> - - Welcome to Dash, {this.credentials.userInfo.name} + + Welcome to Dash, {this.credentials.user_info.name}
{ diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index 0b303eacf..c1ac352b1 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -1,7 +1,5 @@ -/* eslint-disable no-restricted-syntax */ /* eslint-disable no-use-before-define */ import { docs_v1 as docsV1 } from 'googleapis'; -// eslint-disable-next-line node/no-deprecated-api import { isArray } from 'util'; import { EditorState } from 'prosemirror-state'; import { Opt } from '../../../fields/Doc'; @@ -37,7 +35,7 @@ export namespace GoogleApiClientUtils { text: string | string[]; requests: docsV1.Schema$Request[]; } - export type IdHandler = (id: DocumentId) => any; + export type IdHandler = (id: DocumentId) => unknown; export type CreationResult = Opt; export type ReadLinesResult = Opt<{ title?: string; bodyLines?: string[] }>; export type ReadResult = { title: string; body: string }; @@ -145,7 +143,7 @@ export namespace GoogleApiClientUtils { if (paragraphs.length) { const target = paragraphs[paragraphs.length - 1]; if (target.paragraph && target.paragraph.elements) { - length = target.paragraph.elements.length; + const length = target.paragraph.elements.length; if (length) { const final = target.paragraph.elements[length - 1]; return final.endIndex ? final.endIndex - 1 : undefined; @@ -208,13 +206,13 @@ export namespace GoogleApiClientUtils { }); export const setStyle = async (options: UpdateOptions) => { - const replies: any = await update({ + const replies = await update({ documentId: options.documentId, requests: options.requests, }); - if ('errors' in replies) { + if (replies && 'errors' in replies) { console.log('Write operation failed:'); - console.log(replies.errors.map((error: any) => error.message)); + console.log(replies); //.errors.map((error: any) => error.message)); } return replies; }; @@ -229,7 +227,6 @@ export namespace GoogleApiClientUtils { const { mode } = options; if (!(index && mode === WriteMode.Insert)) { const schema = await retrieve({ documentId }); - // eslint-disable-next-line no-cond-assign if (!schema || !(index = Utils.endOf(schema))) { return undefined; } @@ -258,10 +255,10 @@ export namespace GoogleApiClientUtils { return undefined; } requests.push(...options.content.requests); - const replies: any = await update({ documentId, requests }); - if ('errors' in replies) { + const replies = await update({ documentId, requests }); + if (replies && 'errors' in replies) { console.log('Write operation failed:'); - console.log(replies.errors.map((error: any) => error.message)); + console.log(replies); // .errors.map((error: any) => error.message)); } return replies; }; diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index b238f07e9..4b86a8341 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,5 +1,5 @@ /* eslint-disable no-use-before-define */ -import Photos = require('googlephotos'); +import Photos from 'googlephotos'; import { AssertionError } from 'assert'; import { EditorState } from 'prosemirror-state'; import { ClientUtils } from '../../../ClientUtils'; @@ -118,7 +118,7 @@ export namespace GooglePhotos { } export namespace Import { - export type CollectionConstructor = (data: Array, options: DocumentOptions, ...args: any) => Doc; + export type CollectionConstructor = (data: Array, options: DocumentOptions, ...args: unknown[]) => Doc; export const CollectionFromSearch = async (constructor: CollectionConstructor, requested: Opt>): Promise => { await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); @@ -147,7 +147,7 @@ export namespace GooglePhotos { values.forEach(async value => { const searched = (await ContentSearch({ included: [value] }))?.mediaItems?.map(({ id }) => id); searched?.forEach(async id => { - const image = await Cast(idMapping[id], Doc); + const image = await Cast(idMapping[id as string], Doc); if (image) { const key = image[Id]; const tags = tagMapping.get(key); @@ -193,7 +193,7 @@ export namespace GooglePhotos { } export interface SearchResponse { - mediaItems: any[]; + mediaItems: MediaItem[]; nextPageToken: string; } @@ -204,7 +204,7 @@ export namespace GooglePhotos { const found = 0; do { // eslint-disable-next-line no-await-in-loop - const response: any = await photos.mediaItems.search(albumId, pageSize, nextPageTokenStored); + const response = await photos.mediaItems.search(albumId, pageSize, nextPageTokenStored); mediaItems.push(...response.mediaItems); nextPageTokenStored = response.nextPageToken; } while (found); @@ -278,9 +278,9 @@ export namespace GooglePhotos { return undefined; }; - export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise => { + export const WriteMediaItemsToServer = async (body: { mediaItems: MediaItem[] }): Promise => { const uploads = await Networking.PostToServer('/googlePhotosMediaGet', body); - return uploads; + return uploads as UploadInformation[]; }; export const UploadThenFetch = async (sources: Doc[], album?: AlbumReference, descriptionKey = 'caption') => { @@ -320,7 +320,7 @@ export namespace GooglePhotos { }); if (media.length) { const results = await Networking.PostToServer('/googlePhotosMediaPost', { media, album }); - return results; + return results as Opt; } return undefined; }; @@ -331,7 +331,7 @@ export namespace GooglePhotos { if (typeof target === 'string') { description = target; } else if (target instanceof RichTextField) { - description = RichTextUtils.ToPlainText(EditorState.fromJSON(new FormattedTextBox({} as any).config, JSON.parse(target.Data))); + description = RichTextUtils.ToPlainText(EditorState.fromJSON(FormattedTextBox.MakeConfig(undefined, undefined), JSON.parse(target.Data))); } return description; }; diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index f73149fdc..43807397f 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -8,9 +8,9 @@ import { Upload } from '../../../server/SharedMediaTypes'; import { Networking } from '../../Network'; export namespace ImageUtils { - export const ExtractImgInfo = async (document: Doc): Promise => { + export const ExtractImgInfo = async (document: Doc) => { const field = Cast(document.data, ImageField); - return field ? Networking.PostToServer('/inspectImage', { source: field.url.href }) : undefined; + return field ? (Networking.PostToServer('/inspectImage', { source: field.url.href }) as Promise) : undefined; }; export const AssignImgInfo = (document: Doc, data?: Upload.InspectionResults) => { diff --git a/src/client/util/PingManager.ts b/src/client/util/PingManager.ts index 170632836..0e4f8cab0 100644 --- a/src/client/util/PingManager.ts +++ b/src/client/util/PingManager.ts @@ -29,7 +29,7 @@ export class PingManager { }); Networking.PostToServer('/ping', { date: new Date() }) .then(res => { - SnappingManager.SetServerVersion(res.message); + SnappingManager.SetServerVersion((res as { message: string }).message); !this.IsBeating && setIsBeating(true); }) .catch(() => this.IsBeating && setIsBeating(false)); diff --git a/src/client/util/SettingsManager.tsx b/src/client/util/SettingsManager.tsx index 5d041f7b4..6ea242fc3 100644 --- a/src/client/util/SettingsManager.tsx +++ b/src/client/util/SettingsManager.tsx @@ -598,7 +598,8 @@ export class SettingsManager extends React.Component { }); } else { const passwordBundle = { curr_pass: this._curr_password, new_pass: this._new_password, new_confirm: this._new_confirm }; - const { error } = await Networking.PostToServer('/internalResetPassword', passwordBundle); + const reset = await Networking.PostToServer('/internalResetPassword', passwordBundle); + const { error } = reset as { error: { msg: string }[] }; runInAction(() => { this._passwordResultText = error ? 'Error: ' + error[0].msg + '...' : 'Password successfully updated!'; }); diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index b40cd2761..ca830aa6f 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -28,6 +28,7 @@ import { FieldViewProps } from '../nodes/FieldView'; import { DocumentView, DocumentViewProps } from '../nodes/DocumentView'; import { FlashcardPracticeUI } from './FlashcardPracticeUI'; import { OpenWhere, OpenWhereMod } from '../nodes/OpenWhere'; +import { Upload } from '../../../server/SharedMediaTypes'; export enum docSortings { Time = 'time', @@ -407,7 +408,7 @@ export function CollectionSubView() { const imgSrc = img.split('src="')[1].split('"')[0]; const imgOpts = { ...options, _width: 300 }; if (imgSrc.startsWith('data:image') && imgSrc.includes('base64')) { - const result = (await Networking.PostToServer('/uploadRemoteImage', { sources: [imgSrc] })).lastElement(); + const result = ((await Networking.PostToServer('/uploadRemoteImage', { sources: [imgSrc] })) as Upload.ImageInformation[]).lastElement(); const newImgSrc = result.accessPaths.agnostic.client.indexOf('dashblobstore') === -1 // ? ClientUtils.prepend(result.accessPaths.agnostic.client) diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 3c31b584e..b33c267ee 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1291,7 +1291,7 @@ export class CollectionFreeFormView extends CollectionSubView { + addDrawing = (doc: Doc, opts: DrawingOptions, gptRes: string, x?: number, y?: number) => { const docData = doc[DocData]; docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; docData._width = opts.size; @@ -1302,6 +1302,9 @@ export class CollectionFreeFormView extends CollectionSubView { const res = await gptImageCall(prompt); if (res) { - const result = await Networking.PostToServer('/uploadRemoteImage', { sources: res }); + const result = (await Networking.PostToServer('/uploadRemoteImage', { sources: res })) as Upload.FileInformation[]; const source = ClientUtils.prepend(result[0].accessPaths.agnostic.client); return source; } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index b6b0e4a8a..9395863d8 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -36,7 +36,7 @@ import { OverlayView } from '../OverlayView'; import { AnchorMenu } from '../pdf/AnchorMenu'; import { PinDocView, PinProps } from '../PinFuncs'; import { DrawingFillHandler } from '../smartdraw/DrawingFillHandler'; -import { FireflyImageData } from '../smartdraw/FireflyConstants'; +import { FireflyImageData, isFireflyImageData } from '../smartdraw/FireflyConstants'; import { SmartDrawHandler } from '../smartdraw/SmartDrawHandler'; import { StickerPalette } from '../smartdraw/StickerPalette'; import { StyleProp } from '../StyleProp'; @@ -354,7 +354,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { const ext = extname(file); return file.replace(ext, (this._error ? '_o' : this._curSuffix) + ext); })(ImageCast(this.Document[Doc.LayoutFieldKey(this.Document)])?.url.href), - }).then((info: Upload.ImageInformation) => { + }).then(res => { + const info = res as Upload.ImageInformation; const img = Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { title: 'expand:' + this.Document.title }); DocUtils.assignImageInfo(info, img); this._props.addDocTab(img, OpenWhere.addRight); @@ -627,14 +628,15 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { } else SmartDrawHandler.Instance.regenerate([this.Document], undefined, undefined, this._regenInput || StrCast(this.Document.title), true).then( action(newImgs => { - if (newImgs[0] && !(newImgs[0] instanceof Doc)) { - const url = newImgs[0].pathname; + const firstImg = newImgs[0]; + if (isFireflyImageData(firstImg)) { + const url = firstImg.pathname; const imgField = new ImageField(url); this._prevImgs.length === 0 && this._prevImgs.push({ prompt: StrCast(this.dataDoc.ai_firefly_prompt), seed: this.dataDoc.ai_firefly_seed as number, href: this.paths.lastElement(), pathname: field.url.pathname }); - this._prevImgs.unshift({ prompt: newImgs[0].prompt, seed: newImgs[0].seed, pathname: url }); + this._prevImgs.unshift({ prompt: firstImg.prompt, seed: firstImg.seed, pathname: url }); this.dataDoc.ai_firefly_history = JSON.stringify(this._prevImgs); - this.dataDoc.ai_firefly_prompt = newImgs[0].prompt; + this.dataDoc.ai_firefly_prompt = firstImg.prompt; this.dataDoc[this.fieldKey] = imgField; this._regenerateLoading = false; this._regenInput = ''; @@ -688,7 +690,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { const ext = extname(file); return file.replace(ext, (this._error ? '_o' : this._curSuffix) + ext); })(ImageCast(this.Document[Doc.LayoutFieldKey(this.Document)])?.url.href), - }).then((info: Upload.ImageInformation) => { + }).then(res => { + const info = res as Upload.ImageInformation; const img = Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { title: 'expand:' + this.Document.title }); DocUtils.assignImageInfo(info, img); const genratedDocs = this.Document.generatedDocs @@ -741,7 +744,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { focus = (anchor: Doc, options: FocusViewOptions) => (anchor.type === DocumentType.CONFIG ? undefined : this._ffref.current?.focus(anchor, options)); renderedPixelDimensions = async () => { - const { nativeWidth: width, nativeHeight: height } = await Networking.PostToServer('/inspectImage', { source: this.paths[0] }); + const res = await Networking.PostToServer('/inspectImage', { source: this.paths[0] }); + const { nativeWidth: width, nativeHeight: height } = res as { nativeWidth: number; nativeHeight: number }; return { width, height }; }; savedAnnotations = () => this._savedAnnotations; diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx index 37ffca2d6..e7a6193d4 100644 --- a/src/client/views/nodes/RecordingBox/RecordingView.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx @@ -1,4 +1,3 @@ -/* eslint-disable react/button-has-type */ import * as React from 'react'; import { useEffect, useRef, useState } from 'react'; import { IconContext } from 'react-icons'; @@ -72,7 +71,7 @@ export function RecordingView(props: IRecordingViewProps) { const serverPaths: string[] = (await Networking.UploadFilesToServer(videoFiles.map(file => ({ file })))).map(res => (res.result instanceof Error ? '' : res.result.accessPaths.agnostic.server)); // concat the segments together using post call - const result: Upload.AccessPathInfo | Error = await Networking.PostToServer('/concatVideos', serverPaths); + const result = (await Networking.PostToServer('/concatVideos', serverPaths)) as Upload.AccessPathInfo | Error; !(result instanceof Error) ? props.setResult(result, concatPres || undefined) : console.error('video conversion failed'); })(); } diff --git a/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts b/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts index e8ef3fbfe..290c48d6c 100644 --- a/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts @@ -38,10 +38,10 @@ export class CreateCSVTool extends BaseTool { async execute(args: ParametersType): Promise { try { console.log('Creating CSV file:', args.filename, ' with data:', args.csvData); - const { fileUrl, id } = await Networking.PostToServer('/createCSV', { + const { fileUrl, id } = (await Networking.PostToServer('/createCSV', { filename: args.filename, data: args.csvData, - }); + })) as { fileUrl: string; id: string }; this._handleCSVResult(fileUrl, args.filename, id, args.csvData); diff --git a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts index dc6140871..e92d87dfd 100644 --- a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts +++ b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts @@ -37,9 +37,9 @@ export class ImageCreationTool extends BaseTool { console.log(`Generating image for prompt: ${image_prompt}`); // Create an array of promises, each one handling a search for a query try { - const { result, url } = await Networking.PostToServer('/generateImage', { + const { result, url } = (await Networking.PostToServer('/generateImage', { image_prompt, - }); + })) as { result: Upload.FileInformation & Upload.InspectionResults; url: string }; console.log('Image generation result:', result); this._createImage(result, { text: RTFCast(image_prompt) }); return url diff --git a/src/client/views/nodes/chatbot/tools/RAGTool.ts b/src/client/views/nodes/chatbot/tools/RAGTool.ts index 2db61c768..ef374ed22 100644 --- a/src/client/views/nodes/chatbot/tools/RAGTool.ts +++ b/src/client/views/nodes/chatbot/tools/RAGTool.ts @@ -75,7 +75,7 @@ export class RAGTool extends BaseTool { async getFormattedChunks(relevantChunks: RAGChunk[]): Promise { try { - const { formattedChunks } = await Networking.PostToServer('/formatChunks', { relevantChunks }); + const { formattedChunks } = await Networking.PostToServer('/formatChunks', { relevantChunks }) as { formattedChunks: Observation[]} if (!formattedChunks) { throw new Error('Failed to format chunks'); diff --git a/src/client/views/nodes/chatbot/tools/SearchTool.ts b/src/client/views/nodes/chatbot/tools/SearchTool.ts index 5fc6ab768..6a11407a5 100644 --- a/src/client/views/nodes/chatbot/tools/SearchTool.ts +++ b/src/client/views/nodes/chatbot/tools/SearchTool.ts @@ -41,15 +41,15 @@ export class SearchTool extends BaseTool { // Create an array of promises, each one handling a search for a query const searchPromises = queries.map(async query => { try { - const { results } = await Networking.PostToServer('/getWebSearchResults', { + const { results } = (await Networking.PostToServer('/getWebSearchResults', { query, max_results: this._max_results, - }); + })) as { results: { url: string; snippet: string }[] }; const data = results.map((result: { url: string; snippet: string }) => { const id = uuidv4(); this._addLinkedUrlDoc(result.url, id); return { - type: 'text', + type: 'text' as const, text: `${result.url}${result.snippet}`, }; }); @@ -58,7 +58,7 @@ export class SearchTool extends BaseTool { console.log(error); return [ { - type: 'text', + type: 'text' as const, text: `An error occurred while performing the web search for query: ${query}`, }, ]; diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.tsx b/src/client/views/pdf/GPTPopup/GPTPopup.tsx index efddfb841..f4ab2f41c 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.tsx +++ b/src/client/views/pdf/GPTPopup/GPTPopup.tsx @@ -21,11 +21,13 @@ import { DictationButton } from '../../DictationButton'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { TagItem } from '../../TagsView'; import { ChatSortField, docSortings } from '../../collections/CollectionSubView'; -import { DocumentView } from '../../nodes/DocumentView'; +import { DocumentView, DocumentViewInternal } from '../../nodes/DocumentView'; import { SmartDrawHandler } from '../../smartdraw/SmartDrawHandler'; import { AnchorMenu } from '../AnchorMenu'; import './GPTPopup.scss'; import { FireflyImageDimensions } from '../../smartdraw/FireflyConstants'; +import { Upload } from '../../../../server/SharedMediaTypes'; +import { OpenWhere } from '../../nodes/OpenWhere'; export enum GPTPopupMode { SUMMARY, // summary of seleted document text @@ -210,7 +212,12 @@ export class GPTPopup extends ObservableReactComponent { // ); // } else∂ return SmartDrawHandler.CreateWithFirefly(imgDesc, FireflyImageDimensions.Square, 0) - .then(action(() => (this._userPrompt = ''))) + .then( + action(doc => { + doc instanceof Doc && DocumentViewInternal.addDocTabFunc(doc, OpenWhere.addRight); + this._userPrompt = ''; + }) + ) .catch(e => { alert(e); return undefined; @@ -259,7 +266,7 @@ export class GPTPopup extends ObservableReactComponent { .then(imageUrls => imageUrls?.[0] ? Networking.PostToServer('/uploadRemoteImage', { sources: [imageUrls[0]] }).then(res => { - const source = ClientUtils.prepend(res[0].accessPaths.agnostic.client); + const source = ClientUtils.prepend((res as Upload.FileInformation[])[0].accessPaths.agnostic.client); return this.setImgUrls([[imageUrls[0]!, source]]); }) : undefined diff --git a/src/client/views/smartdraw/FireflyConstants.ts b/src/client/views/smartdraw/FireflyConstants.ts index 1f1781617..8cc9e36a5 100644 --- a/src/client/views/smartdraw/FireflyConstants.ts +++ b/src/client/views/smartdraw/FireflyConstants.ts @@ -5,6 +5,11 @@ export interface FireflyImageData { href?: string; } +export function isFireflyImageData(obj: unknown): obj is FireflyImageData { + const tobj = obj as FireflyImageData; + return typeof obj === 'object' && obj !== null && typeof tobj.pathname === 'string' && typeof tobj.prompt === 'string' && typeof tobj.seed === 'number'; +} + export enum FireflyImageDimensions { Square = 'square', Landscape = 'landscape', diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 9d67d111b..532391ac6 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -23,10 +23,10 @@ import { SVGToBezier, SVGType } from '../../util/bezierFit'; import { InkingStroke } from '../InkingStroke'; import { ObservableReactComponent } from '../ObservableReactComponent'; import { MarqueeView } from '../collections/collectionFreeForm'; -import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveInkColor, ActiveInkDash, ActiveInkFillColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; -import { OpenWhere } from '../nodes/OpenWhere'; +import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveInkColor, ActiveInkDash, ActiveInkFillColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../nodes/DocumentView'; import { FireflyDimensionsMap, FireflyImageData, FireflyImageDimensions } from './FireflyConstants'; import './SmartDrawHandler.scss'; +import { Upload } from '../../../server/SharedMediaTypes'; export interface DrawingOptions { text: string; @@ -60,7 +60,6 @@ export class SmartDrawHandler extends ObservableReactComponent { private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; private _lastResponse: string = ''; private _selectedDocs: Doc[] = []; - private _errorOccurredOnce = false; @observable private _display: boolean = false; @observable private _pageX: number = 0; @@ -95,7 +94,7 @@ export class SmartDrawHandler extends ObservableReactComponent { CollectionFreeForm, FormattedTextBox, StickerPalette) to define how a drawing document should be added or removed in their respective locations (to the freeform canvas, to the sticker palette's preview, etc.) */ - public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction; + public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string, x?: number, y?: number) => void = unimplementedFunction; public RemoveDrawing: (useLastContainer: boolean, doc?: Doc) => void = unimplementedFunction; /** * This creates the ink document that represents a drawing, so it goes through the strokes that make up the drawing, @@ -206,16 +205,15 @@ export class SmartDrawHandler extends ObservableReactComponent { this._isLoading = true; this._canInteract = false; if (this.ShowRegenerate) { - await this.regenerate(this._selectedDocs); - runInAction(() => { - this._selectedDocs = []; - this._regenInput = ''; - this._showEditBox = false; - }); + await this.regenerate(this._selectedDocs).then( + action(() => { + this._selectedDocs = []; + this._regenInput = ''; + this._showEditBox = false; + }) + ); } else { - runInAction(() => { - this._showOptions = false; - }); + this._showOptions = false; try { if (this._generateImage) { await this.createImageWithFirefly(this._userInput); @@ -224,18 +222,8 @@ export class SmartDrawHandler extends ObservableReactComponent { await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); } this.hideSmartDrawHandler(); - - runInAction(() => { - this.ShowRegenerate = true; - }); } catch (err) { - if (this._errorOccurredOnce) { - console.error('GPT call failed', err); - this._errorOccurredOnce = false; - } else { - this._errorOccurredOnce = true; - await this.handleSendClick(); - } + console.error('GPT call failed', err); } } runInAction(() => { @@ -256,7 +244,6 @@ export class SmartDrawHandler extends ObservableReactComponent { const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); drawingDoc && this._selectedDocs.push(drawingDoc); - this._errorOccurredOnce = false; return strokeData; } else { console.error('GPT call failed'); @@ -270,7 +257,10 @@ export class SmartDrawHandler extends ObservableReactComponent { */ createImageWithFirefly = (input: string, seed?: number): Promise => { this._lastInput.text = input; - return SmartDrawHandler.CreateWithFirefly(input, this._imgDims, seed); + return SmartDrawHandler.CreateWithFirefly(input, this._imgDims, seed).then(doc => { + doc instanceof Doc && this.AddDrawing(doc, this._lastInput, input, this._pageX, this._pageY); + return doc; + }); }; /** * Calls Firefly API to create an image based on user input */ @@ -281,9 +271,11 @@ export class SmartDrawHandler extends ObservableReactComponent { public static ReCreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise { const dims = FireflyDimensionsMap[imgDims]; return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed }) - .then(img => { - if (img.error) { - alert('recreate image failed: ' + img.error); + .then(res => { + const img = res as Upload.FileInformation; + const error = res as { error: string }; + if ('error' in error) { + alert('recreate image failed: ' + error.error); return undefined; } return { prompt: input, seed, pathname: img.accessPaths.agnostic.client }; @@ -296,21 +288,22 @@ export class SmartDrawHandler extends ObservableReactComponent { public static CreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise { const dims = FireflyDimensionsMap[imgDims]; return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed }) - .then(img => { - if (img.error) { - alert('create image failed: ' + img.error); + .then(res => { + const img = res as Upload.FileInformation; + const error = res as { error: string }; + if ('error' in error) { + alert('create image failed: ' + error.error); return undefined; } - const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1]; + const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)?.[1]; const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { title: input.match(/^(.*?)~~~.*$/)?.[1] || input, nativeWidth: dims.width, nativeHeight: dims.height, ai: 'firefly', - ai_firefly_seed: newseed, + ai_firefly_seed: +(newseed ?? 0), ai_firefly_prompt: input, }); - DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); return imgDoc; }) .catch(e => { @@ -331,26 +324,21 @@ export class SmartDrawHandler extends ObservableReactComponent { return Promise.all( drawingDocs.map(async doc => { switch (doc.type) { - case DocumentType.IMG: - if (this._regenInput) { - // if (this._selectedDoc) { - const newPrompt = doc.ai_firefly_prompt ? `${doc.ai_firefly_prompt} ~~~ ${this._regenInput}` : this._regenInput; - return changeInPlace ? this.recreateImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed)) : this.createImageWithFirefly(newPrompt, NumCast(doc?.ai_firefly_seed)); - // } - } - return changeInPlace - ? this.recreateImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), NumCast(doc?.ai_firefly_seed)) - : this.createImageWithFirefly(this._lastInput.text || StrCast(doc.ai_firefly_prompt), NumCast(doc?.ai_firefly_seed)); + case DocumentType.IMG: { + const func = changeInPlace ? this.recreateImageWithFirefly : this.createImageWithFirefly; + const newPrompt = doc.ai_firefly_prompt ? `${doc.ai_firefly_prompt} ~~~ ${this._regenInput}` : this._regenInput; + return this._regenInput ? func(newPrompt, NumCast(doc?.ai_firefly_seed)) : func(this._lastInput.text || StrCast(doc.ai_firefly_prompt)); + } case DocumentType.COL: { try { - let res; - if (this._regenInput) { - const prompt = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; - res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); - this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; - } else { - res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); - } + const res = await (async () => { + if (this._regenInput) { + const prompt = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; + this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + return gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); + } + return gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + })(); if (res) { const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, doc); -- cgit v1.2.3-70-g09d2 From 215ad40efa2e343e290d18bffbc55884829f1a0d Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 4 Mar 2025 00:52:53 -0500 Subject: fixed up smartDrawHandler a bit to support svg's better. you can now drop in a .svg file from the filesystem - still some unfinished business (arcs, background/foreground color inversion) --- src/client/documents/DocUtils.ts | 57 +++- src/client/util/bezierFit.ts | 302 +++++++++++++++++---- src/client/views/collections/CollectionSubView.tsx | 17 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 36 +-- src/client/views/pdf/AnchorMenu.tsx | 2 +- src/client/views/smartdraw/SmartDrawHandler.tsx | 72 +++-- 6 files changed, 363 insertions(+), 123 deletions(-) (limited to 'src/client/views/collections/CollectionSubView.tsx') diff --git a/src/client/documents/DocUtils.ts b/src/client/documents/DocUtils.ts index 18c8d97d4..7f22e9376 100644 --- a/src/client/documents/DocUtils.ts +++ b/src/client/documents/DocUtils.ts @@ -10,7 +10,7 @@ import { DateField } from '../../fields/DateField'; import { Doc, DocListCast, Field, FieldResult, FieldType, LinkedTo, Opt, StrListCast } from '../../fields/Doc'; import { DocData } from '../../fields/DocSymbols'; import { Id } from '../../fields/FieldSymbols'; -import { InkDataFieldName, InkField } from '../../fields/InkField'; +import { InkData, InkDataFieldName, InkField } from '../../fields/InkField'; import { List, ListFieldName } from '../../fields/List'; import { ProxyField } from '../../fields/Proxy'; import { RichTextField } from '../../fields/RichTextField'; @@ -33,6 +33,10 @@ import { TaskCompletionBox } from '../views/nodes/TaskCompletedBox'; import { DocumentType } from './DocumentTypes'; import { Docs, DocumentOptions } from './Documents'; import { DocumentView } from '../views/nodes/DocumentView'; +import { INode, parse } from 'svgson'; +import { SVGToBezier, SVGType } from '../util/bezierFit'; +import { SmartDrawHandler } from '../views/smartdraw/SmartDrawHandler'; +import { PointData } from '../../pen-gestures/GestureTypes'; export namespace DocUtils { function HasFunctionFilter(val: string) { @@ -780,6 +784,57 @@ export namespace DocUtils { return generatedDocuments; } + export async function openSVGfile(file: File, options: DocumentOptions) { + const reader = new FileReader(); + const scale = 1; + const startPoint = { X: (options.x as number) ?? 0, Y: (options.y as number) ?? 0 }; + const buffer = await new Promise((res, rej) => { + reader.onload = event => { + const fileContent = event.target?.result; + // Process the file content here + console.log(fileContent); + typeof fileContent === 'string' ? res(fileContent) : rej(); + }; + + reader.readAsText(file); + }); + const svg = buffer.match(/]*>([\s\S]*?)<\/svg>/g); + if (svg) { + const svgObject = await parse(svg[0]); + const strokeData: [InkData, string, string][] = []; + const tl = { X: Number.MAX_SAFE_INTEGER, Y: Number.MAX_SAFE_INTEGER }; + let last: PointData = { X: 0, Y: 0 }; + const processStroke = (child: INode) => { + child.attributes.d + .split(/[\n]?M/) + .slice(1) + .map((d, ind) => { + const convertedBezier: InkData = SVGToBezier(child.name as SVGType, { ...child, d: '\nM' + d } as unknown as Record, last); + last = convertedBezier.lastElement(); + convertedBezier.forEach(point => { + if (point.X < tl.X) tl.X = point.X; + if (point.Y < tl.Y) tl.Y = point.Y; + }); + strokeData.push([convertedBezier, child.attributes.stroke || 'black', ind === 0 ? child.attributes.fill : child.attributes.fill === 'none' ? child.attributes.fill : DashColor(child.attributes.fill).negate().toString()]); + }); + }; + const processNode = (parent: INode) => { + if (parent.children.length) parent.children.forEach(processNode); + else if (parent.type !== 'text') processStroke(parent); + }; + processNode(svgObject); + + const mapStroke = (pd: PointData): PointData => ({ X: startPoint.X + (pd.X - tl.X) * scale, Y: startPoint.Y + (pd.Y - tl.Y) * scale }); + + return SmartDrawHandler.CreateDrawingDoc( + strokeData.map(sdata => [sdata[0].map(mapStroke), sdata[1], sdata[2]] as [PointData[], string, string]), + { autoColor: true }, + '', + undefined + ); + } + } + export function uploadFileToDoc(file: File, options: DocumentOptions, overwriteDoc: Doc) { const generatedDocuments: Doc[] = []; // Since this file has an overwriteDoc, we can set the client tracking guid to the overwriteDoc's guid. diff --git a/src/client/util/bezierFit.ts b/src/client/util/bezierFit.ts index 7ef370d48..65bd44bf9 100644 --- a/src/client/util/bezierFit.ts +++ b/src/client/util/bezierFit.ts @@ -1,7 +1,5 @@ /* eslint-disable no-use-before-define */ -/* eslint-disable prefer-destructuring */ /* eslint-disable no-param-reassign */ -/* eslint-disable camelcase */ import { Point } from '../../pen-gestures/ndollar'; export enum SVGType { @@ -625,13 +623,130 @@ export function GenerateControlPoints(coordinates: Point[], alpha = 0.1) { return [...firstEnd, ...points, ...lastEnd]; } -export function SVGToBezier(name: SVGType, attributes: any): Point[] { +function convertToAbsolute(pathData: string): string { + const commands = pathData.match(/[a-zA-Z][^a-zA-Z]*/g); + if (!commands) return pathData; + + let currentX = 0; + let currentY = 0; + let startX = 0; + let startY = 0; + + const absoluteCommands = commands.map(command => { + const type = command[0]; + const values = command + .slice(1) + .trim() + .split(/[\s,]+/) + .map(v => +v); + + switch (type) { + case 'M': + currentX = values[0]; + currentY = values[1]; + startX = currentX; + startY = currentY; + return `M${currentX},${currentY}`; + case 'm': + currentX += values[0]; + currentY += values[1]; + startX = currentX; + startY = currentY; + return `M${currentX},${currentY}`; + case 'L': + currentX = values[0]; + currentY = values[1]; + return `L${currentX},${currentY}`; + case 'l': + currentX += values[0]; + currentY += values[1]; + return `L${currentX},${currentY}`; + case 'H': + currentX = values[0]; + return `H${currentX}`; + case 'h': + currentX += values[0]; + return `H${currentX}`; + case 'V': + currentY = values[0]; + return `V${currentY}`; + case 'v': + currentY += values[0]; + return `V${currentY}`; + case 'C': + currentX = values[4]; + currentY = values[5]; + return `C${values.join(',')}`; + case 'c': { + let str = ''; + for (let i = 0; i < values.length; i += 6) { + str += (i === 0 ? 'C':',') + (values[i] + currentX) + + ',' + (values[i + 1] + currentY) + + ',' + (values[i + 2] + currentX) + + ',' + (values[i + 3] + currentY) + + ',' + (values[i + 4] + currentX) + + ',' + (values[i + 5] + currentY); // prettier-ignore + currentX += values[i + 4]; + currentY += values[i + 5]; + } + return str; + } + case 'S': + currentX = values[2]; + currentY = values[3]; + return `S${values.join(',')}`; + case 's': + return `S${values.map((v, i) => (i % 2 === 0 ? (currentX += v) : (currentY += v))).join(',')}`; + case 'Q': + currentX = values[2]; + currentY = values[3]; + return `Q${values.join(',')}`; + case 'q': { + let str = ''; + for (let i = 0; i < values.length; i += 4) { + str += (i === 0 ? 'Q':',') + (values[i] + currentX) + + ',' + (values[i + 1] + currentY) + + ',' + (values[i + 2] + currentX) + + ',' + (values[i + 3] + currentY); // prettier-ignore + currentX += values[i + 2]; + currentY += values[i + 3]; + } + return str; + } + case 'T': + currentX = values[0]; + currentY = values[1]; + return `T${currentX},${currentY}`; + case 't': + currentX += values[0]; + currentY += values[1]; + return `T${currentX},${currentY}`; + case 'A': + currentX = values[5]; + currentY = values[6]; + return `A${values.join(',')}`; + case 'a': + return `A${values.map((v, i) => (i % 2 === 0 ? (currentX += v) : (currentY += v))).join(',')}`; + case 'Z': + case 'z': + currentX = startX; + currentY = startY; + return 'Z'; + default: + return command; + } + }); + + return absoluteCommands.join(' '); +} + +export function SVGToBezier(name: SVGType, attributes: Record, last: { X: number; Y: number }): Point[] { switch (name) { case 'line': { - const x1 = parseInt(attributes.x1); - const x2 = parseInt(attributes.x2); - const y1 = parseInt(attributes.y1); - const y2 = parseInt(attributes.y2); + const x1 = +attributes.x1; + const x2 = +attributes.x2; + const y1 = +attributes.y1; + const y2 = +attributes.y2; return [ { X: x1, Y: y1 }, { X: x1, Y: y1 }, @@ -642,10 +757,10 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { case 'circle': case 'ellipse': { const c = 0.551915024494; - const centerX = parseInt(attributes.cx); - const centerY = parseInt(attributes.cy); - const radiusX = parseInt(attributes.rx) || parseInt(attributes.r); - const radiusY = parseInt(attributes.ry) || parseInt(attributes.r); + const centerX = +attributes.cx; + const centerY = +attributes.cy; + const radiusX = +attributes.rx || +attributes.r; + const radiusY = +attributes.ry || +attributes.r; return [ { X: centerX, Y: centerY + radiusY }, { X: centerX + c * radiusX, Y: centerY + radiusY }, @@ -666,10 +781,10 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { ]; } case 'rect': { - const x = parseInt(attributes.x); - const y = parseInt(attributes.y); - const width = parseInt(attributes.width); - const height = parseInt(attributes.height); + const x = +attributes.x; + const y = +attributes.y; + const width = +attributes.width; + const height = +attributes.height; return [ { X: x, Y: y }, { X: x, Y: y }, @@ -691,41 +806,122 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { } case 'path': { const coordList: Point[] = []; - const [startX, startY] = attributes.d.match(/M(-?\d+\.?\d*),(-?\d+\.?\d*)/).slice(1); - const startPt = { X: parseInt(startX), Y: parseInt(startY) }; - coordList.push(startPt); - const matches: RegExpMatchArray[] = Array.from( - attributes.d.matchAll(/Q(-?\d+\.?\d*),(-?\d+\.?\d*) (-?\d+\.?\d*),(-?\d+\.?\d*)|C(-?\d+\.?\d*),(-?\d+\.?\d*) (-?\d+\.?\d*),(-?\d+\.?\d*) (-?\d+\.?\d*),(-?\d+\.?\d*)|L(-?\d+\.?\d*),(-?\d+\.?\d*)/g) - ); - let lastPt: Point = startPt; - matches.forEach(match => { - if (match[0].startsWith('Q')) { - coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); - coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); - coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); - coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); - lastPt = { X: parseInt(match[3]), Y: parseInt(match[4]) }; - } else if (match[0].startsWith('C')) { - coordList.push({ X: parseInt(match[5]), Y: parseInt(match[6]) }); - coordList.push({ X: parseInt(match[7]), Y: parseInt(match[8]) }); - coordList.push({ X: parseInt(match[9]), Y: parseInt(match[10]) }); - coordList.push({ X: parseInt(match[9]), Y: parseInt(match[10]) }); - lastPt = { X: parseInt(match[9]), Y: parseInt(match[10]) }; - } else { - coordList.push(lastPt); - coordList.push({ X: parseInt(match[11]), Y: parseInt(match[12]) }); - coordList.push({ X: parseInt(match[11]), Y: parseInt(match[12]) }); - coordList.push({ X: parseInt(match[11]), Y: parseInt(match[12]) }); - lastPt = { X: parseInt(match[11]), Y: parseInt(match[12]) }; + let fixedattrs = attributes.d.trim().replace(/([0-9])-/g, '$1,-'); + for (let i = 0; i < 100; i++) { + const test = fixedattrs.replace(/([0-9]?\.[0-9]+)(\.[0-9]+)/g, '$1,$2'); + if (test === fixedattrs) break; + fixedattrs = test; + } + const attrdata = convertToAbsolute(fixedattrs); + const move = attrdata.match(/M(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)/); + const [startX, startY] = move?.slice(1) ?? [last.X + '', last.Y + '']; + const startPt = { X: +startX, Y: +startY }; + let first = true; + let lastCmd = ''; + for (let attr = attrdata.slice(move?.[0].length ?? 0).trim(); attr; ) { + lastCmd = 'AQCLVHZ'.includes(attr[0]) ? attr[0] : lastCmd; + switch (lastCmd) { + case 'Q': { + const match = attr.match(/Q?[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)/); + if (match) { + const prev = first ? startPt : coordList.lastElement(); + const Q = [+match[1], +match[2], +match[3], +match[4]]; + + coordList.push(prev); + coordList.push({ X: prev.X + (2 / 3) * (Q[0] - prev.X), Y: prev.Y + (2 / 3) * (Q[1] - prev.Y) }); + coordList.push({ X: Q[2] + (2 / 3) * (Q[0] - Q[2]), Y: Q[3] + (2 / 3) * (Q[1] - Q[3]) }); + coordList.push({ X: Q[2], Y: Q[3] }); + attr = attr.slice(match[0].length).trim(); + } else { + attr = attr.slice(1).trim(); + alert('error' + attr); + } + break; + } + case 'C': { + const match = attr.match(/C?[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)/); + if (match) { + coordList.push(first ? startPt : coordList.lastElement()); + coordList.push({ X: +match[1], Y: +match[2] }); + coordList.push({ X: +match[3], Y: +match[4] }); + coordList.push({ X: +match[5], Y: +match[6] }); + attr = attr.slice(match[0].length).trim(); + } else { + attr = attr.slice(1).trim(); + alert('error' + attr); + } + break; + } + case 'A': { + const match = attr.match(/A?[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)/); + if (match) { + console.log('SKIPPING arc - not implemented'); + // coordList.push(first ? startPt : coordList.lastElement()); + // coordList.push({ X: +match[1], Y: +match[2] }); + // coordList.push({ X: +match[3], Y: +match[4] }); + // coordList.push({ X: +match[5], Y: +match[6] }); + attr = attr.slice(match[0].length).trim(); + } else { + attr = attr.slice(1).trim(); + alert('error' + attr); + } + break; + } + case 'L': { + const match = attr.match(/L?[, ]?(-?\d*\.?\d*)[, ]?(-?\d*\.?\d*)/); + if (match) { + coordList.push(first ? startPt : coordList.lastElement()); + coordList.push(coordList.lastElement()); + coordList.push({ X: +match[1], Y: +match[2] }); + coordList.push({ X: +match[1], Y: +match[2] }); + attr = attr.slice(match[0].length).trim(); + } else { + attr = attr.slice(1).trim(); + alert('error' + attr); + } + break; + } + case 'H': { + const match = attr.match(/H?[, ]?(-?\d*\.?\d*)/); + if (match) { + coordList.push(first ? startPt : coordList.lastElement()); + coordList.push(coordList.lastElement()); + coordList.push({ X: +match[1], Y: coordList.lastElement().Y }); + coordList.push({ X: +match[1], Y: coordList.lastElement().Y }); + attr = attr.slice(match[0].length).trim(); + } else { + attr = attr.slice(1).trim(); + alert('error' + attr); + } + break; + } + case 'V': { + const match = attr.match(/V?[, ]?(-?\d*\.?\d*)/); + if (match) { + coordList.push(first ? startPt : coordList.lastElement()); + coordList.push(coordList.lastElement()); + coordList.push({ X: coordList.lastElement().X, Y: +match[1] }); + coordList.push({ X: coordList.lastElement().X, Y: +match[1] }); + attr = attr.slice(match[0].length).trim(); + } else { + attr = attr.slice(1).trim(); + alert('error' + attr); + } + break; + } + case 'Z': { + coordList.push(first ? startPt : coordList.lastElement()); + coordList.push(first ? startPt : coordList.lastElement()); + coordList.push(startPt); + coordList.push(startPt); + attr = attr.slice(1).trim(); + break; + } + default: + attr = attr.slice(1).trim(); + debugger; } - }); - const hasZ = attributes.d.match(/Z/); - if (hasZ || attributes.fill) { - coordList.push(lastPt); - coordList.push(startPt); - coordList.push(startPt); - } else { - coordList.pop(); + first = false; } return coordList; } @@ -733,10 +929,10 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { const coords: RegExpMatchArray[] = Array.from(attributes.points.matchAll(/(-?\d+\.?\d*),(-?\d+\.?\d*)/g)); let list: Point[] = []; coords.forEach(coord => { - list.push({ X: parseInt(coord[1]), Y: parseInt(coord[2]) }); - list.push({ X: parseInt(coord[1]), Y: parseInt(coord[2]) }); - list.push({ X: parseInt(coord[1]), Y: parseInt(coord[2]) }); - list.push({ X: parseInt(coord[1]), Y: parseInt(coord[2]) }); + list.push({ X: +coord[1], Y: +coord[2] }); + list.push({ X: +coord[1], Y: +coord[2] }); + list.push({ X: +coord[1], Y: +coord[2] }); + list.push({ X: +coord[1], Y: +coord[2] }); }); const firstPts = list.splice(0, 2); list = list.concat(firstPts); diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index ca830aa6f..655894e40 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -541,12 +541,17 @@ export function CollectionSubView() { DocUtils.uploadYoutubeVideoLoading(files, {}, loading); } else { generatedDocuments.push( - ...files.map(file => { - const loading = Docs.Create.LoadingDocument(file, options); - Doc.addCurrentlyLoading(loading); - DocUtils.uploadFileToDoc(file, {}, loading); - return loading; - }) + ...(await Promise.all( + files.map(async file => { + if (file.name.endsWith('svg')) { + return (await DocUtils.openSVGfile(file, options)) as Doc; + } + const loading = Docs.Create.LoadingDocument(file, options); + Doc.addCurrentlyLoading(loading); + DocUtils.uploadFileToDoc(file, {}, loading); + return loading; + }) + )) ); } if (generatedDocuments.length) { diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index aa9b9b0fa..b3d908da4 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -30,7 +30,7 @@ import { CompileScript } from '../../../util/Scripting'; import { ScriptingGlobals } from '../../../util/ScriptingGlobals'; import { freeformScrollMode, SnappingManager } from '../../../util/SnappingManager'; import { Transform } from '../../../util/Transform'; -import { undoable, undoBatch, UndoManager } from '../../../util/UndoManager'; +import { undoable, UndoManager } from '../../../util/UndoManager'; import { Timeline } from '../../animationtimeline/Timeline'; import { ContextMenu } from '../../ContextMenu'; import { InkingStroke } from '../../InkingStroke'; @@ -1232,7 +1232,6 @@ export class CollectionFreeFormView extends CollectionSubView { const sm = SmartDrawHandler.Instance; - sm.CreateDrawingDoc = this.createDrawingDoc; sm.RemoveDrawing = this.removeDrawing; sm.AddDrawing = this.addDrawing; (regenerate ? sm.displayRegenerate : sm.displaySmartDrawHandler)(x, y, NumCast(this.layoutDoc[this.scaleFieldKey])); @@ -1240,38 +1239,6 @@ export class CollectionFreeFormView extends CollectionSubView { - this._drawing = []; - const xf = this.screenToFreeformContentsXf; - strokeData.forEach((stroke: [InkData, string, string]) => { - const bounds = InkField.getBounds(stroke[0]); - const B = xf.transformBounds(bounds.left, bounds.top, bounds.width, bounds.height); - const inkWidth = ActiveInkWidth() * this.ScreenToLocalBoxXf().Scale; - const inkDoc = Docs.Create.InkDocument( - stroke[0], - { title: 'stroke', - x: B.x - inkWidth / 2, - y: B.y - inkWidth / 2, - _width: B.width + inkWidth, - _height: B.height + inkWidth, - stroke_showLabel: BoolCast(Doc.UserDoc().activeHideTextLabels)}, // prettier-ignore - inkWidth, - opts.autoColor ? stroke[1] : ActiveInkColor(), - ActiveInkBezierApprox(), - stroke[2] === 'none' ? ActiveInkFillColor() : stroke[2], - ActiveInkArrowStart(), - ActiveInkArrowEnd(), - ActiveInkDash(), - ActiveIsInkMask() - ); - this._drawing.push(inkDoc); - }); - return MarqueeView.getCollection(this._drawing, undefined, true, { left: opts.x, top: opts.y, width: 1, height: 1 }); - }; /** * Part of regenerating a drawing--deletes the old drawing. @@ -2007,7 +1974,6 @@ export class CollectionFreeFormView extends CollectionSubView { - SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; SmartDrawHandler.Instance.AddDrawing = this.addDrawing; SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this._downX, this._downY - 10) : SmartDrawHandler.Instance.hideRegenerate(); diff --git a/src/client/views/pdf/AnchorMenu.tsx b/src/client/views/pdf/AnchorMenu.tsx index f7070c780..28371594e 100644 --- a/src/client/views/pdf/AnchorMenu.tsx +++ b/src/client/views/pdf/AnchorMenu.tsx @@ -139,7 +139,7 @@ export class AnchorMenu extends AntimodeMenu { createDrawingAnnotation = action((drawing: Doc, opts: DrawingOptions, gptRes: string) => { this.AddDrawingAnnotation(drawing); const docData = drawing[DocData]; - docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; + docData.title = opts.text?.match(/^(.*?)~~~.*$/)?.[1] || opts.text; docData.ai_drawing_input = opts.text; docData.ai_drawing_complexity = opts.complexity; docData.ai_drawing_colored = opts.autoColor; diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index fbf471900..ca308015d 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -27,16 +27,19 @@ import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveIn import { FireflyDimensionsMap, FireflyImageData, FireflyImageDimensions } from './FireflyConstants'; import './SmartDrawHandler.scss'; import { Upload } from '../../../server/SharedMediaTypes'; +import { PointData } from '../../../pen-gestures/GestureTypes'; export interface DrawingOptions { - text: string; - complexity: number; - size: number; - autoColor: boolean; - x: number; - y: number; + text?: string; + complexity?: number; + size?: number; + autoColor?: boolean; + x?: number; + y?: number; } +type svgparsedData = [PointData[], string, string]; + /** * The SmartDrawHandler allows users to generate drawings with GPT from text input. Users are able to enter * the item to draw, how complex they want the drawing to be, how large the drawing should be, and whether @@ -102,7 +105,7 @@ export class SmartDrawHandler extends ObservableReactComponent { * classes to customize the way the drawing docs get created. For example, the freeform canvas has a different way of * defining document bounds, so CreateDrawingDoc is redefined when that class calls gpt draw functions. */ - public CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions) => { + public static CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions) => { const drawing: Doc[] = []; strokeList.forEach((stroke: [InkData, string, string]) => { const bounds = InkField.getBounds(stroke[0]); @@ -114,7 +117,7 @@ export class SmartDrawHandler extends ObservableReactComponent { y: bounds.top - inkWidth / 2, _width: bounds.width + inkWidth, _height: bounds.height + inkWidth, - stroke_showLabel: !BoolCast(Doc.UserDoc().activeHideTextLabels)}, // prettier-ignore + stroke_showLabel: false}, // prettier-ignore inkWidth, opts.autoColor ? stroke[1] : ActiveInkColor(), ActiveInkBezierApprox(), @@ -188,9 +191,9 @@ export class SmartDrawHandler extends ObservableReactComponent { /** * This allows users to press the return/enter key to send input. */ - handleKeyPress = (event: React.KeyboardEvent) => { - if (event.key === 'Enter') { - this.handleSendClick(); + handleKeyPress = (e: React.KeyboardEvent) => { + if (e.key === 'Enter') { + this.handleSendClick(this._pageX, this._pageY); } }; @@ -200,7 +203,7 @@ export class SmartDrawHandler extends ObservableReactComponent { * what the user sees. */ @action - handleSendClick = async () => { + handleSendClick = async (X: number, Y: number) => { if ((!this.ShowRegenerate && this._userInput == '') || (!this._generateImage && !this._generateDrawing)) return; this._isLoading = true; this._canInteract = false; @@ -213,7 +216,7 @@ export class SmartDrawHandler extends ObservableReactComponent { await this.createImageWithFirefly(this._userInput); } if (this._generateDrawing) { - await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + await this.drawWithGPT({ X, Y }, this._userInput, this._complexity, this._size, this._autoColor); } this.hideSmartDrawHandler(); } catch (err) { @@ -229,14 +232,14 @@ export class SmartDrawHandler extends ObservableReactComponent { /** * Calls GPT API to create a drawing based on user input. */ - drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { + drawWithGPT = async (screenPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { if (input) { - this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; + this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: screenPt.X, y: screenPt.Y }; const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); if (res) { - const strokeData = await this.parseSvg(res, startPt, false, autoColor); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); - drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + const strokeData = await this.parseSvg(res, { X: 0, Y: 0 }, false, autoColor); + const drawingDoc = strokeData && SmartDrawHandler.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res, screenPt.X, screenPt.Y); drawingDoc && this._selectedDocs.push(drawingDoc); return strokeData; } else { @@ -334,9 +337,9 @@ export class SmartDrawHandler extends ObservableReactComponent { return gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); })(); if (res) { - const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + const strokeData = await this.parseSvg(res, { X: this._lastInput.x ?? 0, Y: this._lastInput.y ?? 0 }, true, lastInput?.autoColor || this._autoColor); this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, doc); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + const drawingDoc = strokeData && SmartDrawHandler.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); } else { console.error('GPT call failed'); @@ -356,20 +359,35 @@ export class SmartDrawHandler extends ObservableReactComponent { */ parseSvg = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => { const svg = res.match(/]*>([\s\S]*?)<\/svg>/g); + if (svg) { this._lastResponse = svg[0]; const svgObject = await parse(svg[0]); + console.log(res, svgObject); const svgStrokes: INode[] = svgObject.children; const strokeData: [InkData, string, string][] = []; + + const tl = { X: Number.MAX_SAFE_INTEGER, Y: Number.MAX_SAFE_INTEGER }; + let last: PointData = { X: 0, Y: 0 }; svgStrokes.forEach(child => { - const convertedBezier: InkData = SVGToBezier(child.name as SVGType, child.attributes); + const convertedBezier: InkData = SVGToBezier(child.name as SVGType, child.attributes, last); + last = convertedBezier.lastElement(); strokeData.push([ - convertedBezier.map(point => ({ X: startPoint.X + (point.X - startPoint.X) * this._scale, Y: startPoint.Y + (point.Y - startPoint.Y) * this._scale })), + convertedBezier.map(point => { + if (point.X < tl.X) tl.X = point.X; + if (point.Y < tl.Y) tl.Y = point.Y; + return { X: point.X, Y: point.Y }; + }), (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.stroke : '', (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.fill : '', ]); }); - return { data: strokeData, lastInput: this._lastInput, lastRes: svg[0] }; + const mapStroke = (pd: PointData): PointData => ({ X: startPoint.X + (pd.X - tl.X) * this._scale, Y: startPoint.Y + (pd.Y - tl.Y) * this._scale }); + return { + data: strokeData.map(sdata => [sdata[0].map(mapStroke), sdata[1], sdata[2]] as svgparsedData), + lastInput: this._lastInput, + lastRes: svg[0], + }; } }; @@ -439,7 +457,7 @@ export class SmartDrawHandler extends ObservableReactComponent { }, }} checked={this._generateImage} - onChange={() => this._canInteract && (this._generateImage = !this._generateImage)} + onChange={action(() => this._canInteract && (this._generateImage = !this._generateImage))} /> @@ -566,7 +584,7 @@ export class SmartDrawHandler extends ObservableReactComponent { icon={this._isLoading ? : } iconPlacement="right" color={SettingsManager.userColor} - onClick={this.handleSendClick} + onClick={() => this.handleSendClick(this._pageX, this._pageY)} /> {this._showOptions && ( @@ -598,7 +616,7 @@ export class SmartDrawHandler extends ObservableReactComponent { icon={this._isLoading && this._regenInput !== '' ? : } iconPlacement="right" color={SettingsManager.userColor} - onClick={this.handleSendClick} + onClick={() => this.handleSendClick(this._pageX, this._pageY)} /> ); @@ -644,7 +662,7 @@ export class SmartDrawHandler extends ObservableReactComponent { tooltip="Regenerate" icon={this._isLoading && this._regenInput === '' ? : } color={SettingsManager.userColor} - onClick={this.handleSendClick} + onClick={() => this.handleSendClick(this._pageX, this._pageY)} /> } color={SettingsManager.userColor} onClick={action(() => (this._showEditBox = !this._showEditBox))} /> {this._showEditBox ? this.renderRegenerateEditBox() : null} -- cgit v1.2.3-70-g09d2