diff options
Diffstat (limited to 'src')
26 files changed, 2812 insertions, 34 deletions
diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 55667684e..455352068 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -5,6 +5,9 @@ enum GPTCallType { SUMMARY = 'summary', COMPLETION = 'completion', EDIT = 'edit', + SORT = 'sort', + DESCRIBE = 'describe', + MERMAID = 'mermaid', DATA = 'data', } @@ -15,21 +18,30 @@ type GPTCallOpts = { prompt: string; }; -/** - * Replace completions (deprecated) with chat - */ - const callTypeMap: { [type: string]: GPTCallOpts } = { // newest model: gpt-4 summary: { model: 'gpt-3.5-turbo', maxTokens: 256, temp: 0.5, prompt: 'Summarize the text given in simpler terms.' }, edit: { model: 'gpt-3.5-turbo', maxTokens: 256, temp: 0.5, prompt: 'Reword the text.' }, completion: { model: 'gpt-3.5-turbo', maxTokens: 256, temp: 0.5, prompt: "You are a helpful assistant. Answer the user's prompt." }, + mermaid: { + model: 'gpt-4-turbo', + maxTokens: 2048, + temp: 0, + prompt: "(Heres an example of changing color of a pie chart to help you pie title Example \"Red\": 20 \"Blue\": 50 \"Green\": 30 %%{init: {'theme': 'base', 'themeVariables': {'pie1': '#0000FF', 'pie2': '#00FF00', 'pie3': '#FF0000'}}}%% keep in mind that pie1 is the highest since its sorted in descending order. Heres an example of a mindmap: mindmap root((mindmap)) Origins Long history ::icon(fa fa-book) Popularisation British popular psychology author Tony Buzan Research On effectivness<br/>and features On Automatic creation Uses Creative techniques Strategic planning Argument mapping Tools Pen and paper Mermaid. ", + }, data: { model: 'gpt-3.5-turbo', maxTokens: 256, temp: 0.5, prompt: "You are a helpful resarch assistant. Analyze the user's data to find meaningful patterns and/or correlation. Please only return a JSON with a correlation column 1 propert, a correlation column 2 property, and an analysis property. ", }, + sort: { + model: 'gpt-4o', + maxTokens: 2048, + temp: 0.5, + prompt: "I'm going to give you a list of descriptions. Each one is seperated by ====== on either side. They will vary in length, so make sure to only seperate when you see ======. Sort them into lists by shared content. MAKE SURE EACH DESCRIPTOR IS IN ONLY ONE LIST. Generate only the list with each list seperated by ====== with the elements seperated by ~~~~~~. Try to do around 4 groups, but a little more or less is ok.", + }, + describe: { model: 'gpt-4-vision-preview', maxTokens: 2048, temp: 0, prompt: 'Describe these images in 3-5 words' }, }; let lastCall = ''; @@ -60,9 +72,9 @@ const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: a const response = await openai.chat.completions.create({ model: opts.model, - max_tokens: opts.maxTokens, + messages: messages, temperature: opts.temp, - messages, + max_tokens: opts.maxTokens, }); lastResp = response.choices[0].message.content ?? ''; return lastResp; @@ -71,7 +83,6 @@ const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: a return 'Error connecting with API.'; } }; - const gptImageCall = async (prompt: string, n?: number) => { try { const configuration: ClientOptions = { @@ -85,11 +96,72 @@ const gptImageCall = async (prompt: string, n?: number) => { n: n ?? 1, size: '1024x1024', }); - return response.data.map(data => data.url); + return response.data.map((data: any) => data.url); + // return response.data.data[0].url; } catch (err) { console.error(err); } return undefined; }; -export { gptAPICall, gptImageCall, GPTCallType }; +const gptGetEmbedding = async (src: string): Promise<number[]> => { + try { + const configuration: ClientOptions = { + apiKey: process.env.OPENAI_KEY, + dangerouslyAllowBrowser: true, + }; + const openai = new OpenAI(configuration); + const embeddingResponse = await openai.embeddings.create({ + model: 'text-embedding-3-large', + input: [src], + encoding_format: 'float', + dimensions: 256, + }); + + // Assume the embeddingResponse structure is correct; adjust based on actual API response + const embedding = embeddingResponse.data[0].embedding; + return embedding; + } catch (err) { + console.log(err); + return []; + } +}; + +const gptImageLabel = async (src: string): Promise<string> => { + try { + const configuration: ClientOptions = { + apiKey: process.env.OPENAI_KEY, + dangerouslyAllowBrowser: true, + }; + + const openai = new OpenAI(configuration); + const response = await openai.chat.completions.create({ + model: 'gpt-4o', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Give three to five labels to describe this image.' }, + { + type: 'image_url', + image_url: { + url: `${src}`, + detail: 'low', + }, + }, + ], + }, + ], + }); + if (response.choices[0].message.content) { + return response.choices[0].message.content; + } else { + return 'Missing labels'; + } + } catch (err) { + console.log(err); + return 'Error connecting with API'; + } +}; + +export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding }; diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index b4ad9c17d..8f95068db 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -12,6 +12,7 @@ export enum DocumentType { REC = 'recording', PDF = 'pdf', INK = 'ink', + DIAGRAM = 'diagram', SCREENSHOT = 'screenshot', FONTICON = 'fonticonbox', SEARCH = 'search', // search query @@ -20,6 +21,7 @@ export enum DocumentType { WEBCAM = 'webcam', // webcam CONFIG = 'config', // configuration document intended to specify a view layout configuration, but not be directly rendered (e.g., for saving the page# of a PDF, or view transform of a collection) SCRIPTING = 'script', // script editor + CHAT = 'chat', // chat with GPT about files EQUATION = 'equation', // equation editor FUNCPLOT = 'funcplot', // function plotter MAP = 'map', @@ -60,4 +62,5 @@ export enum CollectionViewType { StackedTimeline = 'stacked timeline', NoteTaking = 'notetaking', Calendar = 'calendar', + Card = 'card', } diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 443bf51f1..e16e0e834 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -195,6 +195,12 @@ export class DocumentOptions { date_range?: STRt = new StrInfo('date range for calendar', false); + chat?: STRt = new StrInfo('fields related to chatBox', false); + chat_history?: STRt = new StrInfo('chat history for chatbox', false); + chat_thread_id?: STRt = new StrInfo('thread id for chatbox', false); + chat_assistant_id?: STRt = new StrInfo('assistant id for chatbox', false); + chat_vector_store_id?: STRt = new StrInfo('assistant id for chatbox', false); + wikiData?: STRt = new StrInfo('WikiData ID related to map location'); description?: STRt = new StrInfo('description of document'); _timecodeToShow?: NUMt = new NumInfo('media timecode when document should appear (e.g., when an annotation shows up as a video plays)', false); @@ -256,6 +262,8 @@ export class DocumentOptions { layout_hideResizeHandles?: BOOLt = new BoolInfo('whether to hide the resize handles when selected'); layout_hideLinkButton?: BOOLt = new BoolInfo('whether the blue link counter button should be hidden'); layout_hideDecorationTitle?: BOOLt = new BoolInfo('whether to suppress the document decortations title when selected'); + _layout_hideContextMenu?: BOOLt = new BoolInfo('whether the context menu can be shown'); + layout_diagramEditor?: STRt = new StrInfo('specify the JSX string for a diagram editor view'); layout_hideContextMenu?: BOOLt = new BoolInfo('whether the context menu can be shown'); layout_borderRounding?: string; _layout_borderRounding?: STRt = new StrInfo('amount of rounding to document view corners'); @@ -483,6 +491,10 @@ export class DocumentOptions { hoverBackgroundColor?: string; // background color of a label when hovered userBackgroundColor?: STRt = new StrInfo('background color associated with a Dash user (seen in header fields of shared documents)'); userColor?: STRt = new StrInfo('color associated with a Dash user (seen in header fields of shared documents)'); + + cardSort?: STRt = new StrInfo('way cards are sorted in deck view'); + cardSort_customField?: STRt = new StrInfo('field key used for sorting cards'); + cardSort_visibleSortGroups?: List<number>; // which sorting values are being filtered (shown) } export const DocOptions = new DocumentOptions(); @@ -750,6 +762,9 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.SCRIPTING), script || undefined, { ...options, layout: fieldKey ? `<ScriptingBox {...props} fieldKey={'${fieldKey}'}/>` /* ScriptingBox.LayoutString(fieldKey) */ : undefined }); } + export function ChatDocument(options?: DocumentOptions) { + return InstanceFromProto(Prototypes.get(DocumentType.CHAT), undefined, { ...(options || {}) }); + } // eslint-disable-next-line default-param-last export function VideoDocument(url: string, options: DocumentOptions = {}, overwriteDoc?: Doc) { return InstanceFromProto(Prototypes.get(DocumentType.VID), new VideoField(url), options, undefined, undefined, undefined, overwriteDoc); @@ -766,6 +781,9 @@ export namespace Docs { export function ComparisonDocument(options: DocumentOptions = { title: 'Comparison Box' }) { return InstanceFromProto(Prototypes.get(DocumentType.COMPARISON), undefined, options); } + export function DiagramDocument(options: DocumentOptions = { title: 'bruh box' }) { + return InstanceFromProto(Prototypes.get(DocumentType.DIAGRAM), undefined, options); + } // eslint-disable-next-line default-param-last export function AudioDocument(url: string, options: DocumentOptions = {}, overwriteDoc?: Doc) { @@ -944,6 +962,10 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.COL), new List(documents), { ...options, _type_collection: CollectionViewType.Carousel3D }); } + export function CardDeckDocument(documents: Array<Doc>, options: DocumentOptions, id?: string) { + return InstanceFromProto(Prototypes.get(DocumentType.COL), new List(documents), { ...options, _type_collection: CollectionViewType.Card }); + } + export function SchemaDocument(schemaHeaders: SchemaHeaderField[], documents: Array<Doc>, options: DocumentOptions) { return InstanceFromProto(Prototypes.get(DocumentType.COL), new List(documents), { schemaHeaders: new List(schemaHeaders), ...options, _type_collection: CollectionViewType.Schema }); } diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index 35f85f792..c2e915b33 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -21,6 +21,7 @@ import { DashboardView } from "../views/DashboardView"; import { OverlayView } from "../views/OverlayView"; import { CollectionTreeView } from "../views/collections/CollectionTreeView"; import { TreeViewType } from "../views/collections/CollectionTreeViewType"; +import { CollectionView } from "../views/collections/CollectionView"; import { Colors } from "../views/global/globalEnums"; import { mediaState } from "../views/nodes/AudioBox"; import { ButtonType, FontIconBox } from "../views/nodes/FontIconBox/FontIconBox"; @@ -361,6 +362,8 @@ pie title Minerals in my tap water {key: "Collection", creator: opts => Docs.Create.FreeformDocument([], opts), opts: { _width: 150, _height: 100, _layout_fitWidth: true }}, {key: "Webpage", creator: opts => Docs.Create.WebDocument("",opts), opts: { _width: 400, _height: 512, _nativeWidth: 850, data_useCors: true, }}, {key: "Comparison", creator: Docs.Create.ComparisonDocument, opts: { _width: 300, _height: 300 }}, + {key: "Diagram", creator: Docs.Create.DiagramDocument, opts: { _width: 300, _height: 300, _type_collection: CollectionViewType.Freeform, layout_diagramEditor: CollectionView.LayoutString("data") }, scripts: { onPaint: `toggleDetail(documentView, "diagramEditor","")`}}, + {key: "Audio", creator: opts => Docs.Create.AudioDocument(nullAudio, opts),opts: { _width: 200, _height: 100, }}, {key: "Audio", creator: opts => Docs.Create.AudioDocument(nullAudio, opts),opts: { _width: 200, _height: 100, _layout_fitWidth: true, }}, {key: "Map", creator: opts => Docs.Create.MapDocument([], opts), opts: { _width: 800, _height: 600, _layout_fitWidth: true, }}, {key: "Screengrab", creator: Docs.Create.ScreenshotDocument, opts: { _width: 400, _height: 200 }}, @@ -368,6 +371,7 @@ pie title Minerals in my tap water {key: "Button", creator: Docs.Create.ButtonDocument, opts: { _width: 150, _height: 50, _xPadding: 10, _yPadding: 10, title_custom: true, waitForDoubleClickToClick: 'never'}, scripts: {onClick: FollowLinkScript()?.script.originalScript ?? ""}}, {key: "Script", creator: opts => Docs.Create.ScriptingDocument(null, opts), opts: { _width: 200, _height: 250, }}, {key: "DataViz", creator: opts => Docs.Create.DataVizDocument("/users/rz/Downloads/addresses.csv", opts), opts: { _width: 300, _height: 300 }}, + {key: "Chat", creator: Docs.Create.ChatDocument, opts: { _width: 300, _height: 300, }}, {key: "Header", creator: headerTemplate, opts: { _width: 300, _height: 120, _header_pointerEvents: "all", _header_height: 50, _header_fontSize: 9,_layout_autoHeightMargins: 50, _layout_autoHeight: true, treeView_HideUnrendered: true}}, {key: "ViewSlide", creator: slideView, opts: { _width: 400, _height: 300, _xMargin: 3, _yMargin: 3,}}, {key: "Trail", creator: Docs.Create.PresDocument, opts: { _width: 400, _height: 30, _type_collection: CollectionViewType.Stacking, dropAction: dropActionType.embed, treeView_HideTitle: true, _layout_fitWidth:true, layout_boxShadow: "0 0" }}, @@ -381,7 +385,10 @@ pie title Minerals in my tap water {key: "Plotly", creator: plotlyView, opts: { _width: 300, _height: 300, }}, ]; - emptyThings.forEach(thing => DocUtils.AssignDocField(doc, "empty"+thing.key, (opts) => thing.creator(opts), {...standardOps(thing.key), ...thing.opts}, undefined, thing.scripts, thing.funcs)); + emptyThings.forEach( + thing =>{ DocUtils.AssignDocField(doc, "empty"+thing.key, (opts) => thing.creator(opts), {...standardOps(thing.key), ...thing.opts}, undefined, thing.scripts, thing.funcs); + console.log(thing.key) + }); return [ { toolTip: "Tap or drag to create a note", title: "Note", icon: "sticky-note", dragFactory: doc.emptyNote as Doc, clickFactory: DocCast(doc.emptyNote)}, @@ -395,8 +402,10 @@ pie title Minerals in my tap water { toolTip: "Tap or drag to create a collection", title: "Col", icon: "folder", dragFactory: doc.emptyCollection as Doc, clickFactory: DocCast(doc.emptyTab)}, { toolTip: "Tap or drag to create a webpage", title: "Web", icon: "globe-asia", dragFactory: doc.emptyWebpage as Doc, clickFactory: DocCast(doc.emptyWebpage)}, { toolTip: "Tap or drag to create a comparison box", title: "Compare", icon: "columns", dragFactory: doc.emptyComparison as Doc, clickFactory: DocCast(doc.emptyComparison)}, + { toolTip: "Tap or drag to create a diagram", title: "Diagram", icon: "tree", dragFactory: doc.emptyDiagram as Doc, clickFactory: DocCast(doc.emptyDiagram)}, { toolTip: "Tap or drag to create an audio recorder", title: "Audio", icon: "microphone", dragFactory: doc.emptyAudio as Doc, clickFactory: DocCast(doc.emptyAudio), openFactoryLocation: OpenWhere.overlay}, { toolTip: "Tap or drag to create a map", title: "Map", icon: "map-marker-alt", dragFactory: doc.emptyMap as Doc, clickFactory: DocCast(doc.emptyMap)}, + { toolTip: "Tap or drag to create a chat assistant", title: "Assistant Chat", icon: "book",dragFactory: doc.emptyChat as Doc, clickFactory: DocCast(doc.emptyChat)}, { toolTip: "Tap or drag to create a screen grabber", title: "Grab", icon: "photo-video", dragFactory: doc.emptyScreengrab as Doc, clickFactory: DocCast(doc.emptyScreengrab), openFactoryLocation: OpenWhere.overlay, funcs: { hidden: "IsNoviceMode()"}}, { toolTip: "Tap or drag to create a WebCam recorder", title: "WebCam", icon: "photo-video", dragFactory: doc.emptyWebCam as Doc, clickFactory: DocCast(doc.emptyWebCam), openFactoryLocation: OpenWhere.overlay, funcs: { hidden: "IsNoviceMode()"}}, { toolTip: "Tap or drag to create a button", title: "Button", icon: "circle", dragFactory: doc.emptyButton as Doc, clickFactory: DocCast(doc.emptyButton)}, @@ -649,12 +658,45 @@ pie title Minerals in my tap water { title: "Center", icon: "align-center", toolTip: "Center Align Stack", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"center", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform ] } + static cardTools(): Button[] { + return [ + { title: "Time", icon:"hourglass-half", toolTip:"Sort by most recent document creation", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"time", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "Type", icon:"eye", toolTip:"Sort by document type", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"docType",funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "Color", icon:"palette", toolTip:"Sort by document color", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"color", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + ] + } + static labelTools(): Button[] { + return [ + { title: "AI", icon:"robot", toolTip:"Add AI labels", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"chat", funcs: {hidden:`showFreeform ("chat", true)`},scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "AIs", icon:"AI Sort", toolTip:"Filter AI labels", subMenu: this.cardGroupTools("chat"), expertMode: false, toolType:CollectionViewType.Card, funcs: {hidden:`!showFreeform("chat", true)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, + { title: "Like", icon:"heart", toolTip:"Add Like labels", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"like", funcs: {hidden:`showFreeform ("like", true)`},scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "Likes", icon:"Likes", toolTip:"Filter likes", width: 10, subMenu: this.cardGroupTools("heart"), expertMode: false, toolType:CollectionViewType.Card, funcs: {hidden:`!showFreeform("like", true)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, + { title: "Star", icon:"star", toolTip:"Add Star labels", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"star", funcs: {hidden:`showFreeform ("star", true)`},scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "Stars", icon:"Stars", toolTip:"Filter stars", width: 80, subMenu: this.cardGroupTools("star"), expertMode: false, toolType:CollectionViewType.Card, funcs: {hidden:`!showFreeform("star", true)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, + { title: "Idea", icon:"satellite", toolTip:"Add Idea labels", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"idea", funcs: {hidden:`showFreeform ("idea", true)`},scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "Ideas", icon:"Ideas", toolTip:"Filter ideas", width: 80, subMenu: this.cardGroupTools("satellite"), expertMode: false, toolType:CollectionViewType.Card, funcs: {hidden:`!showFreeform("idea", true)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, + ] + } + static cardGroupTools(icon: string): Button[] { + return [ + { title: "1", icon, toolTip:"Click to toggle visibility", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"1", funcs: {hidden:`!cardHasLabel(this.toolType)`}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "2", icon, toolTip:"Click to toggle visibility", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"2", funcs: {hidden:`!cardHasLabel(this.toolType)`}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "3", icon, toolTip:"Click to toggle visibility", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"3", funcs: {hidden:`!cardHasLabel(this.toolType)`}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "4", icon, toolTip:"Click to toggle visibility", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"4", funcs: {hidden:`!cardHasLabel(this.toolType)`}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "5", icon, toolTip:"Click to toggle visibility", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"5", funcs: {hidden:`!cardHasLabel(this.toolType)`}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "6", icon, toolTip:"Click to toggle visibility", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"6", funcs: {hidden:`!cardHasLabel(this.toolType)`}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + { title: "7", icon, toolTip:"Click to toggle visibility", btnType: ButtonType.ToggleButton, expertMode: false, toolType:"7", funcs: {hidden:`!cardHasLabel(this.toolType)`}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, + ] + } static viewTools(): Button[] { return [ { title: "Snap", icon: "th", toolTip: "Show Snap Lines", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"snaplines", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform { title: "Grid", icon: "border-all", toolTip: "Show Grid", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"grid", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform { title: "Fit All", icon: "object-group", toolTip: "Fit Docs to View (double click to make sticky)",btnType: ButtonType.ToggleButton, ignoreClick:true, expertMode: false, toolType:"viewAll", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}', onDoubleClick: '{ return showFreeform(this.toolType, _readOnly_, true);}'}}, // Only when floating document is selected in freeform { title: "Clusters", icon: "braille", toolTip: "Show Doc Clusters", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"clusters", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform + { title: "Cards", icon: "brain", toolTip: "Flashcards", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"flashcards", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform + { title: "Arrange", icon:"arrow-down-short-wide",toolTip:"Toggle Auto Arrange", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"arrange", funcs: {hidden: 'IsNoviceMode()'}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform + ] } static textTools():Button[] { @@ -736,8 +778,8 @@ pie title Minerals in my tap water { btnList: new List<string>([CollectionViewType.Freeform, CollectionViewType.Schema, CollectionViewType.Tree, CollectionViewType.Stacking, CollectionViewType.Masonry, CollectionViewType.Multicolumn, CollectionViewType.Multirow, CollectionViewType.Time, CollectionViewType.Carousel, - CollectionViewType.Carousel3D, CollectionViewType.Linear, CollectionViewType.Map, - CollectionViewType.Grid, CollectionViewType.NoteTaking]), + CollectionViewType.Carousel3D, CollectionViewType.Card, CollectionViewType.Linear, CollectionViewType.Map, + CollectionViewType.Grid, CollectionViewType.NoteTaking, ]), title: "Perspective", toolTip: "View", btnType: ButtonType.DropdownList, ignoreClick: true, width: 100, scripts: { script: '{ return setView(value, _readOnly_); }'}}, { title: "Pin", icon: "map-pin", toolTip: "Pin View to Trail", btnType: ButtonType.ClickButton, expertMode: false, width: 30, scripts: { onClick: 'pinWithView(altKey)'}, funcs: {hidden: "IsNoneSelected()"}}, { title: "Header", icon: "heading", toolTip: "Doc Titlebar Color", btnType: ButtonType.ColorButton, expertMode: false, ignoreClick: true, scripts: { script: 'return setHeaderColor(value, _readOnly_)'} }, @@ -752,6 +794,8 @@ pie title Minerals in my tap water { title: "Doc", icon: "Doc", toolTip: "Freeform Doc tools", subMenu: CurrentUserUtils.freeTools(), expertMode: false, toolType:CollectionViewType.Freeform, funcs: {hidden: `!SelectedDocType(this.toolType, this.expertMode, true)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, // Always available { title: "View", icon: "View", toolTip: "View tools", subMenu: CurrentUserUtils.viewTools(), expertMode: false, toolType:CollectionViewType.Freeform, funcs: {hidden: `!SelectedDocType(this.toolType, this.expertMode)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, // Always available { title: "Stack", icon: "View", toolTip: "Stacking tools", subMenu: CurrentUserUtils.stackTools(), expertMode: false, toolType:CollectionViewType.Stacking, funcs: {hidden: `!SelectedDocType(this.toolType, this.expertMode)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, // Always available + { title: "Card", icon: "Sort", toolTip: "Card sort", subMenu: CurrentUserUtils.cardTools(), expertMode: false, toolType:CollectionViewType.Card, funcs: {hidden: `!SelectedDocType(this.toolType, this.expertMode)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, // Always available + { title: "Label", icon: "Label", toolTip: "Assign card labels", subMenu: CurrentUserUtils.labelTools(), expertMode: false, toolType:CollectionViewType.Card, funcs: {hidden: `!SelectedDocType(this.toolType, this.expertMode)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, // Always available { title: "Web", icon: "Web", toolTip: "Web functions", subMenu: CurrentUserUtils.webTools(), expertMode: false, toolType:DocumentType.WEB, funcs: {hidden: `!SelectedDocType(this.toolType, this.expertMode)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, // Only when Web is selected { title: "Video", icon: "Video", toolTip: "Video functions", subMenu: CurrentUserUtils.videoTools(), expertMode: false, toolType:DocumentType.VID, funcs: {hidden: `!SelectedDocType(this.toolType, this.expertMode)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, // Only when video is selected { title: "Image", icon: "Image", toolTip: "Image functions", subMenu: CurrentUserUtils.imageTools(), expertMode: false, toolType:DocumentType.IMG, funcs: {hidden: `!SelectedDocType(this.toolType, this.expertMode)`, linearView_IsOpen: `SelectedDocType(this.toolType, this.expertMode)`} }, // Only when image is selected diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 8968acbbb..43b9a6b39 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -27,8 +27,10 @@ import { CollectionSchemaView } from './collections/collectionSchema/CollectionS import { SchemaRowBox } from './collections/collectionSchema/SchemaRowBox'; import './global/globalScripts'; import { AudioBox } from './nodes/AudioBox'; +import { ChatBox } from './nodes/ChatBox/ChatBox'; import { ComparisonBox } from './nodes/ComparisonBox'; import { DataVizBox } from './nodes/DataVizBox/DataVizBox'; +import { DiagramBox } from './nodes/DiagramBox'; import { DocumentContentsView, HTMLtag } from './nodes/DocumentContentsView'; import { EquationBox } from './nodes/EquationBox'; import { FieldView } from './nodes/FieldView'; @@ -136,6 +138,8 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' }; MapBox, ScreenshotBox, DataVizBox, + ChatBox, + DiagramBox, HTMLtag, ComparisonBox, LoadingBox, diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 6875ffab0..31d88fb87 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -54,6 +54,7 @@ import { CollectionMenu } from './collections/CollectionMenu'; import { TabDocView } from './collections/TabDocView'; import './collections/TreeView.scss'; import { CollectionFreeFormView } from './collections/collectionFreeForm'; +import { ImageLabelHandler } from './collections/collectionFreeForm/ImageLabelHandler'; import { MarqueeOptionsMenu } from './collections/collectionFreeForm/MarqueeOptionsMenu'; import { CollectionLinearView } from './collections/collectionLinear'; import { LinkMenu } from './linking/LinkMenu'; @@ -76,8 +77,8 @@ import { AnchorMenu } from './pdf/AnchorMenu'; import { GPTPopup } from './pdf/GPTPopup/GPTPopup'; import { TopBar } from './topbar/TopBar'; -const _global = (window /* browser */ || global) /* node */ as any; const { LEFT_MENU_WIDTH, TOPBAR_HEIGHT } = require('./global/globalCssVariables.module.scss'); // prettier-ignore +const _global = (window /* browser */ || global) /* node */ as any; @observer export class MainView extends ObservableReactComponent<{}> { @@ -542,6 +543,11 @@ export class MainView extends ObservableReactComponent<{}> { fa.faZ, fa.faArrowsUpToLine, fa.faArrowsDownToLine, + fa.faPalette, + fa.faHourglassHalf, + fa.faRobot, + fa.faSatellite, + fa.faStar, ] ); } @@ -1082,6 +1088,7 @@ export class MainView extends ObservableReactComponent<{}> { <PreviewCursor /> <TaskCompletionBox /> <ContextMenu /> + <ImageLabelHandler /> <AnchorMenu /> <MapAnchorMenu /> <DirectionsAnchorMenu /> diff --git a/src/client/views/collections/CollectionCardDeckView.scss b/src/client/views/collections/CollectionCardDeckView.scss new file mode 100644 index 000000000..a089b248d --- /dev/null +++ b/src/client/views/collections/CollectionCardDeckView.scss @@ -0,0 +1,84 @@ +@import '../global/globalCssVariables.module.scss'; + +.collectionCardView-outer { + height: 100%; + width: 100%; + position: relative; + background-color: white; + overflow: hidden; +} + +.card-wrapper { + display: grid; + grid-template-columns: repeat(10, 1fr); + // width: 100%; + transform-origin: top left; + + position: absolute; + align-items: center; + justify-items: center; + justify-content: center; + + transition: transform 0.3s cubic-bezier(0.455, 0.03, 0.515, 0.955); +} + +.card-button-container { + display: flex; + padding: 3px; + // width: 300px; + background-color: rgb(218, 218, 218); /* Background color of the container */ + border-radius: 50px; /* Rounds the corners of the container */ + transform: translateY(75px); + // box-shadow: 0 4px 8px rgba(0,0,0,0.1); /* Optional: Adds shadow for depth */ + align-items: center; /* Centers buttons vertically */ + justify-content: start; /* Centers buttons horizontally */ +} + +button { + width: 35px; + height: 35px; + border-radius: 50%; + background-color: $dark-gray; + // border-color: $medium-blue; + margin: 5px; // transform: translateY(-50px); +} + +// button:hover { +// transform: translateY(-50px); +// } + +// .card-wrapper::after { +// content: ""; +// width: 100%; /* Forces wrapping */ +// } + +// .card-wrapper > .card-item:nth-child(10n)::after { +// content: ""; +// width: 100%; /* Forces wrapping after every 10th item */ +// } + +// .card-row{ +// display: flex; +// position: absolute; +// align-items: center; +// transition: transform 0.3s cubic-bezier(0.455, 0.03, 0.515, 0.955); + +// } + +.card-item-inactive, +.card-item-active, +.card-item { + position: relative; + transition: transform 0.5s ease-in-out; + display: flex; + flex-direction: column; +} + +.card-item-inactive { + opacity: 0.5; +} + +.card-item-active { + position: absolute; + z-index: 100; +} diff --git a/src/client/views/collections/CollectionCardDeckView.tsx b/src/client/views/collections/CollectionCardDeckView.tsx new file mode 100644 index 000000000..6036a2ead --- /dev/null +++ b/src/client/views/collections/CollectionCardDeckView.tsx @@ -0,0 +1,513 @@ +import { IReactionDisposer, ObservableMap, action, computed, makeObservable, observable, reaction } from 'mobx'; +import { observer } from 'mobx-react'; +import * as React from 'react'; +import { ClientUtils, DashColor, returnFalse, returnZero } from '../../../ClientUtils'; +import { numberRange } from '../../../Utils'; +import { Doc, NumListCast } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; +import { Id } from '../../../fields/FieldSymbols'; +import { BoolCast, Cast, DateCast, NumCast, RTFCast, ScriptCast, StrCast } from '../../../fields/Types'; +import { URLField } from '../../../fields/URLField'; +import { gptImageLabel } from '../../apis/gpt/GPT'; +import { DocumentType } from '../../documents/DocumentTypes'; +import { DragManager } from '../../util/DragManager'; +import { SnappingManager } from '../../util/SnappingManager'; +import { Transform } from '../../util/Transform'; +import { undoable } from '../../util/UndoManager'; +import { StyleProp } from '../StyleProp'; +import { DocumentView } from '../nodes/DocumentView'; +import { GPTPopup, GPTPopupMode } from '../pdf/GPTPopup/GPTPopup'; +import './CollectionCardDeckView.scss'; +import { CollectionSubView } from './CollectionSubView'; + +enum cardSortings { + Time = 'time', + Type = 'type', + Color = 'color', + Custom = 'custom', + None = '', +} +@observer +export class CollectionCardView extends CollectionSubView() { + private _dropDisposer?: DragManager.DragDropDisposer; + private _childDocumentWidth = 600; // target width of a Doc... + private _disposers: { [key: string]: IReactionDisposer } = {}; + private _textToDoc = new Map<string, Doc>(); + + @observable _forceChildXf = false; + @observable _isLoading = false; + @observable _hoveredNodeIndex = -1; + @observable _docRefs = new ObservableMap<Doc, DocumentView>(); + @observable _maxRowCount = 10; + + static getButtonGroup(groupFieldKey: 'chat' | 'star' | 'idea' | 'like', doc: Doc): number | undefined { + return Cast(doc[groupFieldKey], 'number', null); + } + + static imageUrlToBase64 = async (imageUrl: string): Promise<string> => { + try { + const response = await fetch(imageUrl); + const blob = await response.blob(); + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = error => reject(error); + }); + } catch (error) { + console.error('Error:', error); + throw error; + } + }; + + protected createDashEventsTarget = (ele: HTMLDivElement | null) => { + this._dropDisposer?.(); + if (ele) { + this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this.layoutDoc); + } + }; + + constructor(props: any) { + super(props); + makeObservable(this); + } + + componentDidMount(): void { + this._disposers.sort = reaction( + () => ({ cardSort: this.cardSort, field: this.cardSort_customField }), + ({ cardSort, field }) => (cardSort === cardSortings.Custom && field === 'chat' ? this.openChatPopup() : GPTPopup.Instance.setVisible(false)) + ); + } + + componentWillUnmount() { + Object.keys(this._disposers).forEach(key => this._disposers[key]?.()); + this._dropDisposer?.(); + } + + @computed get cardSort_customField() { + return StrCast(this.Document.cardSort_customField) as any as 'chat' | 'star' | 'idea' | 'like'; + } + + @computed get cardSort() { + return StrCast(this.Document.cardSort) as any as cardSortings; + } + /** + * how much to scale down the contents of the view so that everything will fit + */ + @computed get fitContentScale() { + const length = Math.min(this.childDocsWithoutLinks.length, this._maxRowCount); + return (this._childDocumentWidth * length) / this._props.PanelWidth(); + } + + @computed get translateWrapperX() { + let translate = 0; + + if (this.inactiveDocs().length !== this.childDocsWithoutLinks.length && this.inactiveDocs().length < 10) { + translate += this.panelWidth() / 2; + } + return translate; + } + + /** + * The child documents to be rendered-- either all of them except the Links or the docs in the currently active + * custom group + */ + @computed get childDocsWithoutLinks() { + const regularDocs = this.childDocs.filter(l => l.type !== DocumentType.LINK); + const activeGroups = NumListCast(this.Document.cardSort_visibleSortGroups); + + if (activeGroups.length > 0 && this.cardSort === cardSortings.Custom) { + return regularDocs.filter(doc => { + // Get the group number for the current index + const groupNumber = CollectionCardView.getButtonGroup(this.cardSort_customField, doc); + // Check if the group number is in the active groups + return groupNumber !== undefined && activeGroups.includes(groupNumber); + }); + } + + // Default return for non-custom cardSort or other cases, filtering out links + return regularDocs; + } + + /** + * Determines the order in which the cards will be rendered depending on the current sort type + */ + @computed get sortedDocs() { + return this.sort(this.childDocsWithoutLinks, this.cardSort, BoolCast(this.layoutDoc.sortDesc)); + } + + @action + setHoveredNodeIndex = (index: number) => { + if (!DocumentView.SelectedDocs().includes(this.childDocs[index])) { + this._hoveredNodeIndex = index; + } + }; + /** + * Translates the hovered node to the center of the screen + * @param index + * @returns + */ + translateHover = (index: number) => (this._hoveredNodeIndex === index && !DocumentView.SelectedDocs().includes(this.childDocs[index]) ? -50 : 0); + + isSelected = (index: number) => DocumentView.SelectedDocs().includes(this.childDocs[index]); + + /** + * Returns all the documents except the one that's currently selected + */ + inactiveDocs = () => this.childDocsWithoutLinks.filter(d => !DocumentView.SelectedDocs().includes(d)); + + panelWidth = () => this._childDocumentWidth; + panelHeight = (layout: Doc) => () => (this.panelWidth() * NumCast(layout._height)) / NumCast(layout._width); + onChildDoubleClick = () => ScriptCast(this.layoutDoc.onChildDoubleClick); + isContentActive = () => this._props.isSelected() || this._props.isContentActive() || this._props.isAnyChildContentActive(); + isChildContentActive = () => (this.isContentActive() ? true : false); + + /** + * Returns the degree to rotate a card dependind on the amount of cards in their row and their index in said row + * @param amCards + * @param index + * @returns + */ + rotate = (amCards: number, index: number) => { + const possRotate = -30 + index * (30 / ((amCards - (amCards % 2)) / 2)); + const stepMag = Math.abs(-30 + (amCards / 2 - 1) * (30 / ((amCards - (amCards % 2)) / 2))); + + if (amCards % 2 == 0 && possRotate == 0) { + return possRotate + Math.abs(-30 + (index - 1) * (30 / (amCards / 2))); + } + if (amCards % 2 == 0 && index > (amCards + 1) / 2) { + return possRotate + stepMag; + } + + return possRotate; + }; + /** + * Returns the degree to which a card should be translated in the y direction for the arch effect + */ + translateY = (amCards: number, index: number, realIndex: number) => { + const evenOdd = amCards % 2; + const apex = (amCards - evenOdd) / 2; + const stepMag = 200 / ((amCards - evenOdd) / 2) + Math.abs((apex - index) * 25); + + let rowOffset = 0; + if (realIndex > this._maxRowCount - 1) { + rowOffset = 400 * ((realIndex - (realIndex % this._maxRowCount)) / this._maxRowCount); + } + if (evenOdd == 1 || index < apex - 1) { + return Math.abs(stepMag * (apex - index)) - rowOffset; + } + if (index == apex || index == apex - 1) { + return 0 - rowOffset; + } + + return Math.abs(stepMag * (apex - index - 1)) - rowOffset; + }; + + /** + * Translates the selected node to the middle fo the screen + * @param index + * @returns + */ + translateSelected = (index: number): number => { + // if (this.isSelected(index)) { + const middleOfPanel = this._props.PanelWidth() / 2; + const scaledNodeWidth = this.panelWidth() * 1.25; + + // Calculate the position of the node's left edge before scaling + const nodeLeftEdge = index * this.panelWidth(); + // Find the center of the node after scaling + const scaledNodeCenter = nodeLeftEdge + scaledNodeWidth / 2; + + // Calculate the translation needed to align the scaled node's center with the panel's center + const translation = middleOfPanel - scaledNodeCenter - scaledNodeWidth - scaledNodeWidth / 4; + + return translation; + }; + + /** + * Called in the sortedDocsType method. Compares the cards' value in regards to the desired sort type-- earlier cards are move to the + * front, latter cards to the back + * @param docs + * @param sortType + * @param isDesc + * @returns + */ + sort = (docs: Doc[], sortType: cardSortings, isDesc: boolean) => { + if (sortType === cardSortings.None) return docs; + docs.sort((docA, docB) => { + const [typeA, typeB] = (() => { + switch (sortType) { + case cardSortings.Time: + return [DateCast(docA.author_date)?.date ?? Date.now(), + DateCast(docB.author_date)?.date ?? Date.now()]; + case cardSortings.Color: + return [DashColor(StrCast(docA.backgroundColor)).hsv().toString(), // If docA.type is undefined, use an empty string + DashColor(StrCast(docB.backgroundColor)).hsv().toString()]; // If docB.type is undefined, use an empty string + case cardSortings.Custom: + return [CollectionCardView.getButtonGroup(this.cardSort_customField, docA)??0, + CollectionCardView.getButtonGroup(this.cardSort_customField, docB)??0]; + default: return [StrCast(docA.type), // If docA.type is undefined, use an empty string + StrCast(docB.type)]; // If docB.type is undefined, use an empty string + } // prettier-ignore + })(); + + const out = typeA < typeB ? -1 : typeA > typeB ? 1 : 0; + return isDesc ? -out : out; // Reverse the sort order if descending is true + }); + + return docs; + }; + + displayDoc = (doc: Doc, screenToLocalTransform: () => Transform) => { + return ( + <DocumentView + {...this._props} + ref={action((r: DocumentView) => r?.ContentDiv && this._docRefs.set(doc, r))} + Document={doc} + NativeWidth={returnZero} + NativeHeight={returnZero} + fitWidth={returnFalse} + onDoubleClickScript={this.onChildDoubleClick} + renderDepth={this._props.renderDepth + 1} + LayoutTemplate={this._props.childLayoutTemplate} + LayoutTemplateString={this._props.childLayoutString} + ScreenToLocalTransform={screenToLocalTransform} //makes sure the box wrapper thing is in the right spot + isContentActive={this.isChildContentActive} + isDocumentActive={this._props.childDocumentsActive?.() || this.Document._childDocumentsActive ? this._props.isDocumentActive : this.isContentActive} + PanelWidth={this.panelWidth} + PanelHeight={this.panelHeight(doc)} + /> + ); + }; + + /** + * Determines how many cards are in the row of a card at a specific index + * @param index + * @returns + */ + overflowAmCardsCalc = (index: number) => { + if (this.inactiveDocs().length < this._maxRowCount) { + return this.inactiveDocs().length; + } + // 13 - 3 = 10 + const totalCards = this.inactiveDocs().length; + // if 9 or less + if (index < totalCards - (totalCards % 10)) { + return this._maxRowCount; + } + //(3) + return totalCards % 10; + }; + /** + * Determines the index a card is in in a row + * @param realIndex + * @returns + */ + overflowIndexCalc = (realIndex: number) => realIndex % 10; + /** + * Translates the cards in the second rows and beyond over to the right + * @param realIndex + * @param calcIndex + * @param calcRowCards + * @returns + */ + translateOverflowX = (realIndex: number, calcRowCards: number) => (realIndex < this._maxRowCount ? 0 : (10 - calcRowCards) * (this.panelWidth() / 2)); + + /** + * Determines how far to translate a card in the y direction depending on its index, whether or not its being hovered, or if it's selected + * @param isHovered + * @param isSelected + * @param realIndex + * @param amCards + * @param calcRowIndex + * @returns + */ + calculateTranslateY = (isHovered: boolean, isSelected: boolean, realIndex: number, amCards: number, calcRowIndex: number) => { + if (isSelected) return 50 * this.fitContentScale; + const trans = isHovered ? this.translateHover(realIndex) : 0; + return trans + this.translateY(amCards, calcRowIndex, realIndex); + }; + + /** + * Toggles the buttons between on and off when creating custom sort groupings/changing those created by gpt + * @param childPairIndex + * @param buttonID + * @param doc + */ + toggleButton = undoable((buttonID: number, doc: Doc) => this.cardSort_customField && (doc[this.cardSort_customField] = buttonID), 'toggle custom button'); + + /** + * A list of the text content of all the child docs. RTF documents will have just their text and pdf documents will have the first 50 words. + * Image documents are converted to bse64 and gpt generates a description for them. all other documents use their title. This string is + * inputted into the gpt prompt to sort everything together + * @returns + */ + childPairStringList = () => { + const docToText = (doc: Doc) => { + switch (doc.type) { + case DocumentType.PDF: const words = StrCast(doc.text).split(/\s+/); + return words.slice(0, 50).join(' '); // first 50 words of pdf text + case DocumentType.IMG: return this.getImageDesc(doc); + case DocumentType.RTF: return StrCast(RTFCast(doc.text).Text); + default: return StrCast(doc.title); + } // prettier-ignore + }; + const docTextPromises = this.childDocsWithoutLinks.map(async doc => { + const docText = (await docToText(doc)) ?? ''; + this._textToDoc.set(docText.replace(/\n/g, ' ').trim(), doc); + return `======${docText.replace(/\n/g, ' ').trim()}======`; + }); + return Promise.all<string>(docTextPromises); + }; + + /** + * Calls the gpt API to generate descriptions for the images in the view + * @param image + * @returns + */ + getImageDesc = async (image: Doc) => { + if (StrCast(image.description)) return StrCast(image.description); // Return existing description + const href = (image.data as URLField).url.href; + const hrefParts = href.split('.'); + const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; + try { + const hrefBase64 = await CollectionCardView.imageUrlToBase64(hrefComplete); + const response = await gptImageLabel(hrefBase64); + image[DocData].description = response.trim(); + return response; // Return the response from gptImageLabel + } catch (error) { + console.log('bad things have happened'); + } + return ''; + }; + + /** + * Converts the gpt output into a hashmap that can be used for sorting. lists are seperated by ==== while elements within the list are seperated by ~~~~~~ + * @param gptOutput + */ + processGptOutput = (gptOutput: string) => { + // Split the string into individual list items + const listItems = gptOutput.split('======').filter(item => item.trim() !== ''); + listItems.forEach((item, index) => { + // Split the item by '~~~~~~' to get all descriptors + const parts = item.split('~~~~~~').map(part => part.trim()); + + parts.forEach(part => { + // Find the corresponding Doc in the textToDoc map + const doc = this._textToDoc.get(part); + if (doc) { + doc.chat = index; + } + }); + }); + }; + /** + * Opens up the chat popup and starts the process for smart sorting. + */ + openChatPopup = async () => { + GPTPopup.Instance.setVisible(true); + GPTPopup.Instance.setMode(GPTPopupMode.SORT); + const sortDesc = await this.childPairStringList(); // Await the promise to get the string result + GPTPopup.Instance.setCardsDoneLoading(true); // Set dataDoneLoading to true after data is loaded + GPTPopup.Instance.setSortDesc(sortDesc.join()); + GPTPopup.Instance.onSortComplete = (sortResult: string) => this.processGptOutput(sortResult); + }; + + /** + * Renders the buttons to customize sorting depending on which group the card belongs to and the amount of total groups + * @param childPairIndex + * @param doc + * @returns + */ + renderButtons = (doc: Doc, cardSort: cardSortings) => { + if (cardSort !== cardSortings.Custom) return ''; + const amButtons = Math.max(4, this.childDocs?.reduce((set, doc) => this.cardSort_customField && set.add(NumCast(doc[this.cardSort_customField])), new Set<number>()).size ?? 0); + const activeButtonIndex = CollectionCardView.getButtonGroup(this.cardSort_customField, doc); + const totalWidth = amButtons * 35 + amButtons * 2 * 5 + 6; + return ( + <div className="card-button-container" style={{ width: `${totalWidth}px` }}> + {numberRange(amButtons).map(i => ( + <button + key={i} + type="button" + style={{ backgroundColor: activeButtonIndex === i ? '#4476f7' : '#323232' }} // + onClick={() => this.toggleButton(i, doc)} + /> + ))} + </div> + ); + }; + /** + * Actually renders all the cards + */ + renderCards = () => { + const anySelected = this.childDocs.some(doc => DocumentView.SelectedDocs().includes(doc)); + // Map sorted documents to their rendered components + return this.sortedDocs.map((doc, index) => { + const realIndex = this.sortedDocs.filter(sortDoc => !DocumentView.SelectedDocs().includes(sortDoc)).indexOf(doc); + const calcRowIndex = this.overflowIndexCalc(realIndex); + const amCards = this.overflowAmCardsCalc(realIndex); + const isSelected = DocumentView.SelectedDocs().includes(doc); + + const childScreenToLocal = () => { + this._forceChildXf; + const dref = this._docRefs.get(doc); + const { translateX, translateY, scale } = ClientUtils.GetScreenTransform(dref?.ContentDiv); + return new Transform(-translateX + (dref?.centeringX || 0) * scale, + -translateY + (dref?.centeringY || 0) * scale, 1) + .scale(1 / scale).rotate(!isSelected ? -this.rotate(amCards, calcRowIndex) : 0); // prettier-ignore + }; + + return ( + <div + key={doc[Id]} + className={`card-item${isSelected ? '-active' : anySelected ? '-inactive' : ''}`} + onPointerUp={() => { + // this turns off documentDecorations during a transition, then turns them back on afterward. + SnappingManager.SetIsResizing(this.Document[Id]); + setTimeout( + action(() => { + SnappingManager.SetIsResizing(undefined); + this._forceChildXf = !this._forceChildXf; + }), + 700 + ); + }} + style={{ + width: this.panelWidth(), + height: 'max-content', // this.panelHeight(childPair.layout)(), + transform: `translateY(${this.calculateTranslateY(this._hoveredNodeIndex === index, isSelected, realIndex, amCards, calcRowIndex)}px) + translateX(${isSelected ? this.translateSelected(calcRowIndex) : this.translateOverflowX(realIndex, amCards)}px) + rotate(${!isSelected ? this.rotate(amCards, calcRowIndex) : 0}deg) + scale(${isSelected ? 1.25 : 1})`, + }} + onMouseEnter={() => this.setHoveredNodeIndex(index)}> + {this.displayDoc(doc, childScreenToLocal)} + {this.renderButtons(doc, this.cardSort)} + </div> + ); + }); + }; + render() { + return ( + <div + className="collectionCardView-outer" + ref={this.createDashEventsTarget} + style={{ + background: this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.BackgroundColor), + color: this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.Color), + }}> + <div + className="card-wrapper" + style={{ + transform: ` scale(${1 / this.fitContentScale}) translateX(${this.translateWrapperX}px)`, + height: `${100 * this.fitContentScale}%`, + }} + onMouseLeave={() => this.setHoveredNodeIndex(-1)}> + {this.renderCards()} + </div> + </div> + ); + } +} diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index b52c7c54c..5c304b4a9 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -33,6 +33,7 @@ import { CollectionLinearView } from './collectionLinear'; import { CollectionMulticolumnView } from './collectionMulticolumn/CollectionMulticolumnView'; import { CollectionMultirowView } from './collectionMulticolumn/CollectionMultirowView'; import { CollectionSchemaView } from './collectionSchema/CollectionSchemaView'; +import { CollectionCardView } from './CollectionCardDeckView'; @observer export class CollectionView extends ViewBoxAnnotatableComponent<CollectionViewProps>() { @@ -104,6 +105,7 @@ export class CollectionView extends ViewBoxAnnotatableComponent<CollectionViewPr case CollectionViewType.Masonry: return <CollectionStackingView key="collview" {...props} />; case CollectionViewType.Time: return <CollectionTimeView key="collview" {...props} />; case CollectionViewType.Grid: return <CollectionGridView key="collview" {...props} />; + case CollectionViewType.Card: return <CollectionCardView key="collview" {...props} />; case CollectionViewType.Freeform: default: return <CollectionFreeFormView key="collview" {...props} />; } diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss new file mode 100644 index 000000000..e7413bf8e --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss @@ -0,0 +1,44 @@ +#label-handler { + display: flex; + flex-direction: column; + align-items: center; + + > div:first-child { + display: flex; // Puts the input and button on the same row + align-items: center; // Vertically centers items in the flex container + + input { + color: black; + } + + .IconButton { + margin-left: 8px; // Adds space between the input and the icon button + width: 19px; + } + } + + > div:not(:first-of-type) { + display: flex; + flex-direction: column; + align-items: center; // Centers the content vertically in the flex container + width: 100%; + + > div { + display: flex; + justify-content: space-between; // Puts the content and delete button on opposite ends + align-items: center; + width: 100%; + margin-top: 8px; // Adds space between label rows + + p { + text-align: center; // Centers the text of the paragraph + flex-grow: 1; // Allows the paragraph to grow and occupy the available space + } + + .IconButton { + // Styling for the delete button + margin-left: auto; // Pushes the button to the far right + } + } + } +} diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx new file mode 100644 index 000000000..46bc3d946 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx @@ -0,0 +1,120 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { IconButton } from 'browndash-components'; +import { action, makeObservable, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { SettingsManager } from '../../../util/SettingsManager'; +import { ObservableReactComponent } from '../../ObservableReactComponent'; +import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import './ImageLabelHandler.scss'; + +@observer +export class ImageLabelHandler extends ObservableReactComponent<{}> { + static Instance: ImageLabelHandler; + + @observable _display: boolean = false; + @observable _pageX: number = 0; + @observable _pageY: number = 0; + @observable _yRelativeToTop: boolean = true; + @observable _currentLabel: string = ''; + @observable _labelGroups: string[] = []; + + constructor(props: any) { + super(props); + makeObservable(this); + ImageLabelHandler.Instance = this; + console.log('Instantiated label handler!'); + } + + @action + displayLabelHandler = (x: number, y: number) => { + this._pageX = x; + this._pageY = y; + this._display = true; + this._labelGroups = []; + }; + + @action + hideLabelhandler = () => { + this._display = false; + this._labelGroups = []; + }; + + @action + addLabel = (label: string) => { + label = label.toUpperCase().trim(); + if (label.length > 0) { + if (!this._labelGroups.includes(label)) { + this._labelGroups = [...this._labelGroups, label]; + } + } + }; + + @action + removeLabel = (label: string) => { + label = label.toUpperCase(); + this._labelGroups = this._labelGroups.filter(group => group !== label); + }; + + @action + groupImages = () => { + MarqueeOptionsMenu.Instance.groupImages(); + this._display = false; + }; + + render() { + if (this._display) { + return ( + <div + id="label-handler" + className="contextMenu-cont" + style={{ + display: this._display ? '' : 'none', + left: this._pageX, + ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }), + background: SettingsManager.userBackgroundColor, + color: SettingsManager.userColor, + }}> + <div> + <IconButton tooltip={'Cancel'} onPointerDown={this.hideLabelhandler} icon={<FontAwesomeIcon icon="eye-slash" />} color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> + <input aria-label="label-input" id="new-label" type="text" style={{ color: 'black' }} /> + <IconButton + tooltip={'Add Label'} + onPointerDown={() => { + const input = document.getElementById('new-label') as HTMLInputElement; + const newLabel = input.value; + this.addLabel(newLabel); + this._currentLabel = ''; + input.value = ''; + }} + icon={<FontAwesomeIcon icon="plus" />} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + <IconButton tooltip={'Group Images'} onPointerDown={this.groupImages} icon={<FontAwesomeIcon icon="object-group" />} color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> + </div> + <div> + {this._labelGroups.map(group => { + return ( + <div> + <p>{group}</p> + <IconButton + tooltip={'Remove Label'} + onPointerDown={() => { + this.removeLabel(group); + }} + icon={'x'} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + </div> + ); + })} + </div> + </div> + ); + } else { + return <></>; + } + } +} diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index adac5a102..f02cd9d45 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -18,6 +18,8 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { public showMarquee: () => void = unimplementedFunction; public hideMarquee: () => void = unimplementedFunction; public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; + public classifyImages: (e: React.MouseEvent | undefined) => void = unimplementedFunction; + public groupImages: () => void = unimplementedFunction; public isShown = () => this._opacity > 0; constructor(props: any) { super(props); @@ -37,6 +39,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { <IconButton tooltip="Summarize Documents" onPointerDown={this.summarize} icon={<FontAwesomeIcon icon="compress-arrows-alt" />} color={this.userColor} /> <IconButton tooltip="Delete Documents" onPointerDown={this.delete} icon={<FontAwesomeIcon icon="trash-alt" />} color={this.userColor} /> <IconButton tooltip="Pin selected region" onPointerDown={this.pinWithView} icon={<FontAwesomeIcon icon="map-pin" />} color={this.userColor} /> + <IconButton tooltip="Classify Images" onPointerDown={this.classifyImages} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} /> </> ); return this.getElement(buttons); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index b96444024..98100becb 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -4,15 +4,16 @@ import { observer } from 'mobx-react'; import * as React from 'react'; import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils'; import { intersectRect } from '../../../../Utils'; -import { Doc, Opt } from '../../../../fields/Doc'; +import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; import { InkData, InkField, InkTool } from '../../../../fields/InkField'; import { List } from '../../../../fields/List'; import { RichTextField } from '../../../../fields/RichTextField'; import { Cast, FieldValue, NumCast, StrCast } from '../../../../fields/Types'; -import { ImageField } from '../../../../fields/URLField'; +import { ImageField, URLField } from '../../../../fields/URLField'; import { GetEffectiveAcl } from '../../../../fields/util'; +import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; import { CognitiveServices } from '../../../cognitive_services/CognitiveServices'; import { DocUtils } from '../../../documents/DocUtils'; import { DocumentType } from '../../../documents/DocumentTypes'; @@ -29,9 +30,11 @@ import { OpenWhere } from '../../nodes/OpenWhere'; import { pasteImageBitmap } from '../../nodes/WebBoxRenderer'; import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; import { SubCollectionViewProps } from '../CollectionSubView'; +import { ImageLabelHandler } from './ImageLabelHandler'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; - +import { ImageUtility } from '../../nodes/generativeFill/generativeFillUtils/ImageHandler'; +import { CollectionCardView } from '../CollectionCardDeckView'; interface MarqueeViewProps { getContainerTransform: () => Transform; getTransform: () => Transform; @@ -61,11 +64,13 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps } private _commandExecuted = false; + private _selectedDocs: Doc[] = []; @observable _lastX: number = 0; @observable _lastY: number = 0; @observable _downX: number = 0; @observable _downY: number = 0; @observable _visible: boolean = false; // selection rentangle for marquee selection/free hand lasso is visible + @observable _labelsVisibile: boolean = false; @observable _lassoPts: [number, number][] = []; @observable _lassoFreehand: boolean = false; @@ -267,6 +272,8 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps MarqueeOptionsMenu.Instance.hideMarquee = this.hideMarquee; MarqueeOptionsMenu.Instance.jumpTo(e.clientX, e.clientY); MarqueeOptionsMenu.Instance.pinWithView = this.pinWithView; + MarqueeOptionsMenu.Instance.classifyImages = this.classifyImages; + MarqueeOptionsMenu.Instance.groupImages = this.groupImages; document.addEventListener('pointerdown', hideMarquee, true); document.addEventListener('wheel', hideMarquee, true); } else { @@ -419,6 +426,102 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps this.hideMarquee(); }); + /** + * Classifies images and assigns the labels as document fields. + * TODO: Turn into lists of labels instead of individual fields. + */ + @undoBatch + classifyImages = action(async (e: React.MouseEvent | undefined) => { + const selected = this.marqueeSelect(false, DocumentType.IMG); + this._selectedDocs = selected; + + const imagePromises = selected.map(doc => { + const href = (doc['data'] as URLField).url.href; + const hrefParts = href.split('.'); + const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; + return CollectionCardView.imageUrlToBase64(hrefComplete).then(hrefBase64 => + !hrefBase64 + ? undefined + : gptImageLabel(hrefBase64).then(response => { + const labels = response.split('\n'); + doc.image_labels = new List<string>(Array.from(labels!)); + return Promise.all(labels!.map(label => gptGetEmbedding(label))).then(embeddings => { + return { doc, embeddings }; + }); + }) + ); + }); + + const docsAndEmbeddings = await Promise.all(imagePromises); + docsAndEmbeddings + .filter(d => d) + .map(d => d!) + .forEach(docAndEmbedding => { + if (Array.isArray(docAndEmbedding.embeddings)) { + let doc = docAndEmbedding.doc; + for (let i = 0; i < 3; i++) { + doc[`label_embedding_${i + 1}`] = new List<number>(docAndEmbedding.embeddings[i]); + } + } + }); + + if (e) { + ImageLabelHandler.Instance.displayLabelHandler(e.pageX, e.pageY); + } + }); + + /** + * Groups images to most similar labels. + */ + @undoBatch + groupImages = action(async () => { + const labelGroups: string[] = ImageLabelHandler.Instance._labelGroups; + const labelToCollection: Map<string, Doc> = new Map(); + const labelToEmbedding: Map<string, number[]> = new Map(); + var similarity = require('compute-cosine-similarity'); + + // Create new collections associated with each label and get the embeddings for the labels. + for (const label of labelGroups) { + const newCollection = this.getCollection([], undefined, false); + newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2; + newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2; + labelToCollection.set(label, newCollection); + this._props.addDocument?.(newCollection); + const labelEmbedding = await gptGetEmbedding(label); + if (Array.isArray(labelEmbedding)) { + labelToEmbedding.set(label, labelEmbedding); + } + } + + // For each image, loop through the labels, and calculate similarity. Associate it with the + // most similar one. + this._selectedDocs.forEach(doc => { + let mostSimilarLabel: string | undefined; + let maxSimilarity: number = 0; + const embeddingAsList1 = NumListCast(doc.label_embedding_1); + const embeddingAsList2 = NumListCast(doc.label_embedding_2); + const embeddingAsList3 = NumListCast(doc.label_embedding_3); + + labelGroups.forEach(label => { + let curSimilarity1 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList1)); + let curSimilarity2 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList2)); + let curSimilarity3 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList3)); + let maxCurSimilarity = Math.max(curSimilarity1, curSimilarity2, curSimilarity3); + if (maxCurSimilarity >= 0.3 && maxCurSimilarity > maxSimilarity) { + mostSimilarLabel = label; + maxSimilarity = maxCurSimilarity; + } + + console.log('Doc with labels ' + doc.image_labels + 'has similarity score ' + maxCurSimilarity + ' to ' + mostSimilarLabel); + }); + + if (mostSimilarLabel) { + Doc.AddDocToList(labelToCollection.get(mostSimilarLabel)!, undefined, doc); + this._props.removeDocument?.(doc); + } + }); + }); + @undoBatch syntaxHighlight = action((e: KeyboardEvent | React.PointerEvent | undefined) => { const selected = this.marqueeSelect(false); @@ -579,7 +682,10 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps return false; } - marqueeSelect(selectBackgrounds: boolean = false) { + /** + * When this is called, returns the list of documents that have been selected by the marquee box. + */ + marqueeSelect(selectBackgrounds: boolean = false, docType: DocumentType | undefined = undefined) { const selection: Doc[] = []; const selectFunc = (doc: Doc) => { const layoutDoc = Doc.Layout(doc); @@ -590,10 +696,17 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps (this.touchesLine(bounds) || this.boundingShape(bounds)) && selection.push(doc); } }; - this._props - .activeDocuments() - .filter(doc => !doc.z && !doc._lockedPosition) - .map(selectFunc); + if (docType) { + this._props + .activeDocuments() + .filter(doc => !doc.z && !doc._lockedPosition && doc.type === docType) + .map(selectFunc); + } else { + this._props + .activeDocuments() + .filter(doc => !doc.z && !doc._lockedPosition) + .map(selectFunc); + } if (!selection.length && selectBackgrounds) this._props .activeDocuments() diff --git a/src/client/views/global/globalScripts.ts b/src/client/views/global/globalScripts.ts index ea3e7f825..7730ed385 100644 --- a/src/client/views/global/globalScripts.ts +++ b/src/client/views/global/globalScripts.ts @@ -9,6 +9,7 @@ import { ActiveInkWidth, ActiveIsInkMask, Doc, + DocListCast, Opt, SetActiveFillColor, SetActiveInkColor, @@ -35,6 +36,8 @@ import { ImageBox } from '../nodes/ImageBox'; import { VideoBox } from '../nodes/VideoBox'; import { WebBox } from '../nodes/WebBox'; import { RichTextMenu } from '../nodes/formattedText/RichTextMenu'; +import { NumListCast } from '../../../fields/Doc'; +import { List } from '../../../fields/List'; // import { InkTranscription } from '../InkTranscription'; @@ -135,7 +138,12 @@ ScriptingGlobals.add(function toggleOverlay(checkResult?: boolean) { ScriptingGlobals.add(function showFreeform(attr: 'center' | 'grid' | 'snaplines' | 'clusters' | 'viewAll' | 'fitOnce', checkResult?: boolean, persist?: boolean) { const selected = DocumentView.SelectedDocs().lastElement(); // prettier-ignore - const map: Map<'center' |'grid' | 'snaplines' | 'clusters' | 'viewAll' | 'fitOnce', { waitForRender?: boolean, checkResult: (doc:Doc) => any; setDoc: (doc:Doc, dv:DocumentView) => void;}> = new Map([ + const map: Map<'flashcards' | 'center' | 'grid' | 'snaplines' | 'clusters' | 'arrange' | 'viewAll' | 'fitOnce' | 'time' | 'docType' | 'color' | 'links' | 'like' | 'star' | 'idea' | 'chat' | '1' | '2' | '3' | '4', + { + waitForRender?: boolean; + checkResult: (doc: Doc) => any; + setDoc: (doc: Doc, dv: DocumentView) => void; + }> = new Map([ ['grid', { checkResult: (doc:Doc) => BoolCast(doc?._freeform_backgroundGrid, false), setDoc: (doc:Doc) => { doc._freeform_backgroundGrid = !doc._freeform_backgroundGrid; }, @@ -145,8 +153,8 @@ ScriptingGlobals.add(function showFreeform(attr: 'center' | 'grid' | 'snaplines' setDoc: (doc:Doc) => { doc._freeform_snapLines = !doc._freeform_snapLines; }, }], ['viewAll', { - checkResult: (doc:Doc) => BoolCast(doc?._freeform_fitContentsToBox, false), - setDoc: (doc:Doc,dv:DocumentView) => { + checkResult: (doc: Doc) => BoolCast(doc?._freeform_fitContentsToBox, false), + setDoc: (doc: Doc, dv: DocumentView) => { if (persist) doc._freeform_fitContentsToBox = !doc._freeform_fitContentsToBox; else if (doc._freeform_fitContentsToBox) doc._freeform_fitContentsToBox = undefined; else (dv.ComponentView as CollectionFreeFormView)?.fitContentOnce(); @@ -161,7 +169,68 @@ ScriptingGlobals.add(function showFreeform(attr: 'center' | 'grid' | 'snaplines' checkResult: (doc:Doc) => BoolCast(doc?._freeform_useClusters, false), setDoc: (doc:Doc) => { doc._freeform_useClusters = !doc._freeform_useClusters; }, }], - ]); + ['flashcards', { + checkResult: (doc: Doc) => BoolCast(Doc.UserDoc().defaultToFlashcards, false), + setDoc: (doc: Doc, dv: DocumentView) => Doc.UserDoc().defaultToFlashcards = !Doc.UserDoc().defaultToFlashcards, + }], + ['time', { + checkResult: (doc: Doc) => StrCast(doc?.cardSort) === "time", + setDoc: (doc: Doc, dv: DocumentView) => doc.cardSort = "time", + }], + ['docType', { + checkResult: (doc: Doc) => StrCast(doc?.cardSort) === "type", + setDoc: (doc: Doc, dv: DocumentView) => doc.cardSort = "type", + }], + ['color', { + checkResult: (doc: Doc) => StrCast(doc?.cardSort) === "color", + setDoc: (doc: Doc, dv: DocumentView) => doc.cardSort = "color", + }], + ['links', { + checkResult: (doc: Doc) => StrCast(doc?.cardSort) === "links", + setDoc: (doc: Doc, dv: DocumentView) => doc.cardSort = "links", + }], + ['like', { + checkResult: (doc: Doc) => StrCast(doc?.cardSort) === "custom" && StrCast(doc?.cardSort_customField) === "like", + setDoc: (doc: Doc, dv: DocumentView) => { + doc.cardSort = "custom"; + doc.cardSort_customField = "like"; + doc.cardSort_visibleSortGroups = new List<number>(); + } + }], + ['star', { + checkResult: (doc: Doc) => StrCast(doc?.cardSort) === "custom" && StrCast(doc?.cardSort_customField) === "star", + setDoc: (doc: Doc, dv: DocumentView) => { + doc.cardSort = "custom"; + doc.cardSort_customField = "star"; + doc.cardSort_visibleSortGroups = new List<number>(); + } + }], + ['idea', { + checkResult: (doc: Doc) => StrCast(doc?.cardSort) === "custom" && StrCast(doc?.cardSort_customField) === "idea", + setDoc: (doc: Doc, dv: DocumentView) => { + doc.cardSort = "custom"; + doc.cardSort_customField = "idea"; + doc.cardSort_visibleSortGroups = new List<number>(); + } + }], + ['chat', { + checkResult: (doc: Doc) => StrCast(doc?.cardSort) === "custom" && StrCast(doc?.cardSort_customField) === "chat", + setDoc: (doc: Doc, dv: DocumentView) => { + doc.cardSort = "custom"; + doc.cardSort_customField = "chat"; + doc.cardSort_visibleSortGroups = new List<number>(); + }, + }], + ]); + for (let i = 0; i < 8; i++) { + map.set((i + 1 + '') as any, { + checkResult: (doc: Doc) => NumListCast(doc?.cardSort_visibleSortGroups).includes(i), + setDoc: (doc: Doc, dv: DocumentView) => { + const list = NumListCast(doc.cardSort_visibleSortGroups); + doc.cardSort_visibleSortGroups = new List<number>(list.includes(i) ? list.filter(d => d !== i) : [...list, i]); + }, + }); + } if (checkResult) { return map.get(attr)?.checkResult(selected); @@ -172,6 +241,45 @@ ScriptingGlobals.add(function showFreeform(attr: 'center' | 'grid' | 'snaplines' return undefined; }); +ScriptingGlobals.add(function cardHasLabel(label: string) { + const selected = DocumentView.SelectedDocs().lastElement(); + const labelNum = Number(label) - 1; + return labelNum < 4 || (selected && DocListCast(selected[Doc.LayoutFieldKey(selected)]).some(doc => doc[StrCast(selected.cardSort_customField)] == labelNum)); +}, ''); + +// ScriptingGlobals.add(function setCardSortAttr(attr: 'time' | 'docType' | 'color', value: any, checkResult?: boolean) { +// // const editorView = RichTextMenu.Instance?.TextView?.EditorView; +// const selected = SelectionManager.Docs.lastElement(); +// // prettier-ignore +// const map: Map<'time' | 'docType' | 'color', { waitForRender?: boolean, checkResult: (doc:Doc) => any; setDoc: (doc:Doc, dv:DocumentView) => void;}> = new Map([ +// ['time', { +// checkResult: (doc:Doc) => StrCast(doc?.cardSort), +// setDoc: (doc:Doc,dv:DocumentView) => doc.cardSort = "time", +// }], +// ['docType', { +// checkResult: (doc:Doc) => StrCast(doc?.cardSort), +// setDoc: (doc:Doc,dv:DocumentView) => doc.cardSort = "type", +// }], +// ['color', { +// checkResult: (doc:Doc) => StrCast(doc?.cardSort), +// setDoc: (doc:Doc,dv:DocumentView) => doc.cardSort = "color", +// }], +// // ['custom', { +// // checkResult: () => RichTextMenu.Instance.textAlign, +// // setDoc: () => value && editorView?.state ? RichTextMenu.Instance.align(editorView, editorView.dispatch, value):(Doc.UserDoc().textAlign = value), +// // }] +// // , +// ]); + +// if (checkResult) { +// return map.get(attr)?.checkResult(selected); +// } + +// console.log('hey') +// SelectionManager.Views.map(dv => map.get(attr)?.setDoc(dv.layoutDoc, dv)); +// console.log('success') +// }); + // eslint-disable-next-line prefer-arrow-callback ScriptingGlobals.add(function setFontAttr(attr: 'font' | 'fontColor' | 'highlight' | 'fontSize' | 'alignment', value: any, checkResult?: boolean) { const editorView = RichTextMenu.Instance?.TextView?.EditorView; diff --git a/src/client/views/nodes/ChatBox/ChatBox.scss b/src/client/views/nodes/ChatBox/ChatBox.scss new file mode 100644 index 000000000..f1ad3d074 --- /dev/null +++ b/src/client/views/nodes/ChatBox/ChatBox.scss @@ -0,0 +1,228 @@ +$background-color: #f8f9fa; +$text-color: #333; +$input-background: #fff; +$button-color: #007bff; +$button-hover-color: darken($button-color, 10%); +$shadow-color: rgba(0, 0, 0, 0.075); +$border-radius: 8px; + +.chatBox { + display: flex; + flex-direction: column; + width: 100%; /* Adjust the width as needed, could be in percentage */ + height: 100%; /* Adjust the height as needed, could be in percentage */ + background-color: $background-color; + font-family: 'Helvetica Neue', Arial, sans-serif; + //margin: 20px auto; + //overflow: hidden; + + .scroll-box { + flex-grow: 1; + overflow-y: scroll; + overflow-x: hidden; + height: 100%; + padding: 10px; + display: flex; + flex-direction: column-reverse; + + &::-webkit-scrollbar { + width: 8px; + } + &::-webkit-scrollbar-thumb { + background-color: darken($background-color, 10%); + border-radius: $border-radius; + } + + + .chat-content { + display: flex; + flex-direction: column; + } + + .messages { + display: flex; + flex-direction: column; + .message { + padding: 10px; + margin-bottom: 10px; + border-radius: $border-radius; + background-color: lighten($background-color, 5%); + box-shadow: 0 2px 5px $shadow-color; + //display: flex; + align-items: center; + max-width: 70%; + word-break: break-word; + .message-footer { // Assuming this is the container for the toggle button + //max-width: 70%; + + + .toggle-logs-button { + margin-top: 10px; // Padding on sides to align with the text above + width: 95%; + //display: block; // Ensures the button extends the full width of its container + text-align: center; // Centers the text inside the button + //padding: 8px 0; // Adequate padding for touch targets + background-color: $button-color; + color: #fff; + border: none; + border-radius: $border-radius; + cursor: pointer; + //transition: background-color 0.3s; + //margin-top: 10px; // Adds space above the button + box-shadow: 0 2px 4px $shadow-color; // Consistent shadow with other elements + &:hover { + background-color: $button-hover-color; + } + } + .tool-logs { + width: 100%; + background-color: $input-background; + color: $text-color; + margin-top: 5px; + //padding: 10px; + //border-radius: $border-radius; + //box-shadow: inset 0 2px 4px $shadow-color; + //transition: opacity 1s ease-in-out; + font-family: monospace; + overflow-x: auto; + max-height: 150px; // Ensuring it does not grow too large + overflow-y: auto; + } + + } + + .custom-link { + color: lightblue; + text-decoration: underline; + cursor: pointer; + } + &.user { + align-self: flex-end; + background-color: $button-color; + color: #fff; + } + + &.chatbot { + align-self: flex-start; + background-color: $input-background; + color: $text-color; + } + + span { + flex-grow: 1; + padding-right: 10px; + } + + img { + max-width: 50px; + max-height: 50px; + border-radius: 50%; + } + } + } + padding-bottom: 0; + } + + .chat-form { + display: flex; + flex-grow: 1; + //height: 50px; + bottom: 0; + width: 100%; + padding: 10px; + background-color: $input-background; + box-shadow: inset 0 -1px 2px $shadow-color; + + input[type="text"] { + flex-grow: 1; + border: 1px solid darken($input-background, 10%); + border-radius: $border-radius; + padding: 8px 12px; + margin-right: 10px; + } + + button { + padding: 8px 16px; + background-color: $button-color; + color: #fff; + border: none; + border-radius: $border-radius; + cursor: pointer; + transition: background-color 0.3s; + + &:hover { + background-color: $button-hover-color; + } + } + margin-bottom: 0; + } +} + +.initializing-overlay { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: rgba($background-color, 0.95); + display: flex; + justify-content: center; + align-items: center; + font-size: 1.5em; + color: $text-color; + z-index: 10; // Ensure it's above all other content (may be better solution) + + &::before { + content: 'Initializing...'; + font-weight: bold; + } +} + + +.modal { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + justify-content: center; + align-items: center; + background-color: rgba(0, 0, 0, 0.4); + + .modal-content { + background-color: $input-background; + color: $text-color; + padding: 20px; + border-radius: $border-radius; + box-shadow: 0 2px 10px $shadow-color; + display: flex; + flex-direction: column; + align-items: center; + width: auto; + min-width: 300px; + + h4 { + margin-bottom: 15px; + } + + p { + margin-bottom: 20px; + } + + button { + padding: 10px 20px; + background-color: $button-color; + color: #fff; + border: none; + border-radius: $border-radius; + cursor: pointer; + margin: 5px; + transition: background-color 0.3s; + + &:hover { + background-color: $button-hover-color; + } + } + } +} diff --git a/src/client/views/nodes/ChatBox/ChatBox.tsx b/src/client/views/nodes/ChatBox/ChatBox.tsx new file mode 100644 index 000000000..880c332ac --- /dev/null +++ b/src/client/views/nodes/ChatBox/ChatBox.tsx @@ -0,0 +1,609 @@ +import { MathJaxContext } from 'better-react-mathjax'; +import { action, makeObservable, observable, observe, reaction, runInAction } from 'mobx'; +import { observer } from 'mobx-react'; +import OpenAI, { ClientOptions } from 'openai'; +import { ImageFile, Message } from 'openai/resources/beta/threads/messages'; +import { RunStep } from 'openai/resources/beta/threads/runs/steps'; +import * as React from 'react'; +import { Doc } from '../../../../fields/Doc'; +import { Id } from '../../../../fields/FieldSymbols'; +import { CsvCast, DocCast, PDFCast, StrCast } from '../../../../fields/Types'; +import { CsvField } from '../../../../fields/URLField'; +import { Networking } from '../../../Network'; +import { DocUtils } from '../../../documents/DocUtils'; +import { DocumentType } from '../../../documents/DocumentTypes'; +import { Docs } from '../../../documents/Documents'; +import { DocumentManager } from '../../../util/DocumentManager'; +import { LinkManager } from '../../../util/LinkManager'; +import { ViewBoxAnnotatableComponent } from '../../DocComponent'; +import { FieldView, FieldViewProps } from '../FieldView'; +import './ChatBox.scss'; +import MessageComponent from './MessageComponent'; +import { ANNOTATION_LINK_TYPE, ASSISTANT_ROLE, AssistantMessage, DOWNLOAD_TYPE } from './types'; + +@observer +export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { + @observable modalStatus = false; + @observable currentFile = { url: '' }; + @observable history: AssistantMessage[] = []; + @observable.deep current_message: AssistantMessage | undefined = undefined; + + @observable isLoading: boolean = false; + @observable isInitializing: boolean = true; + @observable expandedLogIndex: number | null = null; + @observable linked_docs_to_add: Doc[] = []; + + private openai: OpenAI; + private interim_history: string = ''; + private assistantID: string = ''; + private threadID: string = ''; + private _oldWheel: any; + private vectorStoreID: string = ''; + private mathJaxConfig: any; + private linkedCsvIDs: string[] = []; + + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(ChatBox, fieldKey); + } + constructor(props: FieldViewProps) { + super(props); + makeObservable(this); + this.openai = this.initializeOpenAI(); + this.history = []; + this.threadID = StrCast(this.dataDoc.thread_id); + this.assistantID = StrCast(this.dataDoc.assistant_id); + this.vectorStoreID = StrCast(this.dataDoc.vector_store_id); + this.openai = this.initializeOpenAI(); + if (this.assistantID === '' || this.threadID === '' || this.vectorStoreID === '') { + this.createAssistant(); + } else { + this.retrieveCsvUrls(); + this.isInitializing = false; + } + this.mathJaxConfig = { + loader: { load: ['input/asciimath'] }, + tex: { + inlineMath: [ + ['$', '$'], + ['\\(', '\\)'], + ], + displayMath: [ + ['$$', '$$'], + ['[', ']'], + ], + }, + }; + reaction( + () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, text: msg.text, image: msg.image, tool_logs: msg.tool_logs, links: msg.links })), + serializableHistory => { + this.dataDoc.data = JSON.stringify(serializableHistory); + } + ); + } + + toggleToolLogs = (index: number) => { + this.expandedLogIndex = this.expandedLogIndex === index ? null : index; + }; + + retrieveCsvUrls() { + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + + linkedDocs.forEach(doc => { + const aiFieldId = StrCast(doc[this.Document[Id] + '_ai_field_id']); + if (CsvCast(doc.data)) { + this.linkedCsvIDs.push(StrCast(aiFieldId)); + console.log(this.linkedCsvIDs); + } + }); + } + + initializeOpenAI() { + const configuration: ClientOptions = { + apiKey: process.env.OPENAI_KEY, + dangerouslyAllowBrowser: true, + }; + return new OpenAI(configuration); + } + + onPassiveWheel = (e: WheelEvent) => { + if (this._props.isContentActive()) { + e.stopPropagation(); + } + }; + + createLink = (linkInfo: string, startIndex: number, endIndex: number, linkType: ANNOTATION_LINK_TYPE, annotationIndex: number = 0) => { + const text = this.interim_history; + const subString = this.current_message?.text.substring(startIndex, endIndex) ?? ''; + if (!text) return; + const textToDisplay = `${annotationIndex}`; + let fileInfo = linkInfo; + const fileName = subString.split('/')[subString.split('/').length - 1]; + if (linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE) { + fileInfo = linkInfo + '!!!' + fileName; + } + + const formattedLink = `[${textToDisplay}](${fileInfo}~~~${linkType})`; + console.log(formattedLink); + const newText = text.replace(subString, formattedLink); + runInAction(() => { + this.interim_history = newText; + console.log(newText); + this.current_message?.links?.push({ + start: startIndex, + end: endIndex, + url: linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE ? fileName : linkInfo, + id: linkType === ANNOTATION_LINK_TYPE.DOWNLOAD_FILE ? linkInfo : undefined, + link_type: linkType, + }); + }); + }; + + @action + createAssistant = async () => { + this.isInitializing = true; + try { + const vectorStore = await this.openai.beta.vectorStores.create({ + name: 'Vector Store for Assistant', + }); + const assistant = await this.openai.beta.assistants.create({ + name: 'Document Analyser Assistant', + instructions: ` + You will analyse documents with which you are provided. You will answer questions and provide insights based on the information in the documents. + For writing math formulas: + You have a MathJax render environment. + - Write all in-line equations within a single dollar sign, $, to render them as TeX (this means any time you want to use a dollar sign to represent a dollar sign itself, you must escape it with a backslash: "$"); + - Use a double dollar sign, $$, to render equations on a new line; + Example: $$x^2 + 3x$$ is output for "x² + 3x" to appear as TeX.`, + model: 'gpt-4-turbo', + tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], + tool_resources: { + file_search: { + vector_store_ids: [vectorStore.id], + }, + code_interpreter: { + file_ids: this.linkedCsvIDs, + }, + }, + }); + const thread = await this.openai.beta.threads.create(); + + runInAction(() => { + this.dataDoc.assistant_id = assistant.id; + this.dataDoc.thread_id = thread.id; + this.dataDoc.vector_store_id = vectorStore.id; + this.assistantID = assistant.id; + this.threadID = thread.id; + this.vectorStoreID = vectorStore.id; + this.isInitializing = false; + }); + } catch (error) { + console.error('Initialization failed:', error); + this.isInitializing = false; + } + }; + + @action + runAssistant = async (inputText: string) => { + // Ensure an assistant and thread are created + if (!this.assistantID || !this.threadID || !this.vectorStoreID) { + await this.createAssistant(); + console.log('Assistant and thread created:', this.assistantID, this.threadID); + } + let currentText: string = ''; + let currentToolCallMessage: string = ''; + + // Send the user's input to the assistant + await this.openai.beta.threads.messages.create(this.threadID, { + role: 'user', + content: inputText, + }); + + // Listen to the streaming responses + const stream = this.openai.beta.threads.runs + .stream(this.threadID, { + assistant_id: this.assistantID, + }) + .on('runStepCreated', (runStep: RunStep) => { + currentText = ''; + runInAction(() => { + this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, text: currentText, tool_logs: '', links: [] }; + }); + this.isLoading = true; + }) + .on('toolCallDelta', (toolCallDelta, snapshot) => { + this.isLoading = false; + if (toolCallDelta.type === 'code_interpreter') { + if (toolCallDelta.code_interpreter?.input) { + currentToolCallMessage += toolCallDelta.code_interpreter.input; + runInAction(() => { + if (this.current_message) { + this.current_message.tool_logs = currentToolCallMessage; + } + }); + } + if (toolCallDelta.code_interpreter?.outputs) { + currentToolCallMessage += '\n Code interpreter output:'; + toolCallDelta.code_interpreter.outputs.forEach(output => { + if (output.type === 'logs') { + runInAction(() => { + if (this.current_message) { + this.current_message.tool_logs += '\n|' + output.logs; + } + }); + } + }); + } + } + }) + .on('textDelta', (textDelta, snapshot) => { + this.isLoading = false; + currentText += textDelta.value; + runInAction(() => { + if (this.current_message) { + // this.current_message = {...this.current_message, text: current_text}; + this.current_message.text = currentText; + } + }); + }) + .on('messageDone', async event => { + console.log(event); + const textItem = event.content.find(item => item.type === 'text'); + if (textItem && textItem.type === 'text') { + const { text } = textItem; + console.log(text.value); + try { + runInAction(() => { + this.interim_history = text.value; + }); + } catch (e) { + console.error('Error parsing JSON response:', e); + } + + const { annotations } = text; + console.log('Annotations: ' + annotations); + let index = 0; + annotations.forEach(async annotation => { + console.log(' ' + annotation); + console.log(' ' + annotation.text); + if (annotation.type === 'file_path') { + const { file_path: filePath } = annotation; + const fileToDownload = filePath.file_id; + console.log(fileToDownload); + if (filePath) { + console.log(filePath); + console.log(fileToDownload); + this.createLink(fileToDownload, annotation.start_index, annotation.end_index, ANNOTATION_LINK_TYPE.DOWNLOAD_FILE); + } + } else { + const { file_citation: fileCitation } = annotation; + if (fileCitation) { + const citedFile = await this.openai.files.retrieve(fileCitation.file_id); + const citationUrl = citedFile.filename; + this.createLink(citationUrl, annotation.start_index, annotation.end_index, ANNOTATION_LINK_TYPE.DASH_DOC, index); + index++; + } + } + }); + runInAction(() => { + if (this.current_message) { + console.log('current message: ' + this.current_message.text); + this.current_message.text = this.interim_history; + this.history.push({ ...this.current_message }); + this.current_message = undefined; + } + }); + } + }) + .on('toolCallDone', toolCall => { + runInAction(() => { + if (this.current_message && currentToolCallMessage) { + this.current_message.tool_logs = currentToolCallMessage; + } + }); + }) + .on('imageFileDone', (content: ImageFile, snapshot: Message) => { + console.log('Image file done:', content); + }) + .on('end', () => { + console.log('Streaming done'); + }); + }; + + @action + goToLinkedDoc = async (link: string) => { + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + + const linkedDoc = linkedDocs.find(doc => { + const docUrl = CsvCast(doc.data, PDFCast(doc.data)).url.pathname.replace('/files/pdfs/', '').replace('/files/csvs/', ''); + console.log('URL: ' + docUrl + ' Citation URL: ' + link); + return link === docUrl; + }); + + if (linkedDoc) { + await DocumentManager.Instance.showDocument(DocCast(linkedDoc), { willZoomCentered: true }, () => {}); + } + }; + + @action + askGPT = async (event: React.FormEvent<HTMLFormElement>): Promise<void> => { + event.preventDefault(); + + const textInput = event.currentTarget.elements.namedItem('messageInput') as HTMLInputElement; + const trimmedText = textInput.value.trim(); + + if (!this.assistantID || !this.threadID) { + try { + await this.createAssistant(); + } catch (err) { + console.error('Error:', err); + } + } + + if (trimmedText) { + try { + textInput.value = ''; + runInAction(() => { + this.history.push({ role: ASSISTANT_ROLE.USER, text: trimmedText }); + }); + await this.runAssistant(trimmedText); + this.dataDoc.data = this.history.toString(); + } catch (err) { + console.error('Error:', err); + } + } + }; + + @action + uploadLinks = async (linkedDocs: Doc[]) => { + if (this.isInitializing) { + console.log('Initialization in progress, upload aborted.'); + return; + } + const urls = linkedDocs.map(doc => CsvCast(doc.data, PDFCast(doc.data)).url.pathname); + const csvUrls = urls.filter(url => url.endsWith('.csv')); + console.log(this.assistantID, this.threadID, urls); + + const { openai_file_ids: openaiFileIds } = await Networking.PostToServer('/uploadPDFToVectorStore', { urls, threadID: this.threadID, assistantID: this.assistantID, vector_store_id: this.vectorStoreID }); + + linkedDocs.forEach((doc, i) => { + doc[this.Document[Id] + '_ai_field_id'] = openaiFileIds[i]; + console.log('AI Field ID: ' + openaiFileIds[i]); + }); + + if (csvUrls.length > 0) { + for (let i = 0; i < csvUrls.length; i++) { + this.linkedCsvIDs.push(openaiFileIds[urls.indexOf(csvUrls[i])]); + } + console.log('linked csvs:' + this.linkedCsvIDs); + await this.openai.beta.assistants.update(this.assistantID, { + tools: [{ type: 'file_search' }, { type: 'code_interpreter' }], + tool_resources: { + file_search: { + vector_store_ids: [this.vectorStoreID], + }, + code_interpreter: { + file_ids: this.linkedCsvIDs, + }, + }, + }); + } + }; + + downloadToComputer = (url: string, fileName: string) => { + fetch(url, { method: 'get', mode: 'no-cors', referrerPolicy: 'no-referrer' }) + .then(res => res.blob()) + .then(res => { + const aElement = document.createElement('a'); + aElement.setAttribute('download', fileName); + const href = URL.createObjectURL(res); + aElement.href = href; + aElement.setAttribute('target', '_blank'); + aElement.click(); + URL.revokeObjectURL(href); + }); + }; + + createDocumentInDash = async (url: string) => { + const fileSuffix = url.substring(url.lastIndexOf('.') + 1); + console.log(fileSuffix); + let doc: Doc | null = null; + switch (fileSuffix) { + case 'pdf': + doc = DocCast(await DocUtils.DocumentFromType('pdf', url, {})); + break; + case 'csv': + doc = DocCast(await DocUtils.DocumentFromType('csv', url, {})); + break; + case 'png': + case 'jpg': + case 'jpeg': + doc = DocCast(await DocUtils.DocumentFromType('image', url, {})); + break; + default: + console.error('Unsupported file type:', fileSuffix); + break; + } + if (doc) { + doc && this._props.addDocument?.(doc); + await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + } + }; + + downloadFile = async (fileInfo: string, downloadType: DOWNLOAD_TYPE) => { + try { + console.log(fileInfo); + const [fileId, fileName] = fileInfo.split(/!!!/); + const { file_path: filePath } = await Networking.PostToServer('/downloadFileFromOpenAI', { file_id: fileId, file_name: fileName }); + const fileLink = CsvCast(new CsvField(filePath)).url.href; + if (downloadType === DOWNLOAD_TYPE.DASH) { + this.createDocumentInDash(fileLink); + } else { + this.downloadToComputer(fileLink, fileName); + } + } catch (error) { + console.error('Error downloading file:', error); + } + }; + + handleDownloadToDevice = () => { + this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DEVICE); + this.modalStatus = false; // Close the modal after the action + this.currentFile = { url: '' }; // Reset the current file + }; + + handleAddToDash = () => { + // Assuming `downloadFile` is a method that handles adding to Dash + this.downloadFile(this.currentFile.url, DOWNLOAD_TYPE.DASH); + this.modalStatus = false; // Close the modal after the action + this.currentFile = { url: '' }; // Reset the current file + }; + + renderModal = () => { + if (!this.modalStatus) return null; + + return ( + <div className="modal"> + <div className="modal-content"> + <h4>File Actions</h4> + <p>Choose an action for the file:</p> + <button type="button" onClick={this.handleDownloadToDevice}> + Download to Device + </button> + <button type="button" onClick={this.handleAddToDash}> + Add to Dash + </button> + <button + type="button" + onClick={() => { + this.modalStatus = false; + }}> + Cancel + </button> + </div> + </div> + ); + }; + @action + showModal = () => { + this.modalStatus = true; + }; + + @action + setCurrentFile = (file: { url: string }) => { + this.currentFile = file; + }; + + componentDidMount() { + this._props.setContentViewBox?.(this); + if (this.dataDoc.data) { + try { + const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); + runInAction(() => { + this.history = storedHistory.map((msg: AssistantMessage) => ({ + role: msg.role, + text: msg.text, + quote: msg.quote, + tool_logs: msg.tool_logs, + image: msg.image, + })); + }); + } catch (e) { + console.error('Failed to parse history from dataDoc:', e); + } + } + reaction( + () => { + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + return linkedDocs; + }, + + linked => this.linked_docs_to_add.push(...linked.filter(linkedDoc => !this.linked_docs_to_add.includes(linkedDoc))) + ); + + observe( + // right now this skips during initialization which is necessary because it would be blank + // However, it will upload the same link twice when it is + this.linked_docs_to_add, + change => { + // observe pushes/splices on a user link DB 'data' field (should only happen for local changes) + switch (change.type as any) { + case 'splice': + if ((change as any).addedCount > 0) { + // maybe check here if its already in the urls datadoc array so doesn't add twice + console.log((change as any).added as Doc[]); + this.uploadLinks((change as any).added as Doc[]); + } + // (change as any).removed.forEach((link: any) => remLinkFromDoc(toRealField(link))); + break; + case 'update': // let oldValue = change.oldValue; + default: + } + }, + true + ); + } + + render() { + return ( + <MathJaxContext config={this.mathJaxConfig}> + <div className="chatBox"> + {this.isInitializing && <div className="initializing-overlay">Initializing...</div>} + {this.renderModal()} + <div + className="scroll-box chat-content" + ref={r => { + this._oldWheel?.removeEventListener('wheel', this.onPassiveWheel); + this._oldWheel = r; + r?.addEventListener('wheel', this.onPassiveWheel, { passive: false }); + }}> + <div className="messages"> + {this.history.map((message, index) => ( + <MessageComponent + key={index} + message={message} + toggleToolLogs={this.toggleToolLogs} + expandedLogIndex={this.expandedLogIndex} + index={index} + showModal={this.showModal} + goToLinkedDoc={this.goToLinkedDoc} + setCurrentFile={this.setCurrentFile} + /> + ))} + {!this.current_message ? null : ( + <MessageComponent + key={this.history.length} + message={this.current_message} + toggleToolLogs={this.toggleToolLogs} + expandedLogIndex={this.expandedLogIndex} + index={this.history.length} + showModal={this.showModal} + goToLinkedDoc={this.goToLinkedDoc} + setCurrentFile={this.setCurrentFile} + isCurrent + /> + )} + </div> + </div> + <form onSubmit={this.askGPT} className="chat-form"> + <input type="text" name="messageInput" autoComplete="off" placeholder="Type a message..." /> + <button type="submit">Send</button> + </form> + </div> + </MathJaxContext> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.CHAT, { + layout: { view: ChatBox, dataField: 'data' }, + options: { acl: '', chat: '', chat_history: '', chat_thread_id: '', chat_assistant_id: '', chat_vector_store_id: '' }, +}); diff --git a/src/client/views/nodes/ChatBox/MessageComponent.tsx b/src/client/views/nodes/ChatBox/MessageComponent.tsx new file mode 100644 index 000000000..fced0b4d5 --- /dev/null +++ b/src/client/views/nodes/ChatBox/MessageComponent.tsx @@ -0,0 +1,116 @@ +/* eslint-disable react/require-default-props */ +import React from 'react'; +import { observer } from 'mobx-react'; +import { MathJax, MathJaxContext } from 'better-react-mathjax'; +import ReactMarkdown from 'react-markdown'; +import { TbCircle0Filled, TbCircle1Filled, TbCircle2Filled, TbCircle3Filled, TbCircle4Filled, TbCircle5Filled, TbCircle6Filled, TbCircle7Filled, TbCircle8Filled, TbCircle9Filled } from 'react-icons/tb'; +import { AssistantMessage } from './types'; + +interface MessageComponentProps { + message: AssistantMessage; + toggleToolLogs: (index: number) => void; + expandedLogIndex: number | null; + index: number; + showModal: () => void; + goToLinkedDoc: (url: string) => void; + setCurrentFile: (file: { url: string }) => void; + isCurrent?: boolean; +} + +const MessageComponent: React.FC<MessageComponentProps> = function ({ message, toggleToolLogs, expandedLogIndex, goToLinkedDoc, index, showModal, setCurrentFile, isCurrent = false }) { + // const messageClass = `${message.role} ${isCurrent ? 'current-message' : ''}`; + + const LinkRenderer = ({ href, children }: { href: string; children: React.ReactNode }) => { + // console.log(href + " " + children) + const regex = /([a-zA-Z0-9_.!-]+)~~~(citation|file_path)/; + const matches = href.match(regex); + // console.log(href) + // console.log(matches) + const url = matches ? matches[1] : href; + const linkType = matches ? matches[2] : null; + if (linkType === 'citation') { + switch (children) { + case '0': + children = <TbCircle0Filled />; + break; + case '1': + children = <TbCircle1Filled />; + break; + case '2': + children = <TbCircle2Filled />; + break; + case '3': + children = <TbCircle3Filled />; + break; + case '4': + children = <TbCircle4Filled />; + break; + case '5': + children = <TbCircle5Filled />; + break; + case '6': + children = <TbCircle6Filled />; + break; + case '7': + children = <TbCircle7Filled />; + break; + case '8': + children = <TbCircle8Filled />; + break; + case '9': + children = <TbCircle9Filled />; + break; + default: + break; + } + } + // console.log(linkType) + const style = { + color: 'lightblue', + verticalAlign: linkType === 'citation' ? 'super' : 'baseline', + fontSize: linkType === 'citation' ? 'smaller' : 'inherit', + }; + + return ( + <a + href="#" + onClick={e => { + e.preventDefault(); + if (linkType === 'citation') { + goToLinkedDoc(url); + } else if (linkType === 'file_path') { + showModal(); + setCurrentFile({ url }); + } + }} + style={style}> + {children} + </a> + ); + }; + + return ( + <div className={`message ${message.role}`}> + <MathJaxContext> + <MathJax dynamic hideUntilTypeset="every"> + <ReactMarkdown components={{ a: LinkRenderer }}>{message.text ? message.text : ''}</ReactMarkdown> + </MathJax> + </MathJaxContext> + {message.image && <img src={message.image} alt="" />} + <div className="message-footer"> + {message.tool_logs && ( + <button className="toggle-logs-button" onClick={() => toggleToolLogs(index)}> + {expandedLogIndex === index ? 'Hide Code Interpreter Logs' : 'Show Code Interpreter Logs'} + </button> + )} + {expandedLogIndex === index && ( + <div className="tool-logs"> + <pre>{message.tool_logs}</pre> + </div> + )} + </div> + </div> + ); +}; + +export default observer(MessageComponent); diff --git a/src/client/views/nodes/ChatBox/types.ts b/src/client/views/nodes/ChatBox/types.ts new file mode 100644 index 000000000..8212a7050 --- /dev/null +++ b/src/client/views/nodes/ChatBox/types.ts @@ -0,0 +1,23 @@ +export enum ASSISTANT_ROLE { + USER = 'User', + ASSISTANT = 'Assistant', +} + +export enum ANNOTATION_LINK_TYPE { + DASH_DOC = 'citation', + DOWNLOAD_FILE = 'file_path', +} + +export enum DOWNLOAD_TYPE { + DASH = 'dash', + DEVICE = 'device', +} + +export interface AssistantMessage { + role: ASSISTANT_ROLE; + text: string; + quote?: string; + image?: string; + tool_logs?: string; + links?: { start: number; end: number; url: string; id?: string; link_type: ANNOTATION_LINK_TYPE }[]; +} diff --git a/src/client/views/nodes/DiagramBox.scss b/src/client/views/nodes/DiagramBox.scss new file mode 100644 index 000000000..d2749f1ad --- /dev/null +++ b/src/client/views/nodes/DiagramBox.scss @@ -0,0 +1,88 @@ +.DIYNodeBox { + width: 100%; + height: 100%; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + + .DIYNodeBox-wrapper { + width: 100%; + height: 100%; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + .DIYNodeBox { + /* existing code */ + + .DIYNodeBox-iframe { + height: 100%; + width: 100%; + border: none; + + } + } + + .search-bar { + display: flex; + justify-content: center; + align-items: center; + width: 100%; + padding: 10px; + + input[type="text"] { + flex: 1; + margin-right: 10px; + } + + button { + padding: 5px 10px; + } + } + + .content { + flex: 1; + display: flex; + justify-content: center; + align-items: center; + width:100%; + height:100%; + .diagramBox{ + flex: 1; + display: flex; + justify-content: center; + align-items: center; + width:100%; + height:100%; + svg{ + flex: 1; + display: flex; + justify-content: center; + align-items: center; + width:100%; + height:100%; + } + } + } + + .loading-circle { + position: relative; + width: 50px; + height: 50px; + border-radius: 50%; + border: 3px solid #ccc; + border-top-color: #333; + animation: spin 1s infinite linear; + } + + @keyframes spin { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } + } + } +}
\ No newline at end of file diff --git a/src/client/views/nodes/DiagramBox.tsx b/src/client/views/nodes/DiagramBox.tsx new file mode 100644 index 000000000..fa7e5868a --- /dev/null +++ b/src/client/views/nodes/DiagramBox.tsx @@ -0,0 +1,305 @@ +import { makeObservable, observable, action, reaction } from 'mobx'; +import { observer } from 'mobx-react'; +import * as React from 'react'; +import { ViewBoxAnnotatableComponent, ViewBoxInterface } from '../DocComponent'; +import { StyleProp } from '../StyleProvider'; +import './DiagramBox.scss'; +import { FieldView, FieldViewProps } from './FieldView'; +import { PinProps, PresBox } from './trails'; +import mermaid from 'mermaid'; +import { Doc, DocListCast } from '../../../fields/Doc'; +import { List } from '../../../fields/List'; +import { RichTextField } from '../../../fields/RichTextField'; +import { ContextMenu } from '../ContextMenu'; +import { gptAPICall, GPTCallType } from '../../apis/gpt/GPT'; +import { ChatCompletionMessageParam } from 'openai/resources/chat/completions'; +import OpenAI, { ClientOptions } from 'openai'; +import { line } from 'd3'; +import { InkingStroke } from '../InkingStroke'; +import { DocumentManager } from '../../util/DocumentManager'; +import { C } from '@fullcalendar/core/internal-common'; +import { Docs } from '../../documents/Documents'; +import { NumCast } from '../../../fields/Types'; +import { LinkManager } from '../../util/LinkManager'; +import { CsvCast, DocCast, StrCast } from '../../../fields/Types'; +import { DocumentType } from '../../documents/DocumentTypes'; + +@observer +export class DiagramBox extends ViewBoxAnnotatableComponent<FieldViewProps>() implements ViewBoxInterface { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(DiagramBox, fieldKey); + } + private _ref: React.RefObject<HTMLDivElement> = React.createRef(); + private _dragRef = React.createRef<HTMLDivElement>(); + constructor(props: FieldViewProps) { + super(props); + makeObservable(this); + } + + @observable inputValue = ''; + @observable loading = false; + @observable errorMessage = ''; + @observable mermaidCode = ''; + + @action handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => { + this.inputValue = e.target.value; + }; + async componentDidMount() { + this._props.setContentViewBox?.(this); + mermaid.initialize({ + securityLevel: 'loose', + startOnLoad: true, + flowchart: { useMaxWidth: true, htmlLabels: true, curve: 'cardinal' }, + }); + this.mermaidCode = 'asdasdasd'; + let docArray: Doc[] = DocListCast(this.Document.data); + let mermaidCodeDoc = docArray.filter(doc => doc.type == 'rich text'); + mermaidCodeDoc = mermaidCodeDoc.filter(doc => (doc.text as RichTextField).Text == 'mermaidCodeTitle'); + if (mermaidCodeDoc[0]) { + if (typeof mermaidCodeDoc[0].title == 'string') { + console.log(mermaidCodeDoc[0].title); + if (mermaidCodeDoc[0].title != '') { + this.renderMermaidAsync(mermaidCodeDoc[0].title); + } + } + } + //this will create a text doc far away where the user cant to save the mermaid code, where it will then be accessed when flipped to the diagram box side + //the code is stored in the title since it is much easier to change than in the text + else { + DocumentManager.Instance.AddViewRenderedCb(this.Document, docViewForYourCollection => { + if (docViewForYourCollection && docViewForYourCollection.ComponentView) { + if (docViewForYourCollection.ComponentView.addDocument && docViewForYourCollection.ComponentView.removeDocument) { + let newDoc = Docs.Create.TextDocument('mermaidCodeTitle', { title: '', x: 9999 + NumCast(this.layoutDoc._width), y: 9999 }); + docViewForYourCollection.ComponentView?.addDocument(newDoc); + } + } + }); + } + console.log(this.Document.title); + //this is so that ever time a new doc, text node or ink node, is created, this.createMermaidCode will run which will create a save + reaction( + () => DocListCast(this.Document.data), + docs => { + console.log('reaction happened'); + this.convertDrawingToMermaidCode(); + }, + { fireImmediately: true } + ); + } + renderMermaid = async (str: string) => { + try { + const { svg, bindFunctions } = await this.mermaidDiagram(str); + return { svg, bindFunctions }; + } catch (error) { + console.error('Error rendering mermaid diagram:', error); + return { svg: '', bindFunctions: undefined }; + } + }; + mermaidDiagram = async (str: string) => { + return await mermaid.render('graph' + Date.now(), str); + }; + + async renderMermaidAsync(mermaidCode: string) { + try { + const { svg, bindFunctions } = await this.renderMermaid(mermaidCode); + const dashDiv = document.getElementById('dashDiv' + this.Document.title); + if (dashDiv) { + dashDiv.innerHTML = svg; + if (bindFunctions) { + bindFunctions(dashDiv); + } + } + } catch (error) { + console.error('Error rendering Mermaid:', error); + } + } + @action handleRenderClick = () => { + this.generateMermaidCode(); + }; + @action async generateMermaidCode() { + console.log('Generating Mermaid Code'); + this.loading = true; + let prompt = ''; + // let docArray: Doc[] = DocListCast(this.Document.data); + // let mermaidCodeDoc = docArray.filter(doc => doc.type == 'rich text') + // mermaidCodeDoc=mermaidCodeDoc.filter(doc=>(doc.text as RichTextField).Text=='mermaidCodeTitle') + // if(mermaidCodeDoc[0]){ + // console.log(mermaidCodeDoc[0].title) + // if(typeof mermaidCodeDoc[0].title=='string'){ + // console.log(mermaidCodeDoc[0].title) + // if(mermaidCodeDoc[0].title!=""){ + // prompt="Edit this code "+this.inputValue+": "+mermaidCodeDoc[0].title + // console.log("you have to see me") + // } + // } + // } + // else{ + prompt = 'Write this in mermaid code and only give me the mermaid code: ' + this.inputValue; + console.log('there is no text save'); + //} + let res = await gptAPICall(prompt, GPTCallType.MERMAID); + this.loading = false; + if (res == 'Error connecting with API.') { + // If GPT call failed + console.error('GPT call failed'); + this.errorMessage = 'GPT call failed; please try again.'; + } else if (res != null) { + // If GPT call succeeded, set htmlCode;;; TODO: check if valid html + if (this.isValidCode(res)) { + this.mermaidCode = res; + console.log('GPT call succeeded:' + res); + this.errorMessage = ''; + } else { + console.error('GPT call succeeded but invalid html; please try again.'); + this.errorMessage = 'GPT call succeeded but invalid html; please try again.'; + } + } + this.renderMermaidAsync.call(this, this.removeWords(this.mermaidCode)); + this.loading = false; + } + isValidCode = (html: string) => { + return true; + }; + removeWords(inputStr: string) { + inputStr = inputStr.replace('```mermaid', ''); + return inputStr.replace('```', ''); + } + //method to convert the drawings on collection node side the mermaid code + async convertDrawingToMermaidCode() { + let mermaidCode = ''; + let diagramExists = false; + if (this.Document.data instanceof List) { + let docArray: Doc[] = DocListCast(this.Document.data); + let rectangleArray = docArray.filter(doc => doc.title == 'rectangle' || doc.title == 'circle'); + let lineArray = docArray.filter(doc => doc.title == 'line' || doc.title == 'stroke'); + let textArray = docArray.filter(doc => doc.type == 'rich text'); + const timeoutPromise = () => + new Promise(resolve => { + setTimeout(resolve, 0); + }); + await timeoutPromise(); + let inkStrokeArray = lineArray.map(doc => DocumentManager.Instance.getDocumentView(doc, this.DocumentView?.())).filter(inkView => inkView?.ComponentView instanceof InkingStroke); + console.log(inkStrokeArray.length); + console.log(lineArray.length); + if (inkStrokeArray[0] && inkStrokeArray.length == lineArray.length) { + mermaidCode = 'graph TD;'; + let inkingStrokeArray = inkStrokeArray.map(stroke => stroke?.ComponentView); + for (let i = 0; i < rectangleArray.length; i++) { + const rectangle = rectangleArray[i]; + for (let j = 0; j < lineArray.length; j++) { + let inkScaleX = (inkingStrokeArray[j] as InkingStroke)?.inkScaledData().inkScaleX; + let inkScaleY = (inkingStrokeArray[j] as InkingStroke)?.inkScaledData().inkScaleY; + let inkStrokeXArray = (inkingStrokeArray[j] as InkingStroke) + ?.inkScaledData() + .inkData.map(coord => coord.X) + .map(doc => doc * inkScaleX); + let inkStrokeYArray = (inkingStrokeArray[j] as InkingStroke) + ?.inkScaledData() + .inkData.map(coord => coord.Y) + .map(doc => doc * inkScaleY); + console.log(inkingStrokeArray.length); + console.log(lineArray.length); + //need to minX and minY to since the inkStroke.x and.y is not relative to the doc. so I have to do some calcluations + let minX: number = Math.min(...inkStrokeXArray); + let minY: number = Math.min(...inkStrokeYArray); + let startX = inkStrokeXArray[0] - minX + (lineArray[j]?.x as number); + let startY = inkStrokeYArray[0] - minY + (lineArray[j]?.y as number); + let endX = inkStrokeXArray[inkStrokeXArray.length - 1] - minX + (lineArray[j].x as number); + let endY = inkStrokeYArray[inkStrokeYArray.length - 1] - minY + (lineArray[j].y as number); + if (this.isPointInBox(rectangle, [startX, startY])) { + for (let k = 0; k < rectangleArray.length; k++) { + const rectangle2 = rectangleArray[k]; + if (this.isPointInBox(rectangle2, [endX, endY]) && typeof rectangle.x === 'number' && typeof rectangle2.x === 'number') { + diagramExists = true; + const linkedDocs: Doc[] = LinkManager.Instance.getAllRelatedLinks(lineArray[j]).map(d => DocCast(LinkManager.getOppositeAnchor(d, lineArray[j]))); + console.log(linkedDocs.length); + if (linkedDocs.length != 0) { + let linkedText = (linkedDocs[0].text as RichTextField).Text; + mermaidCode += Math.abs(rectangle.x) + this.getTextInBox(rectangle, textArray) + '-->|' + linkedText + '|' + Math.abs(rectangle2.x) + this.getTextInBox(rectangle2, textArray) + ';'; + } else { + mermaidCode += Math.abs(rectangle.x) + this.getTextInBox(rectangle, textArray) + '-->' + Math.abs(rectangle2.x) + this.getTextInBox(rectangle2, textArray) + ';'; + } + } + } + } + } + } + //this will save the text + DocumentManager.Instance.AddViewRenderedCb(this.Document, docViewForYourCollection => { + if (docViewForYourCollection && docViewForYourCollection.ComponentView) { + if (docViewForYourCollection.ComponentView.addDocument && docViewForYourCollection.ComponentView.removeDocument) { + let docArray: Doc[] = DocListCast(this.Document.data); + docArray = docArray.filter(doc => doc.type == 'rich text'); + let mermaidCodeDoc = docArray.filter(doc => (doc.text as RichTextField).Text == 'mermaidCodeTitle'); + if (mermaidCodeDoc[0]) { + if (diagramExists) { + mermaidCodeDoc[0].title = mermaidCode; + } else { + mermaidCodeDoc[0].title = ''; + } + } + } + } + }); + } + } + } + testInkingStroke = () => { + if (this.Document.data instanceof List) { + let docArray: Doc[] = DocListCast(this.Document.data); + let lineArray = docArray.filter(doc => doc.title == 'line' || doc.title == 'stroke'); + setTimeout(() => { + let inkStrokeArray = lineArray.map(doc => DocumentManager.Instance.getDocumentView(doc, this.DocumentView?.())).filter(inkView => inkView?.ComponentView instanceof InkingStroke); + console.log(inkStrokeArray); + }); + } + }; + getTextInBox = (box: Doc, richTextArray: Doc[]): string => { + for (let i = 0; i < richTextArray.length; i++) { + let textDoc = richTextArray[i]; + if (typeof textDoc.x === 'number' && typeof textDoc.y === 'number' && typeof box.x === 'number' && typeof box.height === 'number' && typeof box.width === 'number' && typeof box.y === 'number') { + if (textDoc.x > box.x && textDoc.x < box.x + box.width && textDoc.y > box.y && textDoc.y < box.y + box.height) { + if (box.title == 'rectangle') { + return '(' + (textDoc.text as RichTextField)?.Text + ')'; + } + if (box.title == 'circle') { + return '((' + (textDoc.text as RichTextField)?.Text + '))'; + } + } + } + } + return '( )'; + }; + isPointInBox = (box: Doc, line: number[]): boolean => { + if (typeof line[0] === 'number' && typeof box.x === 'number' && typeof box.width === 'number' && typeof box.height === 'number' && typeof box.y === 'number' && typeof line[1] === 'number') { + return line[0] < box.x + box.width && line[0] > box.x && line[1] > box.y && line[1] < box.y + box.height; + } else { + return false; + } + }; + + render() { + return ( + <div ref={this._ref} className="DIYNodeBox"> + <div ref={this._dragRef} className="DIYNodeBox-wrapper"> + <div className="search-bar"> + <input type="text" value={this.inputValue} onChange={this.handleInputChange} /> + <button onClick={this.handleRenderClick}>Generate</button> + </div> + <div className="content"> + {this.mermaidCode ? ( + <div id={'dashDiv' + this.Document.title} className="diagramBox"></div> + ) : ( + <div>{this.loading ? <div className="loading-circle"></div> : <div>{this.errorMessage ? this.errorMessage : 'Insert prompt to generate diagram'}</div>}</div> + )} + </div> + </div> + </div> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.DIAGRAM, { + layout: { view: DiagramBox, dataField: 'dadta' }, + options: { _height: 300, _layout_fitWidth: true, _layout_nativeDimEditable: true, _layout_reflowVertical: true, waitForDoubleClickToClick: 'always', systemIcon: 'BsGlobe' }, +}); diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index 18529a429..192c7875e 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -79,7 +79,7 @@ export class DocumentContentsView extends ObservableReactComponent<DocumentConte * Set of all available rendering componets for Docs (e.g., ImageBox, CollectionFreeFormView, etc) */ private static Components: { [key: string]: any }; - public static Init(defaultLayoutString: string, components:{ [key: string]: any}) { + public static Init(defaultLayoutString: string, components: { [key: string]: any }) { DocumentContentsView.DefaultLayoutString = defaultLayoutString; DocumentContentsView.Components = components; } diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.scss b/src/client/views/pdf/GPTPopup/GPTPopup.scss index 48659d0e7..6d8793f82 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.scss +++ b/src/client/views/pdf/GPTPopup/GPTPopup.scss @@ -55,16 +55,29 @@ $highlightedText: #82e0ff; overflow-y: auto; } - .btns-wrapper { + .btns-wrapper-gpt { height: 50px; display: flex; - justify-content: space-between; + justify-content: center; align-items: center; + transform: translateY(30px); + + + .searchBox-input{ + transform: translateY(-15px); + height: 50px; + border-radius: 10px; + border-color: #5b97ff; + } + + .summarizing { display: flex; align-items: center; } + + } button { @@ -111,6 +124,28 @@ $highlightedText: #82e0ff; } } +.loading-spinner { + display: flex; + justify-content: center; + align-items: center; + height: 100px; + font-size: 20px; + font-weight: bold; + color: #666; +} + + + + + +@keyframes spin { + to { + transform: rotate(360deg); + } +} + + + .image-content-wrapper { display: flex; flex-direction: column; diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.tsx b/src/client/views/pdf/GPTPopup/GPTPopup.tsx index 3f6c154bb..8bb2e2844 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.tsx +++ b/src/client/views/pdf/GPTPopup/GPTPopup.tsx @@ -25,6 +25,7 @@ export enum GPTPopupMode { EDIT, IMAGE, DATA, + SORT, } interface GPTPopupProps {} @@ -102,6 +103,14 @@ export class GPTPopup extends ObservableReactComponent<GPTPopupProps> { this.chatMode = false; }; + @observable + private sortDone: boolean = false; // this is so redundant but the og done variable was causing weird unknown problems and im just a girl + + @action + public setSortDone = (done: boolean) => { + this.sortDone = done; + }; + // change what can be a ref into a ref @observable private sidebarId: string = ''; @@ -124,11 +133,48 @@ export class GPTPopup extends ObservableReactComponent<GPTPopupProps> { this.textAnchor = anchor; }; + @observable + public sortDesc: string = ''; + + @action public setSortDesc = (t: string) => { + this.sortDesc = t; + }; + + @observable onSortComplete?: (sortResult: string) => void; + @observable cardsDoneLoading = false; + + @action setCardsDoneLoading(done: boolean) { + console.log(done + 'HI HIHI'); + this.cardsDoneLoading = done; + } + public addDoc: (doc: Doc | Doc[], sidebarKey?: string | undefined) => boolean = () => false; public createFilteredDoc: (axes?: any) => boolean = () => false; public addToCollection: ((doc: Doc | Doc[], annotationKey?: string | undefined) => boolean) | undefined; /** + * Sorts cards in the CollectionCardDeckView + */ + generateSort = async () => { + this.setLoading(true); + this.setSortDone(false); + + try { + const res = await gptAPICall(this.sortDesc, GPTCallType.SORT); + // Trigger the callback with the result + if (this.onSortComplete) { + this.onSortComplete(res || 'Something went wrong :('); + console.log(res); + } + } catch (err) { + console.error(err); + } + + this.setLoading(false); + this.setSortDone(true); + }; + + /** * Generates a Dalle image and uploads it to the server. */ generateImage = async () => { @@ -267,6 +313,59 @@ export class GPTPopup extends ObservableReactComponent<GPTPopupProps> { } }; + sortBox = () => ( + <> + <div> + {this.heading('SORTING')} + {this.loading ? ( + <div className="content-wrapper"> + <div className="loading-spinner"> + <ReactLoading type="spin" color={StrCast(Doc.UserDoc().userVariantColor)} height={30} width={30} /> + <span>Loading...</span> + </div> + </div> + ) : ( + <> + {!this.cardsDoneLoading ? ( + <div className="content-wrapper"> + <div className="loading-spinner"> + <ReactLoading type="spin" color={StrCast(Doc.UserDoc().userVariantColor)} height={30} width={30} /> + <span>Reading Cards...</span> + </div> + </div> + ) : ( + !this.sortDone && ( + <div className="btns-wrapper-gpt"> + <Button + tooltip="Have ChatGPT sort your cards for you!" + text="Sort!" + onClick={this.generateSort} + color={StrCast(Doc.UserDoc().userVariantColor)} + type={Type.TERT} + style={{ + width: '90%', // Almost as wide as the container + textAlign: 'center', + color: '#ffffff', // White text + fontSize: '16px', // Adjust font size as needed + }} + /> + </div> + ) + )} + + {this.sortDone && ( + <div> + <div className="content-wrapper"> + <p>{this.text === 'Something went wrong :(' ? 'Something went wrong :(' : 'Sorting done! Feel free to move things around / regenerate :) !'}</p> + <IconButton tooltip="Generate Again" onClick={() => this.setSortDone(false)} icon={<FontAwesomeIcon icon="redo-alt" size="lg" />} color={StrCast(Doc.UserDoc().userVariantColor)} /> + </div> + </div> + )} + </> + )} + </div> + </> + ); imageBox = () => ( <div style={{ display: 'flex', flexDirection: 'column', gap: '1rem' }}> {this.heading('GENERATED IMAGE')} @@ -419,7 +518,7 @@ export class GPTPopup extends ObservableReactComponent<GPTPopupProps> { render() { return ( <div className="summary-box" style={{ display: this.visible ? 'flex' : 'none' }}> - {this.mode === GPTPopupMode.SUMMARY ? this.summaryBox() : this.mode === GPTPopupMode.DATA ? this.dataAnalysisBox() : this.mode === GPTPopupMode.IMAGE ? this.imageBox() : null} + {this.mode === GPTPopupMode.SUMMARY ? this.summaryBox() : this.mode === GPTPopupMode.DATA ? this.dataAnalysisBox() : this.mode === GPTPopupMode.IMAGE ? this.imageBox() : this.mode === GPTPopupMode.SORT ? this.sortBox() : null} </div> ); } diff --git a/src/fields/Types.ts b/src/fields/Types.ts index 26196d15d..ef79f72e4 100644 --- a/src/fields/Types.ts +++ b/src/fields/Types.ts @@ -5,7 +5,7 @@ import { ProxyField } from './Proxy'; import { RefField } from './RefField'; import { RichTextField } from './RichTextField'; import { ScriptField } from './ScriptField'; -import { CsvField, ImageField, WebField } from './URLField'; +import { CsvField, ImageField, PdfField, WebField } from './URLField'; // eslint-disable-next-line no-use-before-define export type ToConstructor<T extends FieldType> = T extends string ? 'string' : T extends number ? 'number' : T extends boolean ? 'boolean' : T extends List<infer U> ? ListSpec<U> : new (...args: any[]) => T; @@ -122,6 +122,9 @@ export function CsvCast(field: FieldResult, defaultVal: CsvField | null = null) export function WebCast(field: FieldResult, defaultVal: WebField | null = null) { return Cast(field, WebField, defaultVal); } +export function PDFCast(field: FieldResult, defaultVal: PdfField | null = null) { + return Cast(field, PdfField, defaultVal); +} export function ImageCast(field: FieldResult, defaultVal: ImageField | null = null) { return Cast(field, ImageField, defaultVal); } diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts new file mode 100644 index 000000000..82e48167a --- /dev/null +++ b/src/server/ApiManagers/AssistantManager.ts @@ -0,0 +1,131 @@ +import * as fs from 'fs'; +import { createReadStream, writeFile } from 'fs'; +import OpenAI from 'openai'; +import * as path from 'path'; +import { promisify } from 'util'; +import * as uuid from 'uuid'; +import { filesDirectory, publicDirectory } from '..'; +import { Method } from '../RouteManager'; +import ApiManager, { Registration } from './ApiManager'; + +export enum Directory { + parsed_files = 'parsed_files', + images = 'images', + videos = 'videos', + pdfs = 'pdfs', + text = 'text', + pdf_thumbnails = 'pdf_thumbnails', + audio = 'audio', + csv = 'csv', +} + +export function serverPathToFile(directory: Directory, filename: string) { + return path.normalize(`${filesDirectory}/${directory}/${filename}`); +} + +export function pathToDirectory(directory: Directory) { + return path.normalize(`${filesDirectory}/${directory}`); +} + +export function clientPathToFile(directory: Directory, filename: string) { + return `/files/${directory}/${filename}`; +} + +const writeFileAsync = promisify(writeFile); +const readFileAsync = promisify(fs.readFile); + +export default class AssistantManager extends ApiManager { + protected initialize(register: Registration): void { + const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); + + register({ + method: Method.POST, + subscription: '/uploadPDFToVectorStore', + secureHandler: async ({ req, res }) => { + const { urls, threadID, assistantID, vector_store_id } = req.body; + + const csvFilesIds: string[] = []; + const otherFileIds: string[] = []; + const allFileIds: string[] = []; + + const fileProcesses = urls.map(async (source: string) => { + const fullPath = path.join(publicDirectory, source); + const fileData = await openai.files.create({ file: createReadStream(fullPath), purpose: 'assistants' }); + allFileIds.push(fileData.id); + if (source.endsWith('.csv')) { + console.log(source); + csvFilesIds.push(fileData.id); + } else { + openai.beta.vectorStores.files.create(vector_store_id, { file_id: fileData.id }); + otherFileIds.push(fileData.id); + } + }); + try { + await Promise.all(fileProcesses).then(() => { + res.send({ vector_store_id: vector_store_id, openai_file_ids: allFileIds }); + }); + } catch (error) { + res.status(500).send({ error: 'Failed to process files' + error }); + } + }, + }); + + register({ + method: Method.POST, + subscription: '/downloadFileFromOpenAI', + secureHandler: async ({ req, res }) => { + const { file_id, file_name } = req.body; + //let files_directory: string; + let files_directory = '/files/openAIFiles/'; + switch (file_name.split('.').pop()) { + case 'pdf': + files_directory = '/files/pdfs/'; + break; + case 'csv': + files_directory = '/files/csv/'; + break; + case 'png': + case 'jpg': + case 'jpeg': + files_directory = '/files/images/'; + break; + default: + break; + } + + const directory = path.join(publicDirectory, files_directory); + + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory); + } + const file = await openai.files.content(file_id); + const new_file_name = `${uuid.v4()}-${file_name}`; + const file_path = path.join(directory, new_file_name); + const file_array_buffer = await file.arrayBuffer(); + const bufferView = new Uint8Array(file_array_buffer); + try { + const written_file = await writeFileAsync(file_path, bufferView); + console.log(written_file); + console.log(file_path); + console.log(file_array_buffer); + console.log(bufferView); + const file_object = new File([bufferView], file_name); + //DashUploadUtils.upload(file_object, 'openAIFiles'); + res.send({ file_path: path.join(files_directory, new_file_name) }); + /* res.send( { + source: "file", + result: { + accessPaths: { + agnostic: {client: path.join('/files/openAIFiles/', `${uuid.v4()}-${file_name}`)} + }, + rawText: "", + duration: 0, + }, + } ); */ + } catch (error) { + res.status(500).send({ error: 'Failed to write file' + error }); + } + }, + }); + } +} diff --git a/src/server/index.ts b/src/server/index.ts index 1bbf8a105..3151c2975 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -3,7 +3,7 @@ import * as dotenv from 'dotenv'; import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; import { logExecution } from './ActionUtilities'; -import { AdminPrivileges, resolvedPorts } from './SocketData'; +import AssistantManager from './ApiManagers/AssistantManager'; import DataVizManager from './ApiManagers/DataVizManager'; import DeleteManager from './ApiManagers/DeleteManager'; import DownloadManager from './ApiManagers/DownloadManager'; @@ -13,16 +13,17 @@ import SessionManager from './ApiManagers/SessionManager'; import UploadManager from './ApiManagers/UploadManager'; import UserManager from './ApiManagers/UserManager'; import UtilManager from './ApiManagers/UtilManager'; -import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader'; -import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; import { DashSessionAgent } from './DashSession/DashSessionAgent'; import { AppliedSessionAgent } from './DashSession/Session/agents/applied_session_agent'; import { DashStats } from './DashStats'; import { DashUploadUtils } from './DashUploadUtils'; -import { Database } from './database'; import { Logger } from './ProcessFactory'; import RouteManager, { Method, PublicHandler } from './RouteManager'; import RouteSubscriber from './RouteSubscriber'; +import { AdminPrivileges, resolvedPorts } from './SocketData'; +import { GoogleCredentialsLoader, SSL } from './apis/google/CredentialsLoader'; +import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; +import { Database } from './database'; import initializeServer from './server_Initialization'; // import GooglePhotosManager from './ApiManagers/GooglePhotosManager'; @@ -72,6 +73,7 @@ function routeSetter({ addSupervisedRoute, logRegistrationOutcome }: RouteManage new UtilManager(), new GeneralGoogleManager(), /* new GooglePhotosManager(), */ new DataVizManager(), + new AssistantManager(), ]; // initialize API Managers |