aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorbobzel <zzzman@gmail.com>2024-10-17 17:19:25 -0400
committerbobzel <zzzman@gmail.com>2024-10-17 17:19:25 -0400
commit8ac260db2fdffc37ff9b6e91971f287df6a70528 (patch)
treec4bad3d44cb4c374b84834a39f5fc664345784f7 /src
parent3067940f28563d1217056f6eb428d377365077a8 (diff)
parentdd93f5175064850c6c0e47f025cd7bbba1f23106 (diff)
Merge branch 'master' into alyssa-starter
Diffstat (limited to 'src')
-rw-r--r--src/client/documents/DocumentTypes.ts24
-rw-r--r--src/client/util/CurrentUserUtils.ts11
-rw-r--r--src/client/views/collections/CollectionCardDeckView.tsx14
-rw-r--r--src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx23
-rw-r--r--src/client/views/nodes/ComparisonBox.scss3
-rw-r--r--src/client/views/nodes/ComparisonBox.tsx5
-rw-r--r--src/client/views/nodes/FontIconBox/FontIconBox.tsx2
-rw-r--r--src/client/views/nodes/chatbot/agentsystem/Agent.ts82
-rw-r--r--src/client/views/nodes/chatbot/chatboxcomponents/MessageComponent.tsx3
-rw-r--r--src/client/views/nodes/chatbot/tools/BaseTool.ts82
-rw-r--r--src/client/views/nodes/chatbot/tools/CalculateTool.ts30
-rw-r--r--src/client/views/nodes/chatbot/tools/CreateCSVTool.ts45
-rw-r--r--src/client/views/nodes/chatbot/tools/CreateCollectionTool.ts36
-rw-r--r--src/client/views/nodes/chatbot/tools/DataAnalysisTool.ts37
-rw-r--r--src/client/views/nodes/chatbot/tools/GetDocsTool.ts38
-rw-r--r--src/client/views/nodes/chatbot/tools/NoTool.ts23
-rw-r--r--src/client/views/nodes/chatbot/tools/RAGTool.ts33
-rw-r--r--src/client/views/nodes/chatbot/tools/SearchTool.ts65
-rw-r--r--src/client/views/nodes/chatbot/tools/ToolTypes.ts76
-rw-r--r--src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts114
-rw-r--r--src/client/views/nodes/chatbot/tools/WikipediaTool.ts33
-rw-r--r--src/client/views/nodes/chatbot/types/types.ts16
-rw-r--r--src/server/ApiManagers/AssistantManager.ts116
-rw-r--r--src/server/server_Initialization.ts2
24 files changed, 581 insertions, 332 deletions
diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts
index e79207b04..efe73fbbe 100644
--- a/src/client/documents/DocumentTypes.ts
+++ b/src/client/documents/DocumentTypes.ts
@@ -48,22 +48,22 @@ export enum DocumentType {
export enum CollectionViewType {
Invalid = 'invalid',
Freeform = 'freeform',
- Schema = 'schema',
- Docking = 'docking',
- Tree = 'tree',
- Stacking = 'stacking',
- Masonry = 'masonry',
- Multicolumn = 'multicolumn',
- Multirow = 'multirow',
- Time = 'time',
+ Calendar = 'calendar',
+ Card = 'card',
Carousel = 'carousel',
Carousel3D = '3D Carousel',
+ Docking = 'docking',
+ Grid = 'grid',
Linear = 'linear',
Map = 'map',
- Grid = 'grid',
+ Masonry = 'masonry',
+ Multicolumn = 'multicolumn',
+ Multirow = 'multirow',
+ NoteTaking = 'notetaking',
Pile = 'pileup',
+ Schema = 'schema',
+ Stacking = 'stacking',
StackedTimeline = 'stacked timeline',
- NoteTaking = 'notetaking',
- Calendar = 'calendar',
- Card = 'card',
+ Time = 'time',
+ Tree = 'tree',
}
diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts
index 555ccdd88..30c75c659 100644
--- a/src/client/util/CurrentUserUtils.ts
+++ b/src/client/util/CurrentUserUtils.ts
@@ -88,7 +88,7 @@ export class CurrentUserUtils {
const reqdClickOpts:DocumentOptions = {_width: 300, _height:200, isSystem: true};
const reqdTempOpts:{opts:DocumentOptions, script: string}[] = [
{ opts: { title: "Open In Target", targetScriptKey: "onChildClick"}, script: "docCastAsync(documentView?.containerViewPath().lastElement()?.Document.target).then((target) => target && (target.proto.data = new List([this])))"},
- { opts: { title: "Open Detail On Right", targetScriptKey: "onChildDoubleClick"}, script: `openDoc(this.doubleClickView.${OpenWhere.addRight})`}];
+ { opts: { title: "Open Detail On Right", targetScriptKey: "onChildDoubleClick"}, script: `openDoc(this.doubleClickView, "${OpenWhere.addRight}")`}];
const reqdClickList = reqdTempOpts.map(opts => {
const allOpts = {...reqdClickOpts, ...opts.opts};
const clickDoc = tempClicks ? DocListCast(tempClicks.data).find(fdoc => fdoc.title === opts.opts.title): undefined;
@@ -805,11 +805,10 @@ pie title Minerals in my tap water
}
static contextMenuTools(doc:Doc):Button[] {
return [
- { btnList: new List<string>([CollectionViewType.Freeform, CollectionViewType.Schema, CollectionViewType.Tree,
- CollectionViewType.Stacking, CollectionViewType.Masonry, CollectionViewType.Multicolumn,
- CollectionViewType.Multirow, CollectionViewType.Time, CollectionViewType.Carousel,
- CollectionViewType.Carousel3D, CollectionViewType.Card, CollectionViewType.Linear, CollectionViewType.Map,
- CollectionViewType.Calendar, CollectionViewType.Grid, CollectionViewType.NoteTaking, ]),
+ { btnList: new List<string>([CollectionViewType.Freeform, CollectionViewType.Card, CollectionViewType.Carousel,CollectionViewType.Carousel3D,
+ CollectionViewType.Masonry, CollectionViewType.Multicolumn, CollectionViewType.Multirow, CollectionViewType.Linear,
+ CollectionViewType.Map, CollectionViewType.NoteTaking, CollectionViewType.Schema, CollectionViewType.Stacking,
+ CollectionViewType.Calendar, CollectionViewType.Grid, CollectionViewType.Tree, CollectionViewType.Time, ]),
title: "Perspective", toolTip: "View", btnType: ButtonType.DropdownList, ignoreClick: true, width: 100, scripts: { script: '{ return setView(value, _readOnly_); }'}},
{ title: "Pin", icon: "map-pin", toolTip: "Pin View to Trail", btnType: ButtonType.ClickButton, expertMode: false, width: 30, scripts: { onClick: 'pinWithView(altKey)'}, funcs: {hidden: "IsNoneSelected()"}},
{ title: "Header", icon: "heading", toolTip: "Doc Titlebar Color", btnType: ButtonType.ColorButton, expertMode: false, ignoreClick: true, scripts: { script: 'return setHeaderColor(value, _readOnly_)'} },
diff --git a/src/client/views/collections/CollectionCardDeckView.tsx b/src/client/views/collections/CollectionCardDeckView.tsx
index 5faabacf4..b86dad9d7 100644
--- a/src/client/views/collections/CollectionCardDeckView.tsx
+++ b/src/client/views/collections/CollectionCardDeckView.tsx
@@ -4,7 +4,7 @@ import { computedFn } from 'mobx-utils';
import * as React from 'react';
import { ClientUtils, DashColor, imageUrlToBase64, returnFalse, returnNever, returnZero } from '../../../ClientUtils';
import { emptyFunction } from '../../../Utils';
-import { Doc } from '../../../fields/Doc';
+import { Doc, DocListCast, Opt } from '../../../fields/Doc';
import { Animation, DocData } from '../../../fields/DocSymbols';
import { Id } from '../../../fields/FieldSymbols';
import { List } from '../../../fields/List';
@@ -24,6 +24,7 @@ import { DocumentView, DocumentViewProps } from '../nodes/DocumentView';
import { GPTPopup, GPTPopupMode } from '../pdf/GPTPopup/GPTPopup';
import './CollectionCardDeckView.scss';
import { CollectionSubView, SubCollectionViewProps } from './CollectionSubView';
+import { FocusViewOptions } from '../nodes/FocusViewOptions';
enum cardSortings {
Time = 'time',
@@ -342,6 +343,7 @@ export class CollectionCardView extends CollectionSubView() {
fitWidth={returnFalse}
waitForDoubleClickToClick={returnNever}
scriptContext={this}
+ focus={this.focus}
onDoubleClickScript={this.onChildDoubleClick}
onClickScript={this._curDoc === doc ? undefined : this._clickScript}
dontCenter="y" // Don't center it vertically, because the grid it's in is already doing that and we don't want to do it twice.
@@ -593,6 +595,16 @@ export class CollectionCardView extends CollectionSubView() {
}
});
+ focus = action((anchor: Doc, options: FocusViewOptions): Opt<number> => {
+ const docs = DocListCast(this.Document[this.fieldKey ?? Doc.LayoutFieldKey(this.Document)]);
+ if (anchor.type !== DocumentType.CONFIG && !docs.includes(anchor)) return undefined;
+ options.didMove = true;
+ const target = DocCast(anchor.annotationOn) ?? anchor;
+ const index = docs.indexOf(target);
+ index !== -1 && (this._curDoc = target);
+ return undefined;
+ });
+
/**
* Actually renders all the cards
*/
diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
index 534f67927..6d51ecac6 100644
--- a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
+++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
@@ -8,7 +8,7 @@ import { observer } from 'mobx-react';
import React from 'react';
import { DivHeight, lightOrDark, returnTrue, setupMoveUpEvents } from '../../../../ClientUtils';
import { emptyFunction } from '../../../../Utils';
-import { Doc, Opt } from '../../../../fields/Doc';
+import { Doc, DocListCast, Opt } from '../../../../fields/Doc';
import { DocData } from '../../../../fields/DocSymbols';
import { List } from '../../../../fields/List';
import { DocCast, ImageCast, NumCast, StrCast } from '../../../../fields/Types';
@@ -54,7 +54,7 @@ export class UniqueFaceBox extends ViewBoxBaseComponent<FieldViewProps>() {
@observable _headerRef: HTMLDivElement | null = null;
@observable _listRef: HTMLDivElement | null = null;
- observer = new ResizeObserver(a => {
+ observer = new ResizeObserver(() => {
this._props.setHeight?.(
(this.props.Document._face_showImages ? 20 : 0) + //
(!this._headerRef ? 0 : DivHeight(this._headerRef)) +
@@ -97,9 +97,9 @@ export class UniqueFaceBox extends ViewBoxBaseComponent<FieldViewProps>() {
const faceMatcher = new FaceMatcher([labeledFaceDescriptor], 1);
const faceAnno =
FaceRecognitionHandler.ImageDocFaceAnnos(imgDoc).reduce(
- (prev, faceAnno) => {
- const match = faceMatcher.matchDescriptor(new Float32Array(Array.from(faceAnno.faceDescriptor as List<number>)));
- return match.distance < prev.dist ? { dist: match.distance, faceAnno } : prev;
+ (prev, fAnno) => {
+ const match = faceMatcher.matchDescriptor(new Float32Array(Array.from(fAnno.faceDescriptor as List<number>)));
+ return match.distance < prev.dist ? { dist: match.distance, faceAnno: fAnno } : prev;
},
{ dist: 1, faceAnno: undefined as Opt<Doc> }
).faceAnno ?? imgDoc;
@@ -108,10 +108,18 @@ export class UniqueFaceBox extends ViewBoxBaseComponent<FieldViewProps>() {
if (faceAnno) {
faceAnno.face && FaceRecognitionHandler.UniqueFaceRemoveFaceImage(faceAnno, DocCast(faceAnno.face));
FaceRecognitionHandler.UniqueFaceAddFaceImage(faceAnno, this.Document);
- faceAnno.face = this.Document;
+ faceAnno[DocData].face = this.Document[DocData];
}
}
});
+ de.complete.docDragData?.droppedDocuments
+ ?.filter(doc => DocCast(doc.face)?.type === DocumentType.UFACE)
+ .forEach(faceAnno => {
+ const imgDoc = faceAnno;
+ faceAnno.face && FaceRecognitionHandler.UniqueFaceRemoveFaceImage(imgDoc, DocCast(faceAnno.face));
+ FaceRecognitionHandler.UniqueFaceAddFaceImage(faceAnno, this.Document);
+ faceAnno[DocData].face = this.Document[DocData];
+ });
e.stopPropagation();
return true;
}
@@ -189,7 +197,8 @@ export class UniqueFaceBox extends ViewBoxBaseComponent<FieldViewProps>() {
this,
e,
() => {
- DragManager.StartDocumentDrag([e.target as HTMLElement], new DragManager.DocumentDragData([doc], dropActionType.embed), e.clientX, e.clientY);
+ const dragDoc = DocListCast(doc.data_annotations).find(a => a.face === this.Document[DocData]) ?? this.Document;
+ DragManager.StartDocumentDrag([e.target as HTMLElement], new DragManager.DocumentDragData([dragDoc], dropActionType.embed), e.clientX, e.clientY);
return true;
},
emptyFunction,
diff --git a/src/client/views/nodes/ComparisonBox.scss b/src/client/views/nodes/ComparisonBox.scss
index d1cc48051..d2ba9796b 100644
--- a/src/client/views/nodes/ComparisonBox.scss
+++ b/src/client/views/nodes/ComparisonBox.scss
@@ -236,6 +236,9 @@
}
}
}
+.comparisonBox-interactive {
+ pointer-events: all;
+}
.comparisonBox-explain {
position: absolute;
diff --git a/src/client/views/nodes/ComparisonBox.tsx b/src/client/views/nodes/ComparisonBox.tsx
index 3c126ea4a..f6c33d6ba 100644
--- a/src/client/views/nodes/ComparisonBox.tsx
+++ b/src/client/views/nodes/ComparisonBox.tsx
@@ -792,8 +792,9 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
const renderMode = new Map<flashcardRevealOp, () => JSX.Element>([
[flashcardRevealOp.FLIP, this.renderAsFlip],
[flashcardRevealOp.SLIDE, this.renderAsBeforeAfter]]); // prettier-ignore
- if (this.isQuizMode) this.renderAsQuiz(this.frontText);
- return (
+ return this.isQuizMode ? (
+ this.renderAsQuiz(this.frontText)
+ ) : (
<div className="comparisonBox" style={{ pointerEvents: this._props.isContentActive() && !this.Document[Animation] ? 'unset' : undefined }} onContextMenu={this.flashcardContextMenu}>
{renderMode.get(this.revealOp)?.() ?? null}
{this.loading ? (
diff --git a/src/client/views/nodes/FontIconBox/FontIconBox.tsx b/src/client/views/nodes/FontIconBox/FontIconBox.tsx
index feaf84b7b..d4898eb3c 100644
--- a/src/client/views/nodes/FontIconBox/FontIconBox.tsx
+++ b/src/client/views/nodes/FontIconBox/FontIconBox.tsx
@@ -187,7 +187,7 @@ export class FontIconBox extends ViewBoxBaseComponent<ButtonProps>() {
} else {
return <Button text="None Selected" type={Type.TERT} color={SnappingManager.userColor} background={SnappingManager.userVariantColor} fillWidth inactive />;
}
- noviceList = [CollectionViewType.Freeform, CollectionViewType.Schema, CollectionViewType.Carousel3D, CollectionViewType.Stacking, CollectionViewType.NoteTaking];
+ noviceList = [CollectionViewType.Freeform, CollectionViewType.Schema, CollectionViewType.Card, CollectionViewType.Carousel3D, CollectionViewType.Carousel, CollectionViewType.Stacking, CollectionViewType.NoteTaking];
} else {
text = script?.script.run({ this: this.Document, value: '', _readOnly_: true }).result as string;
// text = StrCast((RichTextMenu.Instance?.TextView?.EditorView ? RichTextMenu.Instance : Doc.UserDoc()).fontFamily);
diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts
index ccf9caf15..34e7cf5ea 100644
--- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts
+++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts
@@ -11,9 +11,11 @@ import { NoTool } from '../tools/NoTool';
import { RAGTool } from '../tools/RAGTool';
import { SearchTool } from '../tools/SearchTool';
import { WebsiteInfoScraperTool } from '../tools/WebsiteInfoScraperTool';
-import { AgentMessage, AssistantMessage, PROCESSING_TYPE, ProcessingInfo, Tool } from '../types/types';
+import { AgentMessage, AssistantMessage, Observation, PROCESSING_TYPE, ProcessingInfo } from '../types/types';
import { Vectorstore } from '../vectorstore/Vectorstore';
import { getReactPrompt } from './prompts';
+import { BaseTool } from '../tools/BaseTool';
+import { Parameter, ParametersType, Tool } from '../tools/ToolTypes';
dotenv.config();
@@ -24,7 +26,6 @@ dotenv.config();
export class Agent {
// Private properties
private client: OpenAI;
- private tools: Record<string, Tool<any>>; // bcz: need a real type here
private messages: AgentMessage[] = [];
private interMessages: AgentMessage[] = [];
private vectorstore: Vectorstore;
@@ -36,6 +37,7 @@ export class Agent {
private processingNumber: number = 0;
private processingInfo: ProcessingInfo[] = [];
private streamedAnswerParser: StreamedAnswerParser = new StreamedAnswerParser();
+ private tools: Record<string, BaseTool<ReadonlyArray<Parameter>>>;
/**
* The constructor initializes the agent with the vector store and toolset, and sets up the OpenAI client.
@@ -108,15 +110,16 @@ export class Agent {
let currentAction: string | undefined;
this.processingInfo = [];
- // Conversation loop (up to maxTurns)
- for (let i = 2; i < maxTurns; i += 2) {
+ let i = 2;
+ while (i < maxTurns) {
console.log(this.interMessages);
console.log(`Turn ${i}/${maxTurns}`);
- // Execute a step in the conversation and get the result
const result = await this.execute(onProcessingUpdate, onAnswerUpdate);
this.interMessages.push({ role: 'assistant', content: result });
+ i += 2;
+
let parsedResult;
try {
// Parse XML result from the assistant
@@ -148,7 +151,7 @@ export class Agent {
{
type: 'text',
text: `<stage number="${i + 1}" role="user">` + builder.build({ action_rules: this.tools[currentAction].getActionRule() }) + `</stage>`,
- },
+ } as Observation,
];
this.interMessages.push({ role: 'user', content: nextPrompt });
break;
@@ -166,8 +169,8 @@ export class Agent {
if (currentAction) {
try {
// Process the action with its input
- const observation = (await this.processAction(currentAction, actionInput.inputs)) as any; // bcz: really need a type here
- const nextPrompt = [{ type: 'text', text: `<stage number="${i + 1}" role="user"> <observation>` }, ...observation, { type: 'text', text: '</observation></stage>' }];
+ const observation = (await this.processAction(currentAction, actionInput.inputs)) as Observation[];
+ const nextPrompt = [{ type: 'text', text: `<stage number="${i + 1}" role="user"> <observation>` }, ...observation, { type: 'text', text: '</observation></stage>' }] as Observation[];
console.log(observation);
this.interMessages.push({ role: 'user', content: nextPrompt });
this.processingNumber++;
@@ -262,16 +265,69 @@ export class Agent {
/**
* Processes a specific action by invoking the appropriate tool with the provided inputs.
- * @param action The action to perform.
- * @param actionInput The inputs for the action.
- * @returns The result of the action.
+ * This method ensures that the action exists and validates the types of `actionInput`
+ * based on the tool's parameter rules. It throws errors for missing required parameters
+ * or mismatched types before safely executing the tool with the validated input.
+ *
+ * Type validation includes checks for:
+ * - `string`, `number`, `boolean`
+ * - `string[]`, `number[]` (arrays of strings or numbers)
+ *
+ * @param action The action to perform. It corresponds to a registered tool.
+ * @param actionInput The inputs for the action, passed as an object where each key is a parameter name.
+ * @returns A promise that resolves to an array of `Observation` objects representing the result of the action.
+ * @throws An error if the action is unknown, if required parameters are missing, or if input types don't match the expected parameter types.
*/
- private async processAction(action: string, actionInput: unknown): Promise<unknown> {
+ private async processAction(action: string, actionInput: Record<string, unknown>): Promise<Observation[]> {
+ // Check if the action exists in the tools list
if (!(action in this.tools)) {
throw new Error(`Unknown action: ${action}`);
}
const tool = this.tools[action];
- return await tool.execute(actionInput);
+
+ // Validate actionInput based on tool's parameter rules
+ for (const paramRule of tool.parameterRules) {
+ const inputValue = actionInput[paramRule.name];
+
+ if (paramRule.required && inputValue === undefined) {
+ throw new Error(`Missing required parameter: ${paramRule.name}`);
+ }
+
+ // If the parameter is defined, check its type
+ if (inputValue !== undefined) {
+ switch (paramRule.type) {
+ case 'string':
+ if (typeof inputValue !== 'string') {
+ throw new Error(`Expected parameter '${paramRule.name}' to be a string.`);
+ }
+ break;
+ case 'number':
+ if (typeof inputValue !== 'number') {
+ throw new Error(`Expected parameter '${paramRule.name}' to be a number.`);
+ }
+ break;
+ case 'boolean':
+ if (typeof inputValue !== 'boolean') {
+ throw new Error(`Expected parameter '${paramRule.name}' to be a boolean.`);
+ }
+ break;
+ case 'string[]':
+ if (!Array.isArray(inputValue) || !inputValue.every(item => typeof item === 'string')) {
+ throw new Error(`Expected parameter '${paramRule.name}' to be an array of strings.`);
+ }
+ break;
+ case 'number[]':
+ if (!Array.isArray(inputValue) || !inputValue.every(item => typeof item === 'number')) {
+ throw new Error(`Expected parameter '${paramRule.name}' to be an array of numbers.`);
+ }
+ break;
+ default:
+ throw new Error(`Unsupported parameter type: ${paramRule.type}`);
+ }
+ }
+ }
+
+ return await tool.execute(actionInput as ParametersType<typeof tool.parameterRules>);
}
}
diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/MessageComponent.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/MessageComponent.tsx
index d48f46963..e463d15bf 100644
--- a/src/client/views/nodes/chatbot/chatboxcomponents/MessageComponent.tsx
+++ b/src/client/views/nodes/chatbot/chatboxcomponents/MessageComponent.tsx
@@ -23,7 +23,6 @@ import ReactMarkdown from 'react-markdown';
*/
interface MessageComponentProps {
message: AssistantMessage;
- index: number;
onFollowUpClick: (question: string) => void;
onCitationClick: (citation: Citation) => void;
updateMessageCitations: (index: number, citations: Citation[]) => void;
@@ -34,7 +33,7 @@ interface MessageComponentProps {
* processing information, and follow-up questions.
* @param {MessageComponentProps} props - The props for the component.
*/
-const MessageComponentBox: React.FC<MessageComponentProps> = ({ message, index, onFollowUpClick, onCitationClick, updateMessageCitations }) => {
+const MessageComponentBox: React.FC<MessageComponentProps> = ({ message, onFollowUpClick, onCitationClick }) => {
// State for managing whether the dropdown is open or closed for processing info
const [dropdownOpen, setDropdownOpen] = useState(false);
diff --git a/src/client/views/nodes/chatbot/tools/BaseTool.ts b/src/client/views/nodes/chatbot/tools/BaseTool.ts
index a77f567a5..58cd514d9 100644
--- a/src/client/views/nodes/chatbot/tools/BaseTool.ts
+++ b/src/client/views/nodes/chatbot/tools/BaseTool.ts
@@ -1,32 +1,78 @@
+import { Observation } from '../types/types';
+import { Parameter, Tool, ParametersType } from './ToolTypes';
+
/**
* @file BaseTool.ts
- * @description This file defines the abstract BaseTool class, which serves as a blueprint
+ * @description This file defines the abstract `BaseTool` class, which serves as a blueprint
* for tool implementations in the AI assistant system. Each tool has a name, description,
- * parameters, and citation rules. The BaseTool class provides a structure for executing actions
+ * parameters, and citation rules. The `BaseTool` class provides a structure for executing actions
* and retrieving action rules for use within the assistant's workflow.
*/
-import { Tool } from '../types/types';
+/**
+ * The `BaseTool` class is an abstract class that implements the `Tool` interface.
+ * It is generic over a type parameter `P`, which extends `ReadonlyArray<Parameter>`.
+ * This means `P` is a readonly array of `Parameter` objects that cannot be modified (immutable).
+ */
+export abstract class BaseTool<P extends ReadonlyArray<Parameter>> implements Tool<P> {
+ // The name of the tool (e.g., "calculate", "searchTool")
+ name: string;
+ // A description of the tool's functionality
+ description: string;
+ // An array of parameter definitions for the tool
+ parameterRules: P;
+ // Guidelines for how to handle citations when using the tool
+ citationRules: string;
+ // A brief summary of the tool's purpose
+ briefSummary: string;
-export abstract class BaseTool<T extends Record<string, unknown> = Record<string, unknown>> implements Tool<T> {
- constructor(
- public name: string,
- public description: string,
- public parameters: Record<string, unknown>,
- public citationRules: string,
- public briefSummary: string
- ) {}
+ /**
+ * Constructs a new `BaseTool` instance.
+ * @param name - The name of the tool.
+ * @param description - A detailed description of what the tool does.
+ * @param parameterRules - A readonly array of parameter definitions (`ReadonlyArray<Parameter>`).
+ * @param citationRules - Rules or guidelines for citations.
+ * @param briefSummary - A short summary of the tool.
+ */
+ constructor(name: string, description: string, parameterRules: P, citationRules: string, briefSummary: string) {
+ this.name = name;
+ this.description = description;
+ this.parameterRules = parameterRules;
+ this.citationRules = citationRules;
+ this.briefSummary = briefSummary;
+ }
- abstract execute(args: T): Promise<unknown>;
+ /**
+ * The `execute` method is abstract and must be implemented by subclasses.
+ * It defines the action the tool performs when executed.
+ * @param args - The arguments for the tool's execution, whose types are inferred from `ParametersType<P>`.
+ * @returns A promise that resolves to an array of `Observation` objects.
+ */
+ abstract execute(args: ParametersType<P>): Promise<Observation[]>;
+ /**
+ * Generates an action rule object that describes the tool's usage.
+ * This is useful for dynamically generating documentation or for tools that need to expose their parameters at runtime.
+ * @returns An object containing the tool's name, description, and parameter definitions.
+ */
getActionRule(): Record<string, unknown> {
return {
- [this.name]: {
- name: this.name,
- citationRules: this.citationRules,
- description: this.description,
- parameters: this.parameters,
- },
+ tool: this.name,
+ description: this.description,
+ parameters: this.parameterRules.reduce(
+ (acc, param) => {
+ // Build an object for each parameter without the 'name' property, since it's used as the key
+ acc[param.name] = {
+ type: param.type,
+ description: param.description,
+ required: param.required,
+ // Conditionally include 'max_inputs' only if it is defined
+ ...(param.max_inputs !== undefined && { max_inputs: param.max_inputs }),
+ } as Omit<P[number], 'name'>; // Type assertion to exclude the 'name' property
+ return acc;
+ },
+ {} as Record<string, Omit<P[number], 'name'>> // Initialize the accumulator as an empty object
+ ),
};
}
}
diff --git a/src/client/views/nodes/chatbot/tools/CalculateTool.ts b/src/client/views/nodes/chatbot/tools/CalculateTool.ts
index 77ab1b39b..e96c9a98a 100644
--- a/src/client/views/nodes/chatbot/tools/CalculateTool.ts
+++ b/src/client/views/nodes/chatbot/tools/CalculateTool.ts
@@ -1,26 +1,32 @@
+import { Observation } from '../types/types';
+import { ParametersType } from './ToolTypes';
import { BaseTool } from './BaseTool';
-export class CalculateTool extends BaseTool<{ expression: string }> {
+const calculateToolParams = [
+ {
+ name: 'expression',
+ type: 'string',
+ description: 'The mathematical expression to evaluate',
+ required: true,
+ },
+] as const;
+
+type CalculateToolParamsType = typeof calculateToolParams;
+
+export class CalculateTool extends BaseTool<CalculateToolParamsType> {
constructor() {
super(
'calculate',
'Perform a calculation',
- {
- expression: {
- type: 'string',
- description: 'The mathematical expression to evaluate',
- required: 'true',
- max_inputs: '1',
- },
- },
+ calculateToolParams, // Use the reusable param config here
'Provide a mathematical expression to calculate that would work with JavaScript eval().',
'Runs a calculation and returns the number - uses JavaScript so be sure to use floating point syntax if necessary'
);
}
- async execute(args: { expression: string }): Promise<unknown> {
- // Note: Using eval() can be dangerous. Consider using a safer alternative.
- const result = eval(args.expression);
+ async execute(args: ParametersType<CalculateToolParamsType>): Promise<Observation[]> {
+ // TypeScript will ensure 'args.expression' is a string based on the param config
+ const result = eval(args.expression); // Be cautious with eval(), as it can be dangerous. Consider using a safer alternative.
return [{ type: 'text', text: result.toString() }];
}
}
diff --git a/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts b/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts
index d3ded0de0..b321d98ba 100644
--- a/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts
+++ b/src/client/views/nodes/chatbot/tools/CreateCSVTool.ts
@@ -1,40 +1,47 @@
import { BaseTool } from './BaseTool';
import { Networking } from '../../../../Network';
+import { Observation } from '../types/types';
+import { ParametersType } from './ToolTypes';
-export class CreateCSVTool extends BaseTool<{ csvData: string; filename: string }> {
+const createCSVToolParams = [
+ {
+ name: 'csvData',
+ type: 'string',
+ description: 'A string of comma-separated values representing the CSV data.',
+ required: true,
+ },
+ {
+ name: 'filename',
+ type: 'string',
+ description: 'The base name of the CSV file to be created. Should end in ".csv".',
+ required: true,
+ },
+] as const;
+
+type CreateCSVToolParamsType = typeof createCSVToolParams;
+
+export class CreateCSVTool extends BaseTool<CreateCSVToolParamsType> {
private _handleCSVResult: (url: string, filename: string, id: string, data: string) => void;
constructor(handleCSVResult: (url: string, title: string, id: string, data: string) => void) {
super(
'createCSV',
'Creates a CSV file from raw CSV data and saves it to the server',
- {
- type: 'object',
- properties: {
- csvData: {
- type: 'string',
- description: 'A string of comma-separated values representing the CSV data.',
- },
- filename: {
- type: 'string',
- description: 'The base name of the CSV file to be created. Should end in ".csv".',
- },
- },
- required: ['csvData', 'filename'],
- },
+ createCSVToolParams,
'Provide a CSV string and a filename to create a CSV file.',
'Creates a CSV file from the provided CSV string and saves it to the server with a unique identifier, returning the file URL and UUID.'
);
this._handleCSVResult = handleCSVResult;
}
- async execute(args: { csvData: string; filename: string }): Promise<unknown> {
+ async execute(args: ParametersType<CreateCSVToolParamsType>): Promise<Observation[]> {
try {
console.log('Creating CSV file:', args.filename, ' with data:', args.csvData);
- // Post the raw CSV data to the createCSV endpoint on the server
- const { fileUrl, id } = await Networking.PostToServer('/createCSV', { filename: args.filename, data: args.csvData });
+ const { fileUrl, id } = await Networking.PostToServer('/createCSV', {
+ filename: args.filename,
+ data: args.csvData,
+ });
- // Handle the result by invoking the callback
this._handleCSVResult(fileUrl, args.filename, id, args.csvData);
return [
diff --git a/src/client/views/nodes/chatbot/tools/CreateCollectionTool.ts b/src/client/views/nodes/chatbot/tools/CreateCollectionTool.ts
deleted file mode 100644
index 1e479a62c..000000000
--- a/src/client/views/nodes/chatbot/tools/CreateCollectionTool.ts
+++ /dev/null
@@ -1,36 +0,0 @@
-import { DocCast } from '../../../../../fields/Types';
-import { DocServer } from '../../../../DocServer';
-import { Docs } from '../../../../documents/Documents';
-import { DocumentView } from '../../DocumentView';
-import { OpenWhere } from '../../OpenWhere';
-import { BaseTool } from './BaseTool';
-
-export class GetDocsContentTool extends BaseTool<{ title: string; document_ids: string[] }> {
- private _docView: DocumentView;
- constructor(docView: DocumentView) {
- super(
- 'retrieveDocs',
- 'Retrieves the contents of all Documents that the user is interacting with in Dash ',
- {
- title: {
- type: 'string',
- description: 'the title of the collection that you will be making',
- required: 'true',
- max_inputs: '1',
- },
- },
- 'Provide a mathematical expression to calculate that would work with JavaScript eval().',
- 'Runs a calculation and returns the number - uses JavaScript so be sure to use floating point syntax if necessary'
- );
- this._docView = docView;
- }
-
- async execute(args: { title: string; document_ids: string[] }): Promise<unknown> {
- // Note: Using eval() can be dangerous. Consider using a safer alternative.
- const docs = args.document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id)));
- const collection = Docs.Create.FreeformDocument(docs, { title: args.title });
- this._docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add
- return [{ type: 'text', text: 'Collection created in Dash called ' + args.title }];
- }
-}
-//export function create_collection(docView: DocumentView, document_ids: string[], title: string): string {}
diff --git a/src/client/views/nodes/chatbot/tools/DataAnalysisTool.ts b/src/client/views/nodes/chatbot/tools/DataAnalysisTool.ts
index 2e663fed1..d9b75219d 100644
--- a/src/client/views/nodes/chatbot/tools/DataAnalysisTool.ts
+++ b/src/client/views/nodes/chatbot/tools/DataAnalysisTool.ts
@@ -1,22 +1,29 @@
+import { Observation } from '../types/types';
+import { ParametersType } from './ToolTypes';
import { BaseTool } from './BaseTool';
-export class DataAnalysisTool extends BaseTool<{ csv_file_name: string | string[] }> {
+const dataAnalysisToolParams = [
+ {
+ name: 'csv_file_names',
+ type: 'string[]',
+ description: 'List of names of the CSV files to analyze',
+ required: true,
+ max_inputs: 3,
+ },
+] as const;
+
+type DataAnalysisToolParamsType = typeof dataAnalysisToolParams;
+
+export class DataAnalysisTool extends BaseTool<DataAnalysisToolParamsType> {
private csv_files_function: () => { filename: string; id: string; text: string }[];
constructor(csv_files: () => { filename: string; id: string; text: string }[]) {
super(
'dataAnalysis',
- 'Analyzes, and provides insights, from one or more CSV files',
- {
- csv_file_name: {
- type: 'string',
- description: 'Name(s) of the CSV file(s) to analyze',
- required: 'true',
- max_inputs: '3',
- },
- },
+ 'Analyzes and provides insights from one or more CSV files',
+ dataAnalysisToolParams,
'Provide the name(s) of up to 3 CSV files to analyze based on the user query and whichever available CSV files may be relevant.',
- 'Provides the full CSV file text for your analysis based on the user query and the available CSV file(s). '
+ 'Provides the full CSV file text for your analysis based on the user query and the available CSV file(s).'
);
this.csv_files_function = csv_files;
}
@@ -33,9 +40,9 @@ export class DataAnalysisTool extends BaseTool<{ csv_file_name: string | string[
return file?.id;
}
- async execute(args: { csv_file_name: string | string[] }): Promise<unknown> {
- const filenames = Array.isArray(args.csv_file_name) ? args.csv_file_name : [args.csv_file_name];
- const results = [];
+ async execute(args: ParametersType<DataAnalysisToolParamsType>): Promise<Observation[]> {
+ const filenames = args.csv_file_names;
+ const results: Observation[] = [];
for (const filename of filenames) {
const fileContent = this.getFileContent(filename);
@@ -44,7 +51,7 @@ export class DataAnalysisTool extends BaseTool<{ csv_file_name: string | string[
if (fileContent && fileID) {
results.push({
type: 'text',
- text: `<chunk chunk_id=${fileID} chunk_type=csv>${fileContent}</chunk>`,
+ text: `<chunk chunk_id="${fileID}" chunk_type="csv">${fileContent}</chunk>`,
});
} else {
results.push({
diff --git a/src/client/views/nodes/chatbot/tools/GetDocsTool.ts b/src/client/views/nodes/chatbot/tools/GetDocsTool.ts
index 903f3f69c..26756522c 100644
--- a/src/client/views/nodes/chatbot/tools/GetDocsTool.ts
+++ b/src/client/views/nodes/chatbot/tools/GetDocsTool.ts
@@ -1,29 +1,47 @@
-import { DocCast } from '../../../../../fields/Types';
+import { Observation } from '../types/types';
+import { ParametersType } from './ToolTypes';
+import { BaseTool } from './BaseTool';
import { DocServer } from '../../../../DocServer';
import { Docs } from '../../../../documents/Documents';
import { DocumentView } from '../../DocumentView';
import { OpenWhere } from '../../OpenWhere';
-import { BaseTool } from './BaseTool';
+import { DocCast } from '../../../../../fields/Types';
-export class GetDocsTool extends BaseTool<{ title: string; document_ids: string[] }> {
+const getDocsToolParams = [
+ {
+ name: 'title',
+ type: 'string',
+ description: 'Title of the collection being created from retrieved documents',
+ required: true,
+ },
+ {
+ name: 'document_ids',
+ type: 'string[]',
+ description: 'List of document IDs to retrieve',
+ required: true,
+ },
+] as const;
+
+type GetDocsToolParamsType = typeof getDocsToolParams;
+
+export class GetDocsTool extends BaseTool<GetDocsToolParamsType> {
private _docView: DocumentView;
+
constructor(docView: DocumentView) {
super(
'retrieveDocs',
'Retrieves the contents of all Documents that the user is interacting with in Dash',
- {},
+ getDocsToolParams,
'No need to provide anything. Just run the tool and it will retrieve the contents of all Documents that the user is interacting with in Dash.',
- 'Returns the the documents in Dash in JSON form. This will include the title of the document, the location in the FreeFormDocument, and the content of the document, any applicable data fields, the layout of the document, etc.'
+ 'Returns the documents in Dash in JSON form.'
);
this._docView = docView;
}
- async execute(args: { title: string; document_ids: string[] }): Promise<unknown> {
- // Note: Using eval() can be dangerous. Consider using a safer alternative.
+ async execute(args: ParametersType<GetDocsToolParamsType>): Promise<Observation[]> {
const docs = args.document_ids.map(doc_id => DocCast(DocServer.GetCachedRefField(doc_id)));
const collection = Docs.Create.FreeformDocument(docs, { title: args.title });
- this._docView._props.addDocTab(collection, OpenWhere.addRight); //in future, create popup prompting user where to add
- return [{ type: 'text', text: 'Collection created in Dash called ' + args.title }];
+ this._docView._props.addDocTab(collection, OpenWhere.addRight);
+ return [{ type: 'text', text: `Collection created in Dash called ${args.title}` }];
}
}
-//export function create_collection(docView: DocumentView, document_ids: string[], title: string): string {}
diff --git a/src/client/views/nodes/chatbot/tools/NoTool.ts b/src/client/views/nodes/chatbot/tools/NoTool.ts
index edd3160ec..a92e3fa23 100644
--- a/src/client/views/nodes/chatbot/tools/NoTool.ts
+++ b/src/client/views/nodes/chatbot/tools/NoTool.ts
@@ -1,19 +1,18 @@
-// tools/NoTool.ts
import { BaseTool } from './BaseTool';
+import { Observation } from '../types/types';
+import { ParametersType } from './ToolTypes';
-export class NoTool extends BaseTool<Record<string, unknown>> {
+const noToolParams = [] as const;
+
+type NoToolParamsType = typeof noToolParams;
+
+export class NoTool extends BaseTool<NoToolParamsType> {
constructor() {
- super(
- 'no_tool',
- 'Use this when no external tool or action is required to answer the question.',
- {},
- 'When using the "no_tool" action, simply provide an empty <action_input> element. The observation will always be "No tool used. Proceed with answering the question."',
- 'Use when no external tool or action is required to answer the question.'
- );
+ super('noTool', 'A placeholder tool that performs no action', noToolParams, 'This tool does not require any input or perform any action.', 'Does nothing.');
}
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
- async execute(args: object): Promise<unknown> {
- return [{ type: 'text', text: 'No tool used. Proceed with answering the question.' }];
+ async execute(args: ParametersType<NoToolParamsType>): Promise<Observation[]> {
+ // Since there are no parameters, args will be an empty object
+ return [{ type: 'text', text: 'This tool does nothing.' }];
}
}
diff --git a/src/client/views/nodes/chatbot/tools/RAGTool.ts b/src/client/views/nodes/chatbot/tools/RAGTool.ts
index 4cc2f26ff..482069f36 100644
--- a/src/client/views/nodes/chatbot/tools/RAGTool.ts
+++ b/src/client/views/nodes/chatbot/tools/RAGTool.ts
@@ -1,20 +1,26 @@
import { Networking } from '../../../../Network';
-import { RAGChunk } from '../types/types';
+import { Observation, RAGChunk } from '../types/types';
+import { ParametersType } from './ToolTypes';
import { Vectorstore } from '../vectorstore/Vectorstore';
import { BaseTool } from './BaseTool';
-export class RAGTool extends BaseTool {
+const ragToolParams = [
+ {
+ name: 'hypothetical_document_chunk',
+ type: 'string',
+ description: "A detailed prompt representing an ideal chunk to embed and compare against document vectors to retrieve the most relevant content for answering the user's query.",
+ required: true,
+ },
+] as const;
+
+type RAGToolParamsType = typeof ragToolParams;
+
+export class RAGTool extends BaseTool<RAGToolParamsType> {
constructor(private vectorstore: Vectorstore) {
super(
'rag',
'Perform a RAG search on user documents',
- {
- hypothetical_document_chunk: {
- type: 'string',
- description: "A detailed prompt representing an ideal chunk to embed and compare against document vectors to retrieve the most relevant content for answering the user's query.",
- required: 'true',
- },
- },
+ ragToolParams,
`
When using the RAG tool, the structure must adhere to the format described in the ReAct prompt. Below are additional guidelines specifically for RAG-based responses:
@@ -51,18 +57,17 @@ export class RAGTool extends BaseTool {
</follow_up_questions>
</answer>
`,
-
`Performs a RAG (Retrieval-Augmented Generation) search on user documents and returns a set of document chunks (text or images) to provide a grounded response based on user documents.`
);
}
- async execute(args: { hypothetical_document_chunk: string }): Promise<unknown> {
+ async execute(args: ParametersType<RAGToolParamsType>): Promise<Observation[]> {
const relevantChunks = await this.vectorstore.retrieve(args.hypothetical_document_chunk);
- const formatted_chunks = await this.getFormattedChunks(relevantChunks);
- return formatted_chunks;
+ const formattedChunks = await this.getFormattedChunks(relevantChunks);
+ return formattedChunks;
}
- async getFormattedChunks(relevantChunks: RAGChunk[]): Promise<unknown> {
+ async getFormattedChunks(relevantChunks: RAGChunk[]): Promise<Observation[]> {
try {
const { formattedChunks } = await Networking.PostToServer('/formatChunks', { relevantChunks });
diff --git a/src/client/views/nodes/chatbot/tools/SearchTool.ts b/src/client/views/nodes/chatbot/tools/SearchTool.ts
index 3a4668422..fd5144dd6 100644
--- a/src/client/views/nodes/chatbot/tools/SearchTool.ts
+++ b/src/client/views/nodes/chatbot/tools/SearchTool.ts
@@ -1,53 +1,68 @@
import { v4 as uuidv4 } from 'uuid';
import { Networking } from '../../../../Network';
import { BaseTool } from './BaseTool';
+import { Observation } from '../types/types';
+import { ParametersType } from './ToolTypes';
-export class SearchTool extends BaseTool<{ query: string | string[] }> {
+const searchToolParams = [
+ {
+ name: 'query',
+ type: 'string[]',
+ description: 'The search query or queries to use for finding websites',
+ required: true,
+ max_inputs: 3,
+ },
+] as const;
+
+type SearchToolParamsType = typeof searchToolParams;
+
+export class SearchTool extends BaseTool<SearchToolParamsType> {
private _addLinkedUrlDoc: (url: string, id: string) => void;
private _max_results: number;
+
constructor(addLinkedUrlDoc: (url: string, id: string) => void, max_results: number = 5) {
super(
'searchTool',
'Search the web to find a wide range of websites related to a query or multiple queries',
- {
- query: {
- type: 'string',
- description: 'The search query or queries to use for finding websites',
- required: 'true',
- max_inputs: '3',
- },
- },
- 'Provide up to 3 search queries to find a broad range of websites. This tool is intended to help you identify relevant websites, but not to be used for providing the final answer. Use this information to determine which specific website to investigate further.',
- 'Returns a list of websites and their overviews based on the search queries, helping to identify which websites might contain relevant information.'
+ searchToolParams,
+ 'Provide up to 3 search queries to find a broad range of websites.',
+ 'Returns a list of websites and their overviews based on the search queries.'
);
this._addLinkedUrlDoc = addLinkedUrlDoc;
this._max_results = max_results;
}
- async execute(args: { query: string | string[] }): Promise<unknown> {
- const queries = Array.isArray(args.query) ? args.query : [args.query];
- const allResults = [];
+ async execute(args: ParametersType<SearchToolParamsType>): Promise<Observation[]> {
+ const queries = args.query;
- for (const query of queries) {
+ // Create an array of promises, each one handling a search for a query
+ const searchPromises = queries.map(async query => {
try {
- const { results } = await Networking.PostToServer('/getWebSearchResults', { query, max_results: this._max_results });
- const data: { type: string; text: string }[] = results.map((result: { url: string; snippet: string }) => {
+ const { results } = await Networking.PostToServer('/getWebSearchResults', {
+ query,
+ max_results: this._max_results,
+ });
+ const data = results.map((result: { url: string; snippet: string }) => {
const id = uuidv4();
return {
type: 'text',
- text: `<chunk chunk_id="${id}" chunk_type="text">
- <url>${result.url}</url>
- <overview>${result.snippet}</overview>
- </chunk>`,
+ text: `<chunk chunk_id="${id}" chunk_type="text"><url>${result.url}</url><overview>${result.snippet}</overview></chunk>`,
};
});
- allResults.push(...data);
+ return data;
} catch (error) {
console.log(error);
- allResults.push({ type: 'text', text: `An error occurred while performing the web search for query: ${query}` });
+ return [
+ {
+ type: 'text',
+ text: `An error occurred while performing the web search for query: ${query}`,
+ },
+ ];
}
- }
+ });
+
+ const allResultsArrays = await Promise.all(searchPromises);
- return allResults;
+ return allResultsArrays.flat();
}
}
diff --git a/src/client/views/nodes/chatbot/tools/ToolTypes.ts b/src/client/views/nodes/chatbot/tools/ToolTypes.ts
new file mode 100644
index 000000000..d47a38952
--- /dev/null
+++ b/src/client/views/nodes/chatbot/tools/ToolTypes.ts
@@ -0,0 +1,76 @@
+import { Observation } from '../types/types';
+
+/**
+ * The `Tool` interface represents a generic tool in the system.
+ * It is generic over a type parameter `P`, which extends `ReadonlyArray<Parameter>`.
+ * @template P - An array of `Parameter` objects defining the tool's parameters.
+ */
+export interface Tool<P extends ReadonlyArray<Parameter>> {
+ // The name of the tool (e.g., "calculate", "searchTool")
+ name: string;
+ // A description of the tool's functionality
+ description: string;
+ // An array of parameter definitions for the tool
+ parameterRules: P;
+ // Guidelines for how to handle citations when using the tool
+ citationRules: string;
+ // A brief summary of the tool's purpose
+ briefSummary: string;
+ /**
+ * Executes the tool's main functionality.
+ * @param args - The arguments for execution, with types inferred from `ParametersType<P>`.
+ * @returns A promise that resolves to an array of `Observation` objects.
+ */
+ execute: (args: ParametersType<P>) => Promise<Observation[]>;
+ /**
+ * Generates an action rule object that describes the tool's usage.
+ * @returns An object representing the tool's action rules.
+ */
+ getActionRule: () => Record<string, unknown>;
+}
+
+/**
+ * The `Parameter` type defines the structure of a parameter configuration.
+ */
+export type Parameter = {
+ // The type of the parameter; constrained to the types 'string', 'number', 'boolean', 'string[]', 'number[]'
+ readonly type: 'string' | 'number' | 'boolean' | 'string[]' | 'number[]';
+ // The name of the parameter
+ readonly name: string;
+ // A description of the parameter
+ readonly description: string;
+ // Indicates whether the parameter is required
+ readonly required: boolean;
+ // (Optional) The maximum number of inputs (useful for array types)
+ readonly max_inputs?: number;
+};
+
+/**
+ * A utility type that maps string representations of types to actual TypeScript types.
+ * This is used to convert the `type` field of a `Parameter` into a concrete TypeScript type.
+ */
+type TypeMap = {
+ string: string;
+ number: number;
+ boolean: boolean;
+ 'string[]': string[];
+ 'number[]': number[];
+};
+
+/**
+ * The `ParamType` type maps a `Parameter`'s `type` field to the corresponding TypeScript type.
+ * If the `type` field matches a key in `TypeMap`, it returns the associated type.
+ * Otherwise, it returns `unknown`.
+ * @template P - A `Parameter` object.
+ */
+export type ParamType<P extends Parameter> = P['type'] extends keyof TypeMap ? TypeMap[P['type']] : unknown;
+
+/**
+ * The `ParametersType` type transforms an array of `Parameter` objects into an object type
+ * where each key is the parameter's name, and the value is the corresponding TypeScript type.
+ * This is used to define the types of the arguments passed to the `execute` method of a tool.
+ * @template P - An array of `Parameter` objects.
+ */
+export type ParametersType<P extends ReadonlyArray<Parameter>> = {
+ [K in P[number] as K['name']]: ParamType<K>;
+};
diff --git a/src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts b/src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts
index 1efb389b8..f2e3863a6 100644
--- a/src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts
+++ b/src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts
@@ -1,83 +1,99 @@
import { v4 as uuidv4 } from 'uuid';
import { Networking } from '../../../../Network';
import { BaseTool } from './BaseTool';
+import { Observation } from '../types/types';
+import { ParametersType } from './ToolTypes';
-export class WebsiteInfoScraperTool extends BaseTool<{ url: string | string[] }> {
+const websiteInfoScraperToolParams = [
+ {
+ name: 'urls',
+ type: 'string[]',
+ description: 'The URLs of the websites to scrape',
+ required: true,
+ max_inputs: 3,
+ },
+] as const;
+
+type WebsiteInfoScraperToolParamsType = typeof websiteInfoScraperToolParams;
+
+export class WebsiteInfoScraperTool extends BaseTool<WebsiteInfoScraperToolParamsType> {
private _addLinkedUrlDoc: (url: string, id: string) => void;
constructor(addLinkedUrlDoc: (url: string, id: string) => void) {
super(
'websiteInfoScraper',
'Scrape detailed information from specific websites relevant to the user query',
- {
- url: {
- type: 'string',
- description: 'The URL(s) of the website(s) to scrape',
- required: true,
- max_inputs: 3,
- },
- },
+ websiteInfoScraperToolParams,
`
- Your task is to provide a comprehensive response to the user's prompt using the content scraped from relevant websites. Ensure you follow these guidelines for structuring your response:
+ Your task is to provide a comprehensive response to the user's prompt using the content scraped from relevant websites. Ensure you follow these guidelines for structuring your response:
- 1. Grounded Text Tag Structure:
- - Wrap all text derived from the scraped website(s) in <grounded_text> tags.
- - **Do not include non-sourced information** in <grounded_text> tags.
- - Use a single <grounded_text> tag for content derived from a single website. If citing multiple websites, create new <grounded_text> tags for each.
- - Ensure each <grounded_text> tag has a citation index corresponding to the scraped URL.
+ 1. Grounded Text Tag Structure:
+ - Wrap all text derived from the scraped website(s) in <grounded_text> tags.
+ - **Do not include non-sourced information** in <grounded_text> tags.
+ - Use a single <grounded_text> tag for content derived from a single website. If citing multiple websites, create new <grounded_text> tags for each.
+ - Ensure each <grounded_text> tag has a citation index corresponding to the scraped URL.
- 2. Citation Tag Structure:
- - Create a <citation> tag for each distinct piece of information used from the website(s).
- - Each <citation> tag must reference a URL chunk using the chunk_id attribute.
- - For URL-based citations, leave the citation content empty, but reference the chunk_id and type as 'url'.
+ 2. Citation Tag Structure:
+ - Create a <citation> tag for each distinct piece of information used from the website(s).
+ - Each <citation> tag must reference a URL chunk using the chunk_id attribute.
+ - For URL-based citations, leave the citation content empty, but reference the chunk_id and type as 'url'.
- 3. Structural Integrity Checks:
- - Ensure all opening and closing tags are matched properly.
- - Verify that all citation_index attributes in <grounded_text> tags correspond to valid citations.
- - Do not over-cite—cite only the most relevant parts of the websites.
+ 3. Structural Integrity Checks:
+ - Ensure all opening and closing tags are matched properly.
+ - Verify that all citation_index attributes in <grounded_text> tags correspond to valid citations.
+ - Do not over-cite—cite only the most relevant parts of the websites.
- Example Usage:
+ Example Usage:
- <answer>
- <grounded_text citation_index="1">
- Based on data from the World Bank, economic growth has stabilized in recent years, following a surge in investments.
- </grounded_text>
- <grounded_text citation_index="2">
- According to information retrieved from the International Monetary Fund, the inflation rate has been gradually decreasing since 2020.
- </grounded_text>
+ <answer>
+ <grounded_text citation_index="1">
+ Based on data from the World Bank, economic growth has stabilized in recent years, following a surge in investments.
+ </grounded_text>
+ <grounded_text citation_index="2">
+ According to information retrieved from the International Monetary Fund, the inflation rate has been gradually decreasing since 2020.
+ </grounded_text>
- <citations>
- <citation index="1" chunk_id="1234" type="url"></citation>
- <citation index="2" chunk_id="5678" type="url"></citation>
- </citations>
+ <citations>
+ <citation index="1" chunk_id="1234" type="url"></citation>
+ <citation index="2" chunk_id="5678" type="url"></citation>
+ </citations>
- <follow_up_questions>
- <question>What are the long-term economic impacts of increased investments on GDP?</question>
- <question>How might inflation trends affect future monetary policy?</question>
- <question>Are there additional factors that could influence economic growth beyond investments and inflation?</question>
- </follow_up_questions>
- </answer>
- `,
+ <follow_up_questions>
+ <question>What are the long-term economic impacts of increased investments on GDP?</question>
+ <question>How might inflation trends affect future monetary policy?</question>
+ <question>Are there additional factors that could influence economic growth beyond investments and inflation?</question>
+ </follow_up_questions>
+ </answer>
+ `,
'Returns the text content of the webpages for further analysis and grounding.'
);
this._addLinkedUrlDoc = addLinkedUrlDoc;
}
- async execute(args: { url: string | string[] }): Promise<unknown> {
- const urls = Array.isArray(args.url) ? args.url : [args.url];
- const results = [];
+ async execute(args: ParametersType<WebsiteInfoScraperToolParamsType>): Promise<Observation[]> {
+ const urls = args.urls;
- for (const url of urls) {
+ // Create an array of promises, each one handling a website scrape for a URL
+ const scrapingPromises = urls.map(async url => {
try {
const { website_plain_text } = await Networking.PostToServer('/scrapeWebsite', { url });
const id = uuidv4();
this._addLinkedUrlDoc(url, id);
- results.push({ type: 'text', text: `<chunk chunk_id=${id} chunk_type=url>\n${website_plain_text}\n</chunk>\n` });
+ return {
+ type: 'text',
+ text: `<chunk chunk_id="${id}" chunk_type="url">\n${website_plain_text}\n</chunk>`,
+ } as Observation;
} catch (error) {
console.log(error);
- results.push({ type: 'text', text: `An error occurred while scraping the website: ${url}` });
+ return {
+ type: 'text',
+ text: `An error occurred while scraping the website: ${url}`,
+ } as Observation;
}
- }
+ });
+
+ // Wait for all scraping promises to resolve
+ const results = await Promise.all(scrapingPromises);
return results;
}
diff --git a/src/client/views/nodes/chatbot/tools/WikipediaTool.ts b/src/client/views/nodes/chatbot/tools/WikipediaTool.ts
index 692dff749..4fcffe2ed 100644
--- a/src/client/views/nodes/chatbot/tools/WikipediaTool.ts
+++ b/src/client/views/nodes/chatbot/tools/WikipediaTool.ts
@@ -1,33 +1,46 @@
import { v4 as uuidv4 } from 'uuid';
import { Networking } from '../../../../Network';
import { BaseTool } from './BaseTool';
+import { Observation } from '../types/types';
+import { ParametersType } from './ToolTypes';
-export class WikipediaTool extends BaseTool<{ title: string }> {
+const wikipediaToolParams = [
+ {
+ name: 'title',
+ type: 'string',
+ description: 'The title of the Wikipedia article to search',
+ required: true,
+ },
+] as const;
+
+type WikipediaToolParamsType = typeof wikipediaToolParams;
+
+export class WikipediaTool extends BaseTool<WikipediaToolParamsType> {
private _addLinkedUrlDoc: (url: string, id: string) => void;
+
constructor(addLinkedUrlDoc: (url: string, id: string) => void) {
super(
'wikipedia',
'Search Wikipedia and return a summary',
- {
- title: {
- type: 'string',
- description: 'The title of the Wikipedia article to search',
- required: true,
- },
- },
+ wikipediaToolParams,
'Provide simply the title you want to search on Wikipedia and nothing more. If re-using this tool, try a different title for different information.',
'Returns a summary from searching an article title on Wikipedia'
);
this._addLinkedUrlDoc = addLinkedUrlDoc;
}
- async execute(args: { title: string }): Promise<unknown> {
+ async execute(args: ParametersType<WikipediaToolParamsType>): Promise<Observation[]> {
try {
const { text } = await Networking.PostToServer('/getWikipediaSummary', { title: args.title });
const id = uuidv4();
const url = `https://en.wikipedia.org/wiki/${args.title.replace(/ /g, '_')}`;
this._addLinkedUrlDoc(url, id);
- return [{ type: 'text', text: `<chunk chunk_id=${id} chunk_type=csv}> ${text} </chunk>` }];
+ return [
+ {
+ type: 'text',
+ text: `<chunk chunk_id="${id}" chunk_type="text"> ${text} </chunk>`,
+ },
+ ];
} catch (error) {
console.log(error);
return [{ type: 'text', text: 'An error occurred while fetching the article.' }];
diff --git a/src/client/views/nodes/chatbot/types/types.ts b/src/client/views/nodes/chatbot/types/types.ts
index 2bc7f4952..7abad85f0 100644
--- a/src/client/views/nodes/chatbot/types/types.ts
+++ b/src/client/views/nodes/chatbot/types/types.ts
@@ -1,3 +1,5 @@
+import { AnyLayer } from 'react-map-gl';
+
export enum ASSISTANT_ROLE {
USER = 'user',
ASSISTANT = 'assistant',
@@ -112,17 +114,9 @@ export interface AI_Document {
type: string;
}
-export interface Tool<T extends Record<string, unknown> = Record<string, unknown>> {
- name: string;
- description: string;
- parameters: Record<string, unknown>;
- citationRules: string;
- briefSummary: string;
- execute: (args: T) => Promise<unknown>;
- getActionRule: () => Record<string, unknown>;
-}
-
export interface AgentMessage {
role: 'system' | 'user' | 'assistant';
- content: string | { type: string; text?: string; image_url?: { url: string } }[];
+ content: string | Observation[];
}
+
+export type Observation = { type: 'text'; text: string } | { type: 'image_url'; image_url: { url: string } };
diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts
index b7d4191ca..8447a4934 100644
--- a/src/server/ApiManagers/AssistantManager.ts
+++ b/src/server/ApiManagers/AssistantManager.ts
@@ -15,7 +15,6 @@ import * as fs from 'fs';
import { writeFile } from 'fs';
import { google } from 'googleapis';
import { JSDOM } from 'jsdom';
-import OpenAI from 'openai';
import * as path from 'path';
import * as puppeteer from 'puppeteer';
import { promisify } from 'util';
@@ -307,33 +306,35 @@ export default class AssistantManager extends ApiManager {
// If the result contains image or table chunks, save the base64 data as image files
if (result.chunks && Array.isArray(result.chunks)) {
- for (const chunk of result.chunks) {
- if (chunk.metadata && (chunk.metadata.type === 'image' || chunk.metadata.type === 'table')) {
- const files_directory = '/files/chunk_images/';
- const directory = path.join(publicDirectory, files_directory);
-
- // Ensure the directory exists or create it
- if (!fs.existsSync(directory)) {
- fs.mkdirSync(directory);
+ await Promise.all(
+ result.chunks.map(chunk => {
+ if (chunk.metadata && (chunk.metadata.type === 'image' || chunk.metadata.type === 'table')) {
+ const files_directory = '/files/chunk_images/';
+ const directory = path.join(publicDirectory, files_directory);
+
+ // Ensure the directory exists or create it
+ if (!fs.existsSync(directory)) {
+ fs.mkdirSync(directory);
+ }
+
+ const fileName = path.basename(chunk.metadata.file_path); // Get the file name from the path
+ const filePath = path.join(directory, fileName); // Create the full file path
+
+ // Check if the chunk contains base64 encoded data
+ if (chunk.metadata.base64_data) {
+ // Decode the base64 data and write it to a file
+ const buffer = Buffer.from(chunk.metadata.base64_data, 'base64');
+ fs.promises.writeFile(filePath, buffer).then(() => {
+ // Update the file path in the chunk's metadata
+ chunk.metadata.file_path = path.join(files_directory, fileName);
+ chunk.metadata.base64_data = undefined; // Remove the base64 data from the metadata
+ });
+ } else {
+ console.warn(`No base64_data found for chunk: ${fileName}`);
+ }
}
-
- const fileName = path.basename(chunk.metadata.file_path); // Get the file name from the path
- const filePath = path.join(directory, fileName); // Create the full file path
-
- // Check if the chunk contains base64 encoded data
- if (chunk.metadata.base64_data) {
- // Decode the base64 data and write it to a file
- const buffer = Buffer.from(chunk.metadata.base64_data, 'base64');
- await fs.promises.writeFile(filePath, buffer);
-
- // Update the file path in the chunk's metadata
- chunk.metadata.file_path = path.join(files_directory, fileName);
- chunk.metadata.base64_data = undefined; // Remove the base64 data from the metadata
- } else {
- console.warn(`No base64_data found for chunk: ${fileName}`);
- }
- }
- }
+ })
+ );
result.status = 'completed';
} else {
result.status = 'pending';
@@ -355,39 +356,42 @@ export default class AssistantManager extends ApiManager {
// Initialize an array to hold the formatted content
const content: { type: string; text?: string; image_url?: { url: string } }[] = [{ type: 'text', text: '<chunks>' }];
- for (const chunk of relevantChunks) {
- // Format each chunk by adding its metadata and content
- content.push({
- type: 'text',
- text: `<chunk chunk_id=${chunk.id} chunk_type="${chunk.metadata.type}">`,
- });
-
- // If the chunk is an image or table, read the corresponding file and encode it as base64
- if (chunk.metadata.type === 'image' || chunk.metadata.type === 'table') {
- try {
- const filePath = serverPathToFile(Directory.chunk_images, chunk.metadata.file_path); // Get the file path
- const imageBuffer = await readFileAsync(filePath); // Read the image file
- const base64Image = imageBuffer.toString('base64'); // Convert the image to base64
-
- // Add the base64-encoded image to the content array
- if (base64Image) {
- content.push({
- type: 'image_url',
- image_url: {
- url: `data:image/jpeg;base64,${base64Image}`,
- },
+ await Promise.all(
+ relevantChunks.map((chunk: { id: string; metadata: { type: string; text: TimeRanges; file_path: string } }) => {
+ // Format each chunk by adding its metadata and content
+ content.push({
+ type: 'text',
+ text: `<chunk chunk_id=${chunk.id} chunk_type="${chunk.metadata.type}">`,
+ });
+
+ // If the chunk is an image or table, read the corresponding file and encode it as base64
+ if (chunk.metadata.type === 'image' || chunk.metadata.type === 'table') {
+ try {
+ const filePath = serverPathToFile(Directory.chunk_images, chunk.metadata.file_path); // Get the file path
+ readFileAsync(filePath).then(imageBuffer => {
+ const base64Image = imageBuffer.toString('base64'); // Convert the image to base64
+
+ // Add the base64-encoded image to the content array
+ if (base64Image) {
+ content.push({
+ type: 'image_url',
+ image_url: {
+ url: `data:image/jpeg;base64,${base64Image}`,
+ },
+ });
+ } else {
+ console.log(`Failed to encode image for chunk ${chunk.id}`);
+ }
});
- } else {
- console.log(`Failed to encode image for chunk ${chunk.id}`);
+ } catch (error) {
+ console.error(`Error reading image file for chunk ${chunk.id}:`, error);
}
- } catch (error) {
- console.error(`Error reading image file for chunk ${chunk.id}:`, error);
}
- }
- // Add the chunk's text content to the formatted content
- content.push({ type: 'text', text: `${chunk.metadata.text}\n</chunk>\n` });
- }
+ // Add the chunk's text content to the formatted content
+ content.push({ type: 'text', text: `${chunk.metadata.text}\n</chunk>\n` });
+ })
+ );
content.push({ type: 'text', text: '</chunks>' });
diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
index 0cf9a6e58..4dcb32f8b 100644
--- a/src/server/server_Initialization.ts
+++ b/src/server/server_Initialization.ts
@@ -130,7 +130,7 @@ function proxyServe(req: any, requrl: string, response: any) {
const htmlText = htmlInputText
.toString('utf8')
.replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>')
- .replace(/(src|href)=(['"])(https?[^\2\n]*)\1/g, refToCors) // replace src or href='http(s)://...' or href="http(s)://.."
+ .replace(/(src|href)=(['"])(https?[^\n]*)\1/g, refToCors) // replace src or href='http(s)://...' or href="http(s)://.."
// .replace(/= *"\/([^"]*)"/g, relpathToCors)
.replace(/data-srcset="[^"]*"/g, '')
.replace(/srcset="[^"]*"/g, '')