updated from master

1 files changed, 86 insertions, 9 deletions

diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts
index 8a2c91269..1894bb4df 100644
--- a/src/client/apis/gpt/GPT.ts
+++ b/src/client/apis/gpt/GPT.ts
@@ -16,10 +16,10 @@ enum GPTCallType {
     PRONUNCIATION = 'pronunciation',
     DRAW = 'draw',
     COLOR = 'color',
-    RUBRIC = 'rubric',
-    TYPE = 'type',
-    SUBSET = 'subset',
-    INFO = 'info',
+    RUBRIC = 'rubric', // needs to be filled in below
+    TYPE = 'type', // needs to be filled in below
+    SUBSET = 'subset', // needs to be filled in below
+    INFO = 'info', // needs to be filled in below
     TEMPLATE = 'template',
     VIZSUM = 'vizsum',
     VIZSUM2 = 'vizsum2',
@@ -42,7 +42,7 @@ const callTypeMap: { [type: string]: GPTCallOpts } = {
         model: 'gpt-4o',
         maxTokens: 2048,
         temp: 0.7,
-        prompt: 'Create a stack of flashcards out of this text with each question and answer labeled as question and answer. For some questions, ask "what is this image of" but tailored to stacks theme and the image and write a keyword that represents the image and label it "keyword". Otherwise, write none. Do not label each flashcard and do not include asterisks.',
+        prompt: 'Create a stack of at least 10 flashcards out of this text with each question and answer labeled as question and answer. Each flashcard should have a title that represents the question in just a few words and label it "title". For some questions, ask "what is this image of" but tailored to stacks theme and the image and write a keyword that represents the image and label it "keyword". Otherwise, write none. Do not label each flashcard and do not include asterisks.',
     },
 
     completion: { model: 'gpt-4-turbo', maxTokens: 256, temp: 0.5, prompt: "You are a helpful assistant. Answer the user's prompt." },
@@ -65,7 +65,12 @@ const callTypeMap: { [type: string]: GPTCallOpts } = {
         prompt: "The user is going to give you a list of descriptions. Each one is separated by `======` on either side. Descriptions will vary in length, so make sure to only separate when you see `======`. Sort them by the user's specifications. Make sure each description is only in the list once. Each item should be separated by `======`. Immediately afterward, surrounded by `------` on BOTH SIDES, provide some insight into your reasoning for the way you sorted (and mention nothing about the formatting details given in this description). It is VERY important that you format it exactly as described, ensuring the proper number of `=` and `-` (6 of each) and NO commas",
     },
     describe: { model: 'gpt-4-vision-preview', maxTokens: 2048, temp: 0, prompt: 'Describe these images in 3-5 words' },
-    flashcard: { model: 'gpt-4-turbo', maxTokens: 512, temp: 0.5, prompt: 'Make flashcards out of this text with each question and answer labeled as question and answer. Do not label each flashcard and do not include asterisks: ' },
+    flashcard: {
+        model: 'gpt-4-turbo',
+        maxTokens: 512,
+        temp: 0.5,
+        prompt: 'Make flashcards out of this text with each question and answer labeled as question and answer. Create a title for each question and asnwer that is labeled as "title".  Do not label each flashcard and do not include asterisks: ',
+    },
     chatcard: { model: 'gpt-4-turbo', maxTokens: 512, temp: 0.5, prompt: 'Answer the following question as a short flashcard response. Do not include a label.' },
     quiz: {
         model: 'gpt-4-turbo',
@@ -116,6 +121,39 @@ const callTypeMap: { [type: string]: GPTCallOpts } = {
         temp: 0.5,
         prompt: 'You will be coloring drawings. You will be given what the drawing is, then a list of descriptions for parts of the drawing. Based on each description, respond with the stroke and fill color that it should be. Follow the rules: 1. Avoid using black for stroke color 2. Make the stroke color 1-3 shades darker than the fill color 3. Use the same colors when possible. Format as {#abcdef #abcdef}, making sure theres a color for each description, and do not include any additional text.',
     },
+    type: {
+        model: 'gpt-4-turbo',
+        maxTokens: 1024,
+        temp: 0,
+        prompt: `I'm going to provide you with a question. 
+                 Based on the question, is the user asking you to 
+                     1. Assigns docs with tags(like star / heart etc)/labels, 
+                     2. Filter docs, 
+                     3. Provide information about a specific doc 
+                     4. Provide a specific doc based on a question/information 
+                     5. Provide general information 
+                     6. Put cards in a specific order. 
+                Answer with only the number for 2-6. For number one, provide the number (1) and the appropriate tag`,
+    },
+    subset: {
+        model: 'gpt-4-turbo',
+        maxTokens: 1024,
+        temp: 0,
+        prompt: "I'm going to give you a list of descriptions. Each one is separated by `======` on either side. Descriptions will vary in length, so make sure to only separate when you see `======`. Based on the question the user asks, provide a subset of the given descriptions that best matches the user's specifications. Make sure each description is only in the list once. Each item should be separated by `======`. Immediately afterward, surrounded by `------` on BOTH SIDES, provide some insight into your reasoning in the 2nd person (and mention nothing about the formatting details given in this description). It is VERY important that you format it exactly as described, ensuring the proper number of `=` and `-` (6 of each) and no commas",
+    },
+
+    info: {
+        model: 'gpt-4-turbo',
+        maxTokens: 1024,
+        temp: 0,
+        prompt: "Answer the user's question with a short (<100 word) response. If a particular document is selected I will provide that information (which may help with your response)",
+    },
+    rubric: {
+        model: 'gpt-4-turbo',
+        maxTokens: 1024,
+        temp: 0,
+        prompt: "BRIEFLY (<25 words) provide a definition for the following term. It will be used as a rubric to evaluate the user's understanding of the topic",
+    },
 };
 let lastCall = '';
 let lastResp = '';
@@ -126,8 +164,12 @@ let lastResp = '';
  * @returns AI Output
  */
 const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: string, dontCache?: boolean) => {
-    const inputText = [GPTCallType.SUMMARY, GPTCallType.FLASHCARD, GPTCallType.QUIZ, GPTCallType.STACK].includes(callType) ? inputTextIn + '.' : inputTextIn;
+    const inputText = inputTextIn + ([GPTCallType.SUMMARY, GPTCallType.FLASHCARD, GPTCallType.QUIZ, GPTCallType.STACK].includes(callType) ? '.' : '');
     const opts = callTypeMap[callType];
+    if (!opts) {
+        console.log('The query type:' + callType + ' requires a configuration.');
+        return 'Error connecting with API.';
+    }
     if (lastCall === inputText && dontCache !== true) return lastResp;
     try {
         lastCall = inputText;
@@ -244,6 +286,41 @@ const gptHandwriting = async (src: string): Promise<string> => {
     }
 };
 
+const gptDescribeImage = async (image: string): Promise<string> => {
+    try {
+        const response = await openai.chat.completions.create({
+            model: 'gpt-4o',
+            temperature: 0,
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Very briefly identify what this drawing is and list all the drawing elements and their location within the image. Do not include anything about the drawing style.`,
+                        },
+                        {
+                            type: 'image_url',
+                            image_url: {
+                                url: `${image}`,
+                                detail: 'low',
+                            },
+                        },
+                    ],
+                },
+            ],
+        });
+        if (response.choices[0].message.content) {
+            console.log('GPT DESCRIPTION', response.choices[0].message.content);
+            return response.choices[0].message.content;
+        }
+        return 'Unknown drawing';
+    } catch (err) {
+        console.log(err);
+        return 'Error connecting with API';
+    }
+};
+
 const gptDrawingColor = async (image: string, coords: string[]): Promise<string> => {
     try {
         const response = await openai.chat.completions.create({
@@ -271,11 +348,11 @@ const gptDrawingColor = async (image: string, coords: string[]): Promise<string>
         if (response.choices[0].message.content) {
             return response.choices[0].message.content;
         }
-        return 'Missing labels';
+        return 'Unknown drawing';
     } catch (err) {
         console.log(err);
         return 'Error connecting with API';
     }
 };
 
-export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDrawingColor };
+export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDescribeImage, gptDrawingColor };