1 files changed, 81 insertions, 3 deletions

diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts
index 8b101bfe3..88352110b 100644
--- a/src/client/apis/gpt/GPT.ts
+++ b/src/client/apis/gpt/GPT.ts
@@ -14,6 +14,8 @@ enum GPTCallType {
     DATA = 'data',
     STACK = 'stack',
     PRONUNCIATION = 'pronunciation',
+    DRAW = 'draw',
+    COLOR = 'color',
     RUBRIC = 'rubric',
     TYPE = 'type',
     SUBSET = 'subset',
@@ -101,6 +103,18 @@ const callTypeMap: { [type: string]: GPTCallOpts } = {
         temp: 0,
         prompt: "Answer the user's question with a short (<100 word) response. If a particular document is selected I will provide that information (which may help with your response)",
     },
+    draw: {
+        model: 'gpt-4o',
+        maxTokens: 1024,
+        temp: 0.8,
+        prompt: 'Given an item, a level of complexity from 1-10, and a size in pixels, generate a detailed and colored line drawing representation of it. Make sure every element has the stroke field filled out. More complex drawings will have much more detail and strokes. The drawing should be in SVG format with no additional text or comments. For path coordinates, make sure you format with a comma between numbers, like M100,200 C150,250 etc. The only supported commands are line, ellipse, circle, rect, polygon, and path with M, Q, C, and L so only use those.',
+    },
+    color: {
+        model: 'gpt-4o',
+        maxTokens: 1024,
+        temp: 0.5,
+        prompt: 'You will be coloring drawings. You will be given what the drawing is, then a list of descriptions for parts of the drawing. Based on each description, respond with the stroke and fill color that it should be. Follow the rules: 1. Avoid using black for stroke color 2. Make the stroke color 1-3 shades darker than the fill color 3. Use the same colors when possible. Format as {#abcdef #abcdef}, making sure theres a color for each description, and do not include any additional text.',
+    },
 };
 
 let lastCall = '';
@@ -111,10 +125,10 @@ let lastResp = '';
  * @param inputText Text to process
  * @returns AI Output
  */
-const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: any) => {
+const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: any, dontCache?: boolean) => {
     const inputText = [GPTCallType.SUMMARY, GPTCallType.FLASHCARD, GPTCallType.QUIZ, GPTCallType.STACK].includes(callType) ? inputTextIn + '.' : inputTextIn;
     const opts: GPTCallOpts = callTypeMap[callType];
-    if (lastCall === inputText) return lastResp;
+    if (lastCall === inputText && dontCache !== true) return lastResp;
     try {
         lastCall = inputText;
 
@@ -199,5 +213,69 @@ const gptImageLabel = async (src: string, prompt: string): Promise<string> => {
         return 'Error connecting with API';
     }
 };
+const gptHandwriting = async (src: string): Promise<string> => {
+    try {
+        const response = await openai.chat.completions.create({
+            model: 'gpt-4o',
+            temperature: 0,
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        { type: 'text', text: 'What is this does this handwriting say. Only return the text' },
+                        {
+                            type: 'image_url',
+                            image_url: {
+                                url: `${src}`,
+                                detail: 'low',
+                            },
+                        },
+                    ],
+                },
+            ],
+        });
+        if (response.choices[0].message.content) {
+            return response.choices[0].message.content;
+        }
+        return 'Missing labels';
+    } catch (err) {
+        console.log(err);
+        return 'Error connecting with API';
+    }
+};
+
+const gptDrawingColor = async (image: string, coords: string[]): Promise<string> => {
+    try {
+        const response = await openai.chat.completions.create({
+            model: 'gpt-4o',
+            temperature: 0,
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Identify what the drawing in the image represents in 1-5 words. Then, given a list of a list of coordinates, where each list is the coordinates for one stroke of the drawing, determine which part of the drawing it is. Return just what the item it is, followed by ~~~ then only your descriptions in a list like [description, description, ...]. Here are the coordinates: ${coords}`,
+                        },
+                        {
+                            type: 'image_url',
+                            image_url: {
+                                url: `${image}`,
+                                detail: 'low',
+                            },
+                        },
+                    ],
+                },
+            ],
+        });
+        if (response.choices[0].message.content) {
+            return response.choices[0].message.content;
+        }
+        return 'Missing labels';
+    } catch (err) {
+        console.log(err);
+        return 'Error connecting with API';
+    }
+};
 
-export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding };
+export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDrawingColor };