created image regeneration with dialogue

1 files changed, 37 insertions, 2 deletions

diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts
index 03380e4d6..9241eb120 100644
--- a/src/client/apis/gpt/GPT.ts
+++ b/src/client/apis/gpt/GPT.ts
@@ -249,6 +249,41 @@ const gptHandwriting = async (src: string): Promise<string> => {
     }
 };
 
+const gptDescribeImage = async (image: string): Promise<string> => {
+    try {
+        const response = await openai.chat.completions.create({
+            model: 'gpt-4o',
+            temperature: 0,
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Identify what this drawing is, naming as many elements and their location in the drawing as possible`,
+                        },
+                        {
+                            type: 'image_url',
+                            image_url: {
+                                url: `${image}`,
+                                detail: 'low',
+                            },
+                        },
+                    ],
+                },
+            ],
+        });
+        if (response.choices[0].message.content) {
+            console.log('GPT DESCRIPTION', response.choices[0].message.content);
+            return response.choices[0].message.content;
+        }
+        return 'Unknown drawing';
+    } catch (err) {
+        console.log(err);
+        return 'Error connecting with API';
+    }
+};
+
 const gptDrawingColor = async (image: string, coords: string[]): Promise<string> => {
     try {
         const response = await openai.chat.completions.create({
@@ -276,11 +311,11 @@ const gptDrawingColor = async (image: string, coords: string[]): Promise<string>
         if (response.choices[0].message.content) {
             return response.choices[0].message.content;
         }
-        return 'Missing labels';
+        return 'Unknown drawing';
     } catch (err) {
         console.log(err);
         return 'Error connecting with API';
     }
 };
 
-export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDrawingColor };
+export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDescribeImage, gptDrawingColor };