ab to do some silly thing

4 files changed, 111 insertions, 15 deletions

diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts
index 1f50ad0b8..7c53ee83e 100644
--- a/src/client/apis/gpt/GPT.ts
+++ b/src/client/apis/gpt/GPT.ts
@@ -5,8 +5,8 @@ enum GPTCallType {
     SUMMARY = 'summary',
     COMPLETION = 'completion',
     EDIT = 'edit',
-    // MERMAID='mermaid'
-    SORT = 'sort'
+    SORT = 'sort',
+    DESCRIBE = 'describe'
 }
 
 type GPTCallOpts = {
@@ -20,7 +20,10 @@ const callTypeMap: { [type: string]: GPTCallOpts } = {
     summary: { model: 'gpt-3.5-turbo-instruct', maxTokens: 256, temp: 0.5, prompt: 'Summarize this text in simpler terms: ' },
     edit: { model: 'gpt-3.5-turbo-instruct', maxTokens: 256, temp: 0.5, prompt: 'Reword this: ' },
     completion: { model: 'gpt-3.5-turbo-instruct', maxTokens: 256, temp: 0.5, prompt: '' },
-    sort:{model:'gpt-4-turbo',maxTokens:2048,temp:0,prompt:"I'm going to give you a list of strings. Sort them into lists by shared content. The format of should be: '[string1, string2], [string 3, string4][string5, string6]' where the actual numbers of strings in each category / total categorys is arbritray and up to your interpretation"}
+    sort:{model:'gpt-4-turbo',maxTokens:2048,temp:0.5,prompt:"I'm going to give you a list of strings. Sort them into lists by shared content. The format of should be: [string1, string2], [string 3, string4], [string5, string6] where the actual numbers of strings in each category / total categorys is arbritray and up to your interpretation"},
+    describe:{model:'gpt-4-vision-preview',maxTokens:2048,temp:0,prompt:"Describe these images in 3-5 words"},
+
+
 };
 
 
@@ -30,6 +33,7 @@ const callTypeMap: { [type: string]: GPTCallOpts } = {
  * @param inputText Text to process
  * @returns AI Output
  */
+
 const gptAPICall = async (inputText: string, callType: GPTCallType) => {
     if (callType === GPTCallType.SUMMARY) inputText += '.';
     const opts: GPTCallOpts = callTypeMap[callType];
@@ -59,6 +63,44 @@ const gptAPICall = async (inputText: string, callType: GPTCallType) => {
     }
 };
 
+
+const gptImageLabel = async (imgUrl: string): Promise<string> => {
+    try {
+        const configuration: ClientOptions = {
+            apiKey: "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",
+            dangerouslyAllowBrowser: true,
+        };
+
+        const openai = new OpenAI(configuration);
+        const response = await openai.chat.completions.create({
+            model: 'gpt-4o',
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        { type: 'text', text: 'Describe this image in 3-5 words' },
+                        {
+                            type: 'image_url',
+                            image_url: {
+                                url: `${imgUrl}`,
+                            },
+                        },
+                    ],
+                },
+            ],
+        });
+        if (response.choices[0].message.content) {
+            return response.choices[0].message.content;
+        } else {
+            return ':(';
+        }
+    } catch (err) {
+        console.log(err);
+        return 'Error connecting with API';
+    }
+};
+
+
 const gptImageCall = async (prompt: string, n?: number) => {
     try {
         const configuration: ClientOptions = {
@@ -80,4 +122,4 @@ const gptImageCall = async (prompt: string, n?: number) => {
     }
 };
 
-export { gptAPICall, gptImageCall, GPTCallType };
\ No newline at end of file
+export { gptAPICall, gptImageCall, gptImageLabel, GPTCallType };
\ No newline at end of file
diff --git a/src/client/views/collections/CollectionCardDeckView.tsx b/src/client/views/collections/CollectionCardDeckView.tsx
index 561b9f806..e7e2cba4a 100644
--- a/src/client/views/collections/CollectionCardDeckView.tsx
+++ b/src/client/views/collections/CollectionCardDeckView.tsx
@@ -18,9 +18,9 @@ import { DocumentType } from '../../documents/DocumentTypes';
 import { forEach } from 'lodash';
 import { SnappingManager } from '../../util/SnappingManager';
 import { List } from '../../../fields/List';
-import { gptAPICall } from '../../apis/gpt/GPT';
+import { gptAPICall, gptImageLabel } from '../../apis/gpt/GPT';
 import { GPTCallType } from '../../apis/gpt/GPT';
-import { ImageField } from '../../../fields/URLField';
+import { ImageField, PdfField, URLField } from '../../../fields/URLField';
 
 @observer
 export class CollectionCardView extends CollectionSubView() {
@@ -253,7 +253,7 @@ export class CollectionCardView extends CollectionSubView() {
                 return this.sort(sorted, 'custom', desc);
 
             case 'chat':
-                this.smartSort()
+                this.getImageDesc()
                 docs = this.myChildLayoutPairs;
                 return {docs}
 
@@ -493,28 +493,83 @@ export class CollectionCardView extends CollectionSubView() {
 
 
         // const
-        const imgurlperchance = Cast(this.childDocs[1].data, ImageField, null)?.url;
+        const imgurlperchance = Cast(this.childDocs[1].data, ImageField, null)?.url.href;
+        const perchance = this.imageUrlToBase64(imgurlperchance)
+        // const imgurlperchance = Cast(this.childDocs[1].data, PdfField, null)?.url.;
 
-        console.log('Print Front of cards: ' + RTFCast(DocCast(this.dataDoc[this.fieldKey + '_0']).text)?.Text);
-        console.log('Print Back of cards: ' + RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text);
 
+        // console.log('Print Front of cards: ' + RTFCast(DocCast(this.dataDoc[this.fieldKey + '_0']).text)?.Text);
+        // console.log('Print Back of cards: ' + RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text);
+
+        // const perchance = StrCast(this.childDocs[0].text);
+
+
+
+        // const pdf = (StrCast(this.myChildLayoutPairs[0].layout.text)); //pdf
+        // const queryText = (RTFCast(this.myChildLayoutPairs[0].layout.text)); //everything else
 
-        // const queryText = (RTFCast(this.myChildLayoutPairs[0].layout.text)).Text;
         // const queryText = RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text;
 
-        console.log(imgurlperchance ?? "sad")
+        console.log(perchance ?? "sad")
     }
 
     childPairStringList () {
         let string = ""
         for (let i=0; i < this.myChildLayoutPairs.length; i++){
+
+            
+            let text = StrCast((RTFCast(this.myChildLayoutPairs[i].layout.text)).Text)
             string += StrCast((RTFCast(this.myChildLayoutPairs[i].layout.text)).Text) + ", "
+            this.textToDoc.has()
         }
         return string
     }
 
     @observable isLoading = false
 
+    imageUrlToBase64 = async (imageUrl: string): Promise<string> => {
+        try {
+            const response = await fetch(imageUrl);
+            const blob = await response.blob();
+    
+            return new Promise((resolve, reject) => {
+                const reader = new FileReader();
+                reader.readAsDataURL(blob); 
+                reader.onloadend = () => resolve(reader.result as string); 
+                reader.onerror = error => reject(error); 
+            });
+        } catch (error) {
+            console.error('Error:', error);
+            throw error; 
+        }
+    };
+
+    textToDoc = new Map<string, Doc>()
+    gptProccessedImages = new Set<Doc>()
+
+    @action async getImageDesc(){
+        const images = this.childDocs.filter(d => d.type == DocumentType.IMG).filter(d=> !this.gptProccessedImages.has(d));
+        images.map(doc => {
+            let href = (doc['data'] as URLField).url.href;
+            let hrefParts = href.split('.');
+            let hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
+            return this.imageUrlToBase64(hrefComplete).then(hrefBase64 => {
+                return gptImageLabel(hrefBase64).then(response => {
+                    this.textToDoc.set(response, doc)
+                    this.gptProccessedImages.add(doc)
+                    console.log(response);
+                    
+                });
+            });
+        });
+    }
+
+   
+
+    //a set of all the images that have already been given desc
+    //a map from the text to the Doc (for everything)
+    
+
     @action async smartSort() {
         this.isLoading = true;
         console.log("loading")
@@ -536,7 +591,6 @@ export class CollectionCardView extends CollectionSubView() {
             // If GPT call failed
             console.error('GPT call failed');
         } else if (res != null) {
-            // If GPT call succeeded, set htmlCode;;; TODO: check if valid html
            console.log(res)
         }
         this.isLoading = false;
diff --git a/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx b/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx
index 6a956f2ac..731ea1235 100644
--- a/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx
+++ b/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx
@@ -51,7 +51,7 @@ export const FInfotoColType: { [key: string]: ColumnType } = {
     enumeration: ColumnType.Enumeration,
 };
 
-const defaultColumnKeys: string[] = ['title', 'type', 'author', 'author_date', 'text'];
+const  defaultColumnKeys: string[] = ['title', 'type', 'author', 'author_date', 'text'];
 
 @observer
 export class CollectionSchemaView extends CollectionSubView() {
diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx
index aaff2a342..b87e29d4e 100644
--- a/src/client/views/pdf/PDFViewer.tsx
+++ b/src/client/views/pdf/PDFViewer.tsx
@@ -30,7 +30,7 @@ const _global = (window /* browser */ || global) /* node */ as any;
 
 //pdfjsLib.GlobalWorkerOptions.workerSrc = `/assets/pdf.worker.js`;
 // The workerSrc property shall be specified.
-Pdfjs.GlobalWorkerOptions.workerSrc = 'https://unpkg.com/pdfjs-dist@4.1.392/build/pdf.worker.mjs';
+Pdfjs.GlobalWorkerOptions.workerSrc = 'https://unpkg.com/pdfjs-dist@4.2.67/build/pdf.worker.mjs';
 
 interface IViewerProps extends FieldViewProps {
     pdfBox: PDFBox;