aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/client/apis/gpt/GPT.ts50
-rw-r--r--src/client/views/collections/CollectionCardDeckView.tsx72
-rw-r--r--src/client/views/collections/collectionSchema/CollectionSchemaView.tsx2
-rw-r--r--src/client/views/pdf/PDFViewer.tsx2
4 files changed, 111 insertions, 15 deletions
diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts
index 1f50ad0b8..7c53ee83e 100644
--- a/src/client/apis/gpt/GPT.ts
+++ b/src/client/apis/gpt/GPT.ts
@@ -5,8 +5,8 @@ enum GPTCallType {
SUMMARY = 'summary',
COMPLETION = 'completion',
EDIT = 'edit',
- // MERMAID='mermaid'
- SORT = 'sort'
+ SORT = 'sort',
+ DESCRIBE = 'describe'
}
type GPTCallOpts = {
@@ -20,7 +20,10 @@ const callTypeMap: { [type: string]: GPTCallOpts } = {
summary: { model: 'gpt-3.5-turbo-instruct', maxTokens: 256, temp: 0.5, prompt: 'Summarize this text in simpler terms: ' },
edit: { model: 'gpt-3.5-turbo-instruct', maxTokens: 256, temp: 0.5, prompt: 'Reword this: ' },
completion: { model: 'gpt-3.5-turbo-instruct', maxTokens: 256, temp: 0.5, prompt: '' },
- sort:{model:'gpt-4-turbo',maxTokens:2048,temp:0,prompt:"I'm going to give you a list of strings. Sort them into lists by shared content. The format of should be: '[string1, string2], [string 3, string4][string5, string6]' where the actual numbers of strings in each category / total categorys is arbritray and up to your interpretation"}
+ sort:{model:'gpt-4-turbo',maxTokens:2048,temp:0.5,prompt:"I'm going to give you a list of strings. Sort them into lists by shared content. The format of should be: [string1, string2], [string 3, string4], [string5, string6] where the actual numbers of strings in each category / total categorys is arbritray and up to your interpretation"},
+ describe:{model:'gpt-4-vision-preview',maxTokens:2048,temp:0,prompt:"Describe these images in 3-5 words"},
+
+
};
@@ -30,6 +33,7 @@ const callTypeMap: { [type: string]: GPTCallOpts } = {
* @param inputText Text to process
* @returns AI Output
*/
+
const gptAPICall = async (inputText: string, callType: GPTCallType) => {
if (callType === GPTCallType.SUMMARY) inputText += '.';
const opts: GPTCallOpts = callTypeMap[callType];
@@ -59,6 +63,44 @@ const gptAPICall = async (inputText: string, callType: GPTCallType) => {
}
};
+
+const gptImageLabel = async (imgUrl: string): Promise<string> => {
+ try {
+ const configuration: ClientOptions = {
+ apiKey: "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",
+ dangerouslyAllowBrowser: true,
+ };
+
+ const openai = new OpenAI(configuration);
+ const response = await openai.chat.completions.create({
+ model: 'gpt-4o',
+ messages: [
+ {
+ role: 'user',
+ content: [
+ { type: 'text', text: 'Describe this image in 3-5 words' },
+ {
+ type: 'image_url',
+ image_url: {
+ url: `${imgUrl}`,
+ },
+ },
+ ],
+ },
+ ],
+ });
+ if (response.choices[0].message.content) {
+ return response.choices[0].message.content;
+ } else {
+ return ':(';
+ }
+ } catch (err) {
+ console.log(err);
+ return 'Error connecting with API';
+ }
+};
+
+
const gptImageCall = async (prompt: string, n?: number) => {
try {
const configuration: ClientOptions = {
@@ -80,4 +122,4 @@ const gptImageCall = async (prompt: string, n?: number) => {
}
};
-export { gptAPICall, gptImageCall, GPTCallType }; \ No newline at end of file
+export { gptAPICall, gptImageCall, gptImageLabel, GPTCallType }; \ No newline at end of file
diff --git a/src/client/views/collections/CollectionCardDeckView.tsx b/src/client/views/collections/CollectionCardDeckView.tsx
index 561b9f806..e7e2cba4a 100644
--- a/src/client/views/collections/CollectionCardDeckView.tsx
+++ b/src/client/views/collections/CollectionCardDeckView.tsx
@@ -18,9 +18,9 @@ import { DocumentType } from '../../documents/DocumentTypes';
import { forEach } from 'lodash';
import { SnappingManager } from '../../util/SnappingManager';
import { List } from '../../../fields/List';
-import { gptAPICall } from '../../apis/gpt/GPT';
+import { gptAPICall, gptImageLabel } from '../../apis/gpt/GPT';
import { GPTCallType } from '../../apis/gpt/GPT';
-import { ImageField } from '../../../fields/URLField';
+import { ImageField, PdfField, URLField } from '../../../fields/URLField';
@observer
export class CollectionCardView extends CollectionSubView() {
@@ -253,7 +253,7 @@ export class CollectionCardView extends CollectionSubView() {
return this.sort(sorted, 'custom', desc);
case 'chat':
- this.smartSort()
+ this.getImageDesc()
docs = this.myChildLayoutPairs;
return {docs}
@@ -493,28 +493,83 @@ export class CollectionCardView extends CollectionSubView() {
// const
- const imgurlperchance = Cast(this.childDocs[1].data, ImageField, null)?.url;
+ const imgurlperchance = Cast(this.childDocs[1].data, ImageField, null)?.url.href;
+ const perchance = this.imageUrlToBase64(imgurlperchance)
+ // const imgurlperchance = Cast(this.childDocs[1].data, PdfField, null)?.url.;
- console.log('Print Front of cards: ' + RTFCast(DocCast(this.dataDoc[this.fieldKey + '_0']).text)?.Text);
- console.log('Print Back of cards: ' + RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text);
+ // console.log('Print Front of cards: ' + RTFCast(DocCast(this.dataDoc[this.fieldKey + '_0']).text)?.Text);
+ // console.log('Print Back of cards: ' + RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text);
+
+ // const perchance = StrCast(this.childDocs[0].text);
+
+
+
+ // const pdf = (StrCast(this.myChildLayoutPairs[0].layout.text)); //pdf
+ // const queryText = (RTFCast(this.myChildLayoutPairs[0].layout.text)); //everything else
- // const queryText = (RTFCast(this.myChildLayoutPairs[0].layout.text)).Text;
// const queryText = RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text;
- console.log(imgurlperchance ?? "sad")
+ console.log(perchance ?? "sad")
}
childPairStringList () {
let string = ""
for (let i=0; i < this.myChildLayoutPairs.length; i++){
+
+
+ let text = StrCast((RTFCast(this.myChildLayoutPairs[i].layout.text)).Text)
string += StrCast((RTFCast(this.myChildLayoutPairs[i].layout.text)).Text) + ", "
+ this.textToDoc.has()
}
return string
}
@observable isLoading = false
+ imageUrlToBase64 = async (imageUrl: string): Promise<string> => {
+ try {
+ const response = await fetch(imageUrl);
+ const blob = await response.blob();
+
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.readAsDataURL(blob);
+ reader.onloadend = () => resolve(reader.result as string);
+ reader.onerror = error => reject(error);
+ });
+ } catch (error) {
+ console.error('Error:', error);
+ throw error;
+ }
+ };
+
+ textToDoc = new Map<string, Doc>()
+ gptProccessedImages = new Set<Doc>()
+
+ @action async getImageDesc(){
+ const images = this.childDocs.filter(d => d.type == DocumentType.IMG).filter(d=> !this.gptProccessedImages.has(d));
+ images.map(doc => {
+ let href = (doc['data'] as URLField).url.href;
+ let hrefParts = href.split('.');
+ let hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
+ return this.imageUrlToBase64(hrefComplete).then(hrefBase64 => {
+ return gptImageLabel(hrefBase64).then(response => {
+ this.textToDoc.set(response, doc)
+ this.gptProccessedImages.add(doc)
+ console.log(response);
+
+ });
+ });
+ });
+ }
+
+
+
+ //a set of all the images that have already been given desc
+ //a map from the text to the Doc (for everything)
+
+
@action async smartSort() {
this.isLoading = true;
console.log("loading")
@@ -536,7 +591,6 @@ export class CollectionCardView extends CollectionSubView() {
// If GPT call failed
console.error('GPT call failed');
} else if (res != null) {
- // If GPT call succeeded, set htmlCode;;; TODO: check if valid html
console.log(res)
}
this.isLoading = false;
diff --git a/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx b/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx
index 6a956f2ac..731ea1235 100644
--- a/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx
+++ b/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx
@@ -51,7 +51,7 @@ export const FInfotoColType: { [key: string]: ColumnType } = {
enumeration: ColumnType.Enumeration,
};
-const defaultColumnKeys: string[] = ['title', 'type', 'author', 'author_date', 'text'];
+const defaultColumnKeys: string[] = ['title', 'type', 'author', 'author_date', 'text'];
@observer
export class CollectionSchemaView extends CollectionSubView() {
diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx
index aaff2a342..b87e29d4e 100644
--- a/src/client/views/pdf/PDFViewer.tsx
+++ b/src/client/views/pdf/PDFViewer.tsx
@@ -30,7 +30,7 @@ const _global = (window /* browser */ || global) /* node */ as any;
//pdfjsLib.GlobalWorkerOptions.workerSrc = `/assets/pdf.worker.js`;
// The workerSrc property shall be specified.
-Pdfjs.GlobalWorkerOptions.workerSrc = 'https://unpkg.com/pdfjs-dist@4.1.392/build/pdf.worker.mjs';
+Pdfjs.GlobalWorkerOptions.workerSrc = 'https://unpkg.com/pdfjs-dist@4.2.67/build/pdf.worker.mjs';
interface IViewerProps extends FieldViewProps {
pdfBox: PDFBox;
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-08 09:35:37 +0000