diff options
| -rw-r--r-- | src/.DS_Store | bin | 10244 -> 14340 bytes | |||
| -rw-r--r-- | src/client/apis/gpt/GPT.ts | 32 | ||||
| -rw-r--r-- | src/client/views/InkTranscription.tsx | 38 |
3 files changed, 65 insertions, 5 deletions
diff --git a/src/.DS_Store b/src/.DS_Store Binary files differindex 75cff7b55..532210ecf 100644 --- a/src/.DS_Store +++ b/src/.DS_Store diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 05007960d..2081f4a5e 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -149,5 +149,35 @@ const gptImageLabel = async (src: string): Promise<string> => { return 'Error connecting with API'; } }; +const gptHandwriting = async (src: string): Promise<string> => { + try { + const response = await openai.chat.completions.create({ + model: 'gpt-4o', + temperature: 0, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'What is this does this handwriting say. Only return the text' }, + { + type: 'image_url', + image_url: { + url: `${src}`, + detail: 'low', + }, + }, + ], + }, + ], + }); + if (response.choices[0].message.content) { + return response.choices[0].message.content; + } + return 'Missing labels'; + } catch (err) { + console.log(err); + return 'Error connecting with API'; + } +}; -export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding }; +export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting }; diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index a1c693e32..f07f7c0a7 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -14,7 +14,9 @@ import { DocumentView } from './nodes/DocumentView'; import { Number } from 'mongoose'; import { NumberArray } from 'd3'; import { ImageField } from '../../fields/URLField'; - +import { gptHandwriting } from '../apis/gpt/GPT'; +import * as fs from 'fs'; +import { URLField } from '../../fields/URLField'; /** * Class component that handles inking in writing mode */ @@ -247,13 +249,25 @@ export class InkTranscription extends React.Component { if (this.currGroup && text) { DocumentView.getDocumentView(this.currGroup)?.ComponentView?.updateIcon?.(); - console.log(this.currGroup.icon); this.currGroup.transcription = text; this.currGroup.title = text; let image = await this.getIcon(); - console.log(this.currGroup.icon, image); + const pathname = image?.url.href as string; + const { href } = (image as URLField).url; + const hrefParts = href.split('.'); + const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; + let response; + try { + const hrefBase64 = await this.imageUrlToBase64(hrefComplete); + response = await gptHandwriting(hrefBase64); + console.log(response); + } catch (error) { + console.log('bad things have happened'); + } + const textBoxText = 'iink: ' + text + '\n' + '\n' + 'ChatGPT: ' + response; if (!this.currGroup.hasTextBox) { - const newDoc = Docs.Create.TextDocument(text, { title: '', x: this.currGroup.x as number, y: (this.currGroup.y as number) + (this.currGroup.height as number) }); + const newDoc = Docs.Create.TextDocument(textBoxText, { title: '', x: this.currGroup.x as number, y: (this.currGroup.y as number) + (this.currGroup.height as number) }); + newDoc.width = 100; this.collectionFreeForm?.addDocument(newDoc); this.currGroup.hasTextBox = true; } @@ -270,6 +284,22 @@ export class InkTranscription extends React.Component { } return undefined; } + imageUrlToBase64 = async (imageUrl: string): Promise<string> => { + try { + const response = await fetch(imageUrl); + const blob = await response.blob(); + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = error => reject(error); + }); + } catch (error) { + console.error('Error:', error); + throw error; + } + }; /** * Creates the ink grouping once the user leaves the writing mode. |