import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Tooltip } from '@mui/material'; import { runInAction } from 'mobx'; import * as React from 'react'; import { returnFalse, setupMoveUpEvents } from '../../ClientUtils'; import { emptyFunction, unimplementedFunction } from '../../Utils'; import { Doc, DocListCast, Opt } from '../../fields/Doc'; import { DocData } from '../../fields/DocSymbols'; import { List } from '../../fields/List'; import { NumCast, StrCast } from '../../fields/Types'; import { Networking } from '../Network'; import { GPTCallType, gptAPICall } from '../apis/gpt/GPT'; import { Docs } from '../documents/Documents'; import { ContextMenu } from './ContextMenu'; import { ContextMenuProps } from './ContextMenuItem'; import { StyleProp } from './StyleProp'; import './StyleProviderQuiz.scss'; import { DocumentViewProps } from './nodes/DocumentView'; import { FieldViewProps } from './nodes/FieldView'; import { ImageBox } from './nodes/ImageBox'; import { ImageUtility } from './nodes/imageEditor/imageEditorUtils/ImageHandler'; import { AnchorMenu } from './pdf/AnchorMenu'; export namespace styleProviderQuiz { enum quizMode { SMART = 'smart', NORMAL = 'normal', NONE = 'none', } async function selectUrlToBase64(blob: Blob): Promise { try { return new Promise((resolve, reject) => { const reader = new FileReader(); reader.readAsDataURL(blob); reader.onloadend = () => resolve(reader.result as string); reader.onerror = error => reject(error); }); } catch (error) { console.error('Error:', error); throw error; } } /** * Creates label boxes over text on the image to be filled in. * @param boxes * @param texts */ async function createBoxes(img: ImageBox, boxes: number[][][], texts: string[]) { img.Document.quizBoxes = new List([]); for (let i = 0; i < boxes.length; i++) { const coords = boxes[i] ? boxes[i] : []; const width = coords[1][0] - coords[0][0]; const height = coords[2][1] - coords[0][1]; const text = texts[i]; const newCol = Docs.Create.LabelDocument({ _width: width, _height: height, _layout_fitWidth: true, title: '', }); const scaling = 1 / (img._props.NativeDimScaling?.() || 1); newCol.x = coords[0][0] + NumCast(img.marqueeref.current?.left) * scaling; newCol.y = coords[0][1] + NumCast(img.marqueeref.current?.top) * scaling; newCol.zIndex = 1000; newCol.forceActive = true; newCol.quiz = text; newCol['$' + Doc.LayoutDataKey(newCol) + '_transform'] = 'none'; Doc.AddDocToList(img.Document, 'quizBoxes', newCol); img.addDocument(newCol); // img._loading = false; } } /** * Calls backend to find any text on an image. Gets the text and the * coordinates of the text and creates label boxes at those locations. * @param quiz * @param i */ async function pushInfo(imgBox: ImageBox, quiz: quizMode, i?: string) { imgBox.Document._quizMode = quiz; const quizBoxes = DocListCast(imgBox.Document.quizBoxes); if (!quizBoxes.length) { runInAction(() => (imgBox.Loading = true)); const response = (await Networking.PostToServer('/labels', { file: i ? i : imgBox.paths[0], drag: i ? 'drag' : 'full', smart: quiz })) as { result: string }; const replacedResponse = response.result.replace(/ '/g, '"').replace(/',/g, '",').replace(/\{'/g, '{"').replace(/':/g, '":').replace(/'\]/g, '"]').replace(/\['/g, '["'); const parsedResponse = JSON.parse(replacedResponse) as { boxes: number[][][]; text: string[] }; if (parsedResponse.boxes.length != 0) { createBoxes(imgBox, parsedResponse.boxes, parsedResponse.text); } runInAction(() => (imgBox.Loading = false)); } else quizBoxes.forEach(box => (box.hidden = false)); } async function createCanvas(img: ImageBox) { const canvas = document.createElement('canvas'); const scaling = 1 / (img._props.NativeDimScaling?.() || 1); const w = AnchorMenu.Instance.marqueeWidth * scaling; const h = AnchorMenu.Instance.marqueeHeight * scaling; canvas.width = w; canvas.height = h; const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions if (ctx) { img.imageRef && ctx.drawImage(img.imageRef, NumCast(img.marqueeref.current?.left) * scaling, NumCast(img.marqueeref.current?.top) * scaling, w, h, 0, 0, w, h); } const blob = await ImageUtility.canvasToBlob(canvas); return selectUrlToBase64(blob); } // /** // * Create flashcards from an image. // */ // async function makeFlashcardsForImage(img: ImageBox) { // img.Loading = true; // try { // const hrefBase64 = await createCanvas(img); // const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this image with each question and answer labeled as "question" and "answer". Do not label each flashcard and do not include asterisks: '); // AnchorMenu.Instance.transferToFlashcard(response, NumCast(img.layoutDoc.x), NumCast(img.layoutDoc.y)); // } catch (error) { // console.log('Error', error); // } // img.Loading = false; // } /** * Calls the createCanvas and pushInfo methods to convert the * image to a form that can be passed to GPT and find the locations * of the text. */ async function makeLabels(img: ImageBox) { try { const hrefBase64 = await createCanvas(img); pushInfo(img, quizMode.NORMAL, hrefBase64); } catch (error) { console.log('Error', error); } } /** * Determines whether two words should be considered * the same, allowing minor typos. * @param str1 * @param str2 * @returns */ function levenshteinDistance(str1: string, str2: string) { const len1 = str1.length; const len2 = str2.length; const dp = Array.from(Array(len1 + 1), () => Array(len2 + 1).fill(0)); if (len1 === 0) return len2; if (len2 === 0) return len1; for (let i = 0; i <= len1; i++) dp[i][0] = i; for (let j = 0; j <= len2; j++) dp[0][j] = j; for (let i = 1; i <= len1; i++) { for (let j = 1; j <= len2; j++) { const cost = str1[i - 1] === str2[j - 1] ? 0 : 1; dp[i][j] = Math.min( dp[i - 1][j] + 1, // deletion dp[i][j - 1] + 1, // insertion dp[i - 1][j - 1] + cost // substitution ); } } return dp[len1][len2]; } /** * Different algorithm for determining string similarity. * @param str1 * @param str2 * @returns */ function jaccardSimilarity(str1: string, str2: string) { const set1 = new Set(str1.split(' ')); const set2 = new Set(str2.split(' ')); const intersection = new Set([...set1].filter(x => set2.has(x))); const union = new Set([...set1, ...set2]); return intersection.size / union.size; } /** * Averages the jaccardSimilarity and levenshteinDistance scores * to determine string similarity for the labelboxes answers and * the users response. * @param str1 * @param str2 * @returns */ function stringSimilarity(str1: string, str2: string) { const levenshteinDist = levenshteinDistance(str1, str2); const levenshteinScore = 1 - levenshteinDist / Math.max(str1.length, str2.length); const jaccardScore = jaccardSimilarity(str1, str2); // Combine the scores with a higher weight on Jaccard similarity return 0.5 * levenshteinScore + 0.5 * jaccardScore; } /** * Returns whether two strings are similar * @param input * @param target * @returns */ function compareWords(input: string, target: string) { const distance = stringSimilarity(input.toLowerCase(), target.toLowerCase()); return distance >= 0.7; } /** * GPT returns a hex color for what color the label box should be based on * the correctness of the users answer. * @param inputString * @returns */ function extractHexAndSentences(inputString: string) { // Regular expression to match a hexadecimal number at the beginning followed by a period and sentences const regex = /^#([0-9A-Fa-f]+)\.\s*(.+)$/; const match = inputString.replace('\n', ' ').match(regex); if (match) { const hexNumber = match[1]; const sentences = match[2].trim(); return { hexNumber, sentences }; } else { return { error: 'The input string does not match the expected format.' }; } } function imgQuizBoxes(img: ImageBox) { return DocListCast(img.Document.quizBoxes); } function imgQuizMode(img: ImageBox) { return StrCast(img.Document._quizMode); } /** * Check whether the contents of the label boxes on an image are correct. */ function check(img: ImageBox) { //this._loading = true; imgQuizBoxes(img).forEach(async doc => { const input = StrCast(doc.$title); if (imgQuizMode(img) == quizMode.SMART && input) { const questionText = 'Question: What was labeled in this image?'; const rubricText = ' Rubric: ' + StrCast(doc.quiz); const queryText = questionText + ' UserAnswer: ' + input + '. ' + rubricText + '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."'; const response = await gptAPICall(queryText, GPTCallType.QUIZDOC); const hexSent = extractHexAndSentences(response); doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); doc.backgroundColor = '#' + hexSent.hexNumber; } else { const match = compareWords(input, StrCast(doc.quiz).trim()); if (input) { doc.backgroundColor = match ? '#11c249' : '#eb2d2d'; } } }); //this._loading = false; } function redo(img: ImageBox) { imgQuizBoxes(img).forEach(doc => { doc.$title = ''; doc.$backgroundColor = '#e4e4e4'; }); } /** * Get rid of all the label boxes on the images. */ function exitQuizMode(img: ImageBox) { img.Document._quizMode = quizMode.NONE; DocListCast(img.Document.quizBoxes).forEach(box => { box.hidden = true; }); } export function quizStyleProvider(doc: Opt, props: Opt, property: string) { const editLabelAnswer = (qdoc: Doc) => { // when click the pencil, set the text to the quiz content. when click off, set the quiz text to that and set textbox to nothing. if (!qdoc._editLabel) { qdoc.title = StrCast(qdoc.quiz); } else { qdoc.quiz = StrCast(qdoc.title); qdoc.title = ''; } qdoc._editLabel = !qdoc._editLabel; }; const editAnswer = (qdoc: Opt) => { return ( {qdoc?._editLabel ? 'save' : 'edit correct answer'} }>
setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => qdoc && editLabelAnswer(qdoc))}>
); }; const answerIcon = (qdoc: Opt) => { return ( {StrCast(qdoc?.quiz ?? '')} }>
); }; const checkIcon = (img: ImageBox) => ( Check}>
check(img)}>
); const redoIcon = (img: ImageBox) => ( Redo}>
redo(img)}>
); const imgBox = props?.DocumentView?.().ComponentView as ImageBox; switch (property) { case StyleProp.Decorations: { if (doc?.quiz) { // this should only be set on Labels that are part of an image quiz return ( <> {editAnswer(doc?.[DocData])} {answerIcon(doc)} ); } else if (imgBox?.Document._quizMode && imgBox.Document._quizMode !== quizMode.NONE) { return ( <> {checkIcon(imgBox)} {redoIcon(imgBox)} ); } } break; case StyleProp.ContextMenuItems: if (imgBox) { const quizes: ContextMenuProps[] = []; quizes.push({ description: 'Smart Check', event: doc?.quizMode == quizMode.NONE ? () => pushInfo(imgBox, quizMode.SMART) : () => exitQuizMode(imgBox), icon: 'pen-to-square', }); quizes.push({ description: 'Normal', event: doc?.quizMode == quizMode.NONE ? () => pushInfo(imgBox, quizMode.NORMAL) : () => exitQuizMode(imgBox), icon: 'pencil', }); ContextMenu.Instance?.addItem({ description: 'Quiz Mode', subitems: quizes, icon: 'file-pen' }); } break; case StyleProp.AnchorMenuItems: AnchorMenu.Instance.makeLabels = imgBox ? () => makeLabels(props?.DocumentView?.().ComponentView as ImageBox) : unimplementedFunction; } return undefined; } }