From 11244a698bce594982ee5dca55b9695bb774451c Mon Sep 17 00:00:00 2001 From: bobzel Date: Mon, 7 Oct 2024 18:13:38 -0400 Subject: moved all quiz code out of LabelBox and ImageBox and into StyleProviderQuiz. changed quizBoxes and quizMode to be stored as Doc metadata. Extended styles to cover contextMenuItems. remove this.setListening() from comparisonBox until contextMenu selectedVal is fixed. --- src/client/views/StyleProviderQuiz.tsx | 391 +++++++++++++++++++++++++++++++++ 1 file changed, 391 insertions(+) create mode 100644 src/client/views/StyleProviderQuiz.tsx (limited to 'src/client/views/StyleProviderQuiz.tsx') diff --git a/src/client/views/StyleProviderQuiz.tsx b/src/client/views/StyleProviderQuiz.tsx new file mode 100644 index 000000000..8973ada95 --- /dev/null +++ b/src/client/views/StyleProviderQuiz.tsx @@ -0,0 +1,391 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Tooltip } from '@mui/material'; +import axios from 'axios'; +import * as React from 'react'; +import { returnFalse, setupMoveUpEvents } from '../../ClientUtils'; +import { emptyFunction } from '../../Utils'; +import { Doc, DocListCast, Opt } from '../../fields/Doc'; +import { DocData } from '../../fields/DocSymbols'; +import { List } from '../../fields/List'; +import { NumCast, StrCast } from '../../fields/Types'; +import { GPTCallType, gptAPICall, gptImageLabel } from '../apis/gpt/GPT'; +import { Docs } from '../documents/Documents'; +import { ContextMenu } from './ContextMenu'; +import { ContextMenuProps } from './ContextMenuItem'; +import { StyleProp } from './StyleProp'; +import { AnchorMenu } from './pdf/AnchorMenu'; +import { DocumentViewProps } from './nodes/DocumentView'; +import { FieldViewProps } from './nodes/FieldView'; +import { ImageBox } from './nodes/ImageBox'; +import { ImageUtility } from './nodes/generativeFill/generativeFillUtils/ImageHandler'; +import './StyleProviderQuiz.scss'; + +export namespace styleProviderQuiz { + enum quizMode { + SMART = 'smart', + NORMAL = 'normal', + NONE = 'none', + } + + async function selectUrlToBase64(blob: Blob): Promise { + try { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = error => reject(error); + }); + } catch (error) { + console.error('Error:', error); + throw error; + } + } + /** + * Creates label boxes over text on the image to be filled in. + * @param boxes + * @param texts + */ + async function createBoxes(img: ImageBox, boxes: [[[number, number]]], texts: [string]) { + img.Document._quizBoxes = new List([]); + for (let i = 0; i < boxes.length; i++) { + const coords = boxes[i] ? boxes[i] : []; + const width = coords[1][0] - coords[0][0]; + const height = coords[2][1] - coords[0][1]; + const text = texts[i]; + + const newCol = Docs.Create.LabelDocument({ + _width: width, + _height: height, + _layout_fitWidth: true, + title: '', + }); + const scaling = 1 / (img._props.NativeDimScaling?.() || 1); + newCol.x = coords[0][0] + NumCast(img.marqueeref.current?.left) * scaling; + newCol.y = coords[0][1] + NumCast(img.marqueeref.current?.top) * scaling; + + newCol.zIndex = 1000; + newCol.forceActive = true; + newCol.quiz = text; + newCol[DocData].textTransform = 'none'; + Doc.AddDocToList(img.Document, '_quizBoxes', newCol); + img.addDocument(newCol); + // img._loading = false; + } + } + + /** + * Calls backend to find any text on an image. Gets the text and the + * coordinates of the text and creates label boxes at those locations. + * @param quiz + * @param i + */ + async function pushInfo(imgBox: ImageBox, quiz: quizMode, i?: string) { + imgBox.Document._quizMode = quiz; + const quizBoxes = DocListCast(imgBox.Document.quizBoxes); + if (!quizBoxes.length) { + // this._loading = true; + + const img = { + file: i ? i : imgBox.paths[0], + drag: i ? 'drag' : 'full', + smart: quiz, + }; + const response = await axios.post('http://localhost:105/labels/', img, { + headers: { + 'Content-Type': 'application/json', + }, + }); + if (response.data['boxes'].length != 0) { + createBoxes(imgBox, response.data['boxes'], response.data['text']); + } else { + // this._loading = false; + } + } else quizBoxes.forEach(box => (box.hidden = false)); + } + + async function createCanvas(img: ImageBox) { + const canvas = document.createElement('canvas'); + const scaling = 1 / (img._props.NativeDimScaling?.() || 1); + const w = AnchorMenu.Instance.marqueeWidth * scaling; + const h = AnchorMenu.Instance.marqueeHeight * scaling; + canvas.width = w; + canvas.height = h; + const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions + if (ctx) { + img.imageRef && ctx.drawImage(img.imageRef, NumCast(img.marqueeref.current?.left) * scaling, NumCast(img.marqueeref.current?.top) * scaling, w, h, 0, 0, w, h); + } + const blob = await ImageUtility.canvasToBlob(canvas); + return selectUrlToBase64(blob); + } + /** + * Create flashcards from an image. + */ + async function getImageDesc(img: ImageBox) { + // this._loading = true; + try { + const hrefBase64 = await createCanvas(img); + const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this image with each question and answer labeled as "question" and "answer". Do not label each flashcard and do not include asterisks: '); + AnchorMenu.Instance.transferToFlashcard(response, NumCast(img.layoutDoc['x']), NumCast(img.layoutDoc['y'])); + } catch (error) { + console.log('Error', error); + } + // this._loading = false; + } + + /** + * Calls the createCanvas and pushInfo methods to convert the + * image to a form that can be passed to GPT and find the locations + * of the text. + */ + async function makeLabels(img: ImageBox) { + try { + const hrefBase64 = await createCanvas(img); + pushInfo(img, quizMode.NORMAL, hrefBase64); + } catch (error) { + console.log('Error', error); + } + } + + /** + * Determines whether two words should be considered + * the same, allowing minor typos. + * @param str1 + * @param str2 + * @returns + */ + function levenshteinDistance(str1: string, str2: string) { + const len1 = str1.length; + const len2 = str2.length; + const dp = Array.from(Array(len1 + 1), () => Array(len2 + 1).fill(0)); + + if (len1 === 0) return len2; + if (len2 === 0) return len1; + + for (let i = 0; i <= len1; i++) dp[i][0] = i; + for (let j = 0; j <= len2; j++) dp[0][j] = j; + + for (let i = 1; i <= len1; i++) { + for (let j = 1; j <= len2; j++) { + const cost = str1[i - 1] === str2[j - 1] ? 0 : 1; + dp[i][j] = Math.min( + dp[i - 1][j] + 1, // deletion + dp[i][j - 1] + 1, // insertion + dp[i - 1][j - 1] + cost // substitution + ); + } + } + + return dp[len1][len2]; + } + + /** + * Different algorithm for determining string similarity. + * @param str1 + * @param str2 + * @returns + */ + function jaccardSimilarity(str1: string, str2: string) { + const set1 = new Set(str1.split(' ')); + const set2 = new Set(str2.split(' ')); + + const intersection = new Set([...set1].filter(x => set2.has(x))); + const union = new Set([...set1, ...set2]); + + return intersection.size / union.size; + } + + /** + * Averages the jaccardSimilarity and levenshteinDistance scores + * to determine string similarity for the labelboxes answers and + * the users response. + * @param str1 + * @param str2 + * @returns + */ + function stringSimilarity(str1: string, str2: string) { + const levenshteinDist = levenshteinDistance(str1, str2); + const levenshteinScore = 1 - levenshteinDist / Math.max(str1.length, str2.length); + + const jaccardScore = jaccardSimilarity(str1, str2); + + // Combine the scores with a higher weight on Jaccard similarity + return 0.5 * levenshteinScore + 0.5 * jaccardScore; + } + /** + * Returns whether two strings are similar + * @param input + * @param target + * @returns + */ + function compareWords(input: string, target: string) { + const distance = stringSimilarity(input.toLowerCase(), target.toLowerCase()); + return distance >= 0.7; + } + + /** + * GPT returns a hex color for what color the label box should be based on + * the correctness of the users answer. + * @param inputString + * @returns + */ + function extractHexAndSentences(inputString: string) { + // Regular expression to match a hexadecimal number at the beginning followed by a period and sentences + const regex = /^#([0-9A-Fa-f]+)\.\s*(.+)$/s; + const match = inputString.match(regex); + + if (match) { + const hexNumber = match[1]; + const sentences = match[2].trim(); + return { hexNumber, sentences }; + } else { + return { error: 'The input string does not match the expected format.' }; + } + } + /** + * Check whether the contents of the label boxes on an image are correct. + */ + function check(img: ImageBox) { + //this._loading = true; + img.quizBoxes.forEach(async doc => { + const input = StrCast(doc[DocData].title); + if (img.quizMode == quizMode.SMART && input) { + const questionText = 'Question: What was labeled in this image?'; + const rubricText = ' Rubric: ' + StrCast(doc.quiz); + const queryText = + questionText + + ' UserAnswer: ' + + input + + '. ' + + rubricText + + '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."'; + const response = await gptAPICall(queryText, GPTCallType.QUIZ); + const hexSent = extractHexAndSentences(response); + doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); + doc.backgroundColor = '#' + hexSent.hexNumber; + } else { + const match = compareWords(input, StrCast(doc.quiz).trim()); + doc.backgroundColor = match ? '#11c249' : '#eb2d2d'; + } + }); + //this._loading = false; + } + + function redo(img: ImageBox) { + img.quizBoxes.forEach(doc => { + doc[DocData].title = ''; + doc.backgroundColor = '#e4e4e4'; + }); + } + + /** + * Get rid of all the label boxes on the images. + */ + function exitQuizMode(img: ImageBox) { + img.Document._quizMode = quizMode.NONE; + DocListCast(img.Document._quizBoxes).forEach(box => { + box.hidden = true; + }); + } + + export function quizStyleProvider(doc: Opt, props: Opt, property: string) { + const editLabelAnswer = (qdoc: Doc) => { + // when click the pencil, set the text to the quiz content. when click off, set the quiz text to that and set textbox to nothing. + if (!qdoc._editLabel) { + qdoc.title = StrCast(qdoc.quiz); + } else { + qdoc.quiz = StrCast(qdoc.title); + qdoc.title = ''; + } + qdoc._editLabel = !qdoc._editLabel; + }; + const editAnswer = (qdoc: Opt) => { + return ( + + {qdoc?._editLabel ? 'save' : 'edit correct answer'} + + }> +
setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => qdoc && editLabelAnswer(qdoc))}> + +
+ + ); + }; + const answerIcon = (qdoc: Opt) => { + return ( + + {StrCast(qdoc?.quiz ?? '')} + + }> +
+ + +
+ + ); + }; + const checkIcon = (img: ImageBox) => ( + Check}> +
check(img)}> + +
+ + ); + const redoIcon = (img: ImageBox) => ( + Redo}> +
redo(img)}> + +
+ + ); + + const imgBox = props?.DocumentView?.().ComponentView as ImageBox; + switch (property) { + case StyleProp.Decorations: + { + if (doc?.quiz) { + // this should only be set on Labels that are part of an image quiz + return ( + <> + {editAnswer(doc?.[DocData])} + {answerIcon(doc)} + + ); + } else if (imgBox?.Document._quizMode && imgBox.Document._quizMode !== quizMode.NONE) { + return ( + <> + {checkIcon(imgBox)} + {redoIcon(imgBox)} + + ); + } + } + break; + case StyleProp.ContextMenuItems: + if (imgBox) { + const quizes: ContextMenuProps[] = []; + quizes.push({ + description: 'Smart Check', + event: doc?.quizMode == quizMode.NONE ? () => pushInfo(imgBox, quizMode.SMART) : () => exitQuizMode(imgBox), + icon: 'pen-to-square', + }); + quizes.push({ + description: 'Normal', + event: doc?.quizMode == quizMode.NONE ? () => pushInfo(imgBox, quizMode.NORMAL) : () => exitQuizMode(imgBox), + icon: 'pencil', + }); + ContextMenu.Instance?.addItem({ description: 'Quiz Mode', subitems: quizes, icon: 'file-pen' }); + } + break; + case StyleProp.AnchorMenuItems: + if (imgBox) { + AnchorMenu.Instance.gptFlashcards = () => getImageDesc(imgBox); + AnchorMenu.Instance.makeLabels = () => makeLabels(props?.DocumentView?.().ComponentView as ImageBox); + } + } + return undefined; + } +} -- cgit v1.2.3-70-g09d2 From 504b8059ff4e162e089177e366a312dd583d5cff Mon Sep 17 00:00:00 2001 From: bobzel Date: Mon, 7 Oct 2024 21:47:05 -0400 Subject: moved some more quiz functions to styleProviderQuiz --- src/client/views/StyleProviderQuiz.tsx | 21 ++++++++++++++------- src/client/views/nodes/ImageBox.tsx | 12 ++---------- 2 files changed, 16 insertions(+), 17 deletions(-) (limited to 'src/client/views/StyleProviderQuiz.tsx') diff --git a/src/client/views/StyleProviderQuiz.tsx b/src/client/views/StyleProviderQuiz.tsx index 8973ada95..1f2ad1485 100644 --- a/src/client/views/StyleProviderQuiz.tsx +++ b/src/client/views/StyleProviderQuiz.tsx @@ -83,7 +83,7 @@ export namespace styleProviderQuiz { imgBox.Document._quizMode = quiz; const quizBoxes = DocListCast(imgBox.Document.quizBoxes); if (!quizBoxes.length) { - // this._loading = true; + imgBox.Loading = true; const img = { file: i ? i : imgBox.paths[0], @@ -98,7 +98,7 @@ export namespace styleProviderQuiz { if (response.data['boxes'].length != 0) { createBoxes(imgBox, response.data['boxes'], response.data['text']); } else { - // this._loading = false; + imgBox.Loading = false; } } else quizBoxes.forEach(box => (box.hidden = false)); } @@ -121,7 +121,7 @@ export namespace styleProviderQuiz { * Create flashcards from an image. */ async function getImageDesc(img: ImageBox) { - // this._loading = true; + img.Loading = true; try { const hrefBase64 = await createCanvas(img); const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this image with each question and answer labeled as "question" and "answer". Do not label each flashcard and do not include asterisks: '); @@ -129,7 +129,7 @@ export namespace styleProviderQuiz { } catch (error) { console.log('Error', error); } - // this._loading = false; + img.Loading = false; } /** @@ -241,14 +241,21 @@ export namespace styleProviderQuiz { return { error: 'The input string does not match the expected format.' }; } } + function imgQuizBoxes(img: ImageBox) { + return DocListCast(img.Document.quizBoxes); + } + function imgQuizMode(img: ImageBox) { + return StrCast(img.Document._quizMode); + } + /** * Check whether the contents of the label boxes on an image are correct. */ function check(img: ImageBox) { //this._loading = true; - img.quizBoxes.forEach(async doc => { + imgQuizBoxes(img).forEach(async doc => { const input = StrCast(doc[DocData].title); - if (img.quizMode == quizMode.SMART && input) { + if (imgQuizMode(img) == quizMode.SMART && input) { const questionText = 'Question: What was labeled in this image?'; const rubricText = ' Rubric: ' + StrCast(doc.quiz); const queryText = @@ -271,7 +278,7 @@ export namespace styleProviderQuiz { } function redo(img: ImageBox) { - img.quizBoxes.forEach(doc => { + imgQuizBoxes(img).forEach(doc => { doc[DocData].title = ''; doc.backgroundColor = '#e4e4e4'; }); diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 0b474076b..0827eb062 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -80,12 +80,12 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { private _annotationLayer: React.RefObject = React.createRef(); imageRef: HTMLImageElement | null = null; //