1 files changed, 391 insertions, 0 deletions
diff --git a/src/client/views/StyleProviderQuiz.tsx b/src/client/views/StyleProviderQuiz.tsx
new file mode 100644
index 000000000..8973ada95
--- /dev/null
+++ b/src/client/views/StyleProviderQuiz.tsx
@@ -0,0 +1,391 @@
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import { Tooltip } from '@mui/material';
+import axios from 'axios';
+import * as React from 'react';
+import { returnFalse, setupMoveUpEvents } from '../../ClientUtils';
+import { emptyFunction } from '../../Utils';
+import { Doc, DocListCast, Opt } from '../../fields/Doc';
+import { DocData } from '../../fields/DocSymbols';
+import { List } from '../../fields/List';
+import { NumCast, StrCast } from '../../fields/Types';
+import { GPTCallType, gptAPICall, gptImageLabel } from '../apis/gpt/GPT';
+import { Docs } from '../documents/Documents';
+import { ContextMenu } from './ContextMenu';
+import { ContextMenuProps } from './ContextMenuItem';
+import { StyleProp } from './StyleProp';
+import { AnchorMenu } from './pdf/AnchorMenu';
+import { DocumentViewProps } from './nodes/DocumentView';
+import { FieldViewProps } from './nodes/FieldView';
+import { ImageBox } from './nodes/ImageBox';
+import { ImageUtility } from './nodes/generativeFill/generativeFillUtils/ImageHandler';
+import './StyleProviderQuiz.scss';
+
+export namespace styleProviderQuiz {
+    enum quizMode {
+        SMART = 'smart',
+        NORMAL = 'normal',
+        NONE = 'none',
+    }
+
+    async function selectUrlToBase64(blob: Blob): Promise<string> {
+        try {
+            return new Promise((resolve, reject) => {
+                const reader = new FileReader();
+                reader.readAsDataURL(blob);
+                reader.onloadend = () => resolve(reader.result as string);
+                reader.onerror = error => reject(error);
+            });
+        } catch (error) {
+            console.error('Error:', error);
+            throw error;
+        }
+    }
+    /**
+     * Creates label boxes over text on the image to be filled in.
+     * @param boxes
+     * @param texts
+     */
+    async function createBoxes(img: ImageBox, boxes: [[[number, number]]], texts: [string]) {
+        img.Document._quizBoxes = new List<Doc>([]);
+        for (let i = 0; i < boxes.length; i++) {
+            const coords = boxes[i] ? boxes[i] : [];
+            const width = coords[1][0] - coords[0][0];
+            const height = coords[2][1] - coords[0][1];
+            const text = texts[i];
+
+            const newCol = Docs.Create.LabelDocument({
+                _width: width,
+                _height: height,
+                _layout_fitWidth: true,
+                title: '',
+            });
+            const scaling = 1 / (img._props.NativeDimScaling?.() || 1);
+            newCol.x = coords[0][0] + NumCast(img.marqueeref.current?.left) * scaling;
+            newCol.y = coords[0][1] + NumCast(img.marqueeref.current?.top) * scaling;
+
+            newCol.zIndex = 1000;
+            newCol.forceActive = true;
+            newCol.quiz = text;
+            newCol[DocData].textTransform = 'none';
+            Doc.AddDocToList(img.Document, '_quizBoxes', newCol);
+            img.addDocument(newCol);
+            //   img._loading = false;
+        }
+    }
+
+    /**
+     * Calls backend to find any text on an image. Gets the text and the
+     * coordinates of the text and creates label boxes at those locations.
+     * @param quiz
+     * @param i
+     */
+    async function pushInfo(imgBox: ImageBox, quiz: quizMode, i?: string) {
+        imgBox.Document._quizMode = quiz;
+        const quizBoxes = DocListCast(imgBox.Document.quizBoxes);
+        if (!quizBoxes.length) {
+            //    this._loading = true;
+
+            const img = {
+                file: i ? i : imgBox.paths[0],
+                drag: i ? 'drag' : 'full',
+                smart: quiz,
+            };
+            const response = await axios.post('http://localhost:105/labels/', img, {
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+            });
+            if (response.data['boxes'].length != 0) {
+                createBoxes(imgBox, response.data['boxes'], response.data['text']);
+            } else {
+                // this._loading = false;
+            }
+        } else quizBoxes.forEach(box => (box.hidden = false));
+    }
+
+    async function createCanvas(img: ImageBox) {
+        const canvas = document.createElement('canvas');
+        const scaling = 1 / (img._props.NativeDimScaling?.() || 1);
+        const w = AnchorMenu.Instance.marqueeWidth * scaling;
+        const h = AnchorMenu.Instance.marqueeHeight * scaling;
+        canvas.width = w;
+        canvas.height = h;
+        const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions
+        if (ctx) {
+            img.imageRef && ctx.drawImage(img.imageRef, NumCast(img.marqueeref.current?.left) * scaling, NumCast(img.marqueeref.current?.top) * scaling, w, h, 0, 0, w, h);
+        }
+        const blob = await ImageUtility.canvasToBlob(canvas);
+        return selectUrlToBase64(blob);
+    }
+    /**
+     * Create flashcards from an image.
+     */
+    async function getImageDesc(img: ImageBox) {
+        // this._loading = true;
+        try {
+            const hrefBase64 = await createCanvas(img);
+            const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this image with each question and answer labeled as "question" and "answer". Do not label each flashcard and do not include asterisks: ');
+            AnchorMenu.Instance.transferToFlashcard(response, NumCast(img.layoutDoc['x']), NumCast(img.layoutDoc['y']));
+        } catch (error) {
+            console.log('Error', error);
+        }
+        // this._loading = false;
+    }
+
+    /**
+     * Calls the createCanvas and pushInfo methods to convert the
+     * image to a form that can be passed to GPT and find the locations
+     * of the text.
+     */
+    async function makeLabels(img: ImageBox) {
+        try {
+            const hrefBase64 = await createCanvas(img);
+            pushInfo(img, quizMode.NORMAL, hrefBase64);
+        } catch (error) {
+            console.log('Error', error);
+        }
+    }
+
+    /**
+     * Determines whether two words should be considered
+     * the same, allowing minor typos.
+     * @param str1
+     * @param str2
+     * @returns
+     */
+    function levenshteinDistance(str1: string, str2: string) {
+        const len1 = str1.length;
+        const len2 = str2.length;
+        const dp = Array.from(Array(len1 + 1), () => Array(len2 + 1).fill(0));
+
+        if (len1 === 0) return len2;
+        if (len2 === 0) return len1;
+
+        for (let i = 0; i <= len1; i++) dp[i][0] = i;
+        for (let j = 0; j <= len2; j++) dp[0][j] = j;
+
+        for (let i = 1; i <= len1; i++) {
+            for (let j = 1; j <= len2; j++) {
+                const cost = str1[i - 1] === str2[j - 1] ? 0 : 1;
+                dp[i][j] = Math.min(
+                    dp[i - 1][j] + 1, // deletion
+                    dp[i][j - 1] + 1, // insertion
+                    dp[i - 1][j - 1] + cost // substitution
+                );
+            }
+        }
+
+        return dp[len1][len2];
+    }
+
+    /**
+     * Different algorithm for determining string similarity.
+     * @param str1
+     * @param str2
+     * @returns
+     */
+    function jaccardSimilarity(str1: string, str2: string) {
+        const set1 = new Set(str1.split(' '));
+        const set2 = new Set(str2.split(' '));
+
+        const intersection = new Set([...set1].filter(x => set2.has(x)));
+        const union = new Set([...set1, ...set2]);
+
+        return intersection.size / union.size;
+    }
+
+    /**
+     * Averages the jaccardSimilarity and levenshteinDistance scores
+     * to determine string similarity for the labelboxes answers and
+     * the users response.
+     * @param str1
+     * @param str2
+     * @returns
+     */
+    function stringSimilarity(str1: string, str2: string) {
+        const levenshteinDist = levenshteinDistance(str1, str2);
+        const levenshteinScore = 1 - levenshteinDist / Math.max(str1.length, str2.length);
+
+        const jaccardScore = jaccardSimilarity(str1, str2);
+
+        // Combine the scores with a higher weight on Jaccard similarity
+        return 0.5 * levenshteinScore + 0.5 * jaccardScore;
+    }
+    /**
+     * Returns whether two strings are similar
+     * @param input
+     * @param target
+     * @returns
+     */
+    function compareWords(input: string, target: string) {
+        const distance = stringSimilarity(input.toLowerCase(), target.toLowerCase());
+        return distance >= 0.7;
+    }
+
+    /**
+     * GPT returns a hex color for what color the label box should be based on
+     * the correctness of the users answer.
+     * @param inputString
+     * @returns
+     */
+    function extractHexAndSentences(inputString: string) {
+        // Regular expression to match a hexadecimal number at the beginning followed by a period and sentences
+        const regex = /^#([0-9A-Fa-f]+)\.\s*(.+)$/s;
+        const match = inputString.match(regex);
+
+        if (match) {
+            const hexNumber = match[1];
+            const sentences = match[2].trim();
+            return { hexNumber, sentences };
+        } else {
+            return { error: 'The input string does not match the expected format.' };
+        }
+    }
+    /**
+     * Check whether the contents of the label boxes on an image are correct.
+     */
+    function check(img: ImageBox) {
+        //this._loading = true;
+        img.quizBoxes.forEach(async doc => {
+            const input = StrCast(doc[DocData].title);
+            if (img.quizMode == quizMode.SMART && input) {
+                const questionText = 'Question: What was labeled in this image?';
+                const rubricText = ' Rubric: ' + StrCast(doc.quiz);
+                const queryText =
+                    questionText +
+                    ' UserAnswer: ' +
+                    input +
+                    '. ' +
+                    rubricText +
+                    '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."';
+                const response = await gptAPICall(queryText, GPTCallType.QUIZ);
+                const hexSent = extractHexAndSentences(response);
+                doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric');
+                doc.backgroundColor = '#' + hexSent.hexNumber;
+            } else {
+                const match = compareWords(input, StrCast(doc.quiz).trim());
+                doc.backgroundColor = match ? '#11c249' : '#eb2d2d';
+            }
+        });
+        //this._loading = false;
+    }
+
+    function redo(img: ImageBox) {
+        img.quizBoxes.forEach(doc => {
+            doc[DocData].title = '';
+            doc.backgroundColor = '#e4e4e4';
+        });
+    }
+
+    /**
+     * Get rid of all the label boxes on the images.
+     */
+    function exitQuizMode(img: ImageBox) {
+        img.Document._quizMode = quizMode.NONE;
+        DocListCast(img.Document._quizBoxes).forEach(box => {
+            box.hidden = true;
+        });
+    }
+
+    export function quizStyleProvider(doc: Opt<Doc>, props: Opt<FieldViewProps & DocumentViewProps>, property: string) {
+        const editLabelAnswer = (qdoc: Doc) => {
+            // when click the pencil, set the text to the quiz content. when click off, set the quiz text to that and set textbox to nothing.
+            if (!qdoc._editLabel) {
+                qdoc.title = StrCast(qdoc.quiz);
+            } else {
+                qdoc.quiz = StrCast(qdoc.title);
+                qdoc.title = '';
+            }
+            qdoc._editLabel = !qdoc._editLabel;
+        };
+        const editAnswer = (qdoc: Opt<Doc>) => {
+            return (
+                <Tooltip
+                    title={
+                        <div className="answer-tooltip" style={{ minWidth: '150px' }}>
+                            {qdoc?._editLabel ? 'save' : 'edit correct answer'}
+                        </div>
+                    }>
+                    <div className="answer-tool-tip" onPointerDown={e => setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => qdoc && editLabelAnswer(qdoc))}>
+                        <FontAwesomeIcon className="edit-icon" color={qdoc?._editLabel ? 'white' : 'black'} icon="pencil" size="sm" />
+                    </div>
+                </Tooltip>
+            );
+        };
+        const answerIcon = (qdoc: Opt<Doc>) => {
+            return (
+                <Tooltip
+                    title={
+                        <div className="answer-tooltip" style={{ minWidth: '150px' }}>
+                            {StrCast(qdoc?.quiz ?? '')}
+                        </div>
+                    }>
+                    <div className="answer-tool-tip">
+                        <FontAwesomeIcon className="q-icon" icon="circle" color="white" />
+                        <FontAwesomeIcon className="answer-icon" icon="question" />
+                    </div>
+                </Tooltip>
+            );
+        };
+        const checkIcon = (img: ImageBox) => (
+            <Tooltip title={<div className="dash-tooltip">Check</div>}>
+                <div className="check-icon" onPointerDown={() => check(img)}>
+                    <FontAwesomeIcon icon="circle-check" size="lg" />
+                </div>
+            </Tooltip>
+        );
+        const redoIcon = (img: ImageBox) => (
+            <Tooltip title={<div className="dash-tooltip">Redo</div>}>
+                <div className="redo-icon" onPointerDown={() => redo(img)}>
+                    <FontAwesomeIcon icon="redo-alt" size="lg" />
+                </div>
+            </Tooltip>
+        );
+
+        const imgBox = props?.DocumentView?.().ComponentView as ImageBox;
+        switch (property) {
+            case StyleProp.Decorations:
+                {
+                    if (doc?.quiz) {
+                        // this should only be set on Labels that are part of an image quiz
+                        return (
+                            <>
+                                {editAnswer(doc?.[DocData])}
+                                {answerIcon(doc)}
+                            </>
+                        );
+                    } else if (imgBox?.Document._quizMode && imgBox.Document._quizMode !== quizMode.NONE) {
+                        return (
+                            <>
+                                {checkIcon(imgBox)}
+                                {redoIcon(imgBox)}
+                            </>
+                        );
+                    }
+                }
+                break;
+            case StyleProp.ContextMenuItems:
+                if (imgBox) {
+                    const quizes: ContextMenuProps[] = [];
+                    quizes.push({
+                        description: 'Smart Check',
+                        event: doc?.quizMode == quizMode.NONE ? () => pushInfo(imgBox, quizMode.SMART) : () => exitQuizMode(imgBox),
+                        icon: 'pen-to-square',
+                    });
+                    quizes.push({
+                        description: 'Normal',
+                        event: doc?.quizMode == quizMode.NONE ? () => pushInfo(imgBox, quizMode.NORMAL) : () => exitQuizMode(imgBox),
+                        icon: 'pencil',
+                    });
+                    ContextMenu.Instance?.addItem({ description: 'Quiz Mode', subitems: quizes, icon: 'file-pen' });
+                }
+                break;
+            case StyleProp.AnchorMenuItems:
+                if (imgBox) {
+                    AnchorMenu.Instance.gptFlashcards = () => getImageDesc(imgBox);
+                    AnchorMenu.Instance.makeLabels = () => makeLabels(props?.DocumentView?.().ComponentView as ImageBox);
+                }
+        }
+        return undefined;
+    }
+}