aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/client/views/StyleProp.ts2
-rw-r--r--src/client/views/StyleProvider.tsx10
-rw-r--r--src/client/views/StyleProviderQuiz.scss40
-rw-r--r--src/client/views/StyleProviderQuiz.tsx391
-rw-r--r--src/client/views/nodes/ComparisonBox.tsx1
-rw-r--r--src/client/views/nodes/DocumentView.tsx3
-rw-r--r--src/client/views/nodes/FieldView.tsx2
-rw-r--r--src/client/views/nodes/ImageBox.scss41
-rw-r--r--src/client/views/nodes/ImageBox.tsx365
-rw-r--r--src/client/views/nodes/LabelBox.tsx52
10 files changed, 466 insertions, 441 deletions
diff --git a/src/client/views/StyleProp.ts b/src/client/views/StyleProp.ts
index dd5b98cfe..44d3bf757 100644
--- a/src/client/views/StyleProp.ts
+++ b/src/client/views/StyleProp.ts
@@ -21,4 +21,6 @@ export enum StyleProp {
FontFamily = 'fontFamily', // font family of text
FontWeight = 'fontWeight', // font weight of text
Highlighting = 'highlighting', // border highlighting
+ ContextMenuItems = 'contextMenuItems', // menu items to add to context menu
+ AnchorMenuItems = 'anchorMenuItems',
}
diff --git a/src/client/views/StyleProvider.tsx b/src/client/views/StyleProvider.tsx
index 16f6aa40b..44bea57eb 100644
--- a/src/client/views/StyleProvider.tsx
+++ b/src/client/views/StyleProvider.tsx
@@ -21,6 +21,7 @@ import { TreeSort } from './collections/TreeSort';
import { Colors } from './global/globalEnums';
import { DocumentView, DocumentViewProps } from './nodes/DocumentView';
import { FieldViewProps } from './nodes/FieldView';
+import { styleProviderQuiz } from './StyleProviderQuiz';
import { StyleProp } from './StyleProp';
import './StyleProvider.scss';
import { TagsView } from './TagsView';
@@ -89,6 +90,7 @@ export function DefaultStyleProvider(doc: Opt<Doc>, props: Opt<FieldViewProps &
containerViewPath,
childFilters,
hideCaptions,
+ hideFilterStatus,
showTitle,
childFiltersByRanges,
renderDepth,
@@ -110,6 +112,12 @@ export function DefaultStyleProvider(doc: Opt<Doc>, props: Opt<FieldViewProps &
const color = () => styleProvider?.(doc, props, StyleProp.Color) as string;
const opacity = () => styleProvider?.(doc, props, StyleProp.Opacity);
const layoutShowTitle = () => styleProvider?.(doc, props, StyleProp.ShowTitle) as string;
+
+ // bcz: For now, this is how to add custom-stylings (like a Quiz styling) for app-specific purposes. The quiz styling will short-circuit
+ // the regular stylings for items that it controls (eg., things with a quiz field, or images)
+ const quizProp = styleProviderQuiz.quizStyleProvider(doc, props, property);
+ if (quizProp !== undefined) return quizProp;
+
// prettier-ignore
switch (property.split(':')[0]) {
case StyleProp.TreeViewIcon: {
@@ -318,7 +326,7 @@ export function DefaultStyleProvider(doc: Opt<Doc>, props: Opt<FieldViewProps &
: childFilters?.().filter(f => ClientUtils.IsRecursiveFilter(f) && f !== ClientUtils.noDragDocsFilter).length || childFiltersByRanges?.().length
? 'orange' // 'inheritsFilter'
: undefined;
- return !showFilterIcon || props?.hideFilterStatus ? null : (
+ return !showFilterIcon || hideFilterStatus ? null : (
<div className="styleProvider-filter">
<Dropdown
type={Type.TERT}
diff --git a/src/client/views/StyleProviderQuiz.scss b/src/client/views/StyleProviderQuiz.scss
new file mode 100644
index 000000000..2f52c8dec
--- /dev/null
+++ b/src/client/views/StyleProviderQuiz.scss
@@ -0,0 +1,40 @@
+.loading-spinner {
+ position: absolute;
+ display: flex;
+ justify-content: center;
+ align-items: center;
+ height: 100%;
+ width: 100%;
+ // left: 50%;
+ // top: 50%;
+ z-index: 200;
+ font-size: 20px;
+ font-weight: bold;
+ color: #17175e;
+}
+
+.check-icon {
+ position: absolute;
+ right: 40;
+ bottom: 10;
+ color: green;
+ display: inline-block;
+ font-size: 20px;
+ overflow: hidden;
+}
+
+.redo-icon {
+ position: absolute;
+ right: 10;
+ bottom: 10;
+ color: black;
+ display: inline-block;
+ font-size: 20px;
+ overflow: hidden;
+}
+
+@keyframes spin {
+ to {
+ transform: rotate(360deg);
+ }
+}
diff --git a/src/client/views/StyleProviderQuiz.tsx b/src/client/views/StyleProviderQuiz.tsx
new file mode 100644
index 000000000..8973ada95
--- /dev/null
+++ b/src/client/views/StyleProviderQuiz.tsx
@@ -0,0 +1,391 @@
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import { Tooltip } from '@mui/material';
+import axios from 'axios';
+import * as React from 'react';
+import { returnFalse, setupMoveUpEvents } from '../../ClientUtils';
+import { emptyFunction } from '../../Utils';
+import { Doc, DocListCast, Opt } from '../../fields/Doc';
+import { DocData } from '../../fields/DocSymbols';
+import { List } from '../../fields/List';
+import { NumCast, StrCast } from '../../fields/Types';
+import { GPTCallType, gptAPICall, gptImageLabel } from '../apis/gpt/GPT';
+import { Docs } from '../documents/Documents';
+import { ContextMenu } from './ContextMenu';
+import { ContextMenuProps } from './ContextMenuItem';
+import { StyleProp } from './StyleProp';
+import { AnchorMenu } from './pdf/AnchorMenu';
+import { DocumentViewProps } from './nodes/DocumentView';
+import { FieldViewProps } from './nodes/FieldView';
+import { ImageBox } from './nodes/ImageBox';
+import { ImageUtility } from './nodes/generativeFill/generativeFillUtils/ImageHandler';
+import './StyleProviderQuiz.scss';
+
+export namespace styleProviderQuiz {
+ enum quizMode {
+ SMART = 'smart',
+ NORMAL = 'normal',
+ NONE = 'none',
+ }
+
+ async function selectUrlToBase64(blob: Blob): Promise<string> {
+ try {
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.readAsDataURL(blob);
+ reader.onloadend = () => resolve(reader.result as string);
+ reader.onerror = error => reject(error);
+ });
+ } catch (error) {
+ console.error('Error:', error);
+ throw error;
+ }
+ }
+ /**
+ * Creates label boxes over text on the image to be filled in.
+ * @param boxes
+ * @param texts
+ */
+ async function createBoxes(img: ImageBox, boxes: [[[number, number]]], texts: [string]) {
+ img.Document._quizBoxes = new List<Doc>([]);
+ for (let i = 0; i < boxes.length; i++) {
+ const coords = boxes[i] ? boxes[i] : [];
+ const width = coords[1][0] - coords[0][0];
+ const height = coords[2][1] - coords[0][1];
+ const text = texts[i];
+
+ const newCol = Docs.Create.LabelDocument({
+ _width: width,
+ _height: height,
+ _layout_fitWidth: true,
+ title: '',
+ });
+ const scaling = 1 / (img._props.NativeDimScaling?.() || 1);
+ newCol.x = coords[0][0] + NumCast(img.marqueeref.current?.left) * scaling;
+ newCol.y = coords[0][1] + NumCast(img.marqueeref.current?.top) * scaling;
+
+ newCol.zIndex = 1000;
+ newCol.forceActive = true;
+ newCol.quiz = text;
+ newCol[DocData].textTransform = 'none';
+ Doc.AddDocToList(img.Document, '_quizBoxes', newCol);
+ img.addDocument(newCol);
+ // img._loading = false;
+ }
+ }
+
+ /**
+ * Calls backend to find any text on an image. Gets the text and the
+ * coordinates of the text and creates label boxes at those locations.
+ * @param quiz
+ * @param i
+ */
+ async function pushInfo(imgBox: ImageBox, quiz: quizMode, i?: string) {
+ imgBox.Document._quizMode = quiz;
+ const quizBoxes = DocListCast(imgBox.Document.quizBoxes);
+ if (!quizBoxes.length) {
+ // this._loading = true;
+
+ const img = {
+ file: i ? i : imgBox.paths[0],
+ drag: i ? 'drag' : 'full',
+ smart: quiz,
+ };
+ const response = await axios.post('http://localhost:105/labels/', img, {
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ });
+ if (response.data['boxes'].length != 0) {
+ createBoxes(imgBox, response.data['boxes'], response.data['text']);
+ } else {
+ // this._loading = false;
+ }
+ } else quizBoxes.forEach(box => (box.hidden = false));
+ }
+
+ async function createCanvas(img: ImageBox) {
+ const canvas = document.createElement('canvas');
+ const scaling = 1 / (img._props.NativeDimScaling?.() || 1);
+ const w = AnchorMenu.Instance.marqueeWidth * scaling;
+ const h = AnchorMenu.Instance.marqueeHeight * scaling;
+ canvas.width = w;
+ canvas.height = h;
+ const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions
+ if (ctx) {
+ img.imageRef && ctx.drawImage(img.imageRef, NumCast(img.marqueeref.current?.left) * scaling, NumCast(img.marqueeref.current?.top) * scaling, w, h, 0, 0, w, h);
+ }
+ const blob = await ImageUtility.canvasToBlob(canvas);
+ return selectUrlToBase64(blob);
+ }
+ /**
+ * Create flashcards from an image.
+ */
+ async function getImageDesc(img: ImageBox) {
+ // this._loading = true;
+ try {
+ const hrefBase64 = await createCanvas(img);
+ const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this image with each question and answer labeled as "question" and "answer". Do not label each flashcard and do not include asterisks: ');
+ AnchorMenu.Instance.transferToFlashcard(response, NumCast(img.layoutDoc['x']), NumCast(img.layoutDoc['y']));
+ } catch (error) {
+ console.log('Error', error);
+ }
+ // this._loading = false;
+ }
+
+ /**
+ * Calls the createCanvas and pushInfo methods to convert the
+ * image to a form that can be passed to GPT and find the locations
+ * of the text.
+ */
+ async function makeLabels(img: ImageBox) {
+ try {
+ const hrefBase64 = await createCanvas(img);
+ pushInfo(img, quizMode.NORMAL, hrefBase64);
+ } catch (error) {
+ console.log('Error', error);
+ }
+ }
+
+ /**
+ * Determines whether two words should be considered
+ * the same, allowing minor typos.
+ * @param str1
+ * @param str2
+ * @returns
+ */
+ function levenshteinDistance(str1: string, str2: string) {
+ const len1 = str1.length;
+ const len2 = str2.length;
+ const dp = Array.from(Array(len1 + 1), () => Array(len2 + 1).fill(0));
+
+ if (len1 === 0) return len2;
+ if (len2 === 0) return len1;
+
+ for (let i = 0; i <= len1; i++) dp[i][0] = i;
+ for (let j = 0; j <= len2; j++) dp[0][j] = j;
+
+ for (let i = 1; i <= len1; i++) {
+ for (let j = 1; j <= len2; j++) {
+ const cost = str1[i - 1] === str2[j - 1] ? 0 : 1;
+ dp[i][j] = Math.min(
+ dp[i - 1][j] + 1, // deletion
+ dp[i][j - 1] + 1, // insertion
+ dp[i - 1][j - 1] + cost // substitution
+ );
+ }
+ }
+
+ return dp[len1][len2];
+ }
+
+ /**
+ * Different algorithm for determining string similarity.
+ * @param str1
+ * @param str2
+ * @returns
+ */
+ function jaccardSimilarity(str1: string, str2: string) {
+ const set1 = new Set(str1.split(' '));
+ const set2 = new Set(str2.split(' '));
+
+ const intersection = new Set([...set1].filter(x => set2.has(x)));
+ const union = new Set([...set1, ...set2]);
+
+ return intersection.size / union.size;
+ }
+
+ /**
+ * Averages the jaccardSimilarity and levenshteinDistance scores
+ * to determine string similarity for the labelboxes answers and
+ * the users response.
+ * @param str1
+ * @param str2
+ * @returns
+ */
+ function stringSimilarity(str1: string, str2: string) {
+ const levenshteinDist = levenshteinDistance(str1, str2);
+ const levenshteinScore = 1 - levenshteinDist / Math.max(str1.length, str2.length);
+
+ const jaccardScore = jaccardSimilarity(str1, str2);
+
+ // Combine the scores with a higher weight on Jaccard similarity
+ return 0.5 * levenshteinScore + 0.5 * jaccardScore;
+ }
+ /**
+ * Returns whether two strings are similar
+ * @param input
+ * @param target
+ * @returns
+ */
+ function compareWords(input: string, target: string) {
+ const distance = stringSimilarity(input.toLowerCase(), target.toLowerCase());
+ return distance >= 0.7;
+ }
+
+ /**
+ * GPT returns a hex color for what color the label box should be based on
+ * the correctness of the users answer.
+ * @param inputString
+ * @returns
+ */
+ function extractHexAndSentences(inputString: string) {
+ // Regular expression to match a hexadecimal number at the beginning followed by a period and sentences
+ const regex = /^#([0-9A-Fa-f]+)\.\s*(.+)$/s;
+ const match = inputString.match(regex);
+
+ if (match) {
+ const hexNumber = match[1];
+ const sentences = match[2].trim();
+ return { hexNumber, sentences };
+ } else {
+ return { error: 'The input string does not match the expected format.' };
+ }
+ }
+ /**
+ * Check whether the contents of the label boxes on an image are correct.
+ */
+ function check(img: ImageBox) {
+ //this._loading = true;
+ img.quizBoxes.forEach(async doc => {
+ const input = StrCast(doc[DocData].title);
+ if (img.quizMode == quizMode.SMART && input) {
+ const questionText = 'Question: What was labeled in this image?';
+ const rubricText = ' Rubric: ' + StrCast(doc.quiz);
+ const queryText =
+ questionText +
+ ' UserAnswer: ' +
+ input +
+ '. ' +
+ rubricText +
+ '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."';
+ const response = await gptAPICall(queryText, GPTCallType.QUIZ);
+ const hexSent = extractHexAndSentences(response);
+ doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric');
+ doc.backgroundColor = '#' + hexSent.hexNumber;
+ } else {
+ const match = compareWords(input, StrCast(doc.quiz).trim());
+ doc.backgroundColor = match ? '#11c249' : '#eb2d2d';
+ }
+ });
+ //this._loading = false;
+ }
+
+ function redo(img: ImageBox) {
+ img.quizBoxes.forEach(doc => {
+ doc[DocData].title = '';
+ doc.backgroundColor = '#e4e4e4';
+ });
+ }
+
+ /**
+ * Get rid of all the label boxes on the images.
+ */
+ function exitQuizMode(img: ImageBox) {
+ img.Document._quizMode = quizMode.NONE;
+ DocListCast(img.Document._quizBoxes).forEach(box => {
+ box.hidden = true;
+ });
+ }
+
+ export function quizStyleProvider(doc: Opt<Doc>, props: Opt<FieldViewProps & DocumentViewProps>, property: string) {
+ const editLabelAnswer = (qdoc: Doc) => {
+ // when click the pencil, set the text to the quiz content. when click off, set the quiz text to that and set textbox to nothing.
+ if (!qdoc._editLabel) {
+ qdoc.title = StrCast(qdoc.quiz);
+ } else {
+ qdoc.quiz = StrCast(qdoc.title);
+ qdoc.title = '';
+ }
+ qdoc._editLabel = !qdoc._editLabel;
+ };
+ const editAnswer = (qdoc: Opt<Doc>) => {
+ return (
+ <Tooltip
+ title={
+ <div className="answer-tooltip" style={{ minWidth: '150px' }}>
+ {qdoc?._editLabel ? 'save' : 'edit correct answer'}
+ </div>
+ }>
+ <div className="answer-tool-tip" onPointerDown={e => setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => qdoc && editLabelAnswer(qdoc))}>
+ <FontAwesomeIcon className="edit-icon" color={qdoc?._editLabel ? 'white' : 'black'} icon="pencil" size="sm" />
+ </div>
+ </Tooltip>
+ );
+ };
+ const answerIcon = (qdoc: Opt<Doc>) => {
+ return (
+ <Tooltip
+ title={
+ <div className="answer-tooltip" style={{ minWidth: '150px' }}>
+ {StrCast(qdoc?.quiz ?? '')}
+ </div>
+ }>
+ <div className="answer-tool-tip">
+ <FontAwesomeIcon className="q-icon" icon="circle" color="white" />
+ <FontAwesomeIcon className="answer-icon" icon="question" />
+ </div>
+ </Tooltip>
+ );
+ };
+ const checkIcon = (img: ImageBox) => (
+ <Tooltip title={<div className="dash-tooltip">Check</div>}>
+ <div className="check-icon" onPointerDown={() => check(img)}>
+ <FontAwesomeIcon icon="circle-check" size="lg" />
+ </div>
+ </Tooltip>
+ );
+ const redoIcon = (img: ImageBox) => (
+ <Tooltip title={<div className="dash-tooltip">Redo</div>}>
+ <div className="redo-icon" onPointerDown={() => redo(img)}>
+ <FontAwesomeIcon icon="redo-alt" size="lg" />
+ </div>
+ </Tooltip>
+ );
+
+ const imgBox = props?.DocumentView?.().ComponentView as ImageBox;
+ switch (property) {
+ case StyleProp.Decorations:
+ {
+ if (doc?.quiz) {
+ // this should only be set on Labels that are part of an image quiz
+ return (
+ <>
+ {editAnswer(doc?.[DocData])}
+ {answerIcon(doc)}
+ </>
+ );
+ } else if (imgBox?.Document._quizMode && imgBox.Document._quizMode !== quizMode.NONE) {
+ return (
+ <>
+ {checkIcon(imgBox)}
+ {redoIcon(imgBox)}
+ </>
+ );
+ }
+ }
+ break;
+ case StyleProp.ContextMenuItems:
+ if (imgBox) {
+ const quizes: ContextMenuProps[] = [];
+ quizes.push({
+ description: 'Smart Check',
+ event: doc?.quizMode == quizMode.NONE ? () => pushInfo(imgBox, quizMode.SMART) : () => exitQuizMode(imgBox),
+ icon: 'pen-to-square',
+ });
+ quizes.push({
+ description: 'Normal',
+ event: doc?.quizMode == quizMode.NONE ? () => pushInfo(imgBox, quizMode.NORMAL) : () => exitQuizMode(imgBox),
+ icon: 'pencil',
+ });
+ ContextMenu.Instance?.addItem({ description: 'Quiz Mode', subitems: quizes, icon: 'file-pen' });
+ }
+ break;
+ case StyleProp.AnchorMenuItems:
+ if (imgBox) {
+ AnchorMenu.Instance.gptFlashcards = () => getImageDesc(imgBox);
+ AnchorMenu.Instance.makeLabels = () => makeLabels(props?.DocumentView?.().ComponentView as ImageBox);
+ }
+ }
+ return undefined;
+ }
+}
diff --git a/src/client/views/nodes/ComparisonBox.tsx b/src/client/views/nodes/ComparisonBox.tsx
index 9fb8bc4d6..c32bbc803 100644
--- a/src/client/views/nodes/ComparisonBox.tsx
+++ b/src/client/views/nodes/ComparisonBox.tsx
@@ -44,7 +44,6 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
constructor(props: FieldViewProps) {
super(props);
makeObservable(this);
- this.setListening();
}
@observable private _inputValue = '';
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 5054432a9..04a31fd83 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -545,6 +545,9 @@ export class DocumentViewInternal extends DocComponent<FieldViewProps & Document
return;
}
+ const items = this._props.styleProvider?.(this.Document, this._props, StyleProp.ContextMenuItems) as ContextMenuProps[];
+ items?.forEach(item => ContextMenu.Instance.addItem(item));
+
const customScripts = Cast(this.Document.contextMenuScripts, listSpec(ScriptField), []);
StrListCast(this.Document.contextMenuLabels).forEach((label, i) =>
cm.addItem({ description: label, event: () => customScripts[i]?.script.run({ documentView: this, this: this.Document, scriptContext: this._props.scriptContext }), icon: 'sticky-note' })
diff --git a/src/client/views/nodes/FieldView.tsx b/src/client/views/nodes/FieldView.tsx
index ca783d034..170966471 100644
--- a/src/client/views/nodes/FieldView.tsx
+++ b/src/client/views/nodes/FieldView.tsx
@@ -14,6 +14,7 @@ import { DocumentView } from './DocumentView';
import { FocusViewOptions } from './FocusViewOptions';
import { OpenWhere } from './OpenWhere';
import { WebField } from '../../../fields/URLField';
+import { ContextMenuProps } from '../ContextMenuItem';
export type FocusFuncType = (doc: Doc, options: FocusViewOptions) => Opt<number>;
export type StyleProviderFuncType = (
@@ -23,6 +24,7 @@ export type StyleProviderFuncType = (
property: string
) =>
| Opt<FieldType>
+ | ContextMenuProps[]
| { clipPath: string; jsx: JSX.Element }
| JSX.Element
| JSX.IntrinsicElements
diff --git a/src/client/views/nodes/ImageBox.scss b/src/client/views/nodes/ImageBox.scss
index 4d199b360..3ffda5a35 100644
--- a/src/client/views/nodes/ImageBox.scss
+++ b/src/client/views/nodes/ImageBox.scss
@@ -139,44 +139,3 @@
.imageBox-fadeBlocker-hover {
opacity: 0;
}
-
-.loading-spinner {
- position: absolute;
- display: flex;
- justify-content: center;
- align-items: center;
- height: 100%;
- width: 100%;
- // left: 50%;
- // top: 50%;
- z-index: 200;
- font-size: 20px;
- font-weight: bold;
- color: #17175e;
-}
-
-.check-icon {
- position: absolute;
- right: 40;
- bottom: 10;
- color: green;
- display: inline-block;
- font-size: 20px;
- overflow: hidden;
-}
-
-.redo-icon {
- position: absolute;
- right: 10;
- bottom: 10;
- color: black;
- display: inline-block;
- font-size: 20px;
- overflow: hidden;
-}
-
-@keyframes spin {
- to {
- transform: rotate(360deg);
- }
-}
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index 31f6df2ea..0b474076b 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -17,13 +17,11 @@ import { Cast, ImageCast, NumCast, RTFCast, StrCast } from '../../../fields/Type
import { ImageField } from '../../../fields/URLField';
import { TraceMobx } from '../../../fields/util';
import { emptyFunction } from '../../../Utils';
-import { gptAPICall, GPTCallType, gptImageLabel } from '../../apis/gpt/GPT';
import { Docs } from '../../documents/Documents';
import { DocumentType } from '../../documents/DocumentTypes';
import { DocUtils, FollowLinkScript } from '../../documents/DocUtils';
import { Networking } from '../../Network';
import { DragManager } from '../../util/DragManager';
-import { dropActionType } from '../../util/DropActionTypes';
import { SnappingManager } from '../../util/SnappingManager';
import { undoable, undoBatch } from '../../util/UndoManager';
import { CollectionFreeFormView } from '../collections/collectionFreeForm/CollectionFreeFormView';
@@ -38,16 +36,8 @@ import { StyleProp } from '../StyleProp';
import { DocumentView } from './DocumentView';
import { FieldView, FieldViewProps } from './FieldView';
import { FocusViewOptions } from './FocusViewOptions';
-import { ImageUtility } from './generativeFill/generativeFillUtils/ImageHandler';
import './ImageBox.scss';
import { OpenWhere } from './OpenWhere';
-// import stringSimilarity from 'string-similarity';
-
-enum quizMode {
- SMART = 'smart',
- NORMAL = 'normal',
- NONE = 'none',
-}
export class ImageEditorData {
// eslint-disable-next-line no-use-before-define
@@ -79,25 +69,24 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
public static LayoutString(fieldKey: string) {
return FieldView.LayoutString(ImageBox, fieldKey);
}
+ _ffref = React.createRef<CollectionFreeFormView>();
private _ignoreScroll = false;
private _forcedScroll = false;
private _dropDisposer?: DragManager.DragDropDisposer;
private _disposers: { [name: string]: IReactionDisposer } = {};
private _getAnchor: (savedAnnotations: Opt<ObservableMap<number, HTMLDivElement[]>>, addAsAnnotation: boolean) => Opt<Doc> = () => undefined;
private _overlayIconRef = React.createRef<HTMLDivElement>();
- private _marqueeref = React.createRef<MarqueeAnnotator>();
private _mainCont: React.RefObject<HTMLDivElement> = React.createRef();
private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef();
- private _imageRef: HTMLImageElement | null = null; // <video> ref
- @observable private _quizBoxes: Doc[] = [];
+ imageRef: HTMLImageElement | null = null; // <video> ref
+ marqueeref = React.createRef<MarqueeAnnotator>();
+
@observable private _searchInput = '';
- @observable private _quizMode = quizMode.NONE;
@observable private _savedAnnotations = new ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]>();
@observable private _curSuffix = '';
@observable private _error = '';
@observable private _loading = false;
@observable private _isHovering = false; // flag to switch between primary and alternate images on hover
- _ffref = React.createRef<CollectionFreeFormView>();
constructor(props: FieldViewProps) {
super(props);
@@ -110,6 +99,13 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this), this.Document));
};
+ @computed get quizBoxes() {
+ return DocListCast(this.Document.quizBoxes);
+ }
+ @computed get quizMode() {
+ return StrCast(this.Document._quizMode);
+ }
+
getAnchor = (addAsAnnotation: boolean, pinProps?: PinProps) => {
const visibleAnchor = this._getAnchor?.(this._savedAnnotations, true); // use marquee anchor, otherwise, save zoom/pan as anchor
const anchor =
@@ -315,331 +311,10 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
return cropping;
};
- createCanvas = async () => {
- const canvas = document.createElement('canvas');
- const scaling = 1 / (this._props.NativeDimScaling?.() || 1);
- const w = AnchorMenu.Instance.marqueeWidth * scaling;
- const h = AnchorMenu.Instance.marqueeHeight * scaling;
- canvas.width = w;
- canvas.height = h;
- const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions
- if (ctx) {
- this._imageRef && ctx.drawImage(this._imageRef, NumCast(this._marqueeref.current?.left) * scaling, NumCast(this._marqueeref.current?.top) * scaling, w, h, 0, 0, w, h);
- }
- const blob = await ImageUtility.canvasToBlob(canvas);
- return ImageBox.selectUrlToBase64(blob);
- };
-
- createSnapshotLink = (imagePath: string, downX?: number, downY?: number) => {
- const url = !imagePath.startsWith('/') ? ClientUtils.CorsProxy(imagePath) : imagePath;
- const width = NumCast(this.layoutDoc._width) || 1;
- const height = NumCast(this.layoutDoc._height);
- const imageSnapshot = Docs.Create.ImageDocument(url, {
- _nativeWidth: Doc.NativeWidth(this.layoutDoc),
- _nativeHeight: Doc.NativeHeight(this.layoutDoc),
- x: NumCast(this.layoutDoc.x) + width,
- y: NumCast(this.layoutDoc.y),
- onClick: FollowLinkScript(),
- _width: 150,
- _height: (height / width) * 150,
- title: '--snapshot' + NumCast(this.layoutDoc._layout_currentTimecode) + ' image-',
- });
- Doc.SetNativeWidth(imageSnapshot[DocData], Doc.NativeWidth(this.layoutDoc));
- Doc.SetNativeHeight(imageSnapshot[DocData], Doc.NativeHeight(this.layoutDoc));
- this._props.addDocument?.(imageSnapshot);
- DocUtils.MakeLink(imageSnapshot, this.getAnchor(true), { link_relationship: 'video snapshot' });
- setTimeout(() => downX !== undefined && downY !== undefined && DocumentView.getFirstDocumentView(imageSnapshot)?.startDragging(downX, downY, dropActionType.move, true));
- };
-
- static selectUrlToBase64 = async (blob: Blob): Promise<string> => {
- try {
- return new Promise((resolve, reject) => {
- const reader = new FileReader();
- reader.readAsDataURL(blob);
- reader.onloadend = () => resolve(reader.result as string);
- reader.onerror = error => reject(error);
- });
- } catch (error) {
- console.error('Error:', error);
- throw error;
- }
- };
-
- /**
- * Calls backend to find any text on an image. Gets the text and the
- * coordinates of the text and creates label boxes at those locations.
- * @param quiz
- * @param i
- */
- pushInfo = async (quiz: quizMode, i?: string) => {
- this._quizMode = quiz;
- this._loading = true;
-
- const img = {
- file: i ? i : this.paths[0],
- drag: i ? 'drag' : 'full',
- smart: quiz,
- };
- const response = await axios.post('http://localhost:105/labels/', img, {
- headers: {
- 'Content-Type': 'application/json',
- },
- });
- if (response.data['boxes'].length != 0) {
- this.createBoxes(response.data['boxes'], response.data['text']);
- } else {
- this._loading = false;
- }
- };
-
- /**
- * Creates label boxes over text on the image to be filled in.
- * @param boxes
- * @param texts
- */
- createBoxes = (boxes: [[[number, number]]], texts: [string]) => {
- for (let i = 0; i < boxes.length; i++) {
- const coords = boxes[i] ? boxes[i] : [];
- const width = coords[1][0] - coords[0][0];
- const height = coords[2][1] - coords[0][1];
- const text = texts[i];
-
- const newCol = Docs.Create.LabelDocument({
- _width: width,
- _height: height,
- _layout_fitWidth: true,
- title: '',
- });
- const scaling = 1 / (this._props.NativeDimScaling?.() || 1);
- newCol.x = coords[0][0] + NumCast(this._marqueeref.current?.left) * scaling;
- newCol.y = coords[0][1] + NumCast(this._marqueeref.current?.top) * scaling;
-
- newCol.zIndex = 1000;
- newCol.forceActive = true;
- newCol.quiz = text;
- newCol.showQuiz = false;
- newCol[DocData].textTransform = 'none';
- this._quizBoxes.push(newCol);
- this.addDocument(newCol);
- this._loading = false;
- }
- };
-
- /**
- * Create flashcards from an image.
- */
- getImageDesc = async () => {
- this._loading = true;
- try {
- const hrefBase64 = await this.createCanvas();
- const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this image with each question and answer labeled as "question" and "answer". Do not label each flashcard and do not include asterisks: ');
- AnchorMenu.Instance.transferToFlashcard(response, NumCast(this.layoutDoc['x']), NumCast(this.layoutDoc['y']));
- } catch (error) {
- console.log('Error', error);
- }
- this._loading = false;
- };
-
- /**
- * Calls the createCanvas and pushInfo methods to convert the
- * image to a form that can be passed to GPT and find the locations
- * of the text.
- */
- makeLabels = async () => {
- try {
- const hrefBase64 = await this.createCanvas();
- this.pushInfo(quizMode.NORMAL, hrefBase64);
- } catch (error) {
- console.log('Error', error);
- }
- };
-
- /**
- * Determines whether two words should be considered
- * the same, allowing minor typos.
- * @param str1
- * @param str2
- * @returns
- */
- levenshteinDistance = (str1: string, str2: string) => {
- const len1 = str1.length;
- const len2 = str2.length;
- const dp = Array.from(Array(len1 + 1), () => Array(len2 + 1).fill(0));
-
- if (len1 === 0) return len2;
- if (len2 === 0) return len1;
-
- for (let i = 0; i <= len1; i++) dp[i][0] = i;
- for (let j = 0; j <= len2; j++) dp[0][j] = j;
-
- for (let i = 1; i <= len1; i++) {
- for (let j = 1; j <= len2; j++) {
- const cost = str1[i - 1] === str2[j - 1] ? 0 : 1;
- dp[i][j] = Math.min(
- dp[i - 1][j] + 1, // deletion
- dp[i][j - 1] + 1, // insertion
- dp[i - 1][j - 1] + cost // substitution
- );
- }
- }
-
- return dp[len1][len2];
- };
-
- /**
- * Different algorithm for determining string similarity.
- * @param str1
- * @param str2
- * @returns
- */
- jaccardSimilarity = (str1: string, str2: string) => {
- const set1 = new Set(str1.split(' '));
- const set2 = new Set(str2.split(' '));
-
- const intersection = new Set([...set1].filter(x => set2.has(x)));
- const union = new Set([...set1, ...set2]);
-
- return intersection.size / union.size;
- };
-
- /**
- * Averages the jaccardSimilarity and levenshteinDistance scores
- * to determine string similarity for the labelboxes answers and
- * the users response.
- * @param str1
- * @param str2
- * @returns
- */
- stringSimilarity(str1: string, str2: string) {
- const levenshteinDist = this.levenshteinDistance(str1, str2);
- const levenshteinScore = 1 - levenshteinDist / Math.max(str1.length, str2.length);
-
- const jaccardScore = this.jaccardSimilarity(str1, str2);
-
- // Combine the scores with a higher weight on Jaccard similarity
- return 0.5 * levenshteinScore + 0.5 * jaccardScore;
- }
-
- @computed get checkIcon() {
- return (
- <Tooltip title={<div className="dash-tooltip">Check</div>}>
- <div className="check-icon" onPointerDown={this.check}>
- <FontAwesomeIcon icon="circle-check" size="lg" />
- </div>
- </Tooltip>
- );
- }
-
- @computed get redoIcon() {
- return (
- <Tooltip title={<div className="dash-tooltip">Redo</div>}>
- <div className="redo-icon" onPointerDown={this.redo}>
- <FontAwesomeIcon icon="redo-alt" size="lg" />
- </div>
- </Tooltip>
- );
- }
-
- /**
- * Returns whether two strings are similar
- * @param input
- * @param target
- * @returns
- */
- compareWords = (input: string, target: string) => {
- const distance = this.stringSimilarity(input.toLowerCase(), target.toLowerCase());
- return distance >= 0.7;
- };
-
- /**
- * GPT returns a hex color for what color the label box should be based on
- * the correctness of the users answer.
- * @param inputString
- * @returns
- */
- extractHexAndSentences = (inputString: string) => {
- // Regular expression to match a hexadecimal number at the beginning followed by a period and sentences
- const regex = /^#([0-9A-Fa-f]+)\.\s*(.+)$/s;
- const match = inputString.match(regex);
-
- if (match) {
- const hexNumber = match[1];
- const sentences = match[2].trim();
- return { hexNumber, sentences };
- } else {
- return { error: 'The input string does not match the expected format.' };
- }
- };
-
- /**
- * Check whether the contents of the label boxes on an image are correct.
- */
- check = () => {
- this._loading = true;
- this._quizBoxes.forEach(async doc => {
- const input = StrCast(doc[DocData].title);
- if (this._quizMode == quizMode.SMART && input) {
- const questionText = 'Question: What was labeled in this image?';
- const rubricText = ' Rubric: ' + StrCast(doc.quiz);
- const queryText =
- questionText +
- ' UserAnswer: ' +
- input +
- '. ' +
- rubricText +
- '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."';
- const response = await gptAPICall(queryText, GPTCallType.QUIZ);
- const hexSent = this.extractHexAndSentences(response);
- doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric');
- doc.backgroundColor = '#' + hexSent.hexNumber;
- } else {
- const match = this.compareWords(input, StrCast(doc.quiz));
- doc.backgroundColor = match ? '#11c249' : '#eb2d2d';
- }
- doc.showQuiz = true;
- });
- this._loading = false;
- };
-
- redo = () => {
- this._quizBoxes.forEach(doc => {
- doc[DocData].title = '';
- doc.backgroundColor = '#e4e4e4';
- doc.showQuiz = false;
- });
- };
-
- /**
- * Get rid of all the label boxes on the images.
- */
- exitQuizMode = () => {
- this._quizMode = quizMode.NONE;
- this._quizBoxes.forEach(doc => {
- this.removeDocument?.(doc);
- });
- this._quizBoxes = [];
- };
-
- @action
- setRef = (iref: HTMLImageElement | null) => {
- this._imageRef = iref;
- };
-
specificContextMenu = (): void => {
const field = Cast(this.dataDoc[this.fieldKey], ImageField);
if (field) {
const funcs: ContextMenuProps[] = [];
- const quizes: ContextMenuProps[] = [];
- quizes.push({
- description: 'Smart Check',
- event: this._quizMode == quizMode.NONE ? () => this.pushInfo(quizMode.SMART) : this.exitQuizMode,
- icon: 'pen-to-square',
- });
- quizes.push({
- description: 'Normal',
- event: this._quizMode == quizMode.NONE ? () => this.pushInfo(quizMode.NORMAL) : this.exitQuizMode,
- icon: 'pencil',
- });
funcs.push({ description: 'Rotate Clockwise 90', event: this.rotate, icon: 'redo-alt' });
funcs.push({ description: `Show ${this.layoutDoc._showFullRes ? 'Dynamic Res' : 'Full Res'}`, event: this.resolution, icon: 'expand' });
funcs.push({ description: 'Set Native Pixel Size', event: this.setNativeSize, icon: 'expand-arrows-alt' });
@@ -654,7 +329,6 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
}),
icon: 'pencil-alt',
});
- ContextMenu.Instance?.addItem({ description: 'Quiz Mode', subitems: quizes, icon: 'file-pen' });
ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' });
}
};
@@ -770,7 +444,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
<div className="imageBox-fader" style={{ opacity: backAlpha }}>
<img
alt=""
- ref={this.setRef}
+ ref={action((r: HTMLImageElement | null) => (this.imageRef = r))}
key="paths"
src={srcpath}
style={{ transform, transformOrigin }}
@@ -811,7 +485,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
e,
action(moveEv => {
MarqueeAnnotator.clearAnnotations(this._savedAnnotations);
- this._marqueeref.current?.onInitiateSelection([moveEv.clientX, moveEv.clientY]);
+ this.marqueeref.current?.onInitiateSelection([moveEv.clientX, moveEv.clientY]);
return true;
}),
returnFalse,
@@ -823,12 +497,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
@action
finishMarquee = () => {
this._getAnchor = AnchorMenu.Instance?.GetAnchor;
- AnchorMenu.Instance.gptFlashcards = this.getImageDesc;
+ this._props.styleProvider?.(this.Document, this._props, StyleProp.AnchorMenuItems);
AnchorMenu.Instance.addToCollection = this._props.DocumentView?.()._props.addDocument;
- AnchorMenu.Instance.makeLabels = this.makeLabels;
- AnchorMenu.Instance.marqueeWidth = this._marqueeref.current?.Width ?? 0;
- AnchorMenu.Instance.marqueeHeight = this._marqueeref.current?.Height ?? 0;
- this._marqueeref.current?.onTerminateSelection();
+ AnchorMenu.Instance.marqueeWidth = this.marqueeref.current?.Width ?? 0;
+ AnchorMenu.Instance.marqueeHeight = this.marqueeref.current?.Height ?? 0;
+ this.marqueeref.current?.onTerminateSelection();
this._props.select(false);
};
focus = (anchor: Doc, options: FocusViewOptions) => (anchor.type === DocumentType.CONFIG ? undefined : this._ffref.current?.focus(anchor, options));
@@ -898,7 +571,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
{!this._mainCont.current || !this.DocumentView || !this._annotationLayer.current ? null : (
<MarqueeAnnotator
Document={this.Document}
- ref={this._marqueeref}
+ ref={this.marqueeref}
scrollTop={0}
annotationLayerScrollTop={0}
scaling={returnOne}
@@ -915,8 +588,6 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
// anchorMenuFlashcard={() => this.getImageDesc()}
/>
)}
- {this._quizMode != quizMode.NONE ? this.checkIcon : null}
- {this._quizMode != quizMode.NONE ? this.redoIcon : null}
</div>
);
}
diff --git a/src/client/views/nodes/LabelBox.tsx b/src/client/views/nodes/LabelBox.tsx
index 058932457..07c0a114a 100644
--- a/src/client/views/nodes/LabelBox.tsx
+++ b/src/client/views/nodes/LabelBox.tsx
@@ -1,12 +1,8 @@
-import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
-import { Tooltip } from '@mui/material';
import { Property } from 'csstype';
import { action, computed, makeObservable, observable } from 'mobx';
import { observer } from 'mobx-react';
import * as React from 'react';
import * as textfit from 'textfit';
-import { returnFalse, setupMoveUpEvents } from '../../../ClientUtils';
-import { emptyFunction } from '../../../Utils';
import { Field, FieldType } from '../../../fields/Doc';
import { BoolCast, NumCast, StrCast } from '../../../fields/Types';
import { TraceMobx } from '../../../fields/util';
@@ -49,48 +45,6 @@ export class LabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
return this._props.styleProvider?.(this.Document, this._props, StyleProp.BackgroundColor) as string;
}
- @computed get answerIcon() {
- return (
- <Tooltip
- title={
- <div className="answer-tooltip" style={{ minWidth: '150px' }}>
- {StrCast(this.Document.quiz)}
- </div>
- }>
- <div className="answer-tool-tip">
- <FontAwesomeIcon className="q-icon" icon="circle" color="white" />
- <FontAwesomeIcon className="answer-icon" icon="question" />
- </div>
- </Tooltip>
- );
- }
-
- @computed get editAnswer() {
- return (
- <Tooltip
- title={
- <div className="answer-tooltip" style={{ minWidth: '150px' }}>
- {this._editLabel ? 'save' : 'edit correct answer'}
- </div>
- }>
- <div className="answer-tool-tip" onPointerDown={e => setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => this.editLabelAnswer())}>
- <FontAwesomeIcon className="edit-icon" color={this._editLabel ? 'white' : 'black'} icon="pencil" size="sm" />
- </div>
- </Tooltip>
- );
- }
-
- editLabelAnswer = () => {
- // when click the pencil, set the text to the quiz content. when click off, set the quiz text to that and set textbox to nothing.
- if (!this._editLabel) {
- this.dataDoc.title = StrCast(this.Document.quiz);
- } else {
- this.Document.quiz = this.Title;
- this.dataDoc.title = '';
- }
- this._editLabel = !this._editLabel;
- };
-
componentDidMount() {
this._props.setContentViewBox?.(this);
}
@@ -98,8 +52,6 @@ export class LabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
this._timeout && clearTimeout(this._timeout);
}
- specificContextMenu = (): void => {};
-
drop = (/* e: Event, de: DragManager.DropEvent */) => {
return false;
};
@@ -152,7 +104,7 @@ export class LabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
const boxParams = this.fitTextToBox(undefined); // this causes mobx to trigger re-render when data changes
const label = this.Title.startsWith('#') ? null : this.Title;
return (
- <div key={label?.length} className="labelBox-outerDiv" ref={this.createDropTarget} onContextMenu={this.specificContextMenu} style={{ boxShadow: this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.BoxShadow) as string }}>
+ <div key={label?.length} className="labelBox-outerDiv" ref={this.createDropTarget} style={{ boxShadow: this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.BoxShadow) as string }}>
<div
className="labelBox-mainButton"
style={{
@@ -203,8 +155,6 @@ export class LabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
{label}
</div>
</div>
- {this.Document.showQuiz ? this.answerIcon : null}
- {this.Document.showQuiz ? this.editAnswer : null}
</div>
);
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 12:42:47 +0000