diff options
Diffstat (limited to 'src/client/views/nodes/ImageBox.tsx')
| -rw-r--r-- | src/client/views/nodes/ImageBox.tsx | 354 |
1 files changed, 239 insertions, 115 deletions
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 8068407bb..86da64e5e 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -11,7 +11,6 @@ import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; import { InkTool } from '../../../fields/InkField'; import { ObjectField } from '../../../fields/ObjectField'; -import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; import { TraceMobx } from '../../../fields/util'; import { emptyFunction } from '../../../Utils'; @@ -33,17 +32,27 @@ import { StyleProp } from '../StyleProp'; import { DocumentView } from './DocumentView'; import { FieldView, FieldViewProps } from './FieldView'; import { FocusViewOptions } from './FocusViewOptions'; +import { DocCast, NumCast, RTFCast, StrCast, ImageCast, Cast, toList } from '../../../fields/Types'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; import { URLField } from '../../../fields/URLField'; -import { gptImageLabel } from '../../apis/gpt/GPT'; +import { gptAPICall, GPTCallType, gptImageLabel } from '../../apis/gpt/GPT'; import ReactLoading from 'react-loading'; import { FollowLinkScript } from '../../documents/DocUtils'; import { basename } from 'path'; import { ImageUtility } from './generativeFill/generativeFillUtils/ImageHandler'; import { dropActionType } from '../../util/DropActionTypes'; import { canvasSize } from './generativeFill/generativeFillUtils/generativeFillConstants'; +import Tesseract from 'tesseract.js'; import axios from 'axios'; +import { TupleType } from 'typescript'; +// import stringSimilarity from 'string-similarity'; + +enum quizMode { + SMART = 'smart', + NORMAL = 'normal', + NONE = 'none', +} export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -85,9 +94,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef(); private _imageRef: HTMLImageElement | null = null; // <video> ref + @observable private _quizBoxes: Doc[] = []; @observable private _width: number = 0; @observable private _height: number = 0; @observable private searchInput = ''; + @observable private _quizMode = quizMode.NONE; @observable _savedAnnotations = new ObservableMap<number, HTMLDivElement[]>(); @observable _curSuffix = ''; @observable _error = ''; @@ -239,6 +250,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.dataDoc._freeform_panX_min = this.dataDoc._freeform_panX_min ? nw * NumCast(this.dataDoc._freeform_panX_min) : undefined; this.dataDoc._freeform_panY_max = this.dataDoc._freeform_panY_max ? nw * NumCast(this.dataDoc._freeform_panY_max) : undefined; this.dataDoc._freeform_panY_min = this.dataDoc._freeform_panY_min ? nw * NumCast(this.dataDoc._freeform_panY_min) : undefined; + return nw; }); @undoBatch rotate = action(() => { @@ -302,70 +314,20 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }; createCanvas = async (downX?: number, downY?: number, cb?: (filename: string, x: number | undefined, y: number | undefined) => void) => { - const width = NumCast(this.layoutDoc._width); const canvas = document.createElement('canvas'); - // canvas.width = 640; - // canvas.height = (640 * Doc.NativeHeight(this.layoutDoc)) / (Doc.NativeWidth(this.layoutDoc) || 1); - canvas.width = NumCast(this.layoutDoc._width); - canvas.height = NumCast(this.layoutDoc._height); + const scaling = 1 / (this._props.NativeDimScaling?.() || 1); + const w = AnchorMenu.Instance.marqueeWidth * scaling; + const h = AnchorMenu.Instance.marqueeHeight * scaling; + canvas.width = w; + canvas.height = h; const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions if (ctx) { - // this._imageRef && ctx.drawImage(this._imageRef, 0, 0, canvas.width, canvas.height); - this._imageRef && ctx.drawImage(this._imageRef, NumCast(this._marqueeref.current?.left), NumCast(this._marqueeref.current?.top), this._width, this._height, 0, 0, 1000, 1000); - //this._imageRef && ctx.drawImage(this._imageRef, 0, 0, 2000, 1000, 0, 0, canvas.width, canvas.height); - // console.log(NumCast(this._marqueeref.current?.left) + 100); + this._imageRef && ctx.drawImage(this._imageRef, NumCast(this._marqueeref.current?.left) * scaling, NumCast(this._marqueeref.current?.top) * scaling, w, h, 0, 0, w, h); } + // canvas.style.zIndex = '2000000'; + // document.body.appendChild(canvas); const blob = await ImageUtility.canvasToBlob(canvas); return ImageBox.selectUrlToBase64(blob); - - // if (this._imageRef) { - // const canv = ImageUtility.getCroppedImg(this._imageRef, this._imageRef.width, this._imageRef.height); - // console.log(this._imageRef.width); - // if (canv) { - // const blob = await ImageUtility.canvasToBlob(canv); - // return ImageBox.selectUrlToBase64(blob); - // } - // } - if (!this._imageRef) { - const b = Docs.Create.LabelDocument({ - x: NumCast(this.layoutDoc.x) + width, - y: NumCast(this.layoutDoc.y, 1), - _width: 150, - _height: 50, - // title: (this.layoutDoc._layout_currentTimecode || 0).toString(), - onClick: FollowLinkScript(), - }); - this._props.addDocument?.(b); - DocUtils.MakeLink(b, this.Document, { link_relationship: 'image snapshot' }); - } else { - // convert to desired file format - // const dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png' - // // if you want to preview the captured image, - // const retitled = StrCast(this.Document.title).replace(/[ -.:]/g, ''); - // const encodedFilename = encodeURIComponent(('snapshot' + retitled + '_' + (this.layoutDoc._layout_currentTimecode || 0).toString()).replace(/[./?=]/g, '_')); - // const filename = basename(encodedFilename); - // ClientUtils.convertDataUri(dataUrl, filename).then((returnedFilename: string) => returnedFilename && (cb ?? this.createSnapshotLink)(returnedFilename, downX, downY)); - } - // convert to desired file format - - // const dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png' - // // if you want to preview the captured image, - // const retitled = StrCast(this.Document.title).replace(/[ -.:]/g, ''); - // const encodedFilename = encodeURIComponent(('snapshot' + retitled + '_' + (this.layoutDoc._layout_currentTimecode || 0).toString()).replace(/[./?=]/g, '_')); - // const filename = basename(encodedFilename); - //ClientUtils.convertDataUri(dataUrl, filename).then((returnedFilename: string) => returnedFilename && (cb ?? this.createSnapshotLink)(returnedFilename, downX, downY)); - // } - // const docViewContent = this.DocumentView?.().ContentDiv!; - // if (docViewContent instanceof HTMLCanvasElement) { - // const canvas = docViewContent; - // const img = document.createElement('img'); // create a Image Element - // img.src = canvas.toDataURL(); // image sourcez - // img.style.width = canvas.style.width; - // img.style.height = canvas.style.height; - // const parEle = newCan.parentElement as HTMLElement; - // parEle.removeChild(newCan); - // parEle.appendChild(img); - // } }; createSnapshotLink = (imagePath: string, downX?: number, downY?: number) => { @@ -390,21 +352,6 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { setTimeout(() => downX !== undefined && downY !== undefined && DocumentView.getFirstDocumentView(imageSnapshot)?.startDragging(downX, downY, dropActionType.move, true)); }; - /** - * - if (oldDiv instanceof HTMLCanvasElement) { - const canvas = oldDiv; - const img = document.createElement('img'); // create a Image Element - img.src = canvas.toDataURL(); // image sourcez - img.style.width = canvas.style.width; - img.style.height = canvas.style.height; - const newCan = newDiv as HTMLCanvasElement; - const parEle = newCan.parentElement as HTMLElement; - parEle.removeChild(newCan); - parEle.appendChild(img); - } - */ - static selectUrlToBase64 = async (blob: Blob): Promise<string> => { try { return new Promise((resolve, reject) => { @@ -419,71 +366,242 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } }; - static imageUrlToBase64 = async (imageUrl: string): Promise<string> => { - try { - const response = await fetch(imageUrl); - const blob = await response.blob(); + pushInfo = async (quiz: quizMode, i?: string) => { + this._quizMode = quiz; + this._loading = true; + console.log('JHSDKFJHKSDJFHKSJDHFKJSDHFKJHSDKF'); - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.readAsDataURL(blob); - reader.onloadend = () => resolve(reader.result as string); - reader.onerror = error => reject(error); + const img = { + file: i ? i : this.paths[0], + drag: i ? 'drag' : 'full', + smart: quiz, + }; + const response = await axios.post('http://localhost:105/labels/', img, { + headers: { + 'Content-Type': 'application/json', + }, + }); + + console.log('RESPONSE:'); + console.log(response.data['boxes']); + console.log(response.data['text']); + this.createBoxes(response.data['boxes'], response.data['text']); + }; + + createBoxes = (boxes: [[[number, number]]], texts: [string]) => { + const nscale = NumCast(this._props.PanelWidth()) * NumCast(this.layoutDoc._freeform_scale, 1); + const nw = nscale / NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']); + for (var i = 0; i < boxes.length; i++) { + const coords = boxes[i] ? boxes[i] : []; + const width = coords[1][0] - coords[0][0]; + const height = coords[2][1] - coords[0][1]; + const text = texts[i]; + + const newCol = Docs.Create.TextDocument('', { + _width: width, + //width * NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']), + _height: height, + //height * NumCast(this.dataDoc[this.fieldKey + '_nativeHeight']), + _layout_fitWidth: true, + // _layout_autoHeight: true, }); - } catch (error) { - console.error('Error:', error); - throw error; + const scaling = 1 / (this._props.NativeDimScaling?.() || 1); + newCol.x = coords[0][0] + NumCast(this._marqueeref.current?.left) * scaling; + newCol.y = coords[0][1] + NumCast(this._marqueeref.current?.top) * scaling; + // newCol[DocData].text_fontSize = height + 'px'; + + newCol.zIndex = 1000; + newCol.forceActive = true; + newCol.quiz = text; + newCol.showQuiz = false; + this._quizBoxes.push(newCol); + this.addDocument(newCol); + this._loading = false; } }; getImageDesc = async () => { - // if (StrCast(this.dataDoc.description)) return StrCast(this.dataDoc.description); // Return existing description - const { href } = (this.dataDoc.data as URLField).url; - const hrefParts = href.split('.'); - const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; this._loading = true; try { - // const hrefBase64 = await ImageBox.imageUrlToBase64(hrefComplete); const hrefBase64 = await this.createCanvas(); - const response = await gptImageLabel(hrefBase64, 'Tell me what words you see on this image.'); - //const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this text with each question and answer labeled as question and answer. Do not label each flashcard and do not include asterisks: '); + const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this image with each question and answer labeled as "question" and "answer". Do not label each flashcard and do not include asterisks: '); console.log(response); - // AnchorMenu.Instance.transferToFlashcard(response); - // this.Document[DocData].description = response.trim(); - // return response; // Return the response + AnchorMenu.Instance.transferToFlashcard(response, NumCast(this.layoutDoc['x']), NumCast(this.layoutDoc['y'])); + } catch (error) { + console.log('Error'); + } + this._loading = false; + }; + + makeLabels = async () => { + this._loading = true; + try { + const hrefBase64 = await this.createCanvas(); + this.pushInfo(quizMode.NORMAL, hrefBase64); } catch (error) { console.log('Error'); } + }; + + levenshteinDistance = (a: string, b: string) => { + const an = a.length; + const bn = b.length; + const matrix = []; + + // Ensure non-zero length strings + if (an === 0) return bn; + if (bn === 0) return an; + + // Initialize the matrix + for (let i = 0; i <= an; i++) { + matrix[i] = [i]; + } + for (let j = 0; j <= bn; j++) { + matrix[0][j] = j; + } + + // Populate the matrix + for (let i = 1; i <= an; i++) { + for (let j = 1; j <= bn; j++) { + if (a[i - 1] === b[j - 1]) { + matrix[i][j] = matrix[i - 1][j - 1]; + } else { + matrix[i][j] = Math.min( + matrix[i - 1][j - 1] + 1, // substitution + Math.min( + matrix[i][j - 1] + 1, // insertion + matrix[i - 1][j] + 1 // deletion + ) + ); + } + } + } + + return matrix[an][bn]; + }; + + @computed get checkIcon() { + return ( + <Tooltip title={<div className="dash-tooltip">Check</div>}> + <div className="check-icon" onPointerDown={this.check}> + <FontAwesomeIcon icon="circle-check" size="lg" /> + </div> + </Tooltip> + ); + } + + @computed get redoIcon() { + return ( + <Tooltip title={<div className="dash-tooltip">Redo</div>}> + <div className="redo-icon" onPointerDown={this.redo}> + <FontAwesomeIcon icon="redo-alt" size="lg" /> + </div> + </Tooltip> + ); + } + + compareWords = (input: string, target: string) => { + const distance = this.levenshteinDistance(input.toLowerCase(), target.toLowerCase()); + const threshold = Math.max(input.length, target.length) * 0.2; // Allow up to 20% of the length as difference + return distance <= threshold; + }; + + extractHexAndSentences = (inputString: string) => { + // Regular expression to match a hexadecimal number at the beginning followed by a period and sentences + const regex = /^#([0-9A-Fa-f]+)\.\s*(.+)$/s; + const match = inputString.match(regex); + + if (match) { + const hexNumber = match[1]; + const sentences = match[2].trim(); + return { hexNumber, sentences }; + } else { + return { error: 'The input string does not match the expected format.' }; + } + }; + + check = () => { + this._loading = true; + this._quizBoxes.forEach(async doc => { + const input = StrCast(RTFCast(DocCast(doc).text)?.Text); + console.log('INP: ' + StrCast(input) + '; DOC: ' + StrCast(doc.quiz)); + if (this._quizMode == quizMode.SMART && input) { + const questionText = 'Question: What was labeled in this image?'; + const rubricText = ' Rubric: ' + StrCast(doc.quiz); + // const queryText = 'RealAnswer: ' + StrCast(doc.quiz) + '. UserAnswer: ' + input + '.'; + const queryText = + questionText + + ' UserAnswer: ' + + input + + '. ' + + rubricText + + '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."'; + const response = await gptAPICall(queryText, GPTCallType.QUIZ); + const hexSent = this.extractHexAndSentences(response); + console.log(hexSent.hexNumber); + doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); + doc.backgroundColor = '#' + hexSent.hexNumber; + } else { + const match = this.compareWords(input, StrCast(doc.quiz)); + doc.backgroundColor = match ? '#11c249' : '#eb2d2d'; + } + doc.showQuiz = true; + // console.log(this.compareWords(input, StrCast(doc.quiz)) ? 'Match' : 'No Match'); + }); this._loading = false; - // return ''; + }; + + redo = () => { + this._quizBoxes.forEach(doc => { + DocCast(doc)[DocData].text = ''; + doc.backgroundColor = '#e4e4e4'; + doc.showQuiz = false; + }); + }; + + exitQuizMode = () => { + this._quizMode = quizMode.NONE; + this._quizBoxes.forEach(doc => { + // this._props.removeDocument?.(DocCast(doc)); + // this._props.DocumentView?.()._props.removeDocument?.(doc); + }); + this._quizBoxes = []; }; @action setRef = (iref: HTMLImageElement | null) => { this._imageRef = iref; - // if (iref) { - // this._videoRef!.ontimeupdate = this.updateTimecode; - // // @ts-ignore - // // vref.onfullscreenchange = action((e) => this._fullScreen = vref.webkitDisplayingFullscreen); - // this._disposers.reactionDisposer?.(); - // this._disposers.reactionDisposer = reaction( - // () => NumCast(this.layoutDoc._layout_currentTimecode), - // time => { - // !this._playing && (vref.currentTime = time); - // }, - // { fireImmediately: true } - // ); - - // (!this.dataDoc[this.fieldKey + '_thumbnails'] || StrListCast(this.dataDoc[this.fieldKey + '_thumbnails']).length !== VideoThumbnails.DENSE) && this.getVideoThumbnails(); - // } }; specificContextMenu = (): void => { const field = Cast(this.dataDoc[this.fieldKey], ImageField); if (field) { const funcs: ContextMenuProps[] = []; + const quizes: ContextMenuProps[] = []; // funcs.push({ description: 'Create ai flashcards', event: () => this.getImageDesc(), icon: 'id-card' }); - funcs.push({ description: 'Get Images', event: () => this.handleSelection('Cats'), icon: 'redo-alt' }); + quizes.push({ + description: 'Smart Check', + event: this._quizMode == quizMode.NONE ? () => this.pushInfo(quizMode.SMART) : this.exitQuizMode, + icon: 'pen-to-square', + }); + quizes.push({ + description: 'Normal', + event: this._quizMode == quizMode.NONE ? () => this.pushInfo(quizMode.NORMAL) : this.exitQuizMode, + icon: 'pencil', + }); + // funcs.push({ description: 'Quiz Mode', subitems: optionItems, icon: 'eye' }); + // funcs.push({ + // description: 'Quiz Mode', + // event: !this._quizMode + // ? () => this.pushInfo(false) + // : () => { + // this._quizMode = false; + // }, + // icon: 'redo-alt', + // }); + // funcs.push({ description: 'Get Text', event: this.check, icon: 'redo-alt' }); + // funcs.push({ description: 'Get Labels2', event: this.getImageLabels2, icon: 'redo-alt' }); + // funcs.push({ description: 'Get Labels', event: this.getImageLabels, icon: 'redo-alt' }); funcs.push({ description: 'Rotate Clockwise 90', event: this.rotate, icon: 'redo-alt' }); funcs.push({ description: `Show ${this.layoutDoc._showFullRes ? 'Dynamic Res' : 'Full Res'}`, event: this.resolution, icon: 'expand' }); funcs.push({ description: 'Set Native Pixel Size', event: this.setNativeSize, icon: 'expand-arrows-alt' }); @@ -498,6 +616,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }), icon: 'pencil-alt', }); + ContextMenu.Instance?.addItem({ description: 'Quiz Mode', subitems: quizes, icon: 'file-pen' }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; @@ -669,6 +788,9 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this._getAnchor = AnchorMenu.Instance?.GetAnchor; AnchorMenu.Instance.gptFlashcards = this.getImageDesc; AnchorMenu.Instance.addToCollection = this._props.DocumentView?.()._props.addDocument; + AnchorMenu.Instance.makeLabels = this.makeLabels; + AnchorMenu.Instance.marqueeWidth = this._marqueeref.current?.Width ?? 0; + AnchorMenu.Instance.marqueeHeight = this._marqueeref.current?.Height ?? 0; this._marqueeref.current?.onTerminateSelection(); this._props.select(false); }; @@ -703,7 +825,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { height: this._props.PanelWidth() ? undefined : `100%`, pointerEvents: this.layoutDoc._lockedPosition ? 'none' : undefined, borderRadius, - overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : undefined, + overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : 'hidden', }}> <CollectionFreeFormView ref={this._ffref} @@ -757,6 +879,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // anchorMenuFlashcard={() => this.getImageDesc()} /> )} + {this._quizMode != quizMode.NONE ? this.checkIcon : null} + {this._quizMode != quizMode.NONE ? this.redoIcon : null} </div> ); } |