diff options
Diffstat (limited to 'src/client/views/nodes/ImageBox.tsx')
| -rw-r--r-- | src/client/views/nodes/ImageBox.tsx | 360 |
1 files changed, 358 insertions, 2 deletions
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 68c313480..86da64e5e 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -11,7 +11,6 @@ import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; import { InkTool } from '../../../fields/InkField'; import { ObjectField } from '../../../fields/ObjectField'; -import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; import { TraceMobx } from '../../../fields/util'; import { emptyFunction } from '../../../Utils'; @@ -33,8 +32,27 @@ import { StyleProp } from '../StyleProp'; import { DocumentView } from './DocumentView'; import { FieldView, FieldViewProps } from './FieldView'; import { FocusViewOptions } from './FocusViewOptions'; +import { DocCast, NumCast, RTFCast, StrCast, ImageCast, Cast, toList } from '../../../fields/Types'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; +import { URLField } from '../../../fields/URLField'; +import { gptAPICall, GPTCallType, gptImageLabel } from '../../apis/gpt/GPT'; +import ReactLoading from 'react-loading'; +import { FollowLinkScript } from '../../documents/DocUtils'; +import { basename } from 'path'; +import { ImageUtility } from './generativeFill/generativeFillUtils/ImageHandler'; +import { dropActionType } from '../../util/DropActionTypes'; +import { canvasSize } from './generativeFill/generativeFillUtils/generativeFillConstants'; +import Tesseract from 'tesseract.js'; +import axios from 'axios'; +import { TupleType } from 'typescript'; +// import stringSimilarity from 'string-similarity'; + +enum quizMode { + SMART = 'smart', + NORMAL = 'normal', + NONE = 'none', +} export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -59,6 +77,8 @@ export class ImageEditorData { public static get AddDoc() { return ImageEditorData.imageData.addDoc; } // prettier-ignore public static set AddDoc(addDoc: Opt<(doc: Doc | Doc[], annotationKey?: string) => boolean>) { ImageEditorData.set(this.imageData.open, this.imageData.rootDoc, this.imageData.source, addDoc); } // prettier-ignore } + +const API_URL = 'https://api.unsplash.com/search/photos'; @observer export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { public static LayoutString(fieldKey: string) { @@ -73,9 +93,16 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { private _marqueeref = React.createRef<MarqueeAnnotator>(); private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef(); + private _imageRef: HTMLImageElement | null = null; // <video> ref + @observable private _quizBoxes: Doc[] = []; + @observable private _width: number = 0; + @observable private _height: number = 0; + @observable private searchInput = ''; + @observable private _quizMode = quizMode.NONE; @observable _savedAnnotations = new ObservableMap<number, HTMLDivElement[]>(); @observable _curSuffix = ''; @observable _error = ''; + @observable private _loading = false; @observable _isHovering = false; // flag to switch between primary and alternate images on hover _ffref = React.createRef<CollectionFreeFormView>(); @@ -148,6 +175,32 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { Object.values(this._disposers).forEach(disposer => disposer?.()); } + fetchImages = async () => { + try { + const { data } = await axios.get(`${API_URL}?query=${this.searchInput}&page=1&per_page=${1}&client_id=${process.env.VITE_API_KEY}`); + console.log('data', data); + console.log(data.results); + const imageSnapshot = Docs.Create.ImageDocument(data.results[0].urls.small, { + _nativeWidth: Doc.NativeWidth(this.layoutDoc), + _nativeHeight: Doc.NativeHeight(this.layoutDoc), + x: NumCast(this.layoutDoc.x), + y: NumCast(this.layoutDoc.y), + onClick: FollowLinkScript(), + _width: 150, + _height: 150, + title: '--snapshot' + NumCast(this.layoutDoc._layout_currentTimecode) + ' image-', + }); + this._props.addDocument?.(imageSnapshot); + } catch (error) { + console.log(error); + } + }; + + handleSelection = async (selection: string) => { + this.searchInput = selection; + const images = await this.fetchImages(); + }; + @undoBatch drop = (e: Event, de: DragManager.DropEvent) => { if (de.complete.docDragData) { @@ -197,6 +250,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.dataDoc._freeform_panX_min = this.dataDoc._freeform_panX_min ? nw * NumCast(this.dataDoc._freeform_panX_min) : undefined; this.dataDoc._freeform_panY_max = this.dataDoc._freeform_panY_max ? nw * NumCast(this.dataDoc._freeform_panY_max) : undefined; this.dataDoc._freeform_panY_min = this.dataDoc._freeform_panY_min ? nw * NumCast(this.dataDoc._freeform_panY_min) : undefined; + return nw; }); @undoBatch rotate = action(() => { @@ -259,10 +313,295 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return cropping; }; + createCanvas = async (downX?: number, downY?: number, cb?: (filename: string, x: number | undefined, y: number | undefined) => void) => { + const canvas = document.createElement('canvas'); + const scaling = 1 / (this._props.NativeDimScaling?.() || 1); + const w = AnchorMenu.Instance.marqueeWidth * scaling; + const h = AnchorMenu.Instance.marqueeHeight * scaling; + canvas.width = w; + canvas.height = h; + const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions + if (ctx) { + this._imageRef && ctx.drawImage(this._imageRef, NumCast(this._marqueeref.current?.left) * scaling, NumCast(this._marqueeref.current?.top) * scaling, w, h, 0, 0, w, h); + } + // canvas.style.zIndex = '2000000'; + // document.body.appendChild(canvas); + const blob = await ImageUtility.canvasToBlob(canvas); + return ImageBox.selectUrlToBase64(blob); + }; + + createSnapshotLink = (imagePath: string, downX?: number, downY?: number) => { + const url = !imagePath.startsWith('/') ? ClientUtils.CorsProxy(imagePath) : imagePath; + const width = NumCast(this.layoutDoc._width) || 1; + const height = NumCast(this.layoutDoc._height); + const imageSnapshot = Docs.Create.ImageDocument(url, { + _nativeWidth: Doc.NativeWidth(this.layoutDoc), + _nativeHeight: Doc.NativeHeight(this.layoutDoc), + x: NumCast(this.layoutDoc.x) + width, + y: NumCast(this.layoutDoc.y), + onClick: FollowLinkScript(), + _width: 150, + _height: (height / width) * 150, + title: '--snapshot' + NumCast(this.layoutDoc._layout_currentTimecode) + ' image-', + }); + Doc.SetNativeWidth(imageSnapshot[DocData], Doc.NativeWidth(this.layoutDoc)); + Doc.SetNativeHeight(imageSnapshot[DocData], Doc.NativeHeight(this.layoutDoc)); + this._props.addDocument?.(imageSnapshot); + DocUtils.MakeLink(imageSnapshot, this.getAnchor(true), { link_relationship: 'video snapshot' }); + // link && (DocCast(link.link_anchor_2)[DocData].timecodeToHide = NumCast(DocCast(link.link_anchor_2).timecodeToShow) + 3); // do we need to set an end time? should default to +0.1 + setTimeout(() => downX !== undefined && downY !== undefined && DocumentView.getFirstDocumentView(imageSnapshot)?.startDragging(downX, downY, dropActionType.move, true)); + }; + + static selectUrlToBase64 = async (blob: Blob): Promise<string> => { + try { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = error => reject(error); + }); + } catch (error) { + console.error('Error:', error); + throw error; + } + }; + + pushInfo = async (quiz: quizMode, i?: string) => { + this._quizMode = quiz; + this._loading = true; + console.log('JHSDKFJHKSDJFHKSJDHFKJSDHFKJHSDKF'); + + const img = { + file: i ? i : this.paths[0], + drag: i ? 'drag' : 'full', + smart: quiz, + }; + const response = await axios.post('http://localhost:105/labels/', img, { + headers: { + 'Content-Type': 'application/json', + }, + }); + + console.log('RESPONSE:'); + console.log(response.data['boxes']); + console.log(response.data['text']); + this.createBoxes(response.data['boxes'], response.data['text']); + }; + + createBoxes = (boxes: [[[number, number]]], texts: [string]) => { + const nscale = NumCast(this._props.PanelWidth()) * NumCast(this.layoutDoc._freeform_scale, 1); + const nw = nscale / NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']); + for (var i = 0; i < boxes.length; i++) { + const coords = boxes[i] ? boxes[i] : []; + const width = coords[1][0] - coords[0][0]; + const height = coords[2][1] - coords[0][1]; + const text = texts[i]; + + const newCol = Docs.Create.TextDocument('', { + _width: width, + //width * NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']), + _height: height, + //height * NumCast(this.dataDoc[this.fieldKey + '_nativeHeight']), + _layout_fitWidth: true, + // _layout_autoHeight: true, + }); + const scaling = 1 / (this._props.NativeDimScaling?.() || 1); + newCol.x = coords[0][0] + NumCast(this._marqueeref.current?.left) * scaling; + newCol.y = coords[0][1] + NumCast(this._marqueeref.current?.top) * scaling; + // newCol[DocData].text_fontSize = height + 'px'; + + newCol.zIndex = 1000; + newCol.forceActive = true; + newCol.quiz = text; + newCol.showQuiz = false; + this._quizBoxes.push(newCol); + this.addDocument(newCol); + this._loading = false; + } + }; + + getImageDesc = async () => { + this._loading = true; + try { + const hrefBase64 = await this.createCanvas(); + const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this image with each question and answer labeled as "question" and "answer". Do not label each flashcard and do not include asterisks: '); + console.log(response); + AnchorMenu.Instance.transferToFlashcard(response, NumCast(this.layoutDoc['x']), NumCast(this.layoutDoc['y'])); + } catch (error) { + console.log('Error'); + } + this._loading = false; + }; + + makeLabels = async () => { + this._loading = true; + try { + const hrefBase64 = await this.createCanvas(); + this.pushInfo(quizMode.NORMAL, hrefBase64); + } catch (error) { + console.log('Error'); + } + }; + + levenshteinDistance = (a: string, b: string) => { + const an = a.length; + const bn = b.length; + const matrix = []; + + // Ensure non-zero length strings + if (an === 0) return bn; + if (bn === 0) return an; + + // Initialize the matrix + for (let i = 0; i <= an; i++) { + matrix[i] = [i]; + } + for (let j = 0; j <= bn; j++) { + matrix[0][j] = j; + } + + // Populate the matrix + for (let i = 1; i <= an; i++) { + for (let j = 1; j <= bn; j++) { + if (a[i - 1] === b[j - 1]) { + matrix[i][j] = matrix[i - 1][j - 1]; + } else { + matrix[i][j] = Math.min( + matrix[i - 1][j - 1] + 1, // substitution + Math.min( + matrix[i][j - 1] + 1, // insertion + matrix[i - 1][j] + 1 // deletion + ) + ); + } + } + } + + return matrix[an][bn]; + }; + + @computed get checkIcon() { + return ( + <Tooltip title={<div className="dash-tooltip">Check</div>}> + <div className="check-icon" onPointerDown={this.check}> + <FontAwesomeIcon icon="circle-check" size="lg" /> + </div> + </Tooltip> + ); + } + + @computed get redoIcon() { + return ( + <Tooltip title={<div className="dash-tooltip">Redo</div>}> + <div className="redo-icon" onPointerDown={this.redo}> + <FontAwesomeIcon icon="redo-alt" size="lg" /> + </div> + </Tooltip> + ); + } + + compareWords = (input: string, target: string) => { + const distance = this.levenshteinDistance(input.toLowerCase(), target.toLowerCase()); + const threshold = Math.max(input.length, target.length) * 0.2; // Allow up to 20% of the length as difference + return distance <= threshold; + }; + + extractHexAndSentences = (inputString: string) => { + // Regular expression to match a hexadecimal number at the beginning followed by a period and sentences + const regex = /^#([0-9A-Fa-f]+)\.\s*(.+)$/s; + const match = inputString.match(regex); + + if (match) { + const hexNumber = match[1]; + const sentences = match[2].trim(); + return { hexNumber, sentences }; + } else { + return { error: 'The input string does not match the expected format.' }; + } + }; + + check = () => { + this._loading = true; + this._quizBoxes.forEach(async doc => { + const input = StrCast(RTFCast(DocCast(doc).text)?.Text); + console.log('INP: ' + StrCast(input) + '; DOC: ' + StrCast(doc.quiz)); + if (this._quizMode == quizMode.SMART && input) { + const questionText = 'Question: What was labeled in this image?'; + const rubricText = ' Rubric: ' + StrCast(doc.quiz); + // const queryText = 'RealAnswer: ' + StrCast(doc.quiz) + '. UserAnswer: ' + input + '.'; + const queryText = + questionText + + ' UserAnswer: ' + + input + + '. ' + + rubricText + + '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."'; + const response = await gptAPICall(queryText, GPTCallType.QUIZ); + const hexSent = this.extractHexAndSentences(response); + console.log(hexSent.hexNumber); + doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); + doc.backgroundColor = '#' + hexSent.hexNumber; + } else { + const match = this.compareWords(input, StrCast(doc.quiz)); + doc.backgroundColor = match ? '#11c249' : '#eb2d2d'; + } + doc.showQuiz = true; + // console.log(this.compareWords(input, StrCast(doc.quiz)) ? 'Match' : 'No Match'); + }); + this._loading = false; + }; + + redo = () => { + this._quizBoxes.forEach(doc => { + DocCast(doc)[DocData].text = ''; + doc.backgroundColor = '#e4e4e4'; + doc.showQuiz = false; + }); + }; + + exitQuizMode = () => { + this._quizMode = quizMode.NONE; + this._quizBoxes.forEach(doc => { + // this._props.removeDocument?.(DocCast(doc)); + // this._props.DocumentView?.()._props.removeDocument?.(doc); + }); + this._quizBoxes = []; + }; + + @action + setRef = (iref: HTMLImageElement | null) => { + this._imageRef = iref; + }; + specificContextMenu = (): void => { const field = Cast(this.dataDoc[this.fieldKey], ImageField); if (field) { const funcs: ContextMenuProps[] = []; + const quizes: ContextMenuProps[] = []; + // funcs.push({ description: 'Create ai flashcards', event: () => this.getImageDesc(), icon: 'id-card' }); + quizes.push({ + description: 'Smart Check', + event: this._quizMode == quizMode.NONE ? () => this.pushInfo(quizMode.SMART) : this.exitQuizMode, + icon: 'pen-to-square', + }); + quizes.push({ + description: 'Normal', + event: this._quizMode == quizMode.NONE ? () => this.pushInfo(quizMode.NORMAL) : this.exitQuizMode, + icon: 'pencil', + }); + // funcs.push({ description: 'Quiz Mode', subitems: optionItems, icon: 'eye' }); + // funcs.push({ + // description: 'Quiz Mode', + // event: !this._quizMode + // ? () => this.pushInfo(false) + // : () => { + // this._quizMode = false; + // }, + // icon: 'redo-alt', + // }); + // funcs.push({ description: 'Get Text', event: this.check, icon: 'redo-alt' }); + // funcs.push({ description: 'Get Labels2', event: this.getImageLabels2, icon: 'redo-alt' }); + // funcs.push({ description: 'Get Labels', event: this.getImageLabels, icon: 'redo-alt' }); funcs.push({ description: 'Rotate Clockwise 90', event: this.rotate, icon: 'redo-alt' }); funcs.push({ description: `Show ${this.layoutDoc._showFullRes ? 'Dynamic Res' : 'Full Res'}`, event: this.resolution, icon: 'expand' }); funcs.push({ description: 'Set Native Pixel Size', event: this.setNativeSize, icon: 'expand-arrows-alt' }); @@ -277,6 +616,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }), icon: 'pencil-alt', }); + ContextMenu.Instance?.addItem({ description: 'Quiz Mode', subitems: quizes, icon: 'file-pen' }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; @@ -391,6 +731,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { <div className="imageBox-fader" style={{ opacity: backAlpha }}> <img alt="" + ref={this.setRef} key="paths" src={srcpath} style={{ transform, transformOrigin }} @@ -432,6 +773,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { action(moveEv => { MarqueeAnnotator.clearAnnotations(this._savedAnnotations); this._marqueeref.current?.onInitiateSelection([moveEv.clientX, moveEv.clientY]); + this._width = moveEv.clientX; + this._height = moveEv.clientY; return true; }), returnFalse, @@ -443,6 +786,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @action finishMarquee = () => { this._getAnchor = AnchorMenu.Instance?.GetAnchor; + AnchorMenu.Instance.gptFlashcards = this.getImageDesc; + AnchorMenu.Instance.addToCollection = this._props.DocumentView?.()._props.addDocument; + AnchorMenu.Instance.makeLabels = this.makeLabels; + AnchorMenu.Instance.marqueeWidth = this._marqueeref.current?.Width ?? 0; + AnchorMenu.Instance.marqueeHeight = this._marqueeref.current?.Height ?? 0; this._marqueeref.current?.onTerminateSelection(); this._props.select(false); }; @@ -477,7 +825,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { height: this._props.PanelWidth() ? undefined : `100%`, pointerEvents: this.layoutDoc._lockedPosition ? 'none' : undefined, borderRadius, - overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : undefined, + overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : 'hidden', }}> <CollectionFreeFormView ref={this._ffref} @@ -505,6 +853,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { addDocument={this.addDocument}> {this.content} </CollectionFreeFormView> + {this._loading ? ( + <div className="loading-spinner" style={{ position: 'absolute' }}> + <ReactLoading type="spin" height={50} width={50} color={'blue'} /> + </div> + ) : null} {this.annotationLayer} {!this._mainCont.current || !this.DocumentView || !this._annotationLayer.current ? null : ( <MarqueeAnnotator @@ -523,8 +876,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { marqueeContainer={this._mainCont.current} highlightDragSrcColor="" anchorMenuCrop={this.crop} + // anchorMenuFlashcard={() => this.getImageDesc()} /> )} + {this._quizMode != quizMode.NONE ? this.checkIcon : null} + {this._quizMode != quizMode.NONE ? this.redoIcon : null} </div> ); } |