diff options
Diffstat (limited to 'src/client/views/nodes/ImageBox.tsx')
| -rw-r--r-- | src/client/views/nodes/ImageBox.tsx | 379 |
1 files changed, 373 insertions, 6 deletions
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index d0a7fc6ac..06e7e576b 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -11,7 +11,6 @@ import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; import { InkTool } from '../../../fields/InkField'; import { ObjectField } from '../../../fields/ObjectField'; -import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; import { TraceMobx } from '../../../fields/util'; import { emptyFunction } from '../../../Utils'; @@ -34,8 +33,27 @@ import { StyleProp } from '../StyleProp'; import { DocumentView } from './DocumentView'; import { FieldView, FieldViewProps } from './FieldView'; import { FocusViewOptions } from './FocusViewOptions'; +import { DocCast, NumCast, RTFCast, StrCast, ImageCast, Cast, toList } from '../../../fields/Types'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; +import { URLField } from '../../../fields/URLField'; +import { gptAPICall, GPTCallType, gptImageLabel } from '../../apis/gpt/GPT'; +import ReactLoading from 'react-loading'; +import { FollowLinkScript } from '../../documents/DocUtils'; +import { basename } from 'path'; +import { ImageUtility } from './generativeFill/generativeFillUtils/ImageHandler'; +import { dropActionType } from '../../util/DropActionTypes'; +import { canvasSize } from './generativeFill/generativeFillUtils/generativeFillConstants'; +import Tesseract from 'tesseract.js'; +import axios from 'axios'; +import { TupleType } from 'typescript'; +// import stringSimilarity from 'string-similarity'; + +enum quizMode { + SMART = 'smart', + NORMAL = 'normal', + NONE = 'none', +} export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -60,6 +78,8 @@ export class ImageEditorData { public static get AddDoc() { return ImageEditorData.imageData.addDoc; } // prettier-ignore public static set AddDoc(addDoc: Opt<(doc: Doc | Doc[], annotationKey?: string) => boolean>) { ImageEditorData.set(this.imageData.open, this.imageData.rootDoc, this.imageData.source, addDoc); } // prettier-ignore } + +const API_URL = 'https://api.unsplash.com/search/photos'; @observer export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { public static LayoutString(fieldKey: string) { @@ -74,10 +94,15 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { private _marqueeref = React.createRef<MarqueeAnnotator>(); private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef(); - @observable _savedAnnotations = new ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]>(); - @observable _curSuffix = ''; - @observable _error = ''; - @observable _isHovering = false; // flag to switch between primary and alternate images on hover + private _imageRef: HTMLImageElement | null = null; // <video> ref + @observable private _quizBoxes: Doc[] = []; + @observable private _searchInput = ''; + @observable private _quizMode = quizMode.NONE; + @observable private _savedAnnotations = new ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]>(); + @observable private _curSuffix = ''; + @observable private _error = ''; + @observable private _loading = false; + @observable private _isHovering = false; // flag to switch between primary and alternate images on hover _ffref = React.createRef<CollectionFreeFormView>(); constructor(props: FieldViewProps) { @@ -149,6 +174,32 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { Object.values(this._disposers).forEach(disposer => disposer?.()); } + fetchImages = async () => { + try { + const { data } = await axios.get(`${API_URL}?query=${this._searchInput}&page=1&per_page=${1}&client_id=${process.env.VITE_API_KEY}`); + console.log('data', data); + console.log(data.results); + const imageSnapshot = Docs.Create.ImageDocument(data.results[0].urls.small, { + _nativeWidth: Doc.NativeWidth(this.layoutDoc), + _nativeHeight: Doc.NativeHeight(this.layoutDoc), + x: NumCast(this.layoutDoc.x), + y: NumCast(this.layoutDoc.y), + onClick: FollowLinkScript(), + _width: 150, + _height: 150, + title: '--snapshot' + NumCast(this.layoutDoc._layout_currentTimecode) + ' image-', + }); + this._props.addDocument?.(imageSnapshot); + } catch (error) { + console.log(error); + } + }; + + handleSelection = async (selection: string) => { + this._searchInput = selection; + const images = await this.fetchImages(); + }; + @undoBatch drop = (e: Event, de: DragManager.DropEvent) => { if (de.complete.docDragData) { @@ -198,6 +249,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.dataDoc._freeform_panX_min = this.dataDoc._freeform_panX_min ? nw * NumCast(this.dataDoc._freeform_panX_min) : undefined; this.dataDoc._freeform_panY_max = this.dataDoc._freeform_panY_max ? nw * NumCast(this.dataDoc._freeform_panY_max) : undefined; this.dataDoc._freeform_panY_min = this.dataDoc._freeform_panY_min ? nw * NumCast(this.dataDoc._freeform_panY_min) : undefined; + return nw; }); @undoBatch rotate = action(() => { @@ -260,10 +312,310 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return cropping; }; + createCanvas = async (downX?: number, downY?: number, cb?: (filename: string, x: number | undefined, y: number | undefined) => void) => { + const canvas = document.createElement('canvas'); + const scaling = 1 / (this._props.NativeDimScaling?.() || 1); + const w = AnchorMenu.Instance.marqueeWidth * scaling; + const h = AnchorMenu.Instance.marqueeHeight * scaling; + canvas.width = w; + canvas.height = h; + const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions + if (ctx) { + this._imageRef && ctx.drawImage(this._imageRef, NumCast(this._marqueeref.current?.left) * scaling, NumCast(this._marqueeref.current?.top) * scaling, w, h, 0, 0, w, h); + } + // canvas.style.zIndex = '2000000'; + // document.body.appendChild(canvas); + const blob = await ImageUtility.canvasToBlob(canvas); + return ImageBox.selectUrlToBase64(blob); + }; + + createSnapshotLink = (imagePath: string, downX?: number, downY?: number) => { + const url = !imagePath.startsWith('/') ? ClientUtils.CorsProxy(imagePath) : imagePath; + const width = NumCast(this.layoutDoc._width) || 1; + const height = NumCast(this.layoutDoc._height); + const imageSnapshot = Docs.Create.ImageDocument(url, { + _nativeWidth: Doc.NativeWidth(this.layoutDoc), + _nativeHeight: Doc.NativeHeight(this.layoutDoc), + x: NumCast(this.layoutDoc.x) + width, + y: NumCast(this.layoutDoc.y), + onClick: FollowLinkScript(), + _width: 150, + _height: (height / width) * 150, + title: '--snapshot' + NumCast(this.layoutDoc._layout_currentTimecode) + ' image-', + }); + Doc.SetNativeWidth(imageSnapshot[DocData], Doc.NativeWidth(this.layoutDoc)); + Doc.SetNativeHeight(imageSnapshot[DocData], Doc.NativeHeight(this.layoutDoc)); + this._props.addDocument?.(imageSnapshot); + DocUtils.MakeLink(imageSnapshot, this.getAnchor(true), { link_relationship: 'video snapshot' }); + // link && (DocCast(link.link_anchor_2)[DocData].timecodeToHide = NumCast(DocCast(link.link_anchor_2).timecodeToShow) + 3); // do we need to set an end time? should default to +0.1 + setTimeout(() => downX !== undefined && downY !== undefined && DocumentView.getFirstDocumentView(imageSnapshot)?.startDragging(downX, downY, dropActionType.move, true)); + }; + + static selectUrlToBase64 = async (blob: Blob): Promise<string> => { + try { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = error => reject(error); + }); + } catch (error) { + console.error('Error:', error); + throw error; + } + }; + + pushInfo = async (quiz: quizMode, i?: string) => { + this._quizMode = quiz; + this._loading = true; + console.log('JHSDKFJHKSDJFHKSJDHFKJSDHFKJHSDKF'); + + const img = { + file: i ? i : this.paths[0], + drag: i ? 'drag' : 'full', + smart: quiz, + }; + const response = await axios.post('http://localhost:105/labels/', img, { + headers: { + 'Content-Type': 'application/json', + }, + }); + + console.log('RESPONSE:'); + console.log(response.data['boxes']); + console.log(response.data['text']); + if (response.data['boxes'].length != 0) { + this.createBoxes(response.data['boxes'], response.data['text']); + } else { + this._loading = false; + } + }; + + createBoxes = (boxes: [[[number, number]]], texts: [string]) => { + const nscale = NumCast(this._props.PanelWidth()) * NumCast(this.layoutDoc._freeform_scale, 1); + const nw = nscale / NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']); + for (var i = 0; i < boxes.length; i++) { + const coords = boxes[i] ? boxes[i] : []; + const width = coords[1][0] - coords[0][0]; + const height = coords[2][1] - coords[0][1]; + const text = texts[i]; + + const newCol = Docs.Create.LabelDocument({ + _width: width, + //width * NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']), + _height: height, + //height * NumCast(this.dataDoc[this.fieldKey + '_nativeHeight']), + _layout_fitWidth: true, + title: '', + // _layout_autoHeight: true, + }); + const scaling = 1 / (this._props.NativeDimScaling?.() || 1); + newCol.x = coords[0][0] + NumCast(this._marqueeref.current?.left) * scaling; + newCol.y = coords[0][1] + NumCast(this._marqueeref.current?.top) * scaling; + // newCol[DocData].text_fontSize = height + 'px'; + + newCol.zIndex = 1000; + newCol.forceActive = true; + newCol.quiz = text; + newCol.showQuiz = false; + newCol[DocData].textTransform = 'none'; + this._quizBoxes.push(newCol); + this.addDocument(newCol); + this._loading = false; + } + }; + + getImageDesc = async () => { + this._loading = true; + try { + const hrefBase64 = await this.createCanvas(); + const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this image with each question and answer labeled as "question" and "answer". Do not label each flashcard and do not include asterisks: '); + console.log(response); + AnchorMenu.Instance.transferToFlashcard(response, NumCast(this.layoutDoc['x']), NumCast(this.layoutDoc['y'])); + } catch (error) { + console.log('Error'); + } + this._loading = false; + }; + + makeLabels = async () => { + try { + const hrefBase64 = await this.createCanvas(); + this.pushInfo(quizMode.NORMAL, hrefBase64); + } catch (error) { + console.log('Error'); + } + }; + + levenshteinDistance = (str1: string, str2: string) => { + const len1 = str1.length; + const len2 = str2.length; + const dp = Array.from(Array(len1 + 1), () => Array(len2 + 1).fill(0)); + + if (len1 === 0) return len2; + if (len2 === 0) return len1; + + for (let i = 0; i <= len1; i++) dp[i][0] = i; + for (let j = 0; j <= len2; j++) dp[0][j] = j; + + for (let i = 1; i <= len1; i++) { + for (let j = 1; j <= len2; j++) { + const cost = str1[i - 1] === str2[j - 1] ? 0 : 1; + dp[i][j] = Math.min( + dp[i - 1][j] + 1, // deletion + dp[i][j - 1] + 1, // insertion + dp[i - 1][j - 1] + cost // substitution + ); + } + } + + return dp[len1][len2]; + }; + + jaccardSimilarity = (str1: string, str2: string) => { + const set1 = new Set(str1.split(' ')); + const set2 = new Set(str2.split(' ')); + + const intersection = new Set([...set1].filter(x => set2.has(x))); + const union = new Set([...set1, ...set2]); + + return intersection.size / union.size; + }; + + stringSimilarity(str1: string, str2: string) { + const levenshteinDist = this.levenshteinDistance(str1, str2); + const levenshteinScore = 1 - levenshteinDist / Math.max(str1.length, str2.length); + + const jaccardScore = this.jaccardSimilarity(str1, str2); + + // Combine the scores with a higher weight on Jaccard similarity + return 0.5 * levenshteinScore + 0.5 * jaccardScore; + } + + @computed get checkIcon() { + return ( + <Tooltip title={<div className="dash-tooltip">Check</div>}> + <div className="check-icon" onPointerDown={this.check}> + <FontAwesomeIcon icon="circle-check" size="lg" /> + </div> + </Tooltip> + ); + } + + @computed get redoIcon() { + return ( + <Tooltip title={<div className="dash-tooltip">Redo</div>}> + <div className="redo-icon" onPointerDown={this.redo}> + <FontAwesomeIcon icon="redo-alt" size="lg" /> + </div> + </Tooltip> + ); + } + + compareWords = (input: string, target: string) => { + const distance = this.stringSimilarity(input.toLowerCase(), target.toLowerCase()); + // const threshold = Math.max(input.length, target.length) * 0.2; // Allow up to 20% of the length as difference + return distance >= 0.7; + }; + + extractHexAndSentences = (inputString: string) => { + // Regular expression to match a hexadecimal number at the beginning followed by a period and sentences + const regex = /^#([0-9A-Fa-f]+)\.\s*(.+)$/s; + const match = inputString.match(regex); + + if (match) { + const hexNumber = match[1]; + const sentences = match[2].trim(); + return { hexNumber, sentences }; + } else { + return { error: 'The input string does not match the expected format.' }; + } + }; + + check = () => { + this._loading = true; + this._quizBoxes.forEach(async doc => { + const input = StrCast(doc[DocData].title); + console.log('INP: ' + StrCast(input) + '; DOC: ' + StrCast(doc.quiz)); + if (this._quizMode == quizMode.SMART && input) { + const questionText = 'Question: What was labeled in this image?'; + const rubricText = ' Rubric: ' + StrCast(doc.quiz); + // const queryText = 'RealAnswer: ' + StrCast(doc.quiz) + '. UserAnswer: ' + input + '.'; + const queryText = + questionText + + ' UserAnswer: ' + + input + + '. ' + + rubricText + + '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."'; + const response = await gptAPICall(queryText, GPTCallType.QUIZ); + const hexSent = this.extractHexAndSentences(response); + console.log(hexSent.hexNumber); + doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); + doc.backgroundColor = '#' + hexSent.hexNumber; + } else { + const match = this.compareWords(input, StrCast(doc.quiz)); + doc.backgroundColor = match ? '#11c249' : '#eb2d2d'; + } + doc.showQuiz = true; + // console.log(this.compareWords(input, StrCast(doc.quiz)) ? 'Match' : 'No Match'); + }); + this._loading = false; + }; + + redo = () => { + this._quizBoxes.forEach(doc => { + doc[DocData].title = ''; + doc.backgroundColor = '#e4e4e4'; + doc.showQuiz = false; + }); + }; + + exitQuizMode = () => { + this._quizMode = quizMode.NONE; + this._quizBoxes.forEach(doc => { + // this._props.removeDocument?.(DocCast(doc)); + // this._props.DocumentView?.()._props.removeDocument?.(doc); + this.removeDocument?.(doc); + }); + this._quizBoxes = []; + console.log('remove'); + }; + + @action + setRef = (iref: HTMLImageElement | null) => { + this._imageRef = iref; + }; + specificContextMenu = (): void => { const field = Cast(this.dataDoc[this.fieldKey], ImageField); if (field) { const funcs: ContextMenuProps[] = []; + const quizes: ContextMenuProps[] = []; + // funcs.push({ description: 'Create ai flashcards', event: () => this.getImageDesc(), icon: 'id-card' }); + quizes.push({ + description: 'Smart Check', + event: this._quizMode == quizMode.NONE ? () => this.pushInfo(quizMode.SMART) : this.exitQuizMode, + icon: 'pen-to-square', + }); + quizes.push({ + description: 'Normal', + event: this._quizMode == quizMode.NONE ? () => this.pushInfo(quizMode.NORMAL) : this.exitQuizMode, + icon: 'pencil', + }); + // funcs.push({ description: 'Quiz Mode', subitems: optionItems, icon: 'eye' }); + // funcs.push({ + // description: 'Quiz Mode', + // event: !this._quizMode + // ? () => this.pushInfo(false) + // : () => { + // this._quizMode = false; + // }, + // icon: 'redo-alt', + // }); + // funcs.push({ description: 'Get Text', event: this.check, icon: 'redo-alt' }); + // funcs.push({ description: 'Get Labels2', event: this.getImageLabels2, icon: 'redo-alt' }); + // funcs.push({ description: 'Get Labels', event: this.getImageLabels, icon: 'redo-alt' }); funcs.push({ description: 'Rotate Clockwise 90', event: this.rotate, icon: 'redo-alt' }); funcs.push({ description: `Show ${this.layoutDoc._showFullRes ? 'Dynamic Res' : 'Full Res'}`, event: this.resolution, icon: 'expand' }); funcs.push({ description: 'Set Native Pixel Size', event: this.setNativeSize, icon: 'expand-arrows-alt' }); @@ -278,6 +630,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }), icon: 'pencil-alt', }); + ContextMenu.Instance?.addItem({ description: 'Quiz Mode', subitems: quizes, icon: 'file-pen' }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; @@ -392,6 +745,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { <div className="imageBox-fader" style={{ opacity: backAlpha }}> <img alt="" + ref={this.setRef} key="paths" src={srcpath} style={{ transform, transformOrigin }} @@ -444,6 +798,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @action finishMarquee = () => { this._getAnchor = AnchorMenu.Instance?.GetAnchor; + AnchorMenu.Instance.gptFlashcards = this.getImageDesc; + AnchorMenu.Instance.addToCollection = this._props.DocumentView?.()._props.addDocument; + AnchorMenu.Instance.makeLabels = this.makeLabels; + AnchorMenu.Instance.marqueeWidth = this._marqueeref.current?.Width ?? 0; + AnchorMenu.Instance.marqueeHeight = this._marqueeref.current?.Height ?? 0; this._marqueeref.current?.onTerminateSelection(); this._props.select(false); }; @@ -478,7 +837,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { height: this._props.PanelWidth() ? undefined : `100%`, pointerEvents: this.layoutDoc._lockedPosition ? 'none' : undefined, borderRadius, - overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : undefined, + overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : 'hidden', }}> <CollectionFreeFormView ref={this._ffref} @@ -506,6 +865,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { addDocument={this.addDocument}> {this.content} </CollectionFreeFormView> + {this._loading ? ( + <div className="loading-spinner" style={{ position: 'absolute' }}> + <ReactLoading type="spin" height={50} width={50} color={'blue'} /> + </div> + ) : null} {this.annotationLayer} {!this._mainCont.current || !this.DocumentView || !this._annotationLayer.current ? null : ( <MarqueeAnnotator @@ -524,8 +888,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { marqueeContainer={this._mainCont.current} highlightDragSrcColor="" anchorMenuCrop={this.crop} + // anchorMenuFlashcard={() => this.getImageDesc()} /> )} + {this._quizMode != quizMode.NONE ? this.checkIcon : null} + {this._quizMode != quizMode.NONE ? this.redoIcon : null} </div> ); } |