From ad32c1606395cdc71ba50eb9b51d3a9d3b707ca0 Mon Sep 17 00:00:00 2001
From: alyssaf16 <alyssa_feinberg@brown.edu>
Date: Tue, 18 Jun 2024 15:23:12 -0400
Subject: working on various image flashcard ideas

---
 src/client/views/nodes/ImageBox.tsx | 232 ++++++++++++++++++++++++++++++++++++
 1 file changed, 232 insertions(+)

(limited to 'src/client/views/nodes/ImageBox.tsx')

diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index e4b3a1b9b..ff938df78 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -35,6 +35,15 @@ import { FieldView, FieldViewProps } from './FieldView';
 import { FocusViewOptions } from './FocusViewOptions';
 import './ImageBox.scss';
 import { OpenWhere } from './OpenWhere';
+import { URLField } from '../../../fields/URLField';
+import { gptImageLabel } from '../../apis/gpt/GPT';
+import ReactLoading from 'react-loading';
+import { FollowLinkScript } from '../../documents/DocUtils';
+import { basename } from 'path';
+import { ImageUtility } from './generativeFill/generativeFillUtils/ImageHandler';
+import { dropActionType } from '../../util/DropActionTypes';
+import { canvasSize } from './generativeFill/generativeFillUtils/generativeFillConstants';
+import axios from 'axios';
 
 export class ImageEditorData {
     // eslint-disable-next-line no-use-before-define
@@ -59,6 +68,8 @@ export class ImageEditorData {
     public static get AddDoc()                   { return ImageEditorData.imageData.addDoc; } // prettier-ignore
     public static set AddDoc(addDoc: Opt<(doc: Doc | Doc[], annotationKey?: string) => boolean>) { ImageEditorData.set(this.imageData.open, this.imageData.rootDoc, this.imageData.source, addDoc); } // prettier-ignore
 }
+
+const API_URL = 'https://api.unsplash.com/search/photos';
 @observer
 export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
     public static LayoutString(fieldKey: string) {
@@ -73,9 +84,14 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
     private _marqueeref = React.createRef<MarqueeAnnotator>();
     private _mainCont: React.RefObject<HTMLDivElement> = React.createRef();
     private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef();
+    private _imageRef: HTMLImageElement | null = null; // <video> ref
+    @observable private _width: number = 0;
+    @observable private _height: number = 0;
+    @observable private searchInput = '';
     @observable _savedAnnotations = new ObservableMap<number, HTMLDivElement[]>();
     @observable _curSuffix = '';
     @observable _error = '';
+    @observable private _loading = false;
     @observable _isHovering = false; // flag to switch between primary and alternate images on hover
     _ffref = React.createRef<CollectionFreeFormView>();
 
@@ -148,6 +164,32 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         Object.values(this._disposers).forEach(disposer => disposer?.());
     }
 
+    fetchImages = async () => {
+        try {
+            const { data } = await axios.get(`${API_URL}?query=${this.searchInput}&page=1&per_page=${1}&client_id=${process.env.VITE_API_KEY}`);
+            console.log('data', data);
+            console.log(data.results);
+            const imageSnapshot = Docs.Create.ImageDocument(data.results[0].urls.small, {
+                _nativeWidth: Doc.NativeWidth(this.layoutDoc),
+                _nativeHeight: Doc.NativeHeight(this.layoutDoc),
+                x: NumCast(this.layoutDoc.x),
+                y: NumCast(this.layoutDoc.y),
+                onClick: FollowLinkScript(),
+                _width: 150,
+                _height: 150,
+                title: '--snapshot' + NumCast(this.layoutDoc._layout_currentTimecode) + ' image-',
+            });
+            this._props.addDocument?.(imageSnapshot);
+        } catch (error) {
+            console.log(error);
+        }
+    };
+
+    handleSelection = async (selection: string) => {
+        this.searchInput = selection;
+        const images = await this.fetchImages();
+    };
+
     @undoBatch
     drop = (e: Event, de: DragManager.DropEvent) => {
         if (de.complete.docDragData) {
@@ -259,10 +301,189 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         return cropping;
     };
 
+    createCanvas = async (downX?: number, downY?: number, cb?: (filename: string, x: number | undefined, y: number | undefined) => void) => {
+        const width = NumCast(this.layoutDoc._width);
+        const canvas = document.createElement('canvas');
+        // canvas.width = 640;
+        // canvas.height = (640 * Doc.NativeHeight(this.layoutDoc)) / (Doc.NativeWidth(this.layoutDoc) || 1);
+        canvas.width = NumCast(this.layoutDoc._width);
+        canvas.height = NumCast(this.layoutDoc._height);
+        const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions
+        if (ctx) {
+            // this._imageRef && ctx.drawImage(this._imageRef, 0, 0, canvas.width, canvas.height);
+            this._imageRef && ctx.drawImage(this._imageRef, NumCast(this._marqueeref.current?.left), NumCast(this._marqueeref.current?.top), this._width, this._height, 0, 0, 1000, 1000);
+            //this._imageRef && ctx.drawImage(this._imageRef, 0, 0, 2000, 1000, 0, 0, canvas.width, canvas.height);
+            // console.log(NumCast(this._marqueeref.current?.left) + 100);
+        }
+        const blob = await ImageUtility.canvasToBlob(canvas);
+        return ImageBox.selectUrlToBase64(blob);
+
+        // if (this._imageRef) {
+        //     const canv = ImageUtility.getCroppedImg(this._imageRef, this._imageRef.width, this._imageRef.height);
+        //     console.log(this._imageRef.width);
+        //     if (canv) {
+        //         const blob = await ImageUtility.canvasToBlob(canv);
+        //         return ImageBox.selectUrlToBase64(blob);
+        //     }
+        // }
+        if (!this._imageRef) {
+            const b = Docs.Create.LabelDocument({
+                x: NumCast(this.layoutDoc.x) + width,
+                y: NumCast(this.layoutDoc.y, 1),
+                _width: 150,
+                _height: 50,
+                // title: (this.layoutDoc._layout_currentTimecode || 0).toString(),
+                onClick: FollowLinkScript(),
+            });
+            this._props.addDocument?.(b);
+            DocUtils.MakeLink(b, this.Document, { link_relationship: 'image snapshot' });
+        } else {
+            // convert to desired file format
+            // const dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png'
+            // // if you want to preview the captured image,
+            // const retitled = StrCast(this.Document.title).replace(/[ -.:]/g, '');
+            // const encodedFilename = encodeURIComponent(('snapshot' + retitled + '_' + (this.layoutDoc._layout_currentTimecode || 0).toString()).replace(/[./?=]/g, '_'));
+            // const filename = basename(encodedFilename);
+            // ClientUtils.convertDataUri(dataUrl, filename).then((returnedFilename: string) => returnedFilename && (cb ?? this.createSnapshotLink)(returnedFilename, downX, downY));
+        }
+        // convert to desired file format
+
+        // const dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png'
+        // // if you want to preview the captured image,
+        // const retitled = StrCast(this.Document.title).replace(/[ -.:]/g, '');
+        // const encodedFilename = encodeURIComponent(('snapshot' + retitled + '_' + (this.layoutDoc._layout_currentTimecode || 0).toString()).replace(/[./?=]/g, '_'));
+        // const filename = basename(encodedFilename);
+        //ClientUtils.convertDataUri(dataUrl, filename).then((returnedFilename: string) => returnedFilename && (cb ?? this.createSnapshotLink)(returnedFilename, downX, downY));
+        // }
+        // const docViewContent = this.DocumentView?.().ContentDiv!;
+        // if (docViewContent instanceof HTMLCanvasElement) {
+        //     const canvas = docViewContent;
+        //     const img = document.createElement('img'); // create a Image Element
+        //     img.src = canvas.toDataURL(); // image sourcez
+        //     img.style.width = canvas.style.width;
+        //     img.style.height = canvas.style.height;
+        //     const parEle = newCan.parentElement as HTMLElement;
+        //     parEle.removeChild(newCan);
+        //     parEle.appendChild(img);
+        // }
+    };
+
+    createSnapshotLink = (imagePath: string, downX?: number, downY?: number) => {
+        const url = !imagePath.startsWith('/') ? ClientUtils.CorsProxy(imagePath) : imagePath;
+        const width = NumCast(this.layoutDoc._width) || 1;
+        const height = NumCast(this.layoutDoc._height);
+        const imageSnapshot = Docs.Create.ImageDocument(url, {
+            _nativeWidth: Doc.NativeWidth(this.layoutDoc),
+            _nativeHeight: Doc.NativeHeight(this.layoutDoc),
+            x: NumCast(this.layoutDoc.x) + width,
+            y: NumCast(this.layoutDoc.y),
+            onClick: FollowLinkScript(),
+            _width: 150,
+            _height: (height / width) * 150,
+            title: '--snapshot' + NumCast(this.layoutDoc._layout_currentTimecode) + ' image-',
+        });
+        Doc.SetNativeWidth(imageSnapshot[DocData], Doc.NativeWidth(this.layoutDoc));
+        Doc.SetNativeHeight(imageSnapshot[DocData], Doc.NativeHeight(this.layoutDoc));
+        this._props.addDocument?.(imageSnapshot);
+        DocUtils.MakeLink(imageSnapshot, this.getAnchor(true), { link_relationship: 'video snapshot' });
+        // link && (DocCast(link.link_anchor_2)[DocData].timecodeToHide = NumCast(DocCast(link.link_anchor_2).timecodeToShow) + 3);  // do we need to set an end time? should default to +0.1
+        setTimeout(() => downX !== undefined && downY !== undefined && DocumentView.getFirstDocumentView(imageSnapshot)?.startDragging(downX, downY, dropActionType.move, true));
+    };
+
+    /**
+     * 
+        if (oldDiv instanceof HTMLCanvasElement) {
+            const canvas = oldDiv;
+            const img = document.createElement('img'); // create a Image Element
+            img.src = canvas.toDataURL(); // image sourcez
+            img.style.width = canvas.style.width;
+            img.style.height = canvas.style.height;
+            const newCan = newDiv as HTMLCanvasElement;
+            const parEle = newCan.parentElement as HTMLElement;
+            parEle.removeChild(newCan);
+            parEle.appendChild(img);
+        }
+     */
+
+    static selectUrlToBase64 = async (blob: Blob): Promise<string> => {
+        try {
+            return new Promise((resolve, reject) => {
+                const reader = new FileReader();
+                reader.readAsDataURL(blob);
+                reader.onloadend = () => resolve(reader.result as string);
+                reader.onerror = error => reject(error);
+            });
+        } catch (error) {
+            console.error('Error:', error);
+            throw error;
+        }
+    };
+
+    static imageUrlToBase64 = async (imageUrl: string): Promise<string> => {
+        try {
+            const response = await fetch(imageUrl);
+            const blob = await response.blob();
+
+            return new Promise((resolve, reject) => {
+                const reader = new FileReader();
+                reader.readAsDataURL(blob);
+                reader.onloadend = () => resolve(reader.result as string);
+                reader.onerror = error => reject(error);
+            });
+        } catch (error) {
+            console.error('Error:', error);
+            throw error;
+        }
+    };
+
+    getImageDesc = async () => {
+        // if (StrCast(this.dataDoc.description)) return StrCast(this.dataDoc.description); // Return existing description
+        const { href } = (this.dataDoc.data as URLField).url;
+        const hrefParts = href.split('.');
+        const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
+        this._loading = true;
+        try {
+            // const hrefBase64 = await ImageBox.imageUrlToBase64(hrefComplete);
+            const hrefBase64 = await this.createCanvas();
+            const response = await gptImageLabel(hrefBase64, 'Tell me what words you see on this image.');
+            //const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this text with each question and answer labeled as question and answer. Do not label each flashcard and do not include asterisks: ');
+            console.log(response);
+            // AnchorMenu.Instance.transferToFlashcard(response);
+            // this.Document[DocData].description = response.trim();
+            // return response; // Return the response
+        } catch (error) {
+            console.log('Error');
+        }
+        this._loading = false;
+        // return '';
+    };
+
+    @action
+    setRef = (iref: HTMLImageElement | null) => {
+        this._imageRef = iref;
+        // if (iref) {
+        //     this._videoRef!.ontimeupdate = this.updateTimecode;
+        //     // @ts-ignore
+        //     // vref.onfullscreenchange = action((e) => this._fullScreen = vref.webkitDisplayingFullscreen);
+        //     this._disposers.reactionDisposer?.();
+        //     this._disposers.reactionDisposer = reaction(
+        //         () => NumCast(this.layoutDoc._layout_currentTimecode),
+        //         time => {
+        //             !this._playing && (vref.currentTime = time);
+        //         },
+        //         { fireImmediately: true }
+        //     );
+
+        //     (!this.dataDoc[this.fieldKey + '_thumbnails'] || StrListCast(this.dataDoc[this.fieldKey + '_thumbnails']).length !== VideoThumbnails.DENSE) && this.getVideoThumbnails();
+        // }
+    };
+
     specificContextMenu = (): void => {
         const field = Cast(this.dataDoc[this.fieldKey], ImageField);
         if (field) {
             const funcs: ContextMenuProps[] = [];
+            // funcs.push({ description: 'Create ai flashcards', event: () => this.getImageDesc(), icon: 'id-card' });
+            funcs.push({ description: 'Get Images', event: () => this.handleSelection('Cats'), icon: 'redo-alt' });
             funcs.push({ description: 'Rotate Clockwise 90', event: this.rotate, icon: 'redo-alt' });
             funcs.push({ description: `Show ${this.layoutDoc._showFullRes ? 'Dynamic Res' : 'Full Res'}`, event: this.resolution, icon: 'expand' });
             funcs.push({ description: 'Set Native Pixel Size', event: this.setNativeSize, icon: 'expand-arrows-alt' });
@@ -391,6 +612,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                 <div className="imageBox-fader" style={{ opacity: backAlpha }}>
                     <img
                         alt=""
+                        ref={this.setRef}
                         key="paths"
                         src={srcpath}
                         style={{ transform, transformOrigin }}
@@ -432,6 +654,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                 action(moveEv => {
                     MarqueeAnnotator.clearAnnotations(this._savedAnnotations);
                     this._marqueeref.current?.onInitiateSelection([moveEv.clientX, moveEv.clientY]);
+                    this._width = moveEv.clientX;
+                    this._height = moveEv.clientY;
                     return true;
                 }),
                 returnFalse,
@@ -443,6 +667,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
     @action
     finishMarquee = () => {
         this._getAnchor = AnchorMenu.Instance?.GetAnchor;
+        AnchorMenu.Instance.gptFlashcards = this.getImageDesc;
+        AnchorMenu.Instance.addToCollection = this._props.DocumentView?.()._props.addDocument;
         this._marqueeref.current?.onTerminateSelection();
         this._props.select(false);
     };
@@ -500,6 +726,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                     addDocument={this.addDocument}>
                     {this.content}
                 </CollectionFreeFormView>
+                {this._loading ? (
+                    <div className="loading-spinner" style={{ position: 'absolute' }}>
+                        <ReactLoading type="spin" height={50} width={50} color={'blue'} />
+                    </div>
+                ) : null}
                 {this.annotationLayer}
                 {!this._mainCont.current || !this.DocumentView || !this._annotationLayer.current ? null : (
                     <MarqueeAnnotator
@@ -518,6 +749,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                         marqueeContainer={this._mainCont.current}
                         highlightDragSrcColor=""
                         anchorMenuCrop={this.crop}
+                        // anchorMenuFlashcard={() => this.getImageDesc()}
                     />
                 )}
             </div>
-- 
cgit v1.2.3-70-g09d2