1 files changed, 77 insertions, 1 deletions

diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index 93c07f3a8..ab7605829 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -384,17 +384,93 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         this._loading = false;
     };
 
+    getImageLabels = async () => {
+        this._loading = true;
+        try {
+            const hrefBase64 = await this.createCanvas();
+            // const hw = await gptImageLabel(hrefBase64, 'Find the image dimensions. Return as height and width.');
+            const response = await gptImageLabel(
+                hrefBase64,
+                //'What is the height and width of the image'
+                'For each group of words in the image, find the x-coordinate and ycoordinate of the top left corner. Find the width and height of the group. Return this information in this format with the correct information replacing the underscores: "observed word: _, x: _, y: _, width: _, height: _," No additional text, asterisks and put it all in one line. Divide the x and width by the width of the image. Divide the y and the height by the height of the image.'
+            );
+            // console.log(hw);
+            console.log('RESPONSE: ' + response);
+            this.createTextboxes(response);
+
+            //AnchorMenu.Instance.transferToFlashcard(response, NumCast(this.layoutDoc['x']), NumCast(this.layoutDoc['y']));
+        } catch (error) {
+            console.log('Error');
+        }
+        this._loading = false;
+    };
+
+    createTextboxes = (response: string) => {
+        const groups = response.replace('*', '').toLowerCase().split('observed word: ');
+        groups.shift();
+        for (var i = 0; i < groups.length; i++) {
+            console.log('Group ' + i + ': ' + groups[i]);
+        }
+        // console.log('GROUPS: ' + groups);
+        groups.forEach(
+            group => {
+                const groupArr = group.split(', ');
+                const word = groupArr[0];
+                const x = parseFloat(groupArr[1].substring(3));
+                const y = parseFloat(groupArr[2].substring(3));
+                const width = parseFloat(groupArr[3].substring(7));
+                const height = parseFloat(groupArr[4].substring(8));
+                const scaling = 1 / (this._props.NativeDimScaling?.() || 1);
+                const w = AnchorMenu.Instance.marqueeWidth * scaling;
+                const h = AnchorMenu.Instance.marqueeHeight * scaling;
+
+                const newCol = Docs.Create.TextDocument(word, {
+                    _width: w * width,
+                    //width * NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']),
+                    _height: h * height + 20,
+                    //height * NumCast(this.dataDoc[this.fieldKey + '_nativeHeight']),
+                    _layout_fitWidth: true,
+                    _layout_autoHeight: true,
+                });
+                newCol.x = x * w;
+                newCol.y = y * h;
+                // newCol.x = x * NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']);
+                // newCol.y = y * NumCast(this.dataDoc[this.fieldKey + '_nativeHeight']);
+                newCol.zIndex = 1000;
+                newCol.forceActive = true;
+                newCol.quiz = true;
+                this.addDocument(newCol);
+            }
+            // console.log(group);
+        );
+    };
+
     @action
     setRef = (iref: HTMLImageElement | null) => {
         this._imageRef = iref;
     };
 
+    pushInfo = async () => {
+        const formData = new FormData();
+
+        const newArticle = {
+            file: '/files/audio/6b412a6222d631a7fff8a8320.mp3',
+        };
+        const response = await axios.post('http://localhost:105/recognize/', newArticle, {
+            headers: {
+                'Content-Type': 'application/json',
+            },
+        });
+        console.log('RESPONSE: ' + response.data['transcription']);
+    };
+
     specificContextMenu = (): void => {
         const field = Cast(this.dataDoc[this.fieldKey], ImageField);
         if (field) {
             const funcs: ContextMenuProps[] = [];
             // funcs.push({ description: 'Create ai flashcards', event: () => this.getImageDesc(), icon: 'id-card' });
-            funcs.push({ description: 'Get Images', event: () => this.handleSelection('Cats'), icon: 'redo-alt' });
+            // funcs.push({ description: 'Push info', event: this.pushInfo, icon: 'redo-alt' });
+            funcs.push({ description: 'Get Labels', event: this.getImageLabels, icon: 'redo-alt' });
             funcs.push({ description: 'Rotate Clockwise 90', event: this.rotate, icon: 'redo-alt' });
             funcs.push({ description: `Show ${this.layoutDoc._showFullRes ? 'Dynamic Res' : 'Full Res'}`, event: this.resolution, icon: 'expand' });
             funcs.push({ description: 'Set Native Pixel Size', event: this.setNativeSize, icon: 'expand-arrows-alt' });