merge with master

1 files changed, 155 insertions, 47 deletions

diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index 4c5ad7a7d..0d9c2bb8a 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -1,33 +1,40 @@
 import { library } from '@fortawesome/fontawesome-svg-core';
-import { faImage } from '@fortawesome/free-solid-svg-icons';
-import { action, observable, computed } from 'mobx';
+import { faEye } from '@fortawesome/free-regular-svg-icons';
+import { faAsterisk, faFileAudio, faImage, faPaintBrush } from '@fortawesome/free-solid-svg-icons';
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import { action, computed, observable, runInAction } from 'mobx';
 import { observer } from "mobx-react";
 import Lightbox from 'react-image-lightbox';
 import 'react-image-lightbox/style.css'; // This only needs to be imported once in your app
-import { Doc, HeightSym, WidthSym, DocListCast } from '../../../new_fields/Doc';
+import { Doc, DocListCast, HeightSym, WidthSym } from '../../../new_fields/Doc';
 import { List } from '../../../new_fields/List';
 import { createSchema, listSpec, makeInterface } from '../../../new_fields/Schema';
-import { Cast, FieldValue, NumCast, StrCast, BoolCast } from '../../../new_fields/Types';
-import { ImageField } from '../../../new_fields/URLField';
+import { ComputedField } from '../../../new_fields/ScriptField';
+import { BoolCast, Cast, FieldValue, NumCast, StrCast } from '../../../new_fields/Types';
+import { AudioField, ImageField } from '../../../new_fields/URLField';
+import { RouteStore } from '../../../server/RouteStore';
 import { Utils } from '../../../Utils';
+import { CognitiveServices, Confidence, Service, Tag } from '../../cognitive_services/CognitiveServices';
+import { Docs } from '../../documents/Documents';
 import { DragManager } from '../../util/DragManager';
+import { CompileScript } from '../../util/Scripting';
 import { undoBatch } from '../../util/UndoManager';
 import { ContextMenu } from "../../views/ContextMenu";
 import { ContextMenuProps } from '../ContextMenuItem';
 import { DocComponent } from '../DocComponent';
 import { InkingControl } from '../InkingControl';
 import { positionSchema } from './DocumentView';
+import FaceRectangles from './FaceRectangles';
 import { FieldView, FieldViewProps } from './FieldView';
 import "./ImageBox.scss";
 import React = require("react");
-import { RouteStore } from '../../../server/RouteStore';
-import { Docs } from '../../documents/Documents';
-import { DocServer } from '../../DocServer';
 var requestImageSize = require('../../util/request-image-size');
 var path = require('path');
+const { Howl } = require('howler');
 
 
-library.add(faImage);
+library.add(faImage, faEye as any, faPaintBrush);
+library.add(faFileAudio, faAsterisk);
 
 
 export const pageSchema = createSchema({
@@ -58,7 +65,7 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
     private dropDisposer?: DragManager.DragDropDisposer;
 
 
-    @computed get dataDoc() { return BoolCast(this.props.Document.isTemplate) && this.props.DataDoc ? this.props.DataDoc : this.props.Document; }
+    @computed get dataDoc() { return this.props.DataDoc && (BoolCast(this.props.Document.isTemplate) || BoolCast(this.props.DataDoc.isTemplate) || this.props.DataDoc.layout === this.props.Document) ? this.props.DataDoc : this.props.Document; }
 
 
     protected createDropTarget = (ele: HTMLDivElement) => {
@@ -78,10 +85,14 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
     @computed get extensionDoc() { return Doc.resolvedFieldDataDoc(this.dataDoc, this.props.fieldKey, "Alternates"); }
 
     @undoBatch
+    @action
     drop = (e: Event, de: DragManager.DropEvent) => {
         if (de.data instanceof DragManager.DocumentDragData) {
             de.data.droppedDocuments.forEach(action((drop: Doc) => {
-                if (de.mods === "AltKey" && /*this.dataDoc !== this.props.Document &&*/ drop.data instanceof ImageField) {
+                if (de.mods === "CtrlKey") {
+                    Doc.ApplyTemplateTo(drop, this.props.Document, this.props.DataDoc);
+                    e.stopPropagation();
+                } else if (de.mods === "AltKey" && /*this.dataDoc !== this.props.Document &&*/ drop.data instanceof ImageField) {
                     Doc.GetProto(this.dataDoc)[this.props.fieldKey] = new ImageField(drop.data.url);
                     e.stopPropagation();
                 } else if (de.mods === "CtrlKey") {
@@ -97,8 +108,6 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
                             e.stopPropagation();
                         }
                     }
-                } else if (!this.props.isSelected()) {
-                    e.stopPropagation();
                 }
             }));
             // de.data.removeDocument()  bcz: need to implement
@@ -161,15 +170,15 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
             recorder.ondataavailable = async function (e: any) {
                 const formData = new FormData();
                 formData.append("file", e.data);
-                const res = await fetch(DocServer.prepend(RouteStore.upload), {
+                const res = await fetch(Utils.prepend(RouteStore.upload), {
                     method: 'POST',
                     body: formData
                 });
                 const files = await res.json();
-                const url = DocServer.prepend(files[0]);
+                const url = Utils.prepend(files[0]);
                 // upload to server with known URL 
                 let audioDoc = Docs.Create.AudioDocument(url, { title: "audio test", x: NumCast(self.props.Document.x), y: NumCast(self.props.Document.y), width: 200, height: 32 });
-                audioDoc.embed = true;
+                audioDoc.treeViewExpandedView = "layout";
                 let audioAnnos = Cast(self.extensionDoc.audioAnnotations, listSpec(Doc));
                 if (audioAnnos === undefined) {
                     self.extensionDoc.audioAnnotations = new List([audioDoc]);
@@ -177,45 +186,90 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
                     audioAnnos.push(audioDoc);
                 }
             };
+            runInAction(() => self._audioState = 2);
             recorder.start();
             setTimeout(() => {
                 recorder.stop();
-
+                runInAction(() => self._audioState = 0);
                 gumStream.getAudioTracks()[0].stop();
             }, 5000);
         });
     }
 
+    @undoBatch
+    rotate = action(() => {
+        let proto = Doc.GetProto(this.props.Document);
+        let nw = this.props.Document.nativeWidth;
+        let nh = this.props.Document.nativeHeight;
+        let w = this.props.Document.width;
+        let h = this.props.Document.height;
+        proto.rotation = (NumCast(this.props.Document.rotation) + 90) % 360;
+        proto.nativeWidth = nh;
+        proto.nativeHeight = nw;
+        this.props.Document.width = h;
+        this.props.Document.height = w;
+    });
+
     specificContextMenu = (e: React.MouseEvent): void => {
         let field = Cast(this.Document[this.props.fieldKey], ImageField);
         if (field) {
             let url = field.url.href;
-            let subitems: ContextMenuProps[] = [];
-            subitems.push({ description: "Copy path", event: () => Utils.CopyText(url), icon: "expand-arrows-alt" });
-            subitems.push({ description: "Record 1sec audio", event: this.recordAudioAnnotation, icon: "expand-arrows-alt" });
-            subitems.push({
-                description: "Rotate", event: action(() => {
-                    let proto = Doc.GetProto(this.props.Document);
-                    let nw = this.props.Document.nativeWidth;
-                    let nh = this.props.Document.nativeHeight;
-                    let w = this.props.Document.width;
-                    let h = this.props.Document.height;
-                    proto.rotation = (NumCast(this.props.Document.rotation) + 90) % 360;
-                    proto.nativeWidth = nh;
-                    proto.nativeHeight = nw;
-                    this.props.Document.width = h;
-                    this.props.Document.height = w;
-                }), icon: "expand-arrows-alt"
-            });
-            ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: subitems });
+            let funcs: ContextMenuProps[] = [];
+            funcs.push({ description: "Copy path", event: () => Utils.CopyText(url), icon: "expand-arrows-alt" });
+            funcs.push({ description: "Record 1sec audio", event: this.recordAudioAnnotation, icon: "expand-arrows-alt" });
+            funcs.push({ description: "Rotate", event: this.rotate, icon: "expand-arrows-alt" });
+
+            let modes: ContextMenuProps[] = [];
+            modes.push({ description: "Generate Tags", event: this.generateMetadata, icon: "tag" });
+            modes.push({ description: "Find Faces", event: this.extractFaces, icon: "camera" });
+
+            ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: funcs, icon: "asterisk" });
+            ContextMenu.Instance.addItem({ description: "Analyze...", subitems: modes, icon: "eye" });
         }
     }
 
+    extractFaces = () => {
+        let converter = (results: any) => {
+            let faceDocs = new List<Doc>();
+            results.map((face: CognitiveServices.Image.Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!));
+            return faceDocs;
+        };
+        CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["faces"], this.url, Service.Face, converter);
+    }
+
+    generateMetadata = (threshold: Confidence = Confidence.Excellent) => {
+        let converter = (results: any) => {
+            let tagDoc = new Doc;
+            let tagsList = new List();
+            results.tags.map((tag: Tag) => {
+                tagsList.push(tag.name);
+                let sanitized = tag.name.replace(" ", "_");
+                let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`;
+                let computed = CompileScript(script, { params: { this: "Doc" } });
+                computed.compiled && (tagDoc[sanitized] = new ComputedField(computed));
+            });
+            this.extensionDoc.generatedTags = tagsList;
+            tagDoc.title = "Generated Tags Doc";
+            tagDoc.confidence = threshold;
+            return tagDoc;
+        };
+        CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter);
+    }
+
     @action
     onDotDown(index: number) {
         this.Document.curPage = index;
     }
 
+    @computed get fieldExtensionDoc() {
+        return Doc.resolvedFieldDataDoc(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey, "true");
+    }
+
+    @computed private get url() {
+        let data = Cast(Doc.GetProto(this.props.Document).data, ImageField);
+        return data ? data.url.href : undefined;
+    }
+
     dots(paths: string[]) {
         let nativeWidth = FieldValue(this.Document.nativeWidth, 1);
         let dist = Math.min(nativeWidth / paths.length, 40);
@@ -229,11 +283,15 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
 
     choosePath(url: URL) {
         const lower = url.href.toLowerCase();
-        if (url.protocol === "data" || url.href.indexOf(window.location.origin) === -1 || !(lower.endsWith(".png") || lower.endsWith(".jpg") || lower.endsWith(".jpeg"))) {
+        if (url.protocol === "data") {
             return url.href;
+        } else if (url.href.indexOf(window.location.origin) === -1) {
+            return Utils.CorsProxy(url.href);
+        } else if (!(lower.endsWith(".png") || lower.endsWith(".jpg") || lower.endsWith(".jpeg"))) {
+            return url.href;//Why is this here
         }
         let ext = path.extname(url.href);
-        const suffix = this.props.renderDepth <= 1 ? "_o" : this._curSuffix;
+        const suffix = this.props.renderDepth < 1 ? "_o" : this._curSuffix;
         return url.href.replace(ext, suffix + ext);
     }
 
@@ -254,16 +312,16 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
     _curSuffix = "_m";
 
     resize(srcpath: string, layoutdoc: Doc) {
-        requestImageSize(window.origin + RouteStore.corsProxy + "/" + srcpath)
+        requestImageSize(srcpath)
             .then((size: any) => {
-                let aspect = size.height / size.width;
                 let rotation = NumCast(this.dataDoc.rotation) % 180;
-                if (rotation === 90 || rotation === 270) aspect = 1 / aspect;
-                if (Math.abs(layoutdoc[HeightSym]() / layoutdoc[WidthSym]() - aspect) > 0.01) {
+                let realsize = rotation === 90 || rotation === 270 ? { height: size.width, width: size.height } : size;
+                let aspect = realsize.height / realsize.width;
+                if (Math.abs(NumCast(layoutdoc.height) - realsize.height) > 1 || Math.abs(NumCast(layoutdoc.width) - realsize.width) > 1) {
                     setTimeout(action(() => {
                         layoutdoc.height = layoutdoc[WidthSym]() * aspect;
-                        layoutdoc.nativeHeight = size.height;
-                        layoutdoc.nativeWidth = size.width;
+                        layoutdoc.nativeHeight = realsize.height;
+                        layoutdoc.nativeWidth = realsize.width;
                     }), 0);
                 }
             })
@@ -272,6 +330,48 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
             });
     }
 
+    @observable _audioState = 0;
+
+    @action
+    onPointerEnter = () => {
+        let self = this;
+        let audioAnnos = DocListCast(this.extensionDoc.audioAnnotations);
+        if (audioAnnos.length && this._audioState === 0) {
+            let anno = audioAnnos[Math.floor(Math.random() * audioAnnos.length)];
+            anno.data instanceof AudioField && new Howl({
+                src: [anno.data.url.href],
+                format: ["mp3"],
+                autoplay: true,
+                loop: false,
+                volume: 0.5,
+                onend: function () {
+                    runInAction(() => self._audioState = 0);
+                }
+            });
+            this._audioState = 1;
+        }
+        // else {
+        //     if (this._audioState === 0) {
+        //         this._audioState = 1;
+        //         new Howl({
+        //             src: ["https://www.kozco.com/tech/piano2-CoolEdit.mp3"],
+        //             autoplay: true,
+        //             loop: false,
+        //             volume: 0.5,
+        //             onend: function () {
+        //                 runInAction(() => self._audioState = 0);
+        //             }
+        //         });
+        //     }
+        // }
+    }
+
+    @action
+    audioDown = () => {
+        this.recordAudioAnnotation();
+    }
+
+
     render() {
         // let transform = this.props.ScreenToLocalTransform().inverse();
         let pw = typeof this.props.PanelWidth === "function" ? this.props.PanelWidth() : typeof this.props.PanelWidth === "number" ? (this.props.PanelWidth as any) as number : 50;
@@ -282,7 +382,7 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
         let id = (this.props as any).id; // bcz: used to set id = "isExpander" in templates.tsx
         let nativeWidth = FieldValue(this.Document.nativeWidth, pw);
         let nativeHeight = FieldValue(this.Document.nativeHeight, 0);
-        let paths: string[] = ["http://www.cs.brown.edu/~bcz/noImage.png"];
+        let paths: string[] = [Utils.CorsProxy("http://www.cs.brown.edu/~bcz/noImage.png")];
         // this._curSuffix = "";
         // if (w > 20) {
         Doc.UpdateDocumentExtensionForField(this.dataDoc, this.props.fieldKey);
@@ -295,11 +395,11 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
         if (field instanceof ImageField) paths = [this.choosePath(field.url)];
         paths.push(...altpaths);
         // }
-        let interactive = InkingControl.Instance.selectedTool ? "" : "-interactive";
+        let interactive = InkingControl.Instance.selectedTool || this.props.Document.isBackground ? "" : "-interactive";
         let rotation = NumCast(this.dataDoc.rotation, 0);
         let aspect = (rotation % 180) ? this.dataDoc[HeightSym]() / this.dataDoc[WidthSym]() : 1;
         let shift = (rotation % 180) ? (nativeHeight - nativeWidth / aspect) / 2 : 0;
-        let srcpath = paths[Math.min(paths.length, this.Document.curPage || 0)];
+        let srcpath = paths[Math.min(paths.length - 1, this.Document.curPage || 0)];
 
         if (!this.props.Document.ignoreAspect && !this.props.leaveNativeSize) this.resize(srcpath, this.props.Document);
 
@@ -311,12 +411,20 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
                     key={this._smallRetryCount + (this._mediumRetryCount << 4) + (this._largeRetryCount << 8)} // force cache to update on retrys
                     src={srcpath}
                     style={{ transform: `translate(0px, ${shift}px) rotate(${rotation}deg) scale(${aspect})` }}
-                    // style={{ objectFit: (this.Document.curPage === 0 ? undefined : "contain") }}
                     width={nativeWidth}
                     ref={this._imgRef}
                     onError={this.onError} />
                 {paths.length > 1 ? this.dots(paths) : (null)}
+                <div className="imageBox-audioBackground"
+                    onPointerDown={this.audioDown}
+                    onPointerEnter={this.onPointerEnter}
+                    style={{ height: `calc(${.1 * nativeHeight / nativeWidth * 100}%)` }}
+                >
+                    <FontAwesomeIcon className="imageBox-audioFont"
+                        style={{ color: [DocListCast(this.extensionDoc.audioAnnotations).length ? "blue" : "gray", "green", "red"][this._audioState] }} icon={faFileAudio} size="sm" />
+                </div>
                 {/* {this.lightbox(paths)} */}
+                <FaceRectangles document={this.extensionDoc} color={"#0000FF"} backgroundColor={"#0000FF"} />
             </div>);
     }
 }
\ No newline at end of file