diff options
Diffstat (limited to 'src/client/views/nodes/ImageBox.tsx')
| -rw-r--r-- | src/client/views/nodes/ImageBox.tsx | 202 |
1 files changed, 155 insertions, 47 deletions
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 4c5ad7a7d..0d9c2bb8a 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -1,33 +1,40 @@ import { library } from '@fortawesome/fontawesome-svg-core'; -import { faImage } from '@fortawesome/free-solid-svg-icons'; -import { action, observable, computed } from 'mobx'; +import { faEye } from '@fortawesome/free-regular-svg-icons'; +import { faAsterisk, faFileAudio, faImage, faPaintBrush } from '@fortawesome/free-solid-svg-icons'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { action, computed, observable, runInAction } from 'mobx'; import { observer } from "mobx-react"; import Lightbox from 'react-image-lightbox'; import 'react-image-lightbox/style.css'; // This only needs to be imported once in your app -import { Doc, HeightSym, WidthSym, DocListCast } from '../../../new_fields/Doc'; +import { Doc, DocListCast, HeightSym, WidthSym } from '../../../new_fields/Doc'; import { List } from '../../../new_fields/List'; import { createSchema, listSpec, makeInterface } from '../../../new_fields/Schema'; -import { Cast, FieldValue, NumCast, StrCast, BoolCast } from '../../../new_fields/Types'; -import { ImageField } from '../../../new_fields/URLField'; +import { ComputedField } from '../../../new_fields/ScriptField'; +import { BoolCast, Cast, FieldValue, NumCast, StrCast } from '../../../new_fields/Types'; +import { AudioField, ImageField } from '../../../new_fields/URLField'; +import { RouteStore } from '../../../server/RouteStore'; import { Utils } from '../../../Utils'; +import { CognitiveServices, Confidence, Service, Tag } from '../../cognitive_services/CognitiveServices'; +import { Docs } from '../../documents/Documents'; import { DragManager } from '../../util/DragManager'; +import { CompileScript } from '../../util/Scripting'; import { undoBatch } from '../../util/UndoManager'; import { ContextMenu } from "../../views/ContextMenu"; import { ContextMenuProps } from '../ContextMenuItem'; import { DocComponent } from '../DocComponent'; import { InkingControl } from '../InkingControl'; import { positionSchema } from './DocumentView'; +import FaceRectangles from './FaceRectangles'; import { FieldView, FieldViewProps } from './FieldView'; import "./ImageBox.scss"; import React = require("react"); -import { RouteStore } from '../../../server/RouteStore'; -import { Docs } from '../../documents/Documents'; -import { DocServer } from '../../DocServer'; var requestImageSize = require('../../util/request-image-size'); var path = require('path'); +const { Howl } = require('howler'); -library.add(faImage); +library.add(faImage, faEye as any, faPaintBrush); +library.add(faFileAudio, faAsterisk); export const pageSchema = createSchema({ @@ -58,7 +65,7 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD private dropDisposer?: DragManager.DragDropDisposer; - @computed get dataDoc() { return BoolCast(this.props.Document.isTemplate) && this.props.DataDoc ? this.props.DataDoc : this.props.Document; } + @computed get dataDoc() { return this.props.DataDoc && (BoolCast(this.props.Document.isTemplate) || BoolCast(this.props.DataDoc.isTemplate) || this.props.DataDoc.layout === this.props.Document) ? this.props.DataDoc : this.props.Document; } protected createDropTarget = (ele: HTMLDivElement) => { @@ -78,10 +85,14 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD @computed get extensionDoc() { return Doc.resolvedFieldDataDoc(this.dataDoc, this.props.fieldKey, "Alternates"); } @undoBatch + @action drop = (e: Event, de: DragManager.DropEvent) => { if (de.data instanceof DragManager.DocumentDragData) { de.data.droppedDocuments.forEach(action((drop: Doc) => { - if (de.mods === "AltKey" && /*this.dataDoc !== this.props.Document &&*/ drop.data instanceof ImageField) { + if (de.mods === "CtrlKey") { + Doc.ApplyTemplateTo(drop, this.props.Document, this.props.DataDoc); + e.stopPropagation(); + } else if (de.mods === "AltKey" && /*this.dataDoc !== this.props.Document &&*/ drop.data instanceof ImageField) { Doc.GetProto(this.dataDoc)[this.props.fieldKey] = new ImageField(drop.data.url); e.stopPropagation(); } else if (de.mods === "CtrlKey") { @@ -97,8 +108,6 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD e.stopPropagation(); } } - } else if (!this.props.isSelected()) { - e.stopPropagation(); } })); // de.data.removeDocument() bcz: need to implement @@ -161,15 +170,15 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD recorder.ondataavailable = async function (e: any) { const formData = new FormData(); formData.append("file", e.data); - const res = await fetch(DocServer.prepend(RouteStore.upload), { + const res = await fetch(Utils.prepend(RouteStore.upload), { method: 'POST', body: formData }); const files = await res.json(); - const url = DocServer.prepend(files[0]); + const url = Utils.prepend(files[0]); // upload to server with known URL let audioDoc = Docs.Create.AudioDocument(url, { title: "audio test", x: NumCast(self.props.Document.x), y: NumCast(self.props.Document.y), width: 200, height: 32 }); - audioDoc.embed = true; + audioDoc.treeViewExpandedView = "layout"; let audioAnnos = Cast(self.extensionDoc.audioAnnotations, listSpec(Doc)); if (audioAnnos === undefined) { self.extensionDoc.audioAnnotations = new List([audioDoc]); @@ -177,45 +186,90 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD audioAnnos.push(audioDoc); } }; + runInAction(() => self._audioState = 2); recorder.start(); setTimeout(() => { recorder.stop(); - + runInAction(() => self._audioState = 0); gumStream.getAudioTracks()[0].stop(); }, 5000); }); } + @undoBatch + rotate = action(() => { + let proto = Doc.GetProto(this.props.Document); + let nw = this.props.Document.nativeWidth; + let nh = this.props.Document.nativeHeight; + let w = this.props.Document.width; + let h = this.props.Document.height; + proto.rotation = (NumCast(this.props.Document.rotation) + 90) % 360; + proto.nativeWidth = nh; + proto.nativeHeight = nw; + this.props.Document.width = h; + this.props.Document.height = w; + }); + specificContextMenu = (e: React.MouseEvent): void => { let field = Cast(this.Document[this.props.fieldKey], ImageField); if (field) { let url = field.url.href; - let subitems: ContextMenuProps[] = []; - subitems.push({ description: "Copy path", event: () => Utils.CopyText(url), icon: "expand-arrows-alt" }); - subitems.push({ description: "Record 1sec audio", event: this.recordAudioAnnotation, icon: "expand-arrows-alt" }); - subitems.push({ - description: "Rotate", event: action(() => { - let proto = Doc.GetProto(this.props.Document); - let nw = this.props.Document.nativeWidth; - let nh = this.props.Document.nativeHeight; - let w = this.props.Document.width; - let h = this.props.Document.height; - proto.rotation = (NumCast(this.props.Document.rotation) + 90) % 360; - proto.nativeWidth = nh; - proto.nativeHeight = nw; - this.props.Document.width = h; - this.props.Document.height = w; - }), icon: "expand-arrows-alt" - }); - ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: subitems }); + let funcs: ContextMenuProps[] = []; + funcs.push({ description: "Copy path", event: () => Utils.CopyText(url), icon: "expand-arrows-alt" }); + funcs.push({ description: "Record 1sec audio", event: this.recordAudioAnnotation, icon: "expand-arrows-alt" }); + funcs.push({ description: "Rotate", event: this.rotate, icon: "expand-arrows-alt" }); + + let modes: ContextMenuProps[] = []; + modes.push({ description: "Generate Tags", event: this.generateMetadata, icon: "tag" }); + modes.push({ description: "Find Faces", event: this.extractFaces, icon: "camera" }); + + ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: funcs, icon: "asterisk" }); + ContextMenu.Instance.addItem({ description: "Analyze...", subitems: modes, icon: "eye" }); } } + extractFaces = () => { + let converter = (results: any) => { + let faceDocs = new List<Doc>(); + results.map((face: CognitiveServices.Image.Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!)); + return faceDocs; + }; + CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["faces"], this.url, Service.Face, converter); + } + + generateMetadata = (threshold: Confidence = Confidence.Excellent) => { + let converter = (results: any) => { + let tagDoc = new Doc; + let tagsList = new List(); + results.tags.map((tag: Tag) => { + tagsList.push(tag.name); + let sanitized = tag.name.replace(" ", "_"); + let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`; + let computed = CompileScript(script, { params: { this: "Doc" } }); + computed.compiled && (tagDoc[sanitized] = new ComputedField(computed)); + }); + this.extensionDoc.generatedTags = tagsList; + tagDoc.title = "Generated Tags Doc"; + tagDoc.confidence = threshold; + return tagDoc; + }; + CognitiveServices.Image.Manager.analyzer(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter); + } + @action onDotDown(index: number) { this.Document.curPage = index; } + @computed get fieldExtensionDoc() { + return Doc.resolvedFieldDataDoc(this.props.DataDoc ? this.props.DataDoc : this.props.Document, this.props.fieldKey, "true"); + } + + @computed private get url() { + let data = Cast(Doc.GetProto(this.props.Document).data, ImageField); + return data ? data.url.href : undefined; + } + dots(paths: string[]) { let nativeWidth = FieldValue(this.Document.nativeWidth, 1); let dist = Math.min(nativeWidth / paths.length, 40); @@ -229,11 +283,15 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD choosePath(url: URL) { const lower = url.href.toLowerCase(); - if (url.protocol === "data" || url.href.indexOf(window.location.origin) === -1 || !(lower.endsWith(".png") || lower.endsWith(".jpg") || lower.endsWith(".jpeg"))) { + if (url.protocol === "data") { return url.href; + } else if (url.href.indexOf(window.location.origin) === -1) { + return Utils.CorsProxy(url.href); + } else if (!(lower.endsWith(".png") || lower.endsWith(".jpg") || lower.endsWith(".jpeg"))) { + return url.href;//Why is this here } let ext = path.extname(url.href); - const suffix = this.props.renderDepth <= 1 ? "_o" : this._curSuffix; + const suffix = this.props.renderDepth < 1 ? "_o" : this._curSuffix; return url.href.replace(ext, suffix + ext); } @@ -254,16 +312,16 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD _curSuffix = "_m"; resize(srcpath: string, layoutdoc: Doc) { - requestImageSize(window.origin + RouteStore.corsProxy + "/" + srcpath) + requestImageSize(srcpath) .then((size: any) => { - let aspect = size.height / size.width; let rotation = NumCast(this.dataDoc.rotation) % 180; - if (rotation === 90 || rotation === 270) aspect = 1 / aspect; - if (Math.abs(layoutdoc[HeightSym]() / layoutdoc[WidthSym]() - aspect) > 0.01) { + let realsize = rotation === 90 || rotation === 270 ? { height: size.width, width: size.height } : size; + let aspect = realsize.height / realsize.width; + if (Math.abs(NumCast(layoutdoc.height) - realsize.height) > 1 || Math.abs(NumCast(layoutdoc.width) - realsize.width) > 1) { setTimeout(action(() => { layoutdoc.height = layoutdoc[WidthSym]() * aspect; - layoutdoc.nativeHeight = size.height; - layoutdoc.nativeWidth = size.width; + layoutdoc.nativeHeight = realsize.height; + layoutdoc.nativeWidth = realsize.width; }), 0); } }) @@ -272,6 +330,48 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD }); } + @observable _audioState = 0; + + @action + onPointerEnter = () => { + let self = this; + let audioAnnos = DocListCast(this.extensionDoc.audioAnnotations); + if (audioAnnos.length && this._audioState === 0) { + let anno = audioAnnos[Math.floor(Math.random() * audioAnnos.length)]; + anno.data instanceof AudioField && new Howl({ + src: [anno.data.url.href], + format: ["mp3"], + autoplay: true, + loop: false, + volume: 0.5, + onend: function () { + runInAction(() => self._audioState = 0); + } + }); + this._audioState = 1; + } + // else { + // if (this._audioState === 0) { + // this._audioState = 1; + // new Howl({ + // src: ["https://www.kozco.com/tech/piano2-CoolEdit.mp3"], + // autoplay: true, + // loop: false, + // volume: 0.5, + // onend: function () { + // runInAction(() => self._audioState = 0); + // } + // }); + // } + // } + } + + @action + audioDown = () => { + this.recordAudioAnnotation(); + } + + render() { // let transform = this.props.ScreenToLocalTransform().inverse(); let pw = typeof this.props.PanelWidth === "function" ? this.props.PanelWidth() : typeof this.props.PanelWidth === "number" ? (this.props.PanelWidth as any) as number : 50; @@ -282,7 +382,7 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD let id = (this.props as any).id; // bcz: used to set id = "isExpander" in templates.tsx let nativeWidth = FieldValue(this.Document.nativeWidth, pw); let nativeHeight = FieldValue(this.Document.nativeHeight, 0); - let paths: string[] = ["http://www.cs.brown.edu/~bcz/noImage.png"]; + let paths: string[] = [Utils.CorsProxy("http://www.cs.brown.edu/~bcz/noImage.png")]; // this._curSuffix = ""; // if (w > 20) { Doc.UpdateDocumentExtensionForField(this.dataDoc, this.props.fieldKey); @@ -295,11 +395,11 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD if (field instanceof ImageField) paths = [this.choosePath(field.url)]; paths.push(...altpaths); // } - let interactive = InkingControl.Instance.selectedTool ? "" : "-interactive"; + let interactive = InkingControl.Instance.selectedTool || this.props.Document.isBackground ? "" : "-interactive"; let rotation = NumCast(this.dataDoc.rotation, 0); let aspect = (rotation % 180) ? this.dataDoc[HeightSym]() / this.dataDoc[WidthSym]() : 1; let shift = (rotation % 180) ? (nativeHeight - nativeWidth / aspect) / 2 : 0; - let srcpath = paths[Math.min(paths.length, this.Document.curPage || 0)]; + let srcpath = paths[Math.min(paths.length - 1, this.Document.curPage || 0)]; if (!this.props.Document.ignoreAspect && !this.props.leaveNativeSize) this.resize(srcpath, this.props.Document); @@ -311,12 +411,20 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD key={this._smallRetryCount + (this._mediumRetryCount << 4) + (this._largeRetryCount << 8)} // force cache to update on retrys src={srcpath} style={{ transform: `translate(0px, ${shift}px) rotate(${rotation}deg) scale(${aspect})` }} - // style={{ objectFit: (this.Document.curPage === 0 ? undefined : "contain") }} width={nativeWidth} ref={this._imgRef} onError={this.onError} /> {paths.length > 1 ? this.dots(paths) : (null)} + <div className="imageBox-audioBackground" + onPointerDown={this.audioDown} + onPointerEnter={this.onPointerEnter} + style={{ height: `calc(${.1 * nativeHeight / nativeWidth * 100}%)` }} + > + <FontAwesomeIcon className="imageBox-audioFont" + style={{ color: [DocListCast(this.extensionDoc.audioAnnotations).length ? "blue" : "gray", "green", "red"][this._audioState] }} icon={faFileAudio} size="sm" /> + </div> {/* {this.lightbox(paths)} */} + <FaceRectangles document={this.extensionDoc} color={"#0000FF"} backgroundColor={"#0000FF"} /> </div>); } }
\ No newline at end of file |