diff options
Diffstat (limited to 'src/client/views/nodes/ImageBox.tsx')
| -rw-r--r-- | src/client/views/nodes/ImageBox.tsx | 213 |
1 files changed, 140 insertions, 73 deletions
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 634555012..896ac1685 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -4,11 +4,11 @@ import { faAsterisk, faFileAudio, faImage, faPaintBrush } from '@fortawesome/fre import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, computed, observable, runInAction, trace } from 'mobx'; import { observer } from "mobx-react"; -import { Doc, DocListCast, HeightSym, WidthSym } from '../../../new_fields/Doc'; +import { Doc, DocListCast, HeightSym, WidthSym, DataSym } from '../../../new_fields/Doc'; import { List } from '../../../new_fields/List'; import { createSchema, listSpec, makeInterface } from '../../../new_fields/Schema'; import { ComputedField } from '../../../new_fields/ScriptField'; -import { Cast, NumCast } from '../../../new_fields/Types'; +import { Cast, NumCast, StrCast } from '../../../new_fields/Types'; import { AudioField, ImageField } from '../../../new_fields/URLField'; import { Utils, returnOne, emptyFunction } from '../../../Utils'; import { CognitiveServices, Confidence, Service, Tag } from '../../cognitive_services/CognitiveServices'; @@ -24,9 +24,12 @@ import "./ImageBox.scss"; import React = require("react"); import { CollectionFreeFormView } from '../collections/collectionFreeForm/CollectionFreeFormView'; import { documentSchema } from '../../../new_fields/documentSchemas'; -import { Id } from '../../../new_fields/FieldSymbols'; +import { Id, Copy } from '../../../new_fields/FieldSymbols'; import { TraceMobx } from '../../../new_fields/util'; import { SelectionManager } from '../../util/SelectionManager'; +import { cache } from 'sharp'; +import { ObjectField } from '../../../new_fields/ObjectField'; +import { Networking } from '../../Network'; const requestImageSize = require('../../util/request-image-size'); const path = require('path'); const { Howl } = require('howler'); @@ -39,7 +42,6 @@ library.add(faFileAudio, faAsterisk); export const pageSchema = createSchema({ curPage: "number", fitWidth: "boolean", - rotation: "number", googlePhotosUrl: "string", googlePhotosTags: "string" }); @@ -56,13 +58,22 @@ declare class MediaRecorder { type ImageDocument = makeInterface<[typeof pageSchema, typeof documentSchema]>; const ImageDocument = makeInterface(pageSchema, documentSchema); +const uploadIcons = { + idle: "downarrow.png", + loading: "loading.gif", + success: "greencheck.png", + failure: "redx.png" +}; + @observer export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocument>(ImageDocument) { + protected multiTouchDisposer?: import("../../util/InteractionUtils").InteractionUtils.MultiTouchEventDisposer | undefined; public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ImageBox, fieldKey); } private _imgRef: React.RefObject<HTMLImageElement> = React.createRef(); private _dropDisposer?: DragManager.DragDropDisposer; @observable private _audioState = 0; @observable static _showControls: boolean; + @observable uploadIcon = uploadIcons.idle; protected createDropTarget = (ele: HTMLDivElement) => { this._dropDisposer && this._dropDisposer(); @@ -73,14 +84,21 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum @action drop = (e: Event, de: DragManager.DropEvent) => { if (de.complete.docDragData) { - if (de.altKey && de.complete.docDragData.draggedDocuments.length && de.complete.docDragData.draggedDocuments[0].data instanceof ImageField) { - Doc.GetProto(this.dataDoc)[this.props.fieldKey] = new ImageField(de.complete.docDragData.draggedDocuments[0].data.url); - e.stopPropagation(); + if (de.metaKey) { + de.complete.docDragData.droppedDocuments.forEach(action((drop: Doc) => { + Doc.AddDocToList(this.dataDoc, this.props.fieldKey + "-alternates", drop); + e.stopPropagation(); + })); + } else if (de.altKey || !this.dataDoc[this.props.fieldKey]) { + const layoutDoc = de.complete.docDragData?.draggedDocuments[0]; + const targetField = Doc.LayoutFieldKey(layoutDoc); + if (layoutDoc?.[DataSym][targetField] instanceof ImageField) { + this.dataDoc[this.props.fieldKey] = ObjectField.MakeCopy(layoutDoc[DataSym][targetField] as ImageField); + this.dataDoc[this.props.fieldKey + "-nativeWidth"] = NumCast(layoutDoc[DataSym][targetField + "-nativeWidth"]); + this.dataDoc[this.props.fieldKey + "-nativeHeight"] = NumCast(layoutDoc[DataSym][targetField + "-nativeHeight"]); + e.stopPropagation(); + } } - de.metaKey && de.complete.docDragData.droppedDocuments.forEach(action((drop: Doc) => { - this.extensionDoc && Doc.AddDocToList(Doc.GetProto(this.extensionDoc), "Alternates", drop); - e.stopPropagation(); - })); } } @@ -88,8 +106,7 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum let gumStream: any; let recorder: any; const self = this; - const extensionDoc = this.extensionDoc; - extensionDoc && navigator.mediaDevices.getUserMedia({ + navigator.mediaDevices.getUserMedia({ audio: true }).then(function (stream) { gumStream = stream; @@ -97,18 +114,18 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum recorder.ondataavailable = async function (e: any) { const formData = new FormData(); formData.append("file", e.data); - const res = await fetch(Utils.prepend("/upload"), { + const res = await fetch(Utils.prepend("/uploadFormData"), { method: 'POST', body: formData }); const files = await res.json(); const url = Utils.prepend(files[0].path); // upload to server with known URL - const audioDoc = Docs.Create.AudioDocument(url, { title: "audio test", width: 200, height: 32 }); + const audioDoc = Docs.Create.AudioDocument(url, { title: "audio test", _width: 200, _height: 32 }); audioDoc.treeViewExpandedView = "layout"; - const audioAnnos = Cast(extensionDoc.audioAnnotations, listSpec(Doc)); + const audioAnnos = Cast(this.dataDoc[this.props.fieldKey + "-audioAnnotations"], listSpec(Doc)); if (audioAnnos === undefined) { - extensionDoc.audioAnnotations = new List([audioDoc]); + this.dataDoc[this.props.fieldKey + "-audioAnnotations"] = new List([audioDoc]); } else { audioAnnos.push(audioDoc); } @@ -125,15 +142,15 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum @undoBatch rotate = action(() => { - const nw = this.Document.nativeWidth; - const nh = this.Document.nativeHeight; - const w = this.Document.width; - const h = this.Document.height; - this.Document.rotation = ((this.Document.rotation || 0) + 90) % 360; - this.Document.nativeWidth = nh; - this.Document.nativeHeight = nw; - this.Document.width = h; - this.Document.height = w; + const nw = NumCast(this.Document[this.props.fieldKey + "-nativeWidth"]); + const nh = NumCast(this.Document[this.props.fieldKey + "-nativeHeight"]); + const w = this.Document._width; + const h = this.Document._height; + this.dataDoc[this.props.fieldKey + "-rotation"] = (NumCast(this.dataDoc[this.props.fieldKey + "-rotation"]) + 90) % 360; + this.dataDoc[this.props.fieldKey + "-nativeWidth"] = nh; + this.dataDoc[this.props.fieldKey + "-nativeHeight"] = nw; + this.Document._width = h; + this.Document._height = w; }); specificContextMenu = (e: React.MouseEvent): void => { @@ -159,7 +176,7 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum results.reduce((face: CognitiveServices.Image.Face, faceDocs: List<Doc>) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!), new List<Doc>()); return faceDocs; }; - this.url && this.extensionDoc && CognitiveServices.Image.Appliers.ProcessImage(this.extensionDoc, ["faces"], this.url, Service.Face, converter); + this.url && CognitiveServices.Image.Appliers.ProcessImage(this.dataDoc, [this.props.fieldKey + "-faces"], this.url, Service.Face, converter); } generateMetadata = (threshold: Confidence = Confidence.Excellent) => { @@ -171,12 +188,12 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum const sanitized = tag.name.replace(" ", "_"); tagDoc[sanitized] = ComputedField.MakeFunction(`(${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`); }); - this.extensionDoc && (this.extensionDoc.generatedTags = tagsList); + this.dataDoc[this.props.fieldKey + "-generatedTags"] = tagsList; tagDoc.title = "Generated Tags Doc"; tagDoc.confidence = threshold; return tagDoc; }; - this.url && this.extensionDoc && CognitiveServices.Image.Appliers.ProcessImage(this.extensionDoc, ["generatedTagsDoc"], this.url, Service.ComputerVision, converter); + this.url && CognitiveServices.Image.Appliers.ProcessImage(this.dataDoc, [this.props.fieldKey + "-generatedTagsDoc"], this.url, Service.ComputerVision, converter); } @computed private get url() { @@ -206,39 +223,54 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum if (this._curSuffix === "_m") this._mediumRetryCount++; if (this._curSuffix === "_l") this._largeRetryCount++; } - @action onError = () => { + @action onError = (error: any) => { const timeout = this._curSuffix === "_s" ? this._smallRetryCount : this._curSuffix === "_m" ? this._mediumRetryCount : this._largeRetryCount; if (timeout < 10) { - setTimeout(this.retryPath, Math.min(10000, timeout * 5)); + // setTimeout(this.retryPath, 500); + } + const original = StrCast(this.dataDoc.originalUrl); + if (error.type === "error" && original) { + this.dataDoc[this.props.fieldKey] = new ImageField(original); } } _curSuffix = "_m"; - _resized = ""; resize = (imgPath: string) => { - requestImageSize(imgPath) - .then((size: any) => { - const rotation = NumCast(this.dataDoc.rotation) % 180; - const realsize = rotation === 90 || rotation === 270 ? { height: size.width, width: size.height } : size; - const aspect = realsize.height / realsize.width; - if (this.Document.width && (Math.abs(1 - NumCast(this.Document.height) / NumCast(this.Document.width) / (realsize.height / realsize.width)) > 0.1)) { - setTimeout(action(() => { - if (this.pathInfos.srcpath === imgPath && (!this.layoutDoc.isTemplateDoc || this.dataDoc !== this.layoutDoc)) { - this._resized = imgPath; - this.Document.height = this.Document[WidthSym]() * aspect; - this.Document.nativeHeight = realsize.height; - this.Document.nativeWidth = realsize.width; - } - }), 0); - } else this._resized = imgPath; + const cachedNativeSize = { + width: NumCast(this.dataDoc[this.props.fieldKey + "-nativeWidth"]), + height: NumCast(this.dataDoc[this.props.fieldKey + "-nativeHeight"]) + }; + const cachedImgPath = this.dataDoc[this.props.fieldKey + "-imgPath"]; + if (!cachedNativeSize.width || !cachedNativeSize.height || imgPath !== cachedImgPath) { + (!this.layoutDoc.isTemplateDoc || this.dataDoc !== this.layoutDoc) && requestImageSize(imgPath).then((inquiredSize: any) => { + const rotation = NumCast(this.dataDoc[this.props.fieldKey + "-rotation"]) % 180; + const rotatedNativeSize = rotation === 90 || rotation === 270 ? { height: inquiredSize.width, width: inquiredSize.height } : inquiredSize; + const rotatedAspect = rotatedNativeSize.height / rotatedNativeSize.width; + const docAspect = this.Document[HeightSym]() / this.Document[WidthSym](); + setTimeout(action(() => { + if (this.Document[WidthSym]() && (!cachedNativeSize.width || !cachedNativeSize.height || Math.abs(1 - docAspect / rotatedAspect) > 0.1)) { + this.Document._height = this.Document[WidthSym]() * rotatedAspect; + this.dataDoc[this.props.fieldKey + "-nativeWidth"] = this.Document._nativeWidth = rotatedNativeSize.width; + this.dataDoc[this.props.fieldKey + "-nativeHeight"] = this.Document._nativeHeight = rotatedNativeSize.height; + } + this.dataDoc[this.props.fieldKey + "-imgPath"] = imgPath; + }), 0); }) - .catch((err: any) => console.log(err)); + .catch((err: any) => console.log(err)); + } else if (this.Document._nativeWidth !== cachedNativeSize.width || this.Document._nativeHeight !== cachedNativeSize.height) { + !(this.Document[StrCast(this.props.Document.layoutKey)] instanceof Doc) && setTimeout(() => { + if (!(this.Document[StrCast(this.props.Document.layoutKey)] instanceof Doc)) { + this.Document._nativeWidth = cachedNativeSize.width; + this.Document._nativeHeight = cachedNativeSize.height; + } + }, 0); + } } @action onPointerEnter = () => { const self = this; - const audioAnnos = this.extensionDoc && DocListCast(this.extensionDoc.audioAnnotations); + const audioAnnos = DocListCast(this.dataDoc[this.props.fieldKey + "-audioAnnotations"]); if (audioAnnos && audioAnnos.length && this._audioState === 0) { const anno = audioAnnos[Math.floor(Math.random() * audioAnnos.length)]; anno.data instanceof AudioField && new Howl({ @@ -271,45 +303,78 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum return !tags ? (null) : (<img id={"google-tags"} src={"/assets/google_tags.png"} />); } + @computed + private get considerDownloadIcon() { + const data = this.dataDoc[this.props.fieldKey]; + if (!(data instanceof ImageField)) { + return (null); + } + const primary = data.url.href; + if (primary.includes(window.location.origin)) { + return (null); + } + return ( + <img + id={"upload-icon"} + src={`/assets/${this.uploadIcon}`} + onClick={async () => { + const { dataDoc } = this; + const { success, failure, idle, loading } = uploadIcons; + runInAction(() => this.uploadIcon = loading); + const [{ clientAccessPath }] = await Networking.PostToServer("/uploadRemoteImage", { sources: [primary] }); + dataDoc.originalUrl = primary; + let succeeded = true; + let data: ImageField | undefined; + try { + data = new ImageField(Utils.prepend(clientAccessPath)); + } catch { + succeeded = false; + } + runInAction(() => this.uploadIcon = succeeded ? success : failure); + setTimeout(action(() => { + this.uploadIcon = idle; + if (data) { + dataDoc[this.props.fieldKey] = data; + } + }), 2000); + }} + /> + ); + } + @computed get nativeSize() { const pw = typeof this.props.PanelWidth === "function" ? this.props.PanelWidth() : typeof this.props.PanelWidth === "number" ? (this.props.PanelWidth as any) as number : 50; - const nativeWidth = (this.Document.nativeWidth || pw); - const nativeHeight = (this.Document.nativeHeight || 1); + const nativeWidth = NumCast(this.dataDoc[this.props.fieldKey + "-nativeWidth"], pw); + const nativeHeight = NumCast(this.dataDoc[this.props.fieldKey + "-nativeHeight"], 1); return { nativeWidth, nativeHeight }; } - @computed get pathInfos() { - const extensionDoc = this.extensionDoc!; - const { nativeWidth, nativeHeight } = this.nativeSize; - let paths = [[Utils.CorsProxy("http://www.cs.brown.edu/~bcz/noImage.png"), nativeWidth / nativeHeight]]; + @computed get paths() { + let paths = [Utils.CorsProxy("http://www.cs.brown.edu/~bcz/noImage.png")]; // this._curSuffix = ""; // if (w > 20) { - const alts = DocListCast(extensionDoc.Alternates); - const altpaths = alts.filter(doc => doc.data instanceof ImageField).map(doc => [this.choosePath((doc.data as ImageField).url), doc[WidthSym]() / doc[HeightSym]()]); + const alts = DocListCast(this.dataDoc[this.props.fieldKey + "-alternates"]); + const altpaths = alts.filter(doc => doc.data instanceof ImageField).map(doc => this.choosePath((doc.data as ImageField).url)); const field = this.dataDoc[this.props.fieldKey]; // if (w < 100 && this._smallRetryCount < 10) this._curSuffix = "_s"; // else if (w < 600 && this._mediumRetryCount < 10) this._curSuffix = "_m"; // else if (this._largeRetryCount < 10) this._curSuffix = "_l"; - if (field instanceof ImageField) paths = [[this.choosePath(field.url), nativeWidth / nativeHeight]]; + if (field instanceof ImageField) paths = [this.choosePath(field.url)]; paths.push(...altpaths); - const srcpath = paths[Math.min(paths.length - 1, (this.Document.curPage || 0))][0] as string; - const srcaspect = paths[Math.min(paths.length - 1, (this.Document.curPage || 0))][1] as number; - const fadepath = paths[Math.min(paths.length - 1, 1)][0] as string; - return { srcpath, srcaspect, fadepath }; + return paths; } @computed get content() { TraceMobx(); - const extensionDoc = this.extensionDoc; - if (!extensionDoc) return (null); - const { srcpath, srcaspect, fadepath } = this.pathInfos; + const srcpath = this.paths[NumCast(this.props.Document.curPage, 0)]; + const fadepath = this.paths[Math.min(1, this.paths.length - 1)]; const { nativeWidth, nativeHeight } = this.nativeSize; - const rotation = NumCast(this.Document.rotation, 0); + const rotation = NumCast(this.dataDoc[this.props.fieldKey + "-rotation"]); const aspect = (rotation % 180) ? this.Document[HeightSym]() / this.Document[WidthSym]() : 1; const shift = (rotation % 180) ? (nativeHeight - nativeWidth / aspect) / 2 : 0; - !this.Document.ignoreAspect && this._resized !== srcpath && this.resize(srcpath); + !this.Document.ignoreAspect && this.resize(srcpath); return <div className="imageBox-cont" key={this.props.Document[Id]} ref={this.createDropTarget} onContextMenu={this.specificContextMenu}> <div className="imageBox-fader" > @@ -319,7 +384,7 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum width={nativeWidth} ref={this._imgRef} onError={this.onError} /> - {fadepath === srcpath ? (null) : <div className="imageBox-fadeBlocker" style={{ width: nativeWidth, height: nativeWidth / srcaspect }}> + {fadepath === srcpath ? (null) : <div className="imageBox-fadeBlocker"> <img className="imageBox-fadeaway" key={"fadeaway" + this._smallRetryCount + (this._mediumRetryCount << 4) + (this._largeRetryCount << 8)} // force cache to update on retrys src={fadepath} @@ -334,10 +399,12 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum style={{ height: `calc(${.1 * nativeHeight / nativeWidth * 100}%)` }} > <FontAwesomeIcon className="imageBox-audioFont" - style={{ color: [DocListCast(extensionDoc.audioAnnotations).length ? "blue" : "gray", "green", "red"][this._audioState] }} icon={!DocListCast(extensionDoc.audioAnnotations).length ? "microphone" : faFileAudio} size="sm" /> + style={{ color: [DocListCast(this.dataDoc[this.props.fieldKey + "-audioAnnotations"]).length ? "blue" : "gray", "green", "red"][this._audioState] }} + icon={!DocListCast(this.dataDoc[this.props.fieldKey + "-audioAnnotations"]).length ? "microphone" : faFileAudio} size="sm" /> </div> + {this.considerDownloadIcon} {this.considerGooglePhotosLink()} - <FaceRectangles document={extensionDoc} color={"#0000FF"} backgroundColor={"#0000FF"} /> + <FaceRectangles document={this.dataDoc} color={"#0000FF"} backgroundColor={"#0000FF"} /> </div>; } @@ -349,12 +416,13 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum style={{ transform: `scale(${this.props.ContentScaling()})`, width: `${100 / this.props.ContentScaling()}%`, - height: `${100 / this.props.ContentScaling()}%` + height: `${100 / this.props.ContentScaling()}%`, + pointerEvents: this.props.Document.isBackground ? "none" : undefined }} > <CollectionFreeFormView {...this.props} PanelHeight={this.props.PanelHeight} PanelWidth={this.props.PanelWidth} - annotationsKey={this.annotationsKey} + annotationsKey={this.annotationKey} isAnnotationOverlay={true} focus={this.props.focus} isSelected={this.props.isSelected} @@ -367,7 +435,6 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum addDocument={this.addDocument} CollectionView={undefined} ScreenToLocalTransform={this.props.ScreenToLocalTransform} - ruleProvider={undefined} renderDepth={this.props.renderDepth + 1} ContainingCollectionDoc={this.props.ContainingCollectionDoc} chromeCollapsed={true}> |