import React = require("react"); import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { Tooltip } from "@material-ui/core"; import { action, computed, IReactionDisposer, observable, ObservableMap, reaction, runInAction, untracked } from "mobx"; import { observer } from "mobx-react"; import * as rp from 'request-promise'; import { Doc, DocListCast } from "../../../fields/Doc"; import { documentSchema } from "../../../fields/documentSchemas"; import { InkTool } from "../../../fields/InkField"; import { makeInterface } from "../../../fields/Schema"; import { Cast, NumCast, StrCast } from "../../../fields/Types"; import { AudioField, VideoField } from "../../../fields/URLField"; import { emptyFunction, formatTime, OmitKeys, returnFalse, returnOne, setupMoveUpEvents, Utils } from "../../../Utils"; import { Docs, DocUtils } from "../../documents/Documents"; import { DocumentType } from "../../documents/DocumentTypes"; import { Networking } from "../../Network"; import { CurrentUserUtils } from "../../util/CurrentUserUtils"; import { DocumentManager } from "../../util/DocumentManager"; import { SelectionManager } from "../../util/SelectionManager"; import { SnappingManager } from "../../util/SnappingManager"; import { undoBatch } from "../../util/UndoManager"; import { CollectionFreeFormView } from "../collections/collectionFreeForm/CollectionFreeFormView"; import { CollectionStackedTimeline, TrimScope } from "../collections/CollectionStackedTimeline"; import { ContextMenu } from "../ContextMenu"; import { ContextMenuProps } from "../ContextMenuItem"; import { ViewBoxAnnotatableComponent, ViewBoxAnnotatableProps } from "../DocComponent"; import { DocumentDecorations } from "../DocumentDecorations"; import { MarqueeAnnotator } from "../MarqueeAnnotator"; import { AnchorMenu } from "../pdf/AnchorMenu"; import { StyleProp } from "../StyleProvider"; import { FieldView, FieldViewProps } from './FieldView'; import "./VideoBox.scss"; const path = require('path'); type VideoDocument = makeInterface<[typeof documentSchema]>; const VideoDocument = makeInterface(documentSchema); @observer export class VideoBox extends ViewBoxAnnotatableComponent(VideoDocument) { public static LayoutString(fieldKey: string) { return FieldView.LayoutString(VideoBox, fieldKey); } static async convertDataUri(imageUri: string, returnedFilename: string) { try { const posting = Utils.prepend("/uploadURI"); const returnedUri = await rp.post(posting, { body: { uri: imageUri, name: returnedFilename }, json: true, }); return returnedUri; } catch (e) { console.log("VideoBox :" + e); } } static _youtubeIframeCounter: number = 0; static heightPercent = 60; // height of timeline in percent of height of videoBox. private _disposers: { [name: string]: IReactionDisposer } = {}; private _youtubePlayer: YT.Player | undefined = undefined; private _videoRef: HTMLVideoElement | null = null; private _youtubeIframeId: number = -1; private _youtubeContentCreated = false; private _audioPlayer: HTMLAudioElement | null = null; private _stackedTimeline = React.createRef(); private _mainCont: React.RefObject = React.createRef(); private _annotationLayer: React.RefObject = React.createRef(); private _playRegionTimer: any = null; private _playRegionDuration = 0; @observable static _nativeControls: boolean; @observable _marqueeing: number[] | undefined; @observable _savedAnnotations = new ObservableMap(); @observable _screenCapture = false; @observable _clicking = false; @observable _forceCreateYouTubeIFrame = false; @observable _playTimer?: NodeJS.Timeout = undefined; @observable _fullScreen = false; @observable _playing = false; @computed get links() { return DocListCast(this.dataDoc.links); } @computed get heightPercent() { return NumCast(this.layoutDoc._timelineHeightPercent, 100); } @computed get rawDuration() { return NumCast(this.dataDoc[this.fieldKey + "-duration"]); } @computed get youtubeVideoId() { const field = Cast(this.dataDoc[this.props.fieldKey], VideoField); return field && field.url.href.indexOf("youtube") !== -1 ? ((arr: string[]) => arr[arr.length - 1])(field.url.href.split("/")) : ""; } // returns the path of the audio file @computed get audiopath() { const field = Cast(this.props.Document[this.props.fieldKey + '-audio'], AudioField, null); const vfield = Cast(this.dataDoc[this.fieldKey], VideoField, null); return field?.url.href ?? vfield?.url.href ?? ""; } private get timeline() { return this._stackedTimeline.current; } private get transition() { return this._clicking ? "left 0.5s, width 0.5s, height 0.5s" : ""; } public get player(): HTMLVideoElement | null { return this._videoRef; } componentDidMount() { this.props.setContentView?.(this); // this tells the DocumentView that this AudioBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. if (this.youtubeVideoId) { const youtubeaspect = 400 / 315; const nativeWidth = Doc.NativeWidth(this.layoutDoc); const nativeHeight = Doc.NativeHeight(this.layoutDoc); if (!nativeWidth || !nativeHeight) { if (!nativeWidth) Doc.SetNativeWidth(this.dataDoc, 600); Doc.SetNativeHeight(this.dataDoc, (nativeWidth || 600) / youtubeaspect); this.layoutDoc._height = (this.layoutDoc._width || 0) / youtubeaspect; } } this.player && this.setPlayheadTime(0); } componentWillUnmount() { this.Pause(); Object.keys(this._disposers).forEach(d => this._disposers[d]?.()); } @action public Play = (update: boolean = true) => { this._playing = true; const eleTime = this.player?.currentTime || 0; const start = eleTime >= (this.timeline?.trimEnd || 0) ? this.timeline?.trimStart || 0 : eleTime; try { this._audioPlayer && this.player && (this._audioPlayer.currentTime = this.player?.currentTime); update && this.player && this.playFrom(start, undefined, true); update && this._audioPlayer?.play(); update && this._youtubePlayer?.playVideo(); this._youtubePlayer && !this._playTimer && (this._playTimer = setInterval(this.updateTimecode, 5)); } catch (e) { console.log("Video Play Exception:", e); } this.updateTimecode(); } @action public Seek(time: number) { try { this._youtubePlayer?.seekTo(Math.round(time), true); } catch (e) { console.log("Video Seek Exception:", e); } this.player && (this.player.currentTime = time); this._audioPlayer && (this._audioPlayer.currentTime = time); } @action public Pause = (update: boolean = true) => { this._playing = false; try { update && this.player?.pause(); update && this._audioPlayer?.pause(); update && this._youtubePlayer?.pauseVideo(); this._youtubePlayer && this._playTimer && clearInterval(this._playTimer); this._youtubePlayer?.seekTo(this._youtubePlayer?.getCurrentTime(), true); } catch (e) { console.log("Video Pause Exception:", e); } this._youtubePlayer && SelectionManager.DeselectAll(); // if we don't deselect the player, then we get an annoying YouTube spinner I guess telling us we're paused. this._playTimer = undefined; this.updateTimecode(); } @action public FullScreen = () => { this._fullScreen = true; this.player && this.player.requestFullscreen(); try { this._youtubePlayer && this.props.addDocTab(this.rootDoc, "add"); } catch (e) { console.log("Video FullScreen Exception:", e); } } @action public Snapshot(downX?: number, downY?: number) { const width = (this.layoutDoc._width || 0); const canvas = document.createElement('canvas'); canvas.width = 640; canvas.height = 640 * Doc.NativeHeight(this.layoutDoc) / (Doc.NativeWidth(this.layoutDoc) || 1); const ctx = canvas.getContext('2d');//draw image to canvas. scale to target dimensions if (ctx) { // ctx.rect(0, 0, canvas.width, canvas.height); // ctx.fillStyle = "blue"; // ctx.fill(); this._videoRef && ctx.drawImage(this._videoRef, 0, 0, canvas.width, canvas.height); } if (!this._videoRef) { const b = Docs.Create.LabelDocument({ x: (this.layoutDoc.x || 0) + width, y: (this.layoutDoc.y || 1), _width: 150, _height: 50, title: (this.layoutDoc._currentTimecode || 0).toString(), _isLinkButton: true }); this.props.addDocument?.(b); DocUtils.MakeLink({ doc: b }, { doc: this.rootDoc }, "video snapshot"); Networking.PostToServer("/youtubeScreenshot", { id: this.youtubeVideoId, timecode: this.layoutDoc._currentTimecode }).then(response => { const resolved = response?.accessPaths?.agnostic?.client; if (resolved) { this.props.removeDocument?.(b); this.createRealSummaryLink(resolved); } }); } else { //convert to desired file format const dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png' // if you want to preview the captured image, const retitled = StrCast(this.rootDoc.title).replace(/[ -\.]/g, ""); const filename = path.basename(encodeURIComponent("snapshot" + retitled + "_" + (this.layoutDoc._currentTimecode || 0).toString().replace(/\./, "_"))); VideoBox.convertDataUri(dataUrl, filename).then((returnedFilename: string) => returnedFilename && this.createRealSummaryLink(returnedFilename, downX, downY)); } } createRealSummaryLink = (imagePath: string, downX?: number, downY?: number) => { const url = !imagePath.startsWith("/") ? Utils.CorsProxy(imagePath) : imagePath; const width = this.layoutDoc._width || 1; const height = this.layoutDoc._height || 0; const imageSummary = Docs.Create.ImageDocument(url, { _nativeWidth: Doc.NativeWidth(this.layoutDoc), _nativeHeight: Doc.NativeHeight(this.layoutDoc), x: (this.layoutDoc.x || 0) + width, y: (this.layoutDoc.y || 0), _isLinkButton: true, _width: 150, _height: height / width * 150, title: "--snapshot" + (this.layoutDoc._currentTimecode || 0) + " image-" }); Doc.SetNativeWidth(Doc.GetProto(imageSummary), Doc.NativeWidth(this.layoutDoc)); Doc.SetNativeHeight(Doc.GetProto(imageSummary), Doc.NativeHeight(this.layoutDoc)); this.props.addDocument?.(imageSummary); const link = DocUtils.MakeLink({ doc: imageSummary }, { doc: this.getAnchor() }, "video snapshot"); link && (Doc.GetProto(link.anchor2 as Doc).timecodeToHide = NumCast((link.anchor2 as Doc).timecodeToShow) + 3); setTimeout(() => (downX !== undefined && downY !== undefined) && DocumentManager.Instance.getFirstDocumentView(imageSummary)?.startDragging(downX, downY, "move", true)); } getAnchor = () => { const timecode = Cast(this.layoutDoc._currentTimecode, "number", null); const marquee = AnchorMenu.Instance.GetAnchor?.(); return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow"/* videoStart */, "_timecodeToHide" /* videoEnd */, timecode ? timecode : undefined, undefined, marquee) || this.rootDoc; } videoLoad = () => { const aspect = this.player!.videoWidth / this.player!.videoHeight; Doc.SetNativeWidth(this.dataDoc, this.player!.videoWidth); Doc.SetNativeHeight(this.dataDoc, this.player!.videoHeight); this.layoutDoc._height = (this.layoutDoc._width || 0) / aspect; if (Number.isFinite(this.player!.duration)) { this.dataDoc[this.fieldKey + "-duration"] = this.player!.duration; } } @action updateTimecode = () => { this.player && (this.layoutDoc._currentTimecode = this.player.currentTime); try { this._youtubePlayer && (this.layoutDoc._currentTimecode = this._youtubePlayer.getCurrentTime?.()); } catch (e) { console.log("Video Timecode Exception:", e); } } @action setVideoRef = (vref: HTMLVideoElement | null) => { this._videoRef = vref; if (vref) { this._videoRef!.ontimeupdate = this.updateTimecode; // @ts-ignore vref.onfullscreenchange = action((e) => this._fullScreen = vref.webkitDisplayingFullscreen); this._disposers.reactionDisposer?.(); this._disposers.reactionDisposer = reaction(() => (this.layoutDoc._currentTimecode || 0), time => !this._playing && (vref.currentTime = time), { fireImmediately: true }); } } specificContextMenu = (e: React.MouseEvent): void => { const field = Cast(this.dataDoc[this.props.fieldKey], VideoField); if (field) { const url = field.url.href; const subitems: ContextMenuProps[] = []; subitems.push({ description: "Full Screen", event: this.FullScreen, icon: "expand" }); subitems.push({ description: "Take Snapshot", event: this.Snapshot, icon: "expand-arrows-alt" }); this.rootDoc.type === DocumentType.SCREENSHOT && subitems.push({ description: "Screen Capture", event: (async () => { runInAction(() => this._screenCapture = !this._screenCapture); this._videoRef!.srcObject = !this._screenCapture ? undefined : await (navigator.mediaDevices as any).getDisplayMedia({ video: true }); }), icon: "expand-arrows-alt" }); subitems.push({ description: (this.layoutDoc.dontAutoFollowLinks ? "" : "Don't") + " follow links when encountered", event: () => this.layoutDoc.dontAutoFollowLinks = !this.layoutDoc.dontAutoFollowLinks, icon: "expand-arrows-alt" }); subitems.push({ description: (this.layoutDoc.dontAutoPlayFollowedLinks ? "" : "Don't") + " play when link is selected", event: () => this.layoutDoc.dontAutoPlayFollowedLinks = !this.layoutDoc.dontAutoPlayFollowedLinks, icon: "expand-arrows-alt" }); subitems.push({ description: (this.layoutDoc.autoPlayAnchors ? "Don't auto play" : "Auto play") + " anchors onClick", event: () => this.layoutDoc.autoPlayAnchors = !this.layoutDoc.autoPlayAnchors, icon: "expand-arrows-alt" }); subitems.push({ description: "Toggle Native Controls", event: action(() => VideoBox._nativeControls = !VideoBox._nativeControls), icon: "expand-arrows-alt" }); subitems.push({ description: "Start Trim All", event: () => this.startTrim(TrimScope.All), icon: "expand-arrows-alt" }); subitems.push({ description: "Start Trim Clip", event: () => this.startTrim(TrimScope.Clip), icon: "expand-arrows-alt" }); subitems.push({ description: "Stop Trim", event: () => this.finishTrim(), icon: "expand-arrows-alt" }); subitems.push({ description: "Copy path", event: () => { Utils.CopyText(url); }, icon: "expand-arrows-alt" }); ContextMenu.Instance.addItem({ description: "Options...", subitems: subitems, icon: "video" }); } } @action youtubeIframeLoaded = (e: any) => { if (!this._youtubeContentCreated) { this._forceCreateYouTubeIFrame = !this._forceCreateYouTubeIFrame; return; } else this._youtubeContentCreated = false; this.loadYouTube(e.target); } loadYouTube = (iframe: any) => { let started = true; const onYoutubePlayerStateChange = (event: any) => runInAction(() => { if (started && event.data === YT.PlayerState.PLAYING) { started = false; this._youtubePlayer?.unMute(); //this.Pause(); return; } if (event.data === YT.PlayerState.PLAYING && !this._playing) this.Play(false); if (event.data === YT.PlayerState.PAUSED && this._playing) this.Pause(false); }); const onYoutubePlayerReady = (event: any) => { this._disposers.reactionDisposer?.(); this._disposers.youtubeReactionDisposer?.(); this._disposers.reactionDisposer = reaction(() => this.layoutDoc._currentTimecode, () => !this._playing && this.Seek((this.layoutDoc._currentTimecode || 0))); this._disposers.youtubeReactionDisposer = reaction( () => CurrentUserUtils.SelectedTool === InkTool.None && this.props.isSelected(true) && !SnappingManager.GetIsDragging() && !DocumentDecorations.Instance.Interacting, (interactive) => iframe.style.pointerEvents = interactive ? "all" : "none", { fireImmediately: true }); }; if (typeof (YT) === undefined) setTimeout(() => this.loadYouTube(iframe), 100); else { (YT as any)?.ready(() => { this._youtubePlayer = new YT.Player(`${this.youtubeVideoId + this._youtubeIframeId}-player`, { events: { 'onReady': this.props.dontRegisterView ? undefined : onYoutubePlayerReady, 'onStateChange': this.props.dontRegisterView ? undefined : onYoutubePlayerStateChange, } }); }); } } onPlayDown = () => this._playing ? this.Pause() : this.Play(); onFullDown = (e: React.PointerEvent) => { this.FullScreen(); e.stopPropagation(); e.preventDefault(); } onSnapshotDown = (e: React.PointerEvent) => { setupMoveUpEvents(this, e, (e) => { this.Snapshot(e.clientX, e.clientY); return true; }, emptyFunction, () => this.Snapshot()); } @action onTimelineHdlDown = (e: React.PointerEvent) => { this._clicking = true; setupMoveUpEvents(this, e, action(encodeURIComponent => { this._clicking = false; if (this.props.isContentActive()) { const local = this.props.ScreenToLocalTransform().scale(this.props.scaling?.() || 1).transformPoint(e.clientX, e.clientY); this.layoutDoc._timelineHeightPercent = Math.max(0, Math.min(100, local[1] / this.props.PanelHeight() * 100)); } return false; }), emptyFunction, () => { this.layoutDoc._timelineHeightPercent = this.heightPercent !== 100 ? 100 : VideoBox.heightPercent; setTimeout(action(() => this._clicking = false), 500); }, this.props.isContentActive(), this.props.isContentActive()); } onResetDown = (e: React.PointerEvent) => { setupMoveUpEvents(this, e, e => { this.Seek(Math.max(0, (this.layoutDoc._currentTimecode || 0) + Math.sign(e.movementX) * 0.0333)); e.stopImmediatePropagation(); return false; }, emptyFunction, e => this.layoutDoc._currentTimecode = 0); } @action.bound addDocWithTimecode(doc: Doc | Doc[]): boolean { const docs = doc instanceof Doc ? [doc] : doc; const curTime = NumCast(this.layoutDoc._currentTimecode); docs.forEach(doc => doc._timecodeToHide = (doc._timecodeToShow = curTime) + 1); return this.addDocument(doc); } // play back the video from time @action playFrom = (seekTimeInSeconds: number, endTime?: number, fullPlay: boolean = false) => { clearTimeout(this._playRegionTimer); if (Number.isNaN(this.player?.duration)) { setTimeout(() => this.playFrom(seekTimeInSeconds, endTime), 500); } else if (this.player) { const end = Math.min(this.timeline?.trimEnd ?? this.rawDuration, endTime ?? this.timeline?.trimEnd ?? this.rawDuration); const start = Math.max(this.timeline?.trimStart ?? 0, seekTimeInSeconds); this._playRegionDuration = end - start; if (seekTimeInSeconds >= 0 && (this.timeline?.trimStart || 0) <= end && seekTimeInSeconds <= (this.timeline?.trimEnd || this.rawDuration)) { this.player.currentTime = start; this._audioPlayer && (this._audioPlayer.currentTime = seekTimeInSeconds); this.player.play(); this._audioPlayer?.play(); this._playing = true; this._playRegionTimer = setTimeout( () => { if (fullPlay) this.setPlayheadTime(this.timeline?.trimStart || 0); this.Pause(); }, this._playRegionDuration * 1000); } else { this.Pause(); } } } // hides trim controls and displays new clip @undoBatch finishTrim = action(() => { this.Pause(); this.setPlayheadTime(Math.max(Math.min(this.timeline?.trimEnd || 0, this.player!.currentTime), this.timeline?.trimStart || 0)); this._stackedTimeline.current?.StopTrimming(); }); startTrim = (scope: TrimScope) => { this.Pause(); this._stackedTimeline.current?.StartTrimming(scope); } onClipPointerDown = (e: React.PointerEvent) => { this.heightPercent >= 100 && this.onTimelineHdlDown(e); this.timeline && setupMoveUpEvents(this, e, returnFalse, returnFalse, action((e: PointerEvent, doubleTap?: boolean) => { if (doubleTap) { this.startTrim(TrimScope.All); } else if (this.timeline) { this.Pause(); this.timeline.IsTrimming !== TrimScope.None ? this.finishTrim() : this.startTrim(TrimScope.Clip); } })); } playLink = (doc: Doc) => { const startTime = Math.max(0, (this._stackedTimeline.current?.anchorStart(doc) || 0)); const endTime = this._stackedTimeline.current?.anchorEnd(doc); if (startTime !== undefined) { if (!this.layoutDoc.dontAutoPlayFollowedLinks) endTime ? this.playFrom(startTime, endTime) : this.playFrom(startTime); else this.Seek(startTime); } } marqueeDown = (e: React.PointerEvent) => { if (!e.altKey && e.button === 0 && this.layoutDoc._viewScale === 1 && this.props.isContentActive(true) && ![InkTool.Highlighter, InkTool.Pen].includes(CurrentUserUtils.SelectedTool)) { setupMoveUpEvents(this, e, action(e => { MarqueeAnnotator.clearAnnotations(this._savedAnnotations); this._marqueeing = [e.clientX, e.clientY]; return true; }), returnFalse, () => MarqueeAnnotator.clearAnnotations(this._savedAnnotations), false); } } @action finishMarquee = () => { this._marqueeing = undefined; this.props.select(true); } timelineWhenChildContentsActiveChanged = action((isActive: boolean) => this.props.whenChildContentsActiveChanged(this._isAnyChildContentActive = isActive)); timelineScreenToLocal = () => this.props.ScreenToLocalTransform().scale(this.scaling()).translate(0, -this.heightPercent / 100 * this.props.PanelHeight()); setPlayheadTime = (time: number) => this.player!.currentTime = this.layoutDoc._currentTimecode = time; timelineHeight = () => this.props.PanelHeight() * (100 - this.heightPercent) / 100; playing = () => this._playing; contentFunc = () => [this.youtubeVideoId ? this.youtubeContent : this.content]; scaling = () => this.props.scaling?.() || 1; panelWidth = () => this.props.PanelWidth() * this.heightPercent / 100; panelHeight = () => this.layoutDoc._fitWidth ? this.panelWidth() / (Doc.NativeAspect(this.rootDoc) || 1) : this.props.PanelHeight() * this.heightPercent / 100; screenToLocalTransform = () => { const offset = (this.props.PanelWidth() - this.panelWidth()) / 2 / this.scaling(); return this.props.ScreenToLocalTransform().translate(-offset, 0).scale(100 / this.heightPercent); } marqueeFitScaling = () => (this.props.scaling?.() || 1) * this.heightPercent / 100; marqueeOffset = () => [this.panelWidth() / 2 * (1 - this.heightPercent / 100) / (this.heightPercent / 100), 0]; timelineDocFilter = () => [`_timelineLabel:true,${Utils.noRecursionHack}:x`]; @computed get content() { const field = Cast(this.dataDoc[this.fieldKey], VideoField); const interactive = CurrentUserUtils.SelectedTool !== InkTool.None || !this.props.isSelected() ? "" : "-interactive"; const style = "videoBox-content" + (this._fullScreen ? "-fullScreen" : "") + interactive; return !field ?
Loading
:
{!this.audiopath || this.audiopath === field.url.href ? (null) : }
; } @computed get youtubeContent() { this._youtubeIframeId = VideoBox._youtubeIframeCounter++; this._youtubeContentCreated = this._forceCreateYouTubeIFrame ? true : true; const style = "videoBox-content-YouTube" + (this._fullScreen ? "-fullScreen" : ""); const start = untracked(() => Math.round((this.layoutDoc._currentTimecode || 0))); return