diff options
Diffstat (limited to 'src/client/views/nodes/VideoBox.tsx')
| -rw-r--r-- | src/client/views/nodes/VideoBox.tsx | 146 |
1 files changed, 114 insertions, 32 deletions
diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx index e47b41539..9797178b2 100644 --- a/src/client/views/nodes/VideoBox.tsx +++ b/src/client/views/nodes/VideoBox.tsx @@ -1,6 +1,5 @@ import React = require("react"); import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { Tooltip } from "@material-ui/core"; import { action, computed, IReactionDisposer, observable, ObservableMap, reaction, runInAction, untracked } from "mobx"; import { observer } from "mobx-react"; import * as rp from 'request-promise'; @@ -32,9 +31,24 @@ import { FieldView, FieldViewProps } from './FieldView'; import "./VideoBox.scss"; const path = require('path'); + +/** + * VideoBox + * Main component: VideoBox.tsx + * Supporting Components: CollectionStackedTimeline + * + * VideoBox is a node that supports the playback of video files in Dash. + * When a video file or YouTube video is importeed into Dash, it is immediately rendered as a VideoBox document. + * CollectionStackedTimline handles AudioBox and VideoBox shared behavior, but VideoBox handles playing, pausing, etc because it contains <video> element + * User can trim video: nondestructive, just sets new bounds for playback and rendering timeline + * Like images, users can zoom and pan and it has an overlay layer allowing for annotations on top of the video at different times + */ + + type VideoDocument = makeInterface<[typeof documentSchema]>; const VideoDocument = makeInterface(documentSchema); + @observer export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps & FieldViewProps, VideoDocument>(VideoDocument) { public static LayoutString(fieldKey: string) { return FieldView.LayoutString(VideoBox, fieldKey); } @@ -54,42 +68,45 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp console.log("VideoBox :" + e); } } + static _youtubeIframeCounter: number = 0; - static heightPercent = 80; // height of timeline in percent of height of videoBox. + static heightPercent = 80; // height of video relative to videoBox when timeline is open private _disposers: { [name: string]: IReactionDisposer } = {}; private _youtubePlayer: YT.Player | undefined = undefined; - private _videoRef: HTMLVideoElement | null = null; - private _contentRef: HTMLDivElement | null = null; + private _videoRef: HTMLVideoElement | null = null; // <video> ref + private _contentRef: HTMLDivElement | null = null; // ref to div that wraps video and controls for full screen private _youtubeIframeId: number = -1; private _youtubeContentCreated = false; private _audioPlayer: HTMLAudioElement | null = null; - private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); + private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); // outermost div private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef(); - private _playRegionTimer: any = null; - private _playRegionDuration = 0; - @observable _stackedTimeline: any; - @observable static _nativeControls: boolean; - @observable _marqueeing: number[] | undefined; + private _playRegionTimer: any = null; // timeout for playback + @observable _stackedTimeline: any; // CollectionStackedTimeline ref + @observable static _nativeControls: boolean; // default html controls + @observable _marqueeing: number[] | undefined; // coords for marquee selection @observable _savedAnnotations = new ObservableMap<number, HTMLDivElement[]>(); @observable _screenCapture = false; - @observable _clicking = false; + @observable _clicking = false; // used for transition between showing/hiding timeline @observable _forceCreateYouTubeIFrame = false; @observable _playTimer?: NodeJS.Timeout = undefined; @observable _fullScreen = false; @observable _playing = false; - @observable _finished: boolean = false; + @observable _finished: boolean = false; // has playback reached end of clip @observable _volume: number = 1; @observable _muted: boolean = false; @computed get links() { return DocListCast(this.dataDoc.links); } - @computed get heightPercent() { return NumCast(this.layoutDoc._timelineHeightPercent, 100); } + @computed get heightPercent() { return NumCast(this.layoutDoc._timelineHeightPercent, 100); } // current percent of video relative to VideoBox height // @computed get rawDuration() { return NumCast(this.dataDoc[this.fieldKey + "-duration"]); } @observable rawDuration: number = 0; + @computed get youtubeVideoId() { const field = Cast(this.dataDoc[this.props.fieldKey], VideoField); return field && field.url.href.indexOf("youtube") !== -1 ? ((arr: string[]) => arr[arr.length - 1])(field.url.href.split("/")) : ""; } + + // returns the path of the audio file @computed get audiopath() { const field = Cast(this.props.Document[this.props.fieldKey + '-audio'], AudioField, null); @@ -97,12 +114,14 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp return field?.url.href ?? vfield?.url.href ?? ""; } - private get timeline() { return this._stackedTimeline; } - private get transition() { return this._clicking ? "left 0.5s, width 0.5s, height 0.5s" : ""; } + + @computed private get timeline() { return this._stackedTimeline; } + private get transition() { return this._clicking ? "left 0.5s, width 0.5s, height 0.5s" : ""; } // css transition for hiding/showing timeline public get player(): HTMLVideoElement | null { return this._videoRef; } + componentDidMount() { - this.props.setContentView?.(this); // this tells the DocumentView that this AudioBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. + this.props.setContentView?.(this); // this tells the DocumentView that this VideoBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the VideoBox when making a link. if (this.youtubeVideoId) { const youtubeaspect = 400 / 315; const nativeWidth = Doc.NativeWidth(this.layoutDoc); @@ -122,15 +141,20 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp Object.keys(this._disposers).forEach(d => this._disposers[d]?.()); } + + // plays video @action public Play = (update: boolean = true) => { this._playing = true; const eleTime = this.player?.currentTime || 0; if (this.timeline) { let start = eleTime >= this.timeline.trimEnd || eleTime <= this.timeline.trimStart ? this.timeline.trimStart : eleTime; + if (this._finished) { + // restarts video if reached end on previous play this._finished = false; start = this.timeline.trimStart; } + try { this._audioPlayer && this.player && (this._audioPlayer.currentTime = this.player?.currentTime); update && this.player && this.playFrom(start, undefined, true); @@ -144,6 +168,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp this.updateTimecode(); } + // goes to time @action public Seek(time: number) { try { this._youtubePlayer?.seekTo(Math.round(time), true); @@ -154,6 +179,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp this._audioPlayer && (this._audioPlayer.currentTime = time); } + // pauses video @action public Pause = (update: boolean = true) => { this._playing = false; this.removeCurrentlyPlaying(); @@ -169,9 +195,10 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp this._youtubePlayer && SelectionManager.DeselectAll(); // if we don't deselect the player, then we get an annoying YouTube spinner I guess telling us we're paused. this._playTimer = undefined; this.updateTimecode(); - if (!this._finished) clearTimeout(this._playRegionTimer);; + if (!this._finished) clearTimeout(this._playRegionTimer); // if paused in the middle of playback, prevents restart on next play } + // toggles video full screen @action public FullScreen = () => { if (document.fullscreenElement == this._contentRef) { this._fullScreen = false; @@ -189,6 +216,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + + // creates and links snapshot photo of current video frame @action public Snapshot(downX?: number, downY?: number) { const width = (this.layoutDoc._width || 0); const canvas = document.createElement('canvas'); @@ -231,6 +260,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + // creates link for snapshot createRealSummaryLink = (imagePath: string, downX?: number, downY?: number) => { const url = !imagePath.startsWith("/") ? Utils.CorsProxy(imagePath) : imagePath; const width = this.layoutDoc._width || 1; @@ -249,12 +279,15 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp (downX !== undefined && downY !== undefined) && DocumentManager.Instance.getFirstDocumentView(imageSummary)?.startDragging(downX, downY, "move", true)); } + getAnchor = () => { const timecode = Cast(this.layoutDoc._currentTimecode, "number", null); const marquee = AnchorMenu.Instance.GetAnchor?.(); return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow"/* videoStart */, "_timecodeToHide" /* videoEnd */, timecode ? timecode : undefined, undefined, marquee) || this.rootDoc; } + + // sets video info on load videoLoad = action(() => { const aspect = this.player!.videoWidth / this.player!.videoHeight; Doc.SetNativeWidth(this.dataDoc, this.player!.videoWidth); @@ -265,6 +298,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } }) + + // updates video time @action updateTimecode = () => { this.player && (this.layoutDoc._currentTimecode = this.player.currentTime); @@ -275,6 +310,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + + // sets video element ref @action setVideoRef = (vref: HTMLVideoElement | null) => { this._videoRef = vref; @@ -288,6 +325,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + // set ref for div that wraps video and controls for fullscreen @action setContentRef = (cref: HTMLDivElement | null) => { this._contentRef = cref; @@ -296,6 +334,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + + // context menu specificContextMenu = (e: React.MouseEvent): void => { const field = Cast(this.dataDoc[this.props.fieldKey], VideoField); if (field) { @@ -321,8 +361,11 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + // ref for updating time setAudioRef = (e: HTMLAudioElement | null) => this._audioPlayer = e; + + // renders the video and audio @computed get content() { const field = Cast(this.dataDoc[this.fieldKey], VideoField); const interactive = CurrentUserUtils.SelectedTool !== InkTool.None || !this.props.isSelected() ? "" : "-interactive"; @@ -350,6 +393,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp </div>; } + @action youtubeIframeLoaded = (e: any) => { if (!this._youtubeContentCreated) { this._forceCreateYouTubeIFrame = !this._forceCreateYouTubeIFrame; @@ -359,6 +403,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp this.loadYouTube(e.target); } + loadYouTube = (iframe: any) => { let started = true; const onYoutubePlayerStateChange = (event: any) => runInAction(() => { @@ -392,14 +437,18 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + + // for play button onPlayDown = () => this._playing ? this.Pause() : this.Play(); + // for fullscreen button onFullDown = (e: React.PointerEvent) => { this.FullScreen(); e.stopPropagation(); e.preventDefault(); } + // for snapshot button onSnapshotDown = (e: React.PointerEvent) => { setupMoveUpEvents(this, e, (e) => { this.Snapshot(e.clientX, e.clientY); @@ -407,6 +456,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp }, emptyFunction, () => this.Snapshot()); } + // for show/hide timeline button, transitions between show/hide @action onTimelineHdlDown = (e: React.PointerEvent) => { this._clicking = true; @@ -427,18 +477,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp }, this.props.isContentActive(), this.props.isContentActive()); } - onResetDown = (e: React.PointerEvent) => { - const start = this.timeline?.clipStart || 0; - setupMoveUpEvents(this, e, - e => { - this.Seek(Math.max(start, (this.layoutDoc._currentTimecode || 0) + Math.sign(e.movementX) * 0.0333)); - e.stopImmediatePropagation(); - return false; - }, - emptyFunction, - (e: PointerEvent) => this.layoutDoc._currentTimecode = 0); - } + // removes video from currently playing display @action removeCurrentlyPlaying = () => { if (CollectionStackedTimeline.CurrentlyPlaying) { @@ -447,6 +487,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + // adds video to currently playing display @action addCurrentlyPlaying = () => { if (!CollectionStackedTimeline.CurrentlyPlaying) { @@ -457,6 +498,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + @computed get youtubeContent() { this._youtubeIframeId = VideoBox._youtubeIframeCounter++; this._youtubeContentCreated = this._forceCreateYouTubeIFrame ? true : true; @@ -468,6 +510,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp src={`https://www.youtube.com/embed/${this.youtubeVideoId}?enablejsapi=1&rel=0&showinfo=1&autoplay=0&mute=1&start=${start}&modestbranding=1&controls=${VideoBox._nativeControls ? 1 : 0}`} />; } + + // for annotating, adds doc with time info @action.bound addDocWithTimecode(doc: Doc | Doc[]): boolean { const docs = doc instanceof Doc ? [doc] : doc; @@ -476,7 +520,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp return this.addDocument(doc); } - // play back the video from time + + // play back the audio from seekTimeInSeconds, fullPlay tells whether clip is being played to end vs link range @action playFrom = (seekTimeInSeconds: number, endTime?: number, fullPlay: boolean = false) => { clearTimeout(this._playRegionTimer); @@ -484,9 +529,11 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp setTimeout(() => this.playFrom(seekTimeInSeconds, endTime), 500); } else if (this.player) { + // trimBounds override requested playback bounds const end = Math.min(this.timeline?.trimEnd ?? this.rawDuration, endTime ?? this.timeline?.trimEnd ?? this.rawDuration); const start = Math.max(this.timeline?.trimStart ?? 0, seekTimeInSeconds); - this._playRegionDuration = end - start; + const playRegionDuration = end - start; + // checks if times are within clip range if (seekTimeInSeconds >= 0 && (this.timeline?.trimStart || 0) <= end && seekTimeInSeconds <= (this.timeline?.trimEnd || this.rawDuration)) { this.player.currentTime = start; this._audioPlayer && (this._audioPlayer.currentTime = seekTimeInSeconds); @@ -496,16 +543,20 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp this.addCurrentlyPlaying(); this._playRegionTimer = setTimeout( () => { + // need to keep track of if end of clip is reached so on next play, clip restarts if (fullPlay) this._finished = true; + // removes from currently playing if playback has reached end of range marker else this.removeCurrentlyPlaying(); this.Pause(); - }, this._playRegionDuration * 1000); + }, playRegionDuration * 1000); } else { this.Pause(); } } } - // hides trim controls and displays new clip + + + // ends trim, hides trim controls and displays new clip @undoBatch finishTrim = action(() => { this.Pause(); @@ -513,12 +564,15 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp this.timeline?.StopTrimming(); }); + // displays trim controls to start trimming clip startTrim = (scope: TrimScope) => { this.Pause(); this.timeline?.StartTrimming(scope); } + // for trim button, double click displays full clip, single displays curr trim bounds onClipPointerDown = (e: React.PointerEvent) => { + // if timeline isn't shown, show first then trim this.heightPercent >= 100 && this.onTimelineHdlDown(e); this.timeline && setupMoveUpEvents(this, e, returnFalse, returnFalse, action((e: PointerEvent, doubleTap?: boolean) => { if (doubleTap) { @@ -530,6 +584,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp })); } + + // for volume slider sets volume @action setVolume = (volume: number) => { if (this.player) { @@ -541,6 +597,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + // toggles video mute @action toggleMute = () => { if (this.player) { @@ -549,6 +606,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + + // stretches vertically or horizontally depending on video orientation so video fits full screen fullScreenSize() { if (this._videoRef && this._videoRef.videoHeight / this._videoRef.videoWidth > 1) { return { height: "100%" } @@ -558,10 +617,14 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + + // for zoom slider, sets timeline waveform zoom zoom = (zoom: number) => { this.timeline?.setZoom(zoom); } + + // plays link playLink = (doc: Doc) => { const startTime = Math.max(0, (this._stackedTimeline?.anchorStart(doc) || 0)); const endTime = this.timeline?.anchorEnd(doc); @@ -571,6 +634,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + + // starts marquee selection marqueeDown = (e: React.PointerEvent) => { if (!e.altKey && e.button === 0 && this.layoutDoc._viewScale === 1 && this.props.isContentActive(true) && ![InkTool.Highlighter, InkTool.Pen].includes(CurrentUserUtils.SelectedTool)) { setupMoveUpEvents(this, e, action(e => { @@ -581,6 +646,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } } + // ends marquee selection @action finishMarquee = () => { this._marqueeing = undefined; @@ -588,23 +654,34 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp } timelineWhenChildContentsActiveChanged = action((isActive: boolean) => this.props.whenChildContentsActiveChanged(this._isAnyChildContentActive = isActive)); + timelineScreenToLocal = () => this.props.ScreenToLocalTransform().scale(this.scaling()).translate(0, -this.heightPercent / 100 * this.props.PanelHeight()); + setPlayheadTime = (time: number) => this.player!.currentTime = this.layoutDoc._currentTimecode = time; + timelineHeight = () => this.props.PanelHeight() * (100 - this.heightPercent) / 100; + playing = () => this._playing; contentFunc = () => [this.youtubeVideoId ? this.youtubeContent : this.content]; + scaling = () => this.props.scaling?.() || 1; + panelWidth = () => this.props.PanelWidth() * this.heightPercent / 100; panelHeight = () => this.layoutDoc._fitWidth ? this.panelWidth() / (Doc.NativeAspect(this.rootDoc) || 1) : this.props.PanelHeight() * this.heightPercent / 100; + screenToLocalTransform = () => { const offset = (this.props.PanelWidth() - this.panelWidth()) / 2 / this.scaling(); return this.props.ScreenToLocalTransform().translate(-offset, 0).scale(100 / this.heightPercent); } + marqueeFitScaling = () => (this.props.scaling?.() || 1) * this.heightPercent / 100; marqueeOffset = () => [this.panelWidth() / 2 * (1 - this.heightPercent / 100) / (this.heightPercent / 100), 0]; + timelineDocFilter = () => [`_timelineLabel:true,${Utils.noRecursionHack}:x`]; + + // renders video controls @computed get uIButtons() { const curTime = (this.layoutDoc._currentTimecode || 0) - (this.timeline?.clipStart || 0); return <div className="videoBox-ui" style={this._fullScreen || this.heightPercent == 100 ? { fontSize: "40px", minWidth: "80%" } : {}}> @@ -677,6 +754,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp </>} </div> } + + // renders CollectionStackedTimeline @computed get renderTimeline() { return <div className="videoBox-stackPanel" style={{ transition: this.transition, height: `${100 - this.heightPercent}%` }}> <CollectionStackedTimeline ref={action((r: any) => this._stackedTimeline = r)} {...this.props} @@ -705,9 +784,12 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp /> </div>; } + + // renders annotation layer @computed get annotationLayer() { return <div className="videoBox-annotationLayer" style={{ transition: this.transition, height: `${this.heightPercent}%` }} ref={this._annotationLayer} />; } + render() { const borderRad = this.props.styleProvider?.(this.layoutDoc, this.props, StyleProp.BorderRounding); const borderRadius = borderRad?.includes("px") ? `${Number(borderRad.split("px")[0]) / this.scaling()}px` : borderRad; |