From 39c85293f6c3d385ea64ba0db8c9736dfaaec993 Mon Sep 17 00:00:00 2001 From: mehekj Date: Sun, 20 Mar 2022 15:22:50 -0400 Subject: cleaned up files and added some comments --- src/client/views/AudioWaveform.tsx | 137 ++++--------------- .../collections/CollectionStackedTimeline.tsx | 102 ++++++++++---- src/client/views/nodes/AudioBox.tsx | 108 +++++++++++---- src/client/views/nodes/VideoBox.tsx | 146 ++++++++++++++++----- 4 files changed, 295 insertions(+), 198 deletions(-) (limited to 'src') diff --git a/src/client/views/AudioWaveform.tsx b/src/client/views/AudioWaveform.tsx index 8a3c3c319..525c0ce5a 100644 --- a/src/client/views/AudioWaveform.tsx +++ b/src/client/views/AudioWaveform.tsx @@ -1,16 +1,26 @@ import React = require("react"); import axios from "axios"; -import { action, computed, reaction, IReactionDisposer } from "mobx"; +import { action, computed, IReactionDisposer, reaction } from "mobx"; import { observer } from "mobx-react"; import Waveform from "react-audio-waveform"; import { Doc } from "../../fields/Doc"; import { List } from "../../fields/List"; import { listSpec } from "../../fields/Schema"; -import { Cast, NumCast } from "../../fields/Types"; +import { Cast } from "../../fields/Types"; import { numberRange } from "../../Utils"; import "./AudioWaveform.scss"; import { Colors } from "./global/globalEnums"; -import Color = require("color"); + + +/** + * AudioWaveform + * + * Used in CollectionStackedTimeline to render a canvas with a visual of an audio waveform for AudioBox and VideoBox documents. + * Uses react-audio-waveform package. + * Bins the audio data into audioBuckets which are passed to package to render the lines. + * Calculates new buckets each time a new zoom factor or new set of trim bounds is created and stores it in a field on the layout doc with a title indicating the bounds and zoom for that list (see audioBucketField) + */ + export interface AudioWaveformProps { duration: number; // length of media clip @@ -26,19 +36,24 @@ export interface AudioWaveformProps { @observer export class AudioWaveform extends React.Component { - public static NUMBER_OF_BUCKETS = 100; + public static NUMBER_OF_BUCKETS = 100; // number of buckets data is divided into to draw waveform lines + _disposer: IReactionDisposer | undefined; + @computed get waveHeight() { return Math.max(50, this.props.PanelHeight); } + @computed get clipStart() { return this.props.clipStart; } @computed get clipEnd() { return this.props.clipEnd; } @computed get zoomFactor() { return this.props.zoomFactor; } - @computed get audioBuckets() { return Cast(this.props.layoutDoc[this.audioBucketField(this.clipStart, this.clipEnd, this.zoomFactor)], listSpec("number"), []); } + @computed get audioBuckets() { return Cast(this.props.layoutDoc[this.audioBucketField(this.clipStart, this.clipEnd, this.zoomFactor)], listSpec("number"), []); } audioBucketField = (start: number, end: number, zoomFactor: number) => "audioBuckets/" + "/" + start.toFixed(2).replace(".", "_") + "/" + end.toFixed(2).replace(".", "_") + "/" + (zoomFactor * 10); + componentWillUnmount() { this._disposer?.(); } + componentDidMount() { this._disposer = reaction(() => ({ clipStart: this.clipStart, clipEnd: this.clipEnd, fieldKey: this.audioBucketField(this.clipStart, this.clipEnd, this.zoomFactor), zoomFactor: this.props.zoomFactor }), ({ clipStart, clipEnd, fieldKey, zoomFactor }) => { @@ -49,7 +64,6 @@ export class AudioWaveform extends React.Component { setTimeout(() => this.createWaveformBuckets(fieldKey, clipStart, clipEnd, zoomFactor)); } }, { fireImmediately: true }); - } // decodes the audio file into peaks for generating the waveform @@ -86,16 +100,10 @@ export class AudioWaveform extends React.Component { } ); } + render() { return (
- {/* this.props.PanelWidth} - height={this.props.PanelHeight} - peaks={this.audioBuckets} - color={Colors.MEDIUM_BLUE} - /> */} {
); } -} - - -export interface WaveformProps { - barWidth: number; - width: () => number; - height: () => number; - peaks: number[]; - color: string; -} - -// @observer -// export class Waveform extends React.Component { -// private _canvas: HTMLCanvasElement | null = null; - -// get width() { return this.props.width(); } -// get height() { return this.props.height(); } -// get peaks() { return this.props.peaks; } - -// componentDidMount() { -// this.drawBars(); -// } - -// drawBars() { -// const waveCanvasCtx = this._canvas?.getContext("2d"); - -// if (waveCanvasCtx) { -// const pixelRatio = window.devicePixelRatio; -// console.log(pixelRatio); - -// const displayWidth = Math.round(this.width); -// const displayHeight = Math.round(this.height); -// waveCanvasCtx.canvas.width = this.width; -// waveCanvasCtx.canvas.height = this.height; -// waveCanvasCtx.canvas.style.width = `${displayWidth}px`; -// waveCanvasCtx.canvas.style.height = `${displayHeight}px`; - -// waveCanvasCtx.clearRect(0, 0, this.width, this.height); - -// const hasMinVals = [].some.call(this.peaks, (val) => val < 0); -// let filteredPeaks = this.peaks; -// if (hasMinVals) { -// // If the first value is negative, add 1 to the filtered indices -// let indexOffset = 0; -// if (this.peaks[0] < 0) { -// indexOffset = 1; -// } -// filteredPeaks = [].filter.call( -// this.peaks, -// (_, index) => (index + indexOffset) % 2 == 0 -// ); -// } - -// const $ = 0.5; -// const height = this.height; -// const offsetY = 0; -// const halfH = this.height / 2; -// const length = filteredPeaks.length; -// const bar = this.props.barWidth; -// const gap = 2; -// const step = bar + gap; - -// let absmax = 1; -// absmax = this.absMax(filteredPeaks); - -// const scale = length / this.width; - -// waveCanvasCtx.fillStyle = this.props.color; - -// for (let i = 0; i < this.width; i += step) { -// let h = Math.round(filteredPeaks[Math.floor(i * scale)] / absmax * halfH) -// if (h === 0) { -// h = 1 -// } -// waveCanvasCtx.fillRect(i + $, halfH - h + offsetY, bar + $, h * 2) -// } -// } -// } - -// absMax = (values: number[]) => { -// let max = -Infinity; -// for (const i in values) { -// const num = Math.abs(values[i]); -// if (num > max) { -// max = num; -// } -// } - -// return max; -// } - - -// render() { -// return this.props.peaks ? ( -// { -// this._canvas = instance; -// }} -// /> -// ) : null -// } -// } \ No newline at end of file +} \ No newline at end of file diff --git a/src/client/views/collections/CollectionStackedTimeline.tsx b/src/client/views/collections/CollectionStackedTimeline.tsx index 7d9dc39ae..bd7d0083b 100644 --- a/src/client/views/collections/CollectionStackedTimeline.tsx +++ b/src/client/views/collections/CollectionStackedTimeline.tsx @@ -43,6 +43,19 @@ import { import { LabelBox } from "../nodes/LabelBox"; import "./CollectionStackedTimeline.scss"; + + +/** + * CollectionStackedTimeline + * Main component: CollectionStackedTimeline.tsx + * Supporting Components: AudioWaveform + * + * CollectionStackedTimeline is a collection view used for audio and video nodes to display a timeline of the temporal media documents with an audio waveform and markers for links and annotations + * The actual media is handled in the containing classes (AudioBox, VideoBox) but the timeline deals with rendering and updating timecodes, links, and trimming. + * When trimming there are two pairs of times that are tracked: trimStart and trimEnd are the bounds of the trim controls, clipStart and clipEnd are the actual trimmed playback bounds of the clip + */ + + type PanZoomDocument = makeInterface<[]>; const PanZoomDocument = makeInterface(); export type CollectionStackedTimelineProps = { @@ -60,38 +73,42 @@ export type CollectionStackedTimelineProps = { fieldKey: string; }; +// trimming state: shows full clip, current trim bounds, or not trimming export enum TrimScope { All = 2, Clip = 1, None = 0, } + @observer export class CollectionStackedTimeline extends CollectionSubView< PanZoomDocument, CollectionStackedTimelineProps >(PanZoomDocument) { - @observable static SelectingRegion: CollectionStackedTimeline | undefined; - @observable public static CurrentlyPlaying: Doc[]; + @observable static SelectingRegion: CollectionStackedTimeline | undefined; // timeline selection region + @observable public static CurrentlyPlaying: Doc[]; // tracks all currently playing audio and video docs static RangeScript: ScriptField; static LabelScript: ScriptField; static RangePlayScript: ScriptField; static LabelPlayScript: ScriptField; - private _timeline: HTMLDivElement | null = null; - private _timelineWrapper: HTMLDivElement | null = null; + private _timeline: HTMLDivElement | null = null; // ref to actual timeline div + private _timelineWrapper: HTMLDivElement | null = null; // ref to timeline wrapper div for zooming and scrolling private _markerStart: number = 0; @observable _markerEnd: number | undefined; @observable _trimming: number = TrimScope.None; - @observable _trimStart: number = 0; - @observable _trimEnd: number = 0; + @observable _trimStart: number = 0; // trim controls start pos + @observable _trimEnd: number = 0; // trim controls end pos @observable _zoomFactor: number = 1; @observable _scroll: number = 0; + // ensures that clip doesn't get trimmed so small that controls cannot be adjusted anymore get minTrimLength() { return Math.max(this._timeline?.getBoundingClientRect() ? 0.05 * this.clipDuration : 0, 0.5) } + @computed get trimStart() { return this.IsTrimming !== TrimScope.None ? this._trimStart : this.clipStart; } @computed get trimDuration() { return this.trimEnd - this.trimStart; } @computed get trimEnd() { return this.IsTrimming !== TrimScope.None ? this._trimEnd : this.clipEnd; } @@ -104,6 +121,7 @@ export class CollectionStackedTimeline extends CollectionSubView< @computed get zoomFactor() { return this._zoomFactor } + constructor(props: any) { super(props); // onClick play scripts @@ -135,6 +153,7 @@ export class CollectionStackedTimeline extends CollectionSubView< } } + public get IsTrimming() { return this._trimming; } @action @@ -155,24 +174,31 @@ export class CollectionStackedTimeline extends CollectionSubView< this._zoomFactor = zoom; } + anchorStart = (anchor: Doc) => NumCast(anchor._timecodeToShow, NumCast(anchor[this.props.startTag])); anchorEnd = (anchor: Doc, val: any = null) => NumCast(anchor._timecodeToHide, NumCast(anchor[this.props.endTag], val) ?? null); + + + // converts screen pixel offset to time toTimeline = (screen_delta: number, width: number) => { return Math.max( this.clipStart, Math.min(this.clipEnd, (screen_delta / width) * this.clipDuration + this.clipStart)); } + rangeClickScript = () => CollectionStackedTimeline.RangeScript; rangePlayScript = () => CollectionStackedTimeline.RangePlayScript; - // for creating key anchors with key events + + // handles key events for for creating key anchors, scrubbing, exiting trim @action keyEvents = (e: KeyboardEvent) => { if ( !(e.target instanceof HTMLInputElement) && this.props.isSelected(true) ) { + // if shift pressed scrub 1 second otherwise 1/10th const jump = e.shiftKey ? 1 : 0.1; e.stopPropagation(); switch (e.key) { @@ -196,6 +222,7 @@ export class CollectionStackedTimeline extends CollectionSubView< } break; case "Escape": + // abandons current trim this._trimStart = this.clipStart; this._trimStart = this.clipEnd; this._trimming = TrimScope.None; @@ -210,6 +237,7 @@ export class CollectionStackedTimeline extends CollectionSubView< } } + getLinkData(l: Doc) { let la1 = l.anchor1 as Doc; let la2 = l.anchor2 as Doc; @@ -224,7 +252,8 @@ export class CollectionStackedTimeline extends CollectionSubView< return { la1, la2, linkTime }; } - // starting the drag event for anchor resizing + + // handles dragging selection to create markers @action onPointerDownTimeline = (e: React.PointerEvent): void => { const rect = this._timeline?.getBoundingClientRect(); @@ -299,6 +328,8 @@ export class CollectionStackedTimeline extends CollectionSubView< } + + // for dragging trim start handle @action trimLeft = (e: React.PointerEvent): void => { const rect = this._timeline?.getBoundingClientRect(); @@ -325,6 +356,7 @@ export class CollectionStackedTimeline extends CollectionSubView< ); } + // for dragging trim end handle @action trimRight = (e: React.PointerEvent): void => { const rect = this._timeline?.getBoundingClientRect(); @@ -351,12 +383,15 @@ export class CollectionStackedTimeline extends CollectionSubView< ); } + + // for rendering scrolling when timeline zoomed @action setScroll = (e: React.UIEvent) => { e.stopPropagation(); this._scroll = this._timelineWrapper!.scrollLeft; } + // smooth scrolls to time like when following links overflowed due to zoom @action scrollToTime = (time: number) => { if (this._timelineWrapper) { @@ -371,6 +406,8 @@ export class CollectionStackedTimeline extends CollectionSubView< } } + + // handles dragging and dropping markers in timeline @action internalDocDrop(e: Event, de: DragManager.DropEvent, docDragData: DragManager.DocumentDragData, xp: number) { if (!de.embedKey && this.props.layerProvider?.(this.props.Document) !== false && this.props.Document._isGroup) return false; @@ -396,6 +433,8 @@ export class CollectionStackedTimeline extends CollectionSubView< return false; } + + // creates marker on timeline @undoBatch @action static createAnchor( @@ -430,6 +469,7 @@ export class CollectionStackedTimeline extends CollectionSubView< return anchor; } + @action playOnClick = (anchorDoc: Doc, clientX: number) => { const seekTimeInSeconds = this.anchorStart(anchorDoc) - 0.25; @@ -521,14 +561,20 @@ export class CollectionStackedTimeline extends CollectionSubView< return level; } + dictationHeightPercent = 50; dictationHeight = () => (this.props.PanelHeight() * (100 - this.dictationHeightPercent)) / 100; + @computed get timelineContentHeight() { return this.props.PanelHeight() * this.dictationHeightPercent / 100; } @computed get timelineContentWidth() { return this.props.PanelWidth() * this.zoomFactor - 4 }; // subtract size of container border + dictationScreenToLocalTransform = () => this.props.ScreenToLocalTransform().translate(0, -this.timelineContentHeight); + isContentActive = () => this.props.isSelected() || this.props.isContentActive(); + currentTimecode = () => this.currentTime; + @computed get renderDictation() { const dictation = Cast(this.dataDoc[this.props.dictationKey], Doc, null); return !dictation ? null : ( @@ -565,23 +611,8 @@ export class CollectionStackedTimeline extends CollectionSubView< ); } - @computed get renderAudioWaveform() { - return !this.props.mediaPath ? null : ( -
- -
- ); - } + + // renders selection region on timeline @computed get selectionContainer() { const markerEnd = CollectionStackedTimeline.SelectingRegion === this ? this.currentTime : this._markerEnd; return markerEnd === undefined ? null : ( @@ -668,7 +699,6 @@ export class CollectionStackedTimeline extends CollectionSubView< ); })} {!this.IsTrimming && this.selectionContainer} - {/* {this.renderAudioWaveform} */} ScriptField; @@ -753,20 +789,26 @@ interface StackedTimelineAnchorProps { trimStart: number; trimEnd: number; } + + @observer class StackedTimelineAnchor extends React.Component { _lastTimecode: number; _disposer: IReactionDisposer | undefined; + constructor(props: any) { super(props); this._lastTimecode = this.props.currentTimecode(); } + // updates marker document title to reflect correct timecodes computeTitle = () => { const start = Math.max(NumCast(this.props.mark[this.props.startTag]), this.props.trimStart) - this.props.trimStart; const end = Math.min(NumCast(this.props.mark[this.props.endTag]), this.props.trimEnd) - this.props.trimStart; return `#${formatTime(start)}-${formatTime(end)}`; } + + componentDidMount() { this._disposer = reaction( () => this.props.currentTimecode(), @@ -805,9 +847,12 @@ class StackedTimelineAnchor extends React.Component } ); } + componentWillUnmount() { this._disposer?.(); } + + // starting the drag event for anchor resizing onAnchorDown = (e: React.PointerEvent, anchor: Doc, left: boolean): void => { this.props._timeline?.setPointerCapture(e.pointerId); @@ -851,11 +896,15 @@ class StackedTimelineAnchor extends React.Component ); } + + // context menu contextMenuItems = () => { const resetTitle = { script: ScriptField.MakeFunction(`self.title = "#" + formatToTime(self["${this.props.startTag}"]) + "-" + formatToTime(self["${this.props.endTag}"])`)!, icon: "folder-plus", label: "Reset Title" }; return [resetTitle]; } + + // renders anchor LabelBox renderInner = computedFn(function ( this: StackedTimelineAnchor, mark: Doc, @@ -910,6 +959,7 @@ class StackedTimelineAnchor extends React.Component anchorScreenToLocalXf = () => this.props.ScreenToLocalTransform().translate(-this.props.left, -this.props.top); width = () => this.props.width; height = () => this.props.height; + render() { const inner = this.renderInner( this.props.mark, diff --git a/src/client/views/nodes/AudioBox.tsx b/src/client/views/nodes/AudioBox.tsx index 9351bc3be..f5de31fcb 100644 --- a/src/client/views/nodes/AudioBox.tsx +++ b/src/client/views/nodes/AudioBox.tsx @@ -22,6 +22,22 @@ import { ViewBoxAnnotatableComponent, ViewBoxAnnotatableProps } from "../DocComp import "./AudioBox.scss"; import { FieldView, FieldViewProps } from "./FieldView"; + +/** + * AudioBox + * Main component: AudioBox.tsx + * Supporting Components: CollectionStackedTimeline, AudioWaveform + * + * AudioBox is a node that supports the recording and playback of audio files in Dash. + * When an audio file is importeed into Dash, it is immediately rendered as an AudioBox document. + * When a blank AudioBox node is created in Dash, audio recording controls are displayed and the user can start a recording which can be paused or stopped, and can use dictation to create a text transcript. + * Recording is done using the MediaDevices API to access the user's device microphone (see recordAudioAnnotation below) + * CollectionStackedTimeline handles AudioBox and VideoBox shared behavior, but AudioBox handles playing, pausing, etc because it contains