3 files changed, 238 insertions, 0 deletions
diff --git a/src/client/views/nodes/audio/AudioWaveform.scss b/src/client/views/nodes/audio/AudioWaveform.scss
new file mode 100644
index 000000000..6cbd1759a
--- /dev/null
+++ b/src/client/views/nodes/audio/AudioWaveform.scss
@@ -0,0 +1,17 @@
+.audioWaveform  {
+    position: relative;
+    width: 100%;
+    height: 200%;
+    overflow: hidden;
+    z-index: -1000;
+    bottom: 0;
+    pointer-events: none;
+    div {
+        height: 100% !important;
+        width: 100%  !important;
+    }
+    canvas { 
+        height: 100% !important;
+        width: 100%  !important;
+    }
+}
diff --git a/src/client/views/nodes/audio/AudioWaveform.tsx b/src/client/views/nodes/audio/AudioWaveform.tsx
new file mode 100644
index 000000000..7fd799952
--- /dev/null
+++ b/src/client/views/nodes/audio/AudioWaveform.tsx
@@ -0,0 +1,121 @@
+import axios from 'axios';
+import { computed, IReactionDisposer, makeObservable, reaction } from 'mobx';
+import { observer } from 'mobx-react';
+import * as React from 'react';
+import { Doc, NumListCast } from '../../../../fields/Doc';
+import { List } from '../../../../fields/List';
+import { listSpec } from '../../../../fields/Schema';
+import { Cast } from '../../../../fields/Types';
+import { numberRange } from '../../../../Utils';
+import { ObservableReactComponent } from '../../ObservableReactComponent';
+import { Colors } from './../../global/globalEnums';
+import './AudioWaveform.scss';
+import { WaveCanvas } from './WaveCanvas';
+
+/**
+ * AudioWaveform
+ *
+ * Used in CollectionStackedTimeline to render a canvas with a visual of an audio waveform for AudioBox and VideoBox documents.
+ * Bins the audio data into audioBuckets which are passed to package to render the lines.
+ * Calculates new buckets each time a new zoom factor or new set of trim bounds is created and stores it in a field on the layout doc with a title indicating the bounds and zoom for that list (see audioBucketField)
+ */
+
+export interface AudioWaveformProps {
+    duration: number; // length of media clip
+    rawDuration: number; // length of underlying media data
+    mediaPath: string;
+    layoutDoc: Doc;
+    clipStart: number;
+    clipEnd: number;
+    zoomFactor: number;
+    PanelHeight: number;
+    PanelWidth: number;
+    fieldKey: string;
+    progress?: number;
+}
+
+@observer
+export class AudioWaveform extends ObservableReactComponent<AudioWaveformProps> {
+    public static NUMBER_OF_BUCKETS = 100; // number of buckets data is divided into to draw waveform lines
+    _disposer: IReactionDisposer | undefined;
+
+    constructor(props: any) {
+        super(props);
+        makeObservable(this);
+    }
+
+    get waveHeight() {
+        return Math.max(50, this._props.PanelHeight);
+    }
+
+    get clipStart() {
+        return this._props.clipStart;
+    }
+    get clipEnd() {
+        return this._props.clipEnd;
+    }
+    get zoomFactor() {
+        return this._props.zoomFactor;
+    }
+
+    @computed get audioBuckets() {
+        return NumListCast(this._props.layoutDoc[this.audioBucketField(this.clipStart, this.clipEnd, this.zoomFactor)]);
+    }
+
+    audioBucketField = (start: number, end: number, zoomFactor: number) => this._props.fieldKey + '_audioBuckets/' + '/' + start.toFixed(2).replace('.', '_') + '/' + end.toFixed(2).replace('.', '_') + '/' + zoomFactor * 10;
+
+    componentWillUnmount() {
+        this._disposer?.();
+    }
+
+    componentDidMount() {
+        this._disposer = reaction(
+            () => ({ clipStart: this.clipStart, clipEnd: this.clipEnd, fieldKey: this.audioBucketField(this.clipStart, this.clipEnd, this.zoomFactor), zoomFactor: this._props.zoomFactor }),
+            ({ clipStart, clipEnd, fieldKey, zoomFactor }) => {
+                if (!this._props.layoutDoc[fieldKey] && this._props.layoutDoc.layout_fieldKey != 'layout_icon') {
+                    // setting these values here serves as a "lock" to prevent multiple attempts to create the waveform at nerly the same time.
+                    const waveform = Cast(this._props.layoutDoc[this.audioBucketField(0, this._props.rawDuration, 1)], listSpec('number'));
+                    this._props.layoutDoc[fieldKey] = waveform && new List<number>(waveform.slice((clipStart / this._props.rawDuration) * waveform.length, (clipEnd / this._props.rawDuration) * waveform.length));
+                    setTimeout(() => this.createWaveformBuckets(fieldKey, clipStart, clipEnd, zoomFactor));
+                }
+            },
+            { fireImmediately: true }
+        );
+    }
+
+    // decodes the audio file into peaks for generating the waveform
+    createWaveformBuckets = (fieldKey: string, clipStart: number, clipEnd: number, zoomFactor: number) => {
+        axios({ url: this._props.mediaPath, responseType: 'arraybuffer' }).then(response =>
+            new window.AudioContext().decodeAudioData(response.data, buffer => {
+                const rawDecodedAudioData = buffer.getChannelData(0);
+                const startInd = clipStart / this._props.rawDuration;
+                const endInd = clipEnd / this._props.rawDuration;
+                const decodedAudioData = rawDecodedAudioData.slice(Math.floor(startInd * rawDecodedAudioData.length), Math.floor(endInd * rawDecodedAudioData.length));
+                const numBuckets = Math.floor(AudioWaveform.NUMBER_OF_BUCKETS * zoomFactor);
+
+                const bucketDataSize = Math.floor(decodedAudioData.length / numBuckets);
+                const brange = Array.from(Array(bucketDataSize));
+                const bucketList = numberRange(numBuckets).map((i: number) => brange.reduce((p, x, j) => Math.abs(Math.max(p, decodedAudioData[i * bucketDataSize + j])), 0) / 2);
+                this._props.layoutDoc[fieldKey] = new List<number>(bucketList);
+            })
+        );
+    };
+
+    render() {
+        return (
+            <div className="audioWaveform">
+                <WaveCanvas
+                    color={Colors.LIGHT_GRAY}
+                    progressColor={Colors.MEDIUM_BLUE_ALT}
+                    progress={this._props.progress ?? 1}
+                    barWidth={200 / this.audioBuckets.length}
+                    //gradientColors={this._props.gradientColors}
+                    peaks={this.audioBuckets}
+                    width={(this._props.PanelWidth ?? 0) * window.devicePixelRatio}
+                    height={this.waveHeight * window.devicePixelRatio}
+                    pixelRatio={window.devicePixelRatio}
+                />
+            </div>
+        );
+    }
+}
diff --git a/src/client/views/nodes/audio/WaveCanvas.tsx b/src/client/views/nodes/audio/WaveCanvas.tsx
new file mode 100644
index 000000000..d3f5669a2
--- /dev/null
+++ b/src/client/views/nodes/audio/WaveCanvas.tsx
@@ -0,0 +1,100 @@
+import React from 'react';
+
+interface WaveCanvasProps {
+    barWidth: number;
+    color: string;
+    progress: number;
+    progressColor: string;
+    gradientColors?: { stopPosition: number; color: string }[]; // stopPosition between 0 and 1
+    peaks: number[];
+    width: number;
+    height: number;
+    pixelRatio: number;
+}
+
+export class WaveCanvas extends React.Component<WaveCanvasProps> {
+    // If the first value of peaks is negative, addToIndices will be 1
+    posPeaks = (peaks: number[], addToIndices: number) => peaks.filter((_, index) => (index + addToIndices) % 2 == 0);
+
+    drawBars = (waveCanvasCtx: CanvasRenderingContext2D, width: number, halfH: number, peaks: number[]) => {
+        // Bar wave draws the bottom only as a reflection of the top,
+        // so we don't need negative values
+        const posPeaks = peaks.some(val => val < 0) ? this.posPeaks(peaks, peaks[0] < 0 ? 1 : 0) : peaks;
+
+        // A half-pixel offset makes lines crisp
+        const $ = 0.5 / this.props.pixelRatio;
+        const bar = this.props.barWidth * this.props.pixelRatio;
+        const gap = Math.max(this.props.pixelRatio, 2);
+
+        const max = Math.max(...posPeaks);
+        const scale = posPeaks.length / width;
+
+        for (let i = 0; i < width; i += bar + gap) {
+            if (i > width * this.props.progress) waveCanvasCtx.fillStyle = this.props.color;
+
+            const h = Math.round((posPeaks[Math.floor(i * scale)] / max) * halfH) || 1;
+
+            waveCanvasCtx.fillRect(i + $, halfH - h, bar + $, h * 2);
+        }
+    };
+
+    addNegPeaks = (peaks: number[]) => 
+         peaks.reduce((reflectedPeaks, peak) => reflectedPeaks.push(peak, -peak) ? reflectedPeaks:[], 
+                            [] as number[]); // prettier-ignore
+
+    drawWaves = (waveCanvasCtx: CanvasRenderingContext2D, width: number, halfH: number, peaks: number[]) => {
+        const allPeaks = peaks.some(val => val < 0) ? peaks : this.addNegPeaks(peaks); // add negative peaks to arrays without negative peaks
+
+        // A half-pixel offset makes lines crisp
+        const $ = 0.5 / this.props.pixelRatio;
+        const length = ~~(allPeaks.length / 2); // ~~ is Math.floor for positive numbers.
+
+        const scale = width / length;
+        const absmax = Math.max(...allPeaks.map(peak => Math.abs(peak)));
+
+        waveCanvasCtx.beginPath();
+        waveCanvasCtx.moveTo($, halfH);
+
+        for (var i = 0; i < length; i++) {
+            var h = Math.round((allPeaks[2 * i] / absmax) * halfH);
+            waveCanvasCtx.lineTo(i * scale + $, halfH - h);
+        }
+
+        // Draw the bottom edge going backwards, to make a single closed hull to fill.
+        for (var i = length - 1; i >= 0; i--) {
+            var h = Math.round((allPeaks[2 * i + 1] / absmax) * halfH);
+            waveCanvasCtx.lineTo(i * scale + $, halfH - h);
+        }
+
+        waveCanvasCtx.fill();
+
+        // Always draw a median line
+        waveCanvasCtx.fillRect(0, halfH - $, width, $);
+    };
+
+    updateSize = (width: number, height: number, peaks: number[], waveCanvasCtx: CanvasRenderingContext2D) => {
+        const displayWidth = Math.round(width / this.props.pixelRatio);
+        const displayHeight = Math.round(height / this.props.pixelRatio);
+        waveCanvasCtx.canvas.width = width;
+        waveCanvasCtx.canvas.height = height;
+        waveCanvasCtx.canvas.style.width = `${displayWidth}px`;
+        waveCanvasCtx.canvas.style.height = `${displayHeight}px`;
+
+        waveCanvasCtx.clearRect(0, 0, width, height);
+
+        const gradient = this.props.gradientColors && waveCanvasCtx.createLinearGradient(0, 0, width, 0);
+        gradient && this.props.gradientColors?.forEach(color => gradient.addColorStop(color.stopPosition, color.color));
+        waveCanvasCtx.fillStyle = gradient ?? this.props.progressColor;
+
+        const waveDrawer = this.props.barWidth ? this.drawBars : this.drawWaves;
+        waveDrawer(waveCanvasCtx, width, height / 2, peaks);
+    };
+
+    render() {
+        return this.props.peaks ? (
+            <div style={{ position: 'relative', width: '100%', height: '100%', cursor: 'pointer' }}>
+                <canvas ref={instance => (ctx => ctx && this.updateSize(this.props.width, this.props.height, this.props.peaks, ctx))(instance?.getContext('2d'))} />
+            </div>
+        ) : null;
+    }
+}