From 36028677c61be7ab5937e864e881fa91a3b82d8c Mon Sep 17 00:00:00 2001 From: bobzel Date: Fri, 26 Mar 2021 13:22:13 -0400 Subject: extracted AudioWaveform out of AudioBox in order to use in CollectionStackedTimelineView to share between audioBox and videoBox. --- src/client/util/DictationManager.ts | 2 +- src/client/util/DragManager.ts | 2 +- src/client/util/LinkManager.ts | 20 +++--- src/client/views/AudioWaveform.scss | 17 +++++ src/client/views/AudioWaveform.tsx | 61 +++++++++++++++++ src/client/views/DocumentDecorations.tsx | 2 +- src/client/views/SidebarAnnos.tsx | 2 +- .../collections/CollectionStackedTimeline.scss | 7 ++ .../collections/CollectionStackedTimeline.tsx | 80 +++++++++++++--------- src/client/views/nodes/AudioBox.scss | 18 ----- src/client/views/nodes/AudioBox.tsx | 41 +---------- src/client/views/nodes/DocumentView.tsx | 8 +-- src/client/views/nodes/ScreenshotBox.tsx | 2 +- src/client/views/nodes/VideoBox.tsx | 9 +-- .../views/nodes/formattedText/FormattedTextBox.tsx | 2 +- src/server/DashUploadUtils.ts | 5 +- 16 files changed, 165 insertions(+), 113 deletions(-) create mode 100644 src/client/views/AudioWaveform.scss create mode 100644 src/client/views/AudioWaveform.tsx (limited to 'src') diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index f00cdce1e..a93b2f573 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -89,7 +89,7 @@ export namespace DictationManager { export const listen = async (options?: Partial) => { if (pendingListen instanceof Promise) return pendingListen.then(pl => innerListen(options)); return innerListen(options); - } + }; const innerListen = async (options?: Partial) => { let results: string | undefined; diff --git a/src/client/util/DragManager.ts b/src/client/util/DragManager.ts index 1809a77bf..38d0ecaa6 100644 --- a/src/client/util/DragManager.ts +++ b/src/client/util/DragManager.ts @@ -388,7 +388,7 @@ export namespace DragManager { if (dragElement !== ele) { const children = [Array.from(ele.children), Array.from(dragElement.children)]; while (children[0].length) { - const childs = [children[0].pop(), children[1].pop()] + const childs = [children[0].pop(), children[1].pop()]; if (childs[0]?.children) { children[0].push(...Array.from(childs[0].children)); children[1].push(...Array.from(childs[1]!.children)); diff --git a/src/client/util/LinkManager.ts b/src/client/util/LinkManager.ts index 159011516..3c3d5c3b8 100644 --- a/src/client/util/LinkManager.ts +++ b/src/client/util/LinkManager.ts @@ -33,7 +33,7 @@ export class LinkManager { constructor() { LinkManager._instance = this; setTimeout(() => { - LinkManager.userDocs = [Doc.LinkDBDoc().data as Doc, ...SharingManager.Instance.users.map(user => user.linkDatabase as Doc)]; + LinkManager.userDocs = [Doc.LinkDBDoc().data as Doc, ...SharingManager.Instance.users.map(user => user.linkDatabase)]; const addLinkToDoc = (link: Doc): any => { const a1 = link?.anchor1; const a2 = link?.anchor2; @@ -43,7 +43,7 @@ export class LinkManager { Doc.GetProto(a2)[DirectLinksSym].add(link); Doc.GetProto(link)[DirectLinksSym].add(link); } - } + }; const remLinkFromDoc = (link: Doc): any => { const a1 = link?.anchor1; const a2 = link?.anchor2; @@ -53,23 +53,23 @@ export class LinkManager { Doc.GetProto(a2)[DirectLinksSym].delete(link); Doc.GetProto(link)[DirectLinksSym].delete(link); } - } + }; const watchUserLinks = (userLinks: List) => { const toRealField = (field: Field) => field instanceof ProxyField ? field.value() : field; // see List.ts. data structure is not a simple list of Docs, but a list of ProxyField/Fields observe(userLinks, change => { - switch (change.type) { + switch (change.type as any) { case "splice": (change as any).added.forEach((link: any) => addLinkToDoc(toRealField(link))); (change as any).removed.forEach((link: any) => remLinkFromDoc(toRealField(link))); break; - case "update": let oldValue = change.oldValue; + case "update": //let oldValue = change.oldValue; } }, true); - } + }; observe(LinkManager.userDocs, change => { - switch (change.type) { + switch (change.type as any) { case "splice": (change as any).added.forEach(watchUserLinks); break; - case "update": let oldValue = change.oldValue; + case "update": //let oldValue = change.oldValue; } }, true); }); @@ -82,7 +82,9 @@ export class LinkManager { public deleteAllLinksOnAnchor(anchor: Doc) { LinkManager.Instance.relatedLinker(anchor).forEach(linkDoc => LinkManager.Instance.deleteLink(linkDoc)); } public getAllRelatedLinks(anchor: Doc) { return this.relatedLinker(anchor); } // finds all links that contain the given anchor - public getAllDirectLinks(anchor: Doc): Doc[] { return Array.from(Doc.GetProto(anchor)[DirectLinksSym]); } // finds all links that contain the given anchor + public getAllDirectLinks(anchor: Doc): Doc[] { + return Array.from(Doc.GetProto(anchor)[DirectLinksSym]); + } // finds all links that contain the given anchor public getAllLinks(): Doc[] { return []; }//this.allLinks(); } // allLinks = computedFn(function allLinks(this: any): Doc[] { diff --git a/src/client/views/AudioWaveform.scss b/src/client/views/AudioWaveform.scss new file mode 100644 index 000000000..e20434a25 --- /dev/null +++ b/src/client/views/AudioWaveform.scss @@ -0,0 +1,17 @@ +.audioWaveform { + position: relative; + width: 100%; + height: 100%; + overflow: hidden; + z-index: -1000; + bottom: 0; + pointer-events: none; + div { + height: 100% !important; + width: 100% !important; + } + canvas { + height: 100% !important; + width: 100% !important; + } +} diff --git a/src/client/views/AudioWaveform.tsx b/src/client/views/AudioWaveform.tsx new file mode 100644 index 000000000..7ff9c1071 --- /dev/null +++ b/src/client/views/AudioWaveform.tsx @@ -0,0 +1,61 @@ +import React = require("react"); +import axios from "axios"; +import { action, computed } from "mobx"; +import { observer } from "mobx-react"; +import Waveform from "react-audio-waveform"; +import { Doc } from "../../fields/Doc"; +import { List } from "../../fields/List"; +import { listSpec } from "../../fields/Schema"; +import { Cast } from "../../fields/Types"; +import { numberRange } from "../../Utils"; +import "./AudioWaveform.scss"; + +export interface AudioWaveformProps { + duration: number; + mediaPath: string; + dataDoc: Doc; + PanelHeight: () => number; +} + +@observer +export class AudioWaveform extends React.Component { + public static NUMBER_OF_BUCKETS = 100; + @computed get _waveHeight() { return Math.max(50, this.props.PanelHeight()); } + componentDidMount() { + const audioBuckets = Cast(this.props.dataDoc.audioBuckets, listSpec("number"), []); + if (!audioBuckets.length) { + this.props.dataDoc.audioBuckets = new List([0, 0]); /// "lock" to prevent other views from computing the same data + setTimeout(this.createWaveformBuckets); + } + } + // decodes the audio file into peaks for generating the waveform + createWaveformBuckets = async () => { + axios({ url: this.props.mediaPath, responseType: "arraybuffer" }) + .then(response => { + const context = new window.AudioContext(); + context.decodeAudioData(response.data, + action(buffer => { + const decodedAudioData = buffer.getChannelData(0); + const bucketDataSize = Math.floor(decodedAudioData.length / AudioWaveform.NUMBER_OF_BUCKETS); + const brange = Array.from(Array(bucketDataSize)); + this.props.dataDoc.audioBuckets = new List( + numberRange(AudioWaveform.NUMBER_OF_BUCKETS).map((i: number) => + brange.reduce((p, x, j) => Math.abs(Math.max(p, decodedAudioData[i * bucketDataSize + j])), 0) / 2)); + })); + }); + } + + render() { + const audioBuckets = Cast(this.props.dataDoc.audioBuckets, listSpec("number"), []); + return
+ +
; + } +} \ No newline at end of file diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index da50ba4d8..a92891ee5 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -94,7 +94,7 @@ export class DocumentDecorations extends React.Component<{ boundsLeft: number, b setupMoveUpEvents(this, e, e => this.onBackgroundMove(true, e), (e) => { }, action((e) => { !this._edtingTitle && (this._accumulatedTitle = this._titleControlString.startsWith("#") ? this.selectionTitle : this._titleControlString); this._edtingTitle = true; - this._keyinput.current && setTimeout(this._keyinput.current!.focus); + this._keyinput.current && setTimeout(this._keyinput.current.focus); })); } diff --git a/src/client/views/SidebarAnnos.tsx b/src/client/views/SidebarAnnos.tsx index 87887483f..6b0b928b3 100644 --- a/src/client/views/SidebarAnnos.tsx +++ b/src/client/views/SidebarAnnos.tsx @@ -79,7 +79,7 @@ export class SidebarAnnos extends React.Component { sidebarStyleProvider = (doc: Opt, props: Opt, property: string) => { if (property === StyleProp.ShowTitle) return StrCast(this.props.layoutDoc["sidebar-childShowTitle"], "title"); - return this.props.styleProvider?.(doc, props, property) + return this.props.styleProvider?.(doc, props, property); } render() { const renderTag = (tag: string) => { diff --git a/src/client/views/collections/CollectionStackedTimeline.scss b/src/client/views/collections/CollectionStackedTimeline.scss index cc56831f3..2b9f3d782 100644 --- a/src/client/views/collections/CollectionStackedTimeline.scss +++ b/src/client/views/collections/CollectionStackedTimeline.scss @@ -58,4 +58,11 @@ left: 0; } } + + .collectionStackedTimeline-waveform { + position: absolute; + width: 100%; + top: 0; + left: 0; + } } \ No newline at end of file diff --git a/src/client/views/collections/CollectionStackedTimeline.tsx b/src/client/views/collections/CollectionStackedTimeline.tsx index 66b74277b..2191a3df5 100644 --- a/src/client/views/collections/CollectionStackedTimeline.tsx +++ b/src/client/views/collections/CollectionStackedTimeline.tsx @@ -1,26 +1,26 @@ import React = require("react"); -import { action, computed, IReactionDisposer, observable, runInAction, reaction } from "mobx"; +import { action, computed, IReactionDisposer, observable, reaction, runInAction } from "mobx"; import { observer } from "mobx-react"; -import { Doc, Opt, DocListCast } from "../../../fields/Doc"; +import { computedFn } from "mobx-utils"; +import { Doc, DocListCast } from "../../../fields/Doc"; import { Id } from "../../../fields/FieldSymbols"; import { List } from "../../../fields/List"; import { listSpec, makeInterface } from "../../../fields/Schema"; import { ComputedField, ScriptField } from "../../../fields/ScriptField"; import { Cast, NumCast } from "../../../fields/Types"; -import { emptyFunction, formatTime, OmitKeys, returnFalse, setupMoveUpEvents, StopEvent, returnOne } from "../../../Utils"; +import { emptyFunction, formatTime, OmitKeys, returnFalse, returnOne, setupMoveUpEvents, StopEvent } from "../../../Utils"; import { Docs } from "../../documents/Documents"; +import { LinkManager } from "../../util/LinkManager"; import { Scripting } from "../../util/Scripting"; import { SelectionManager } from "../../util/SelectionManager"; +import { Transform } from "../../util/Transform"; import { undoBatch } from "../../util/UndoManager"; +import { AudioWaveform } from "../AudioWaveform"; import { CollectionSubView } from "../collections/CollectionSubView"; -import { DocumentView, DocAfterFocusFunc, DocFocusFunc, DocumentViewProps } from "../nodes/DocumentView"; +import { LightboxView } from "../LightboxView"; +import { DocAfterFocusFunc, DocFocusFunc, DocumentView, DocumentViewProps } from "../nodes/DocumentView"; import { LabelBox } from "../nodes/LabelBox"; import "./CollectionStackedTimeline.scss"; -import { Transform } from "../../util/Transform"; -import { LinkManager } from "../../util/LinkManager"; -import { computedFn } from "mobx-utils"; -import { LightboxView } from "../LightboxView"; -import { FormattedTextBox } from "../nodes/formattedText/FormattedTextBox"; type PanZoomDocument = makeInterface<[]>; const PanZoomDocument = makeInterface(); @@ -35,6 +35,7 @@ export type CollectionStackedTimelineProps = { isChildActive: () => boolean; startTag: string; endTag: string; + mediaPath: string; }; @observer @@ -232,11 +233,42 @@ export class CollectionStackedTimeline extends CollectionSubView this.props.PanelHeight() / 3; + timelineContentHeight = () => this.props.PanelHeight() * 2 / 3; + @computed get renderDictation() { + const dictation = Cast(this.dataDoc[this.props.fieldKey.replace("annotations", "dictation")], Doc, null); + return !dictation ? (null) :
+ + +
; + } + @computed get renderAudioWaveform() { + return !this.props.mediaPath ? (null) :
+ +
; + } currentTimecode = () => this.currentTime; render() { const timelineContentWidth = this.props.PanelWidth(); - const timelineContentHeight = this.props.PanelHeight() * 2 / 3; - const dictationHeight = this.props.PanelHeight() / 3; const overlaps: { anchorStartTime: number, anchorEndTime: number, level: number }[] = []; const drawAnchors = this.childDocs.map(anchor => ({ level: this.getLevel(anchor, overlaps), anchor })); const maxLevel = overlaps.reduce((m, o) => Math.max(m, o.level), 0) + 2; @@ -247,11 +279,11 @@ export class CollectionStackedTimeline extends CollectionSubView { this.props.playFrom(start, this.anchorEnd(d.anchor)); e.stopPropagation(); }} > ; })} {this.selectionContainer} -
- dictationHeight} - isAnnotationOverlay={true} - select={emptyFunction} - active={returnFalse} - scaling={returnOne} - xMargin={25} - yMargin={10} - whenActiveChanged={emptyFunction} - removeDocument={returnFalse} - moveDocument={returnFalse} - addDocument={returnFalse} - CollectionView={undefined} - renderDepth={this.props.renderDepth + 1}> - -
+ {this.renderAudioWaveform} + {this.renderDictation}
; diff --git a/src/client/views/nodes/AudioBox.scss b/src/client/views/nodes/AudioBox.scss index fc881ca25..3fcb024df 100644 --- a/src/client/views/nodes/AudioBox.scss +++ b/src/client/views/nodes/AudioBox.scss @@ -144,24 +144,6 @@ border-radius: 3px; z-index: 1000; overflow: hidden; - - .waveform { - position: relative; - width: 100%; - height: 100%; - overflow: hidden; - z-index: -1000; - bottom: 0; - pointer-events: none; - div { - height: 100% !important; - width: 100% !important; - } - canvas { - height: 100% !important; - width: 100% !important; - } - } } .audioBox-total-time, diff --git a/src/client/views/nodes/AudioBox.tsx b/src/client/views/nodes/AudioBox.tsx index 2f7a6cfd8..06a27c22a 100644 --- a/src/client/views/nodes/AudioBox.tsx +++ b/src/client/views/nodes/AudioBox.tsx @@ -1,18 +1,15 @@ import React = require("react"); import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import axios from "axios"; import { action, computed, IReactionDisposer, observable, reaction, runInAction } from "mobx"; import { observer } from "mobx-react"; -import Waveform from "react-audio-waveform"; import { DateField } from "../../../fields/DateField"; import { Doc, DocListCast, Opt } from "../../../fields/Doc"; import { documentSchema } from "../../../fields/documentSchemas"; -import { List } from "../../../fields/List"; -import { createSchema, listSpec, makeInterface } from "../../../fields/Schema"; +import { makeInterface } from "../../../fields/Schema"; import { ComputedField } from "../../../fields/ScriptField"; import { Cast, NumCast } from "../../../fields/Types"; import { AudioField, nullAudio } from "../../../fields/URLField"; -import { emptyFunction, formatTime, numberRange, Utils } from "../../../Utils"; +import { emptyFunction, formatTime, Utils } from "../../../Utils"; import { DocUtils } from "../../documents/Documents"; import { Networking } from "../../Network"; import { CurrentUserUtils } from "../../util/CurrentUserUtils"; @@ -35,7 +32,6 @@ const AudioDocument = makeInterface(documentSchema); export class AudioBox extends ViewBoxAnnotatableComponent(AudioDocument) { public static LayoutString(fieldKey: string) { return FieldView.LayoutString(AudioBox, fieldKey); } public static Enabled = false; - public static NUMBER_OF_BUCKETS = 100; static playheadWidth = 30; // width of playhead static heightPercent = 80; // height of timeline in percent of height of audioBox. static Instance: AudioBox; @@ -295,35 +291,6 @@ export class AudioBox extends ViewBoxAnnotatableComponent this.createWaveformBuckets()); - return ; - } - - // decodes the audio file into peaks for generating the waveform - createWaveformBuckets = async () => { - axios({ url: this.path, responseType: "arraybuffer" }) - .then(response => (new (window.AudioContext)()).decodeAudioData(response.data, - action(buffer => { - const decodedAudioData = buffer.getChannelData(0); - const bucketDataSize = Math.floor(decodedAudioData.length / AudioBox.NUMBER_OF_BUCKETS); - const brange = Array.from(Array(bucketDataSize)); - this.dataDoc.audioBuckets = new List( - numberRange(AudioBox.NUMBER_OF_BUCKETS).map(i => - brange.reduce((p, x, j) => Math.abs(Math.max(p, decodedAudioData[i * bucketDataSize + j])), 0) / 2)); - })) - ); - } - playing = () => this.mediaState === "playing"; playLink = (link: Doc) => { const stack = this._stackedTimeline.current; @@ -353,6 +320,7 @@ export class AudioBox extends ViewBoxAnnotatableComponent
-
- {this.waveform} -
{this.renderTimeline}
{this.audio} diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index df769a407..8a2a755ac 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -1102,16 +1102,16 @@ export class DocumentView extends React.Component { render() { TraceMobx(); - const xshift = this.props.Document.isInkMask ? InkingStroke.MaskDim : Math.abs(this.Xshift) <= 0.001 ? this.props.PanelWidth() : undefined; - const yshift = this.props.Document.isInkMask ? InkingStroke.MaskDim : Math.abs(this.Yshift) <= 0.001 ? this.props.PanelHeight() : undefined; + const xshift = () => (this.props.Document.isInkMask ? InkingStroke.MaskDim : Math.abs(this.Xshift) <= 0.001 ? this.props.PanelWidth() : undefined); + const yshift = () => (this.props.Document.isInkMask ? InkingStroke.MaskDim : Math.abs(this.Yshift) <= 0.001 ? this.props.PanelHeight() : undefined); return (
{!this.props.Document || !this.props.PanelWidth() ? (null) : (
[this.content]; videoPanelHeight = () => NumCast(this.dataDoc[this.fieldKey + "-nativeHeight"], 1) / NumCast(this.dataDoc[this.fieldKey + "-nativeWidth"], 1) * this.props.PanelWidth(); - formattedPanelHeight = () => Math.max(0, this.props.PanelHeight() - this.videoPanelHeight()) + formattedPanelHeight = () => Math.max(0, this.props.PanelHeight() - this.videoPanelHeight()); render() { TraceMobx(); return
diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx index 4e03589d6..da05b0c13 100644 --- a/src/client/views/nodes/VideoBox.tsx +++ b/src/client/views/nodes/VideoBox.tsx @@ -297,9 +297,9 @@ export class VideoBox extends ViewBoxAnnotatableComponent Not supported. - {!this.audiopath ? (null) : + {!this.audiopath || this.audiopath === field.url.href ? (null) :