import React = require("react"); import { FieldViewProps, FieldView } from './FieldView'; import { observer } from "mobx-react"; import "./AudioBox.scss"; import { Cast, DateCast, NumCast, FieldValue, ScriptCast } from "../../../fields/Types"; import { AudioField, nullAudio } from "../../../fields/URLField"; import { ViewBoxBaseComponent, ViewBoxAnnotatableComponent } from "../DocComponent"; import { makeInterface, createSchema } from "../../../fields/Schema"; import { documentSchema } from "../../../fields/documentSchemas"; import { Utils, returnTrue, emptyFunction, returnOne, returnTransparent, returnFalse, returnZero } from "../../../Utils"; import { runInAction, observable, reaction, IReactionDisposer, computed, action, trace, toJS } from "mobx"; import { DateField } from "../../../fields/DateField"; import { SelectionManager } from "../../util/SelectionManager"; import { Doc, DocListCast } from "../../../fields/Doc"; import { ContextMenuProps } from "../ContextMenuItem"; import { ContextMenu } from "../ContextMenu"; import { Id } from "../../../fields/FieldSymbols"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { DocumentView } from "./DocumentView"; import { Docs, DocUtils } from "../../documents/Documents"; import { ComputedField, ScriptField } from "../../../fields/ScriptField"; import { Networking } from "../../Network"; import { LinkAnchorBox } from "./LinkAnchorBox"; import { FormattedTextBox } from "./formattedText/FormattedTextBox"; import { RichTextField } from "../../../fields/RichTextField"; import { AudioResizer } from "./AudioResizer"; import { List } from "../../../fields/List"; import { LabelBox } from "./LabelBox"; import { Transform } from "../../util/Transform"; import { Scripting } from "../../util/Scripting"; import { ColorBox } from "./ColorBox"; // testing testing interface Window { MediaRecorder: MediaRecorder; } declare class MediaRecorder { // whatever MediaRecorder has constructor(e: any); } export const audioSchema = createSchema({ playOnSelect: "boolean" }); type AudioDocument = makeInterface<[typeof documentSchema, typeof audioSchema]>; const AudioDocument = makeInterface(documentSchema, audioSchema); @observer export class AudioBox extends ViewBoxAnnotatableComponent(AudioDocument) { public static LayoutString(fieldKey: string) { return FieldView.LayoutString(AudioBox, fieldKey); } public static Enabled = false; static Instance: AudioBox; static RangeScript: ScriptField; static LabelScript: ScriptField; _linkPlayDisposer: IReactionDisposer | undefined; _reactionDisposer: IReactionDisposer | undefined; _scrubbingDisposer: IReactionDisposer | undefined; _ele: HTMLAudioElement | null = null; _recorder: any; _recordStart = 0; _pauseStart = 0; _pauseEnd = 0; _pausedTime = 0; _stream: MediaStream | undefined; _start: number = 0; _hold: boolean = false; _left: boolean = false; _amount: number = 1; _markers: Array = []; private _isPointerDown = false; private _currMarker: any; @observable private _dragging: boolean = false; @observable private _duration = 0; @observable private _rect: Array = []; @observable private _paused: boolean = false; @observable private static _scrubTime = 0; @observable private _repeat: boolean = false; @computed get audioState(): undefined | "recording" | "paused" | "playing" { return this.dataDoc.audioState as (undefined | "recording" | "paused" | "playing"); } set audioState(value) { this.dataDoc.audioState = value; } public static SetScrubTime = (timeInMillisFrom1970: number) => { runInAction(() => AudioBox._scrubTime = 0); runInAction(() => AudioBox._scrubTime = timeInMillisFrom1970); }; @computed get recordingStart() { return Cast(this.dataDoc[this.props.fieldKey + "-recordingStart"], DateField)?.date.getTime(); } async slideTemplate() { return (await Cast((await Cast(Doc.UserDoc().slidesBtn, Doc) as Doc).dragFactory, Doc) as Doc); } constructor(props: Readonly) { super(props); if (!AudioBox.RangeScript) { AudioBox.RangeScript = ScriptField.MakeScript(`scriptContext.playFrom((this.audioStart), (this.audioEnd))`, { scriptContext: "any" })!; } if (!AudioBox.LabelScript) { AudioBox.LabelScript = ScriptField.MakeScript(`scriptContext.playFrom((this.audioStart))`, { scriptContext: "any" })!; } } componentWillUnmount() { this._reactionDisposer?.(); this._linkPlayDisposer?.(); this._scrubbingDisposer?.(); } componentDidMount() { if (!this.dataDoc.markerAmount) { this.dataDoc.markerAmount = 0; } runInAction(() => this.audioState = this.path ? "paused" : undefined); this._linkPlayDisposer = reaction(() => this.layoutDoc.scrollToLinkID, scrollLinkId => { if (scrollLinkId) { DocListCast(this.dataDoc.links).filter(l => l[Id] === scrollLinkId).map(l => { const linkTime = Doc.AreProtosEqual(l.anchor1 as Doc, this.dataDoc) ? NumCast(l.anchor1_timecode) : NumCast(l.anchor2_timecode); setTimeout(() => { this.playFromTime(linkTime); Doc.linkFollowHighlight(l); }, 250); }); Doc.SetInPlace(this.layoutDoc, "scrollToLinkID", undefined, false); } }, { fireImmediately: true }); this._reactionDisposer = reaction(() => SelectionManager.SelectedDocuments(), selected => { const sel = selected.length ? selected[0].props.Document : undefined; this.layoutDoc.playOnSelect && this.recordingStart && sel && sel.creationDate && !Doc.AreProtosEqual(sel, this.props.Document) && this.playFromTime(DateCast(sel.creationDate).date.getTime()); this.layoutDoc.playOnSelect && this.recordingStart && !sel && this.pause(); }); this._scrubbingDisposer = reaction(() => AudioBox._scrubTime, (time) => this.layoutDoc.playOnSelect && this.playFromTime(AudioBox._scrubTime)); } timecodeChanged = () => { const htmlEle = this._ele; if (this.audioState !== "recording" && htmlEle) { htmlEle.duration && htmlEle.duration !== Infinity && runInAction(() => this.dataDoc.duration = htmlEle.duration); DocListCast(this.dataDoc.links).map(l => { let la1 = l.anchor1 as Doc; let linkTime = NumCast(l.anchor2_timecode); if (Doc.AreProtosEqual(la1, this.dataDoc)) { linkTime = NumCast(l.anchor1_timecode); la1 = l.anchor2 as Doc; } if (linkTime > NumCast(this.layoutDoc.currentTimecode) && linkTime < htmlEle.currentTime) { Doc.linkFollowHighlight(la1); } }); this.layoutDoc.currentTimecode = htmlEle.currentTime; } } pause = action(() => { if (this._repeat) { this.playFrom(0); } else { this._ele!.pause(); this.audioState = "paused"; } }); playFromTime = (absoluteTime: number) => { this.recordingStart && this.playFrom((absoluteTime - this.recordingStart) / 1000); } @action playFrom = (seekTimeInSeconds: number, endTime: number = this.dataDoc.duration) => { let play; clearTimeout(play); this._duration = endTime - seekTimeInSeconds; if (this._ele && AudioBox.Enabled) { if (seekTimeInSeconds < 0) { if (seekTimeInSeconds > -1) { setTimeout(() => this.playFrom(0), -seekTimeInSeconds * 1000); } else { console.log("dude"); this.pause(); } } else if (seekTimeInSeconds <= this._ele.duration) { console.log("playing"); this._ele.currentTime = seekTimeInSeconds; this._ele.play(); runInAction(() => this.audioState = "playing"); if (endTime !== this.dataDoc.duration) { play = setTimeout(() => this.pause(), (this._duration) * 1000); } } else { this.pause(); } } } updateRecordTime = () => { if (this.audioState === "recording") { if (this._paused) { setTimeout(this.updateRecordTime, 30); this._pausedTime += (new Date().getTime() - this._recordStart) / 1000; } else { setTimeout(this.updateRecordTime, 30); this.layoutDoc.currentTimecode = (new Date().getTime() - this._recordStart - this.pauseTime) / 1000; } } } recordAudioAnnotation = async () => { this._stream = await navigator.mediaDevices.getUserMedia({ audio: true }); this._recorder = new MediaRecorder(this._stream); this.dataDoc[this.props.fieldKey + "-recordingStart"] = new DateField(new Date()); DocUtils.ActiveRecordings.push(this.props.Document); this._recorder.ondataavailable = async (e: any) => { const [{ result }] = await Networking.UploadFilesToServer(e.data); if (!(result instanceof Error)) { this.props.Document[this.props.fieldKey] = new AudioField(Utils.prepend(result.accessPaths.agnostic.client)); } }; this._recordStart = new Date().getTime(); runInAction(() => this.audioState = "recording"); setTimeout(this.updateRecordTime, 0); this._recorder.start(); setTimeout(() => this._recorder && this.stopRecording(), 60 * 1000); // stop after an hour } specificContextMenu = (e: React.MouseEvent): void => { const funcs: ContextMenuProps[] = []; funcs.push({ description: (this.layoutDoc.playOnSelect ? "Don't play" : "Play") + " when document selected", event: () => this.layoutDoc.playOnSelect = !this.layoutDoc.playOnSelect, icon: "expand-arrows-alt" }); ContextMenu.Instance?.addItem({ description: "Options...", subitems: funcs, icon: "asterisk" }); } stopRecording = action(() => { this._recorder.stop(); this._recorder = undefined; this.dataDoc.duration = (new Date().getTime() - this._recordStart - this.pauseTime) / 1000; this.audioState = "paused"; this._stream?.getAudioTracks()[0].stop(); const ind = DocUtils.ActiveRecordings.indexOf(this.props.Document); ind !== -1 && (DocUtils.ActiveRecordings.splice(ind, 1)); }); recordClick = (e: React.MouseEvent) => { if (e.button === 0 && !e.ctrlKey) { this._recorder ? this.stopRecording() : this.recordAudioAnnotation(); e.stopPropagation(); } } onPlay = (e: any) => { this.playFrom(this._ele!.paused ? this._ele!.currentTime : -1); e.stopPropagation(); } onStop = (e: any) => { this.layoutDoc.playOnSelect = !this.layoutDoc.playOnSelect; e.stopPropagation(); } onFile = (e: any) => { const newDoc = Docs.Create.TextDocument("", { title: "", _chromeStatus: "disabled", x: NumCast(this.props.Document.x), y: NumCast(this.props.Document.y) + NumCast(this.props.Document._height) + 10, _width: NumCast(this.props.Document._width), _height: 2 * NumCast(this.props.Document._height) }); Doc.GetProto(newDoc).recordingSource = this.dataDoc; Doc.GetProto(newDoc).recordingStart = ComputedField.MakeFunction(`self.recordingSource["${this.props.fieldKey}-recordingStart"]`); Doc.GetProto(newDoc).audioState = ComputedField.MakeFunction("self.recordingSource.audioState"); this.props.addDocument?.(newDoc); e.stopPropagation(); } setRef = (e: HTMLAudioElement | null) => { e?.addEventListener("timeupdate", this.timecodeChanged); e?.addEventListener("ended", this.pause); this._ele = e; } @computed get path() { const field = Cast(this.props.Document[this.props.fieldKey], AudioField); const path = (field instanceof AudioField) ? field.url.href : ""; return path === nullAudio ? "" : path; } @computed get audio() { const interactive = this.active() ? "-interactive" : ""; return ; } @action onRepeat = (e: React.MouseEvent) => { this._repeat = !this._repeat; e.stopPropagation(); } @action recordPause = (e: React.MouseEvent) => { this._pauseStart = new Date().getTime(); this._paused = true; this._recorder.pause(); e.stopPropagation(); } @action recordPlay = (e: React.MouseEvent) => { this._pauseEnd = new Date().getTime(); this._paused = false; this._recorder.resume(); e.stopPropagation(); } @computed get pauseTime() { return (this._pauseEnd - this._pauseStart); } @action newMarker(marker: Doc) { if (this.dataDoc[this.annotationKey]) { this.dataDoc[this.annotationKey].push(marker); } else { this.dataDoc[this.annotationKey] = new List([marker]); } } start(marker: number) { console.log("start!"); this._hold = true; this._start = marker; } @action end(marker: number) { console.log("end!"); this._hold = false; //this._markers.push(Docs.Create.LabelDocument({ isLabel: false, audioStart: this._start, audioEnd: marker, _showSidebar: false, _autoHeight: true, annotationOn: this.props.Document })) let newMarker = Docs.Create.LabelDocument({ title: "", isLabel: false, audioStart: this._start, audioEnd: marker, _showSidebar: false, _autoHeight: true, annotationOn: this.props.Document }); if (this.dataDoc[this.annotationKey]) { this.dataDoc[this.annotationKey].push(newMarker); // onClick: ScriptField.MakeScript(`playFrom(${NumCast(this._start)}, ${NumCast(marker)})`) } else { this.dataDoc[this.annotationKey] = new List([newMarker]); } this._start = 0; this._amount++; } onPointerDown = (e: React.PointerEvent, m: any, left: boolean): void => { e.stopPropagation(); e.preventDefault(); this._isPointerDown = true; console.log("click"); this._currMarker = m; const targetele = document.getElementById("timeline"); targetele?.setPointerCapture(e.pointerId); this._left = left; document.removeEventListener("pointermove", this.onPointerMove); document.addEventListener("pointermove", this.onPointerMove); document.removeEventListener("pointerup", this.onPointerUp); document.addEventListener("pointerup", this.onPointerUp); } @action onPointerUp = (e: PointerEvent): void => { e.stopPropagation(); e.preventDefault(); this._isPointerDown = false; this._dragging = false; const rect = (e.target as any).getBoundingClientRect(); this._ele!.currentTime = this.layoutDoc.currentTimecode = (e.clientX - rect.x) / rect.width * NumCast(this.dataDoc.duration); const targetele = document.getElementById("timeline"); targetele?.releasePointerCapture(e.pointerId); document.removeEventListener("pointermove", this.onPointerMove); document.removeEventListener("pointerup", this.onPointerUp); } onPointerMove = async (e: PointerEvent) => { e.stopPropagation(); e.preventDefault(); console.log("drag"); if (!this._isPointerDown) { return; } const rect = await (e.target as any).getBoundingClientRect(); // if (e.target as HTMLElement === document.getElementById("timeline")) { let newTime = (e.clientX - rect.x) / rect.width * NumCast(this.dataDoc.duration); this.changeMarker(this._currMarker, newTime); // } } @action changeMarker = (m: any, time: any) => { for (let i = 0; i < this.dataDoc[this.annotationKey].length; i++) { if (this.isSame(this.dataDoc[this.annotationKey][i], m)) { // this._left ? this._markers[i][0] = time : this._markers[i][1] = time; this._left ? this.dataDoc[this.annotationKey][i].audioStart = time : this.dataDoc[this.annotationKey][i].audioEnd = time; } } } isSame = (m1: any, m2: any) => { if (m1.audioStart === m2.audioStart && m1.audioEnd === m2.audioEnd) { return true; } return false; } isOverlap = (m: any, i: number) => { console.log("called"); let counter = 0; if (i == 0) { this._markers = []; } for (let i = 0; i < this._markers.length; i++) { if ((m.audioEnd > this._markers[i].audioStart && m.audioStart < this._markers[i].audioEnd)) { counter++; console.log(counter); } } console.log(counter); console.log(this.dataDoc.markerAmount); if (this.dataDoc.markerAmount < counter) { this.dataDoc.markerAmount = counter; } this._markers.push(m); return counter; } // isOverlap = (m: any) => { // if (this._markers.length < 1) { // this._markers = new Array(Math.round(this.dataDoc.duration)).fill(0); // } // console.log(this._markers); // let max = 0 // for (let i = Math.round(m.audioStart); i <= Math.round(m.audioEnd); i++) { // this._markers[i] = this._markers[i] + 1; // console.log(this._markers[i]); // if (this._markers[i] > max) { // max = this._markers[i]; // } // } // console.log(max); // if (this.dataDoc.markerAmount < max) { // this.dataDoc.markerAmount = max; // } // return max // } formatTime = (time: number) => { const hours = Math.floor(time / 60 / 60); const minutes = Math.floor(time / 60) - (hours * 60); const seconds = time % 60; return hours.toString().padStart(2, '0') + ':' + minutes.toString().padStart(2, '0') + ':' + seconds.toString().padStart(2, '0'); } @action onHover = () => { this._dragging = true; } @action onLeave = () => { this._dragging = false; } // onMouseOver={this.onHover} onMouseLeave={this.onLeave} change = (e: React.PointerEvent) => { e.stopPropagation(); e.preventDefault(); const rect = (e.target as any).getBoundingClientRect(); const wasPaused = this.audioState === "paused"; this._ele!.currentTime = this.layoutDoc.currentTimecode = (e.clientX - rect.x) / rect.width * NumCast(this.dataDoc.duration); wasPaused && this.pause(); console.log("double!"); } rangeScript = () => AudioBox.RangeScript; labelScript = () => AudioBox.LabelScript; // see if time is encapsulated by comparing time on both sides (for moving onto a new row in the timeline for the markers) render() { trace(); const interactive = this.active() ? "-interactive" : ""; return
{!this.path ?
{/* */} {this.audioState === "recording" ?
e.stopPropagation()}>
{this.formatTime(Math.round(NumCast(this.layoutDoc.currentTimecode)))}
: }
:
{/*
*/}
{ e.stopPropagation(); e.preventDefault(); }} onDoubleClick={e => this.change} onPointerDown={e => { e.stopPropagation(); e.preventDefault(); if (e.button === 0 && !e.ctrlKey) { const rect = (e.target as any).getBoundingClientRect(); if (e.target as HTMLElement !== document.getElementById("current")) { const wasPaused = this.audioState === "paused"; this._ele!.currentTime = this.layoutDoc.currentTimecode = (e.clientX - rect.x) / rect.width * NumCast(this.dataDoc.duration); wasPaused && this.pause(); } } if (e.button === 0 && e.altKey) { this.newMarker(Docs.Create.LabelDocument({ title: "", isLabel: true, audioStart: this._ele!.currentTime, _showSidebar: false, _autoHeight: true, annotationOn: this.props.Document })); } if (e.button === 0 && e.shiftKey) { const rect = (e.target as any).getBoundingClientRect(); this._ele!.currentTime = this.layoutDoc.currentTimecode = (e.clientX - rect.x) / rect.width * NumCast(this.dataDoc.duration); this._hold ? this.end(this._ele!.currentTime) : this.start(this._ele!.currentTime); } }}> {DocListCast(this.dataDoc[this.annotationKey]).map((m, i) => { // let text = Docs.Create.TextDocument("hello", { title: "label", _showSidebar: false, _autoHeight: false }); let rect; (!m.isLabel) ? rect =
{ this.playFrom(NumCast(m.audioStart), NumCast(m.audioEnd)); e.stopPropagation() }} >
this.onPointerDown(e, m, true)}>
{/* */} {/*
{ this.playFrom(NumCast(m.audioStart), NumCast(m.audioEnd)) }}>
*/}
this.onPointerDown(e, m, false)}>
: rect =
; return rect; })} {DocListCast(this.dataDoc.links).map((l, i) => { console.log("hi"); let la1 = l.anchor1 as Doc; let la2 = l.anchor2 as Doc; let linkTime = NumCast(l.anchor2_timecode); if (Doc.AreProtosEqual(la1, this.dataDoc)) { la1 = l.anchor2 as Doc; la2 = l.anchor1 as Doc; linkTime = NumCast(l.anchor1_timecode); } return !linkTime ? (null) :
e.stopPropagation()}> {/*
*/} {/*
*/}
Doc.linkFollowHighlight(la1)} onPointerDown={e => { if (e.button === 0 && !e.ctrlKey) { const wasPaused = this.audioState === "paused"; this.playFrom(linkTime); this.pause(); e.stopPropagation(); e.preventDefault(); } }} />
; })}
{ e.stopPropagation(); e.preventDefault(); console.log("hi"); }} style={{ left: `${NumCast(this.layoutDoc.currentTimecode) / NumCast(this.dataDoc.duration, 1) * 100}%` }} /> {this.audio}
{this.formatTime(Math.round(NumCast(this.layoutDoc.currentTimecode)))}
{this.formatTime(Math.round(NumCast(this.layoutDoc.duration)))}
}
; } } // Scripting.addGlobal(function playFrom(audioDoc: Doc, start: number, end: number) { return audioDoc.playFrom(start, end); })