From ddcab916cf125817bb532b7c34617e50bcc3840d Mon Sep 17 00:00:00 2001 From: Jenny Yu Date: Tue, 26 Apr 2022 01:39:51 -0400 Subject: feat: uploading works --- src/client/Network.ts | 3 - .../views/nodes/RecordingBox/RecordingBox.tsx | 9 +- .../views/nodes/RecordingBox/RecordingView.tsx | 27 +- src/client/views/nodes/ScreenshotBox.tsx | 429 +++++++++++---------- 4 files changed, 240 insertions(+), 228 deletions(-) (limited to 'src') diff --git a/src/client/Network.ts b/src/client/Network.ts index 58f356c90..3597e7b2b 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -19,7 +19,6 @@ export namespace Networking { } export async function UploadFilesToServer(files: File | File[]): Promise[]> { - console.log(files) const formData = new FormData(); if (Array.isArray(files)) { if (!files.length) { @@ -34,8 +33,6 @@ export namespace Networking { body: formData }; const response = await fetch("/uploadFormData", parameters); - console.log(response) - // console.log(response.json()) return response.json(); } diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx index 1e17e2ddd..4c7fdd278 100644 --- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx @@ -1,12 +1,13 @@ import { action, observable } from "mobx"; import { observer } from "mobx-react"; import * as React from "react"; -import { AudioField } from "../../../../fields/URLField"; +import { AudioField, VideoField } from "../../../../fields/URLField"; import { Upload } from "../../../../server/SharedMediaTypes"; import { ViewBoxBaseComponent } from "../../DocComponent"; import { FieldView } from "../FieldView"; import { VideoBox } from "../VideoBox"; import { RecordingView } from './RecordingView'; +import { DocumentType } from "../../../documents/DocumentTypes"; @observer @@ -27,7 +28,11 @@ export class RecordingBox extends ViewBoxBaseComponent() { console.log("Setting result to " + info) this.result = info console.log(this.result.accessPaths.agnostic.client) - this.props.Document[this.fieldKey] = new AudioField(this.result.accessPaths.agnostic.client); + this.dataDoc.type = DocumentType.VID; + this.layoutDoc.layout = VideoBox.LayoutString(this.fieldKey); + this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = undefined; + this.layoutDoc._fitWidth = undefined; + this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.agnostic.client); } render() { diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx index 0d24f3c74..3a79f790f 100644 --- a/src/client/views/nodes/RecordingBox/RecordingView.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx @@ -27,7 +27,6 @@ interface IRecordingViewProps { const MAXTIME = 1000; export function RecordingView(props: IRecordingViewProps) { - const { setResult } = props const [recording, setRecording] = useState(false); const recordingTimerRef = useRef(0); @@ -52,11 +51,11 @@ export function RecordingView(props: IRecordingViewProps) { width: 1280, height: 720, }, - audio: { - echoCancellation: true, - noiseSuppression: true, - sampleRate: 44100 - } + // audio: { + // echoCancellation: true, + // noiseSuppression: true, + // sampleRate: 44100 + // } } useEffect(() => { @@ -75,7 +74,7 @@ export function RecordingView(props: IRecordingViewProps) { type: 'video/webm' }) const blobUrl = URL.createObjectURL(blob) - const videoFile = new File([blob], "video", { lastModified: new Date().getDate() }); + const videoFile = new File(allVideoChunks, "video", { type: allVideoChunks[0].type, lastModified: Date.now() }); videoElementRef.current!.srcObject = null videoElementRef.current!.src = blobUrl @@ -89,12 +88,20 @@ export function RecordingView(props: IRecordingViewProps) { // } // } - Networking.UploadFilesToServer(allVideoChunks) + Networking.UploadFilesToServer(videoFile) .then((data) => { - console.log(data) + const result = data[0].result + if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox + props.setResult(result) + } else { + alert("video conversion failed"); + } }) // uploadVideo() + // this.dataDoc[this.fieldKey + "-duration"] = (new Date().getTime() - this.recordingStart!) / 1000; + + // change to one recording box } @@ -150,6 +157,8 @@ export function RecordingView(props: IRecordingViewProps) { } const startShowingStream = async (mediaConstraints = DEFAULT_MEDIA_CONSTRAINTS) => { const stream = await navigator.mediaDevices.getUserMedia(mediaConstraints) + // const stream = await navigator.mediaDevices.getUserMedia({ video: true }); + videoElementRef.current!.src = "" videoElementRef.current!.srcObject = stream diff --git a/src/client/views/nodes/ScreenshotBox.tsx b/src/client/views/nodes/ScreenshotBox.tsx index dbb567d3a..0c7d7dc31 100644 --- a/src/client/views/nodes/ScreenshotBox.tsx +++ b/src/client/views/nodes/ScreenshotBox.tsx @@ -26,7 +26,7 @@ import { FormattedTextBox } from "./formattedText/FormattedTextBox"; import "./ScreenshotBox.scss"; import { VideoBox } from "./VideoBox"; declare class MediaRecorder { - constructor(e: any, options?: any); // whatever MediaRecorder has + constructor(e: any, options?: any); // whatever MediaRecorder has } // interface VideoTileProps { @@ -107,225 +107,226 @@ declare class MediaRecorder { @observer export class ScreenshotBox extends ViewBoxAnnotatableComponent() { - public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ScreenshotBox, fieldKey); } - private _audioRec: any; - private _videoRec: any; - @observable private _videoRef: HTMLVideoElement | null = null; - @observable _screenCapture = false; - @computed get recordingStart() { return Cast(this.dataDoc[this.props.fieldKey + "-recordingStart"], DateField)?.date.getTime(); } + public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ScreenshotBox, fieldKey); } + private _audioRec: any; + private _videoRec: any; + @observable private _videoRef: HTMLVideoElement | null = null; + @observable _screenCapture = false; + @computed get recordingStart() { return Cast(this.dataDoc[this.props.fieldKey + "-recordingStart"], DateField)?.date.getTime(); } - constructor(props: any) { - super(props); - if (this.rootDoc.videoWall) { - this.rootDoc.nativeWidth = undefined; - this.rootDoc.nativeHeight = undefined; - this.layoutDoc.popOff = 0; - this.layoutDoc.popOut = 1; - } else { - this.setupDictation(); - } - } - getAnchor = () => { - const startTime = Cast(this.layoutDoc._currentTimecode, "number", null) || (this._videoRec ? (Date.now() - (this.recordingStart || 0)) / 1000 : undefined); - return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow", "_timecodeToHide", - startTime, startTime === undefined ? undefined : startTime + 3) - || this.rootDoc; - } + constructor(props: any) { + super(props); + if (this.rootDoc.videoWall) { + this.rootDoc.nativeWidth = undefined; + this.rootDoc.nativeHeight = undefined; + this.layoutDoc.popOff = 0; + this.layoutDoc.popOut = 1; + } else { + this.setupDictation(); + } + } + getAnchor = () => { + const startTime = Cast(this.layoutDoc._currentTimecode, "number", null) || (this._videoRec ? (Date.now() - (this.recordingStart || 0)) / 1000 : undefined); + return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow", "_timecodeToHide", + startTime, startTime === undefined ? undefined : startTime + 3) + || this.rootDoc; + } - videoLoad = () => { - const aspect = this._videoRef!.videoWidth / this._videoRef!.videoHeight; - const nativeWidth = Doc.NativeWidth(this.layoutDoc); - const nativeHeight = Doc.NativeHeight(this.layoutDoc); - if (!nativeWidth || !nativeHeight) { - if (!nativeWidth) Doc.SetNativeWidth(this.dataDoc, 1200); - Doc.SetNativeHeight(this.dataDoc, (nativeWidth || 1200) / aspect); - this.layoutDoc._height = (this.layoutDoc[WidthSym]() || 0) / aspect; - } - } + videoLoad = () => { + const aspect = this._videoRef!.videoWidth / this._videoRef!.videoHeight; + const nativeWidth = Doc.NativeWidth(this.layoutDoc); + const nativeHeight = Doc.NativeHeight(this.layoutDoc); + if (!nativeWidth || !nativeHeight) { + if (!nativeWidth) Doc.SetNativeWidth(this.dataDoc, 1200); + Doc.SetNativeHeight(this.dataDoc, (nativeWidth || 1200) / aspect); + this.layoutDoc._height = (this.layoutDoc[WidthSym]() || 0) / aspect; + } + } - componentDidMount() { - this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = 0; - this.props.setContentView?.(this); // this tells the DocumentView that this ScreenshotBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. - // this.rootDoc.videoWall && reaction(() => ({ width: this.props.PanelWidth(), height: this.props.PanelHeight() }), - // ({ width, height }) => { - // if (this._camera) { - // const angle = -Math.abs(1 - width / height); - // const xz = [0, (this._numScreens - 2) / Math.abs(1 + angle)]; - // this._camera.position.set(this._numScreens / 2 + xz[1] * Math.sin(angle), this._numScreens / 2, xz[1] * Math.cos(angle)); - // this._camera.lookAt(this._numScreens / 2, this._numScreens / 2, 0); - // (this._camera as any).updateProjectionMatrix(); - // } - // }); - } - componentWillUnmount() { - const ind = DocUtils.ActiveRecordings.indexOf(this); - ind !== -1 && (DocUtils.ActiveRecordings.splice(ind, 1)); - } + componentDidMount() { + this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = 0; + this.props.setContentView?.(this); // this tells the DocumentView that this ScreenshotBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. + // this.rootDoc.videoWall && reaction(() => ({ width: this.props.PanelWidth(), height: this.props.PanelHeight() }), + // ({ width, height }) => { + // if (this._camera) { + // const angle = -Math.abs(1 - width / height); + // const xz = [0, (this._numScreens - 2) / Math.abs(1 + angle)]; + // this._camera.position.set(this._numScreens / 2 + xz[1] * Math.sin(angle), this._numScreens / 2, xz[1] * Math.cos(angle)); + // this._camera.lookAt(this._numScreens / 2, this._numScreens / 2, 0); + // (this._camera as any).updateProjectionMatrix(); + // } + // }); + } + componentWillUnmount() { + const ind = DocUtils.ActiveRecordings.indexOf(this); + ind !== -1 && (DocUtils.ActiveRecordings.splice(ind, 1)); + } - specificContextMenu = (e: React.MouseEvent): void => { - const subitems = [{ description: "Screen Capture", event: this.toggleRecording, icon: "expand-arrows-alt" as any }]; - ContextMenu.Instance.addItem({ description: "Options...", subitems, icon: "video" }); - } + specificContextMenu = (e: React.MouseEvent): void => { + const subitems = [{ description: "Screen Capture", event: this.toggleRecording, icon: "expand-arrows-alt" as any }]; + ContextMenu.Instance.addItem({ description: "Options...", subitems, icon: "video" }); + } - @computed get content() { - if (this.rootDoc.videoWall) return (null); - return ; - } + @computed get content() { + if (this.rootDoc.videoWall) return (null); + return ; + } - // _numScreens = 5; - // _camera: Camera | undefined; - // @observable _raised = [] as { coord: Vector2, off: Vector3 }[]; - // @action setRaised = (r: { coord: Vector2, off: Vector3 }[]) => this._raised = r; - @computed get threed() { - // if (this.rootDoc.videoWall) { - // const screens: any[] = []; - // const colors = ["yellow", "red", "orange", "brown", "maroon", "gray"]; - // let count = 0; - // numberRange(this._numScreens).forEach(x => numberRange(this._numScreens).forEach(y => screens.push( - // ))); - // return { - // this._camera = props.camera; - // props.camera.position.set(this._numScreens / 2, this._numScreens / 2, this._numScreens - 2); - // props.camera.lookAt(this._numScreens / 2, this._numScreens / 2, 0); - // }}> - // {/* */} - // - // {screens} - // ; - // } - return (null); - } - toggleRecording = async () => { - if (!this._screenCapture) { - this._audioRec = new MediaRecorder(await navigator.mediaDevices.getUserMedia({ audio: true })); - const aud_chunks: any = []; - this._audioRec.ondataavailable = (e: any) => aud_chunks.push(e.data); - this._audioRec.onstop = async (e: any) => { - const [{ result }] = await Networking.UploadFilesToServer(aud_chunks); - if (!(result instanceof Error)) { - this.dataDoc[this.props.fieldKey + "-audio"] = new AudioField(result.accessPaths.agnostic.client); - } - }; - this._videoRef!.srcObject = await (navigator.mediaDevices as any).getDisplayMedia({ video: true }); - this._videoRec = new MediaRecorder(this._videoRef!.srcObject); - const vid_chunks: any = []; - this._videoRec.onstart = () => this.dataDoc[this.props.fieldKey + "-recordingStart"] = new DateField(new Date()); - this._videoRec.ondataavailable = (e: any) => vid_chunks.push(e.data); - this._videoRec.onstop = async (e: any) => { - const file = new File(vid_chunks, `${this.rootDoc[Id]}.mkv`, { type: vid_chunks[0].type, lastModified: Date.now() }); - const [{ result }] = await Networking.UploadFilesToServer(file); - this.dataDoc[this.fieldKey + "-duration"] = (new Date().getTime() - this.recordingStart!) / 1000; - if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox - this.dataDoc.type = DocumentType.VID; - this.layoutDoc.layout = VideoBox.LayoutString(this.fieldKey); - this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = undefined; - this.layoutDoc._fitWidth = undefined; - this.dataDoc[this.props.fieldKey] = new VideoField(result.accessPaths.agnostic.client); - } else alert("video conversion failed"); - }; - this._audioRec.start(); - this._videoRec.start(); - runInAction(() => { - this._screenCapture = true; - this.dataDoc.mediaState = "recording"; - }); - DocUtils.ActiveRecordings.push(this); - } else { - this._audioRec?.stop(); - this._videoRec?.stop(); - runInAction(() => { - this._screenCapture = false; - this.dataDoc.mediaState = "paused"; - }); - const ind = DocUtils.ActiveRecordings.indexOf(this); - ind !== -1 && (DocUtils.ActiveRecordings.splice(ind, 1)); + // _numScreens = 5; + // _camera: Camera | undefined; + // @observable _raised = [] as { coord: Vector2, off: Vector3 }[]; + // @action setRaised = (r: { coord: Vector2, off: Vector3 }[]) => this._raised = r; + @computed get threed() { + // if (this.rootDoc.videoWall) { + // const screens: any[] = []; + // const colors = ["yellow", "red", "orange", "brown", "maroon", "gray"]; + // let count = 0; + // numberRange(this._numScreens).forEach(x => numberRange(this._numScreens).forEach(y => screens.push( + // ))); + // return { + // this._camera = props.camera; + // props.camera.position.set(this._numScreens / 2, this._numScreens / 2, this._numScreens - 2); + // props.camera.lookAt(this._numScreens / 2, this._numScreens / 2, 0); + // }}> + // {/* */} + // + // {screens} + // ; + // } + return (null); + } + toggleRecording = async () => { + if (!this._screenCapture) { + this._audioRec = new MediaRecorder(await navigator.mediaDevices.getUserMedia({ audio: true })); + const aud_chunks: any = []; + this._audioRec.ondataavailable = (e: any) => aud_chunks.push(e.data); + this._audioRec.onstop = async (e: any) => { + const [{ result }] = await Networking.UploadFilesToServer(aud_chunks); + if (!(result instanceof Error)) { + this.dataDoc[this.props.fieldKey + "-audio"] = new AudioField(result.accessPaths.agnostic.client); + } + }; + this._videoRef!.srcObject = await (navigator.mediaDevices as any).getDisplayMedia({ video: true }); + this._videoRec = new MediaRecorder(this._videoRef!.srcObject); + const vid_chunks: any = []; + this._videoRec.onstart = () => this.dataDoc[this.props.fieldKey + "-recordingStart"] = new DateField(new Date()); + this._videoRec.ondataavailable = (e: any) => vid_chunks.push(e.data); + this._videoRec.onstop = async (e: any) => { + console.log("screenshotbox: upload") + const file = new File(vid_chunks, `${this.rootDoc[Id]}.mkv`, { type: vid_chunks[0].type, lastModified: Date.now() }); + const [{ result }] = await Networking.UploadFilesToServer(file); + this.dataDoc[this.fieldKey + "-duration"] = (new Date().getTime() - this.recordingStart!) / 1000; + if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox + this.dataDoc.type = DocumentType.VID; + this.layoutDoc.layout = VideoBox.LayoutString(this.fieldKey); + this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = undefined; + this.layoutDoc._fitWidth = undefined; + this.dataDoc[this.props.fieldKey] = new VideoField(result.accessPaths.agnostic.client); + } else alert("video conversion failed"); + }; + this._audioRec.start(); + this._videoRec.start(); + runInAction(() => { + this._screenCapture = true; + this.dataDoc.mediaState = "recording"; + }); + DocUtils.ActiveRecordings.push(this); + } else { + this._audioRec?.stop(); + this._videoRec?.stop(); + runInAction(() => { + this._screenCapture = false; + this.dataDoc.mediaState = "paused"; + }); + const ind = DocUtils.ActiveRecordings.indexOf(this); + ind !== -1 && (DocUtils.ActiveRecordings.splice(ind, 1)); - CaptureManager.Instance.open(this.rootDoc); - } - } + CaptureManager.Instance.open(this.rootDoc); + } + } - setupDictation = () => { - if (this.dataDoc[this.fieldKey + "-dictation"]) return; - const dictationText = CurrentUserUtils.GetNewTextDoc("dictation", - NumCast(this.rootDoc.x), NumCast(this.rootDoc.y) + NumCast(this.layoutDoc._height) + 10, - NumCast(this.layoutDoc._width), 2 * NumCast(this.layoutDoc._height)); - dictationText._autoHeight = false; - const dictationTextProto = Doc.GetProto(dictationText); - dictationTextProto.recordingSource = this.dataDoc; - dictationTextProto.recordingStart = ComputedField.MakeFunction(`self.recordingSource["${this.props.fieldKey}-recordingStart"]`); - dictationTextProto.mediaState = ComputedField.MakeFunction("self.recordingSource.mediaState"); - this.dataDoc[this.fieldKey + "-dictation"] = dictationText; - } - contentFunc = () => [this.threed, this.content]; - videoPanelHeight = () => NumCast(this.dataDoc[this.fieldKey + "-nativeHeight"], this.layoutDoc[HeightSym]()) / NumCast(this.dataDoc[this.fieldKey + "-nativeWidth"], this.layoutDoc[WidthSym]()) * this.props.PanelWidth(); - formattedPanelHeight = () => Math.max(0, this.props.PanelHeight() - this.videoPanelHeight()); - render() { - TraceMobx(); - return
-
-
- - {this.contentFunc} -
-
- {!(this.dataDoc[this.fieldKey + "-dictation"] instanceof Doc) ? (null) : - - } -
-
- {!this.props.isSelected() ? (null) :
-
- -
-
} -
; - } + setupDictation = () => { + if (this.dataDoc[this.fieldKey + "-dictation"]) return; + const dictationText = CurrentUserUtils.GetNewTextDoc("dictation", + NumCast(this.rootDoc.x), NumCast(this.rootDoc.y) + NumCast(this.layoutDoc._height) + 10, + NumCast(this.layoutDoc._width), 2 * NumCast(this.layoutDoc._height)); + dictationText._autoHeight = false; + const dictationTextProto = Doc.GetProto(dictationText); + dictationTextProto.recordingSource = this.dataDoc; + dictationTextProto.recordingStart = ComputedField.MakeFunction(`self.recordingSource["${this.props.fieldKey}-recordingStart"]`); + dictationTextProto.mediaState = ComputedField.MakeFunction("self.recordingSource.mediaState"); + this.dataDoc[this.fieldKey + "-dictation"] = dictationText; + } + contentFunc = () => [this.threed, this.content]; + videoPanelHeight = () => NumCast(this.dataDoc[this.fieldKey + "-nativeHeight"], this.layoutDoc[HeightSym]()) / NumCast(this.dataDoc[this.fieldKey + "-nativeWidth"], this.layoutDoc[WidthSym]()) * this.props.PanelWidth(); + formattedPanelHeight = () => Math.max(0, this.props.PanelHeight() - this.videoPanelHeight()); + render() { + TraceMobx(); + return
+
+
+ + {this.contentFunc} +
+
+ {!(this.dataDoc[this.fieldKey + "-dictation"] instanceof Doc) ? (null) : + + } +
+
+ {!this.props.isSelected() ? (null) :
+
+ +
+
} +
; + } } \ No newline at end of file -- cgit v1.2.3-70-g09d2