From c656abad84ac56c3067a48ca6d34d567e3fcdd88 Mon Sep 17 00:00:00 2001 From: Michael Foiani Date: Wed, 8 Jun 2022 00:05:34 -0400 Subject: Took stab at ffmpeg in server. did some reworking of recordingview css. fix smoothness bug for the moving segment that follow cursor. --- src/client/views/nodes/RecordingBox/RecordingBox.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx') diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx index 159271223..68f3b3ad4 100644 --- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx @@ -54,9 +54,9 @@ export class RecordingBox extends ViewBoxBaseComponent() { } render() { - // console.log("Proto[Is]: ", this.rootDoc.proto?.[Id]) + console.log("Proto[Is]: ", this.rootDoc.proto?.[Id]) return
- {!this.result && } + {!this.result && }
; } } -- cgit v1.2.3-70-g09d2 From 48c60bd982734676f972514f7074be6121e7c5df Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 8 Jun 2022 18:24:53 -0400 Subject: big commit. FINALLY got the combining segments to work using ffmpeg using a different workflow. small ui changes as well. --- src/client/Network.ts | 15 +-- .../views/nodes/RecordingBox/ProgressBar.tsx | 20 +++- .../views/nodes/RecordingBox/RecordingBox.tsx | 7 +- .../views/nodes/RecordingBox/RecordingView.tsx | 126 ++++----------------- src/server/ApiManagers/UploadManager.ts | 23 +--- src/server/DashUploadUtils.ts | 59 +++------- 6 files changed, 60 insertions(+), 190 deletions(-) (limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx') diff --git a/src/client/Network.ts b/src/client/Network.ts index 8c1f31488..b26f2458d 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -35,20 +35,7 @@ export namespace Networking { const response = await fetch("/uploadFormData", parameters); return response.json(); } - - export async function UploadSegmentsAndConcatenate(files: File[]): Promise[]> { - console.log("network.ts : uploading segments and concatenating", files); - const formData = new FormData(); - if (!Array.isArray(files) || !files.length) return []; - files.forEach(file => formData.append(Utils.GenerateGuid(), file)); - const parameters = { - method: 'POST', - body: formData - }; - const response = await fetch("/uploadVideosandConcatenate", parameters); - return response.json(); - } - + export async function UploadYoutubeToServer(videoId: string): Promise[]> { const parameters = { method: 'POST', diff --git a/src/client/views/nodes/RecordingBox/ProgressBar.tsx b/src/client/views/nodes/RecordingBox/ProgressBar.tsx index a91656cbc..effc3d8a8 100644 --- a/src/client/views/nodes/RecordingBox/ProgressBar.tsx +++ b/src/client/views/nodes/RecordingBox/ProgressBar.tsx @@ -53,11 +53,6 @@ export function ProgressBar(props: ProgressBarProps) { return [...prevOrdered, { endTime, startTime , order }]; }); } - // }props.videos.map((vid, order) => { - // //const { endTime, startTime } = vid - // // TODO: not tranfer the blobs around - // return { ...vid, order }; - // })) }, [props.videos]); useEffect(() => { @@ -96,7 +91,6 @@ export function ProgressBar(props: ProgressBarProps) { } const onPointerDown = (e: React.PointerEvent) => { - console.log('pointer down') // don't move the videobox element e.stopPropagation() @@ -107,6 +101,20 @@ export function ProgressBar(props: ProgressBarProps) { // don't do anything if null const progressBar = progressBarRef.current if (progressBar == null || clickedSegment.id === progressBar.id) return + + console.log('pointer down') + // if holding shift key, let's remove that segment + if (e.shiftKey) { + // TODO: fix this + const segId = parseInt(clickedSegment.id.split('-')[1]) + const o = ordered[segId].order + setOrdered(prevOrdered => { + return prevOrdered.filter((seg, i) => i !== segId) + }) + + return + } + const ptrId = e.pointerId; progressBar.setPointerCapture(ptrId) diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx index 68f3b3ad4..bd4af0a75 100644 --- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx @@ -30,7 +30,7 @@ export class RecordingBox extends ViewBoxBaseComponent() { Doc.SetNativeHeight(this.dataDoc, 720); } - @observable result: Upload.FileInformation | undefined = undefined + @observable result: Upload.AccessPathInfo | undefined = undefined @observable videoDuration: number | undefined = undefined @action @@ -39,13 +39,14 @@ export class RecordingBox extends ViewBoxBaseComponent() { } @action - setResult = (info: Upload.FileInformation, trackScreen: boolean) => { + setResult = (info: Upload.AccessPathInfo, trackScreen: boolean) => { + if (info == null) return this.result = info this.dataDoc.type = DocumentType.VID; this.dataDoc[this.fieldKey + "-duration"] = this.videoDuration; this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey); - this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.agnostic.client); + this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client); this.dataDoc[this.fieldKey + "-recorded"] = true; // stringify the presenation and store it if (trackScreen) { diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx index a5c2dc85c..8c8728fc3 100644 --- a/src/client/views/nodes/RecordingBox/RecordingView.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx @@ -18,7 +18,7 @@ export interface MediaSegment { } interface IRecordingViewProps { - setResult: (info: Upload.FileInformation, trackScreen: boolean) => void + setResult: (info: Upload.AccessPathInfo, trackScreen: boolean) => void setDuration: (seconds: number) => void id: string } @@ -57,7 +57,7 @@ export function RecordingView(props: IRecordingViewProps) { useEffect(() => { - console.log('in videos useEffect') + // console.log('in videos useEffect', finished) if (finished) { (async () => { @@ -71,89 +71,18 @@ export function RecordingView(props: IRecordingViewProps) { const { name } = videoFile; inputPaths.push(name) }) - - console.log(inputPaths) - const data = await Networking.UploadSegmentsAndConcatenate(videoFiles) - console.log('data', data) - const result = data[0].result - if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox - props.setResult(result, trackScreen) - } else { - alert("video conversion failed"); - } - - + const serverPaths: string[] = (await Networking.UploadFilesToServer(videoFiles)) + .map(res => (res.result instanceof Error) ? '' : res.result.accessPaths.agnostic.server) - // const inputListName = 'order.txt'; - // fs.writeFileSync(inputListName, inputPaths.join('\n')); - // var merge = ffmpeg(); - // merge.input(inputListName) - // .inputOptions(['-f concat', '-safe 0']) - // .outputOptions('-c copy') - // .save('output.mp4') - - // fs.unlinkSync(inputListName); - - // const combined = await DashUploadUtils.combineSegments(videoFiles, inputPaths) - // console.log('combined', combined) - - // const outputFile = new File(['output.mp4'], 'output.mp4', { type: 'video/mp4', lastModified: Date.now() }); - - // const data = await Networking.UploadFilesToServer(combined) - // const result = data[0].result - // if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox - // props.setResult(result, trackScreen) - // } else { - // alert("video conversion failed"); - // } - - - // if (format.includes("x-matroska")) { - // await new Promise(res => ffmpeg(file.path) - // .videoCodec("copy") // this will copy the data instead of reencode it - // .save(file.path.replace(".mkv", ".mp4")) - // .on('end', res)); - // file.path = file.path.replace(".mkv", ".mp4"); - // format = ".mp4"; - // } - // console.log('crossOriginIsolated', crossOriginIsolated) - // props.setDuration(recordingTimer * 100) - - // console.log('Loading ffmpeg-core.js'); - // const ffmpeg = createFFmpeg({ log: true }); - // await ffmpeg.load(); - // console.log('ffmpeg-core.js loaded'); - - // let allVideoChunks: any = []; - // const inputPaths: string[] = []; - // // write each segment into it's indexed file - // videos.forEach(async (vid, i) => { - // const vidName = `segvideo${i}.mkv` - // inputPaths.push(vidName) - // const videoFile = new File(vid.videoChunks, vidName, { type: allVideoChunks[0].type, lastModified: Date.now() }); - // ffmpeg.FS('writeFile', vidName, await fetchFile(videoFile)); - // // }) - // ffmpeg.FS('writeFile', 'order.txt', inputPaths.join('\n')); - - // console.log('concat') - // await ffmpeg.run('-f', 'concat', '-safe', '0', '-i', 'order.txt', 'ouput.mp4'); - - // const { buffer } = ffmpeg.FS('readFile', 'output.mp4'); - // const concatVideo = new File([buffer], 'concat.mp4', { type: "video/mp4" }); - - // const data = await Networking.UploadFilesToServer(concatVideo) - // const result = data[0].result - // if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox - // props.setResult(result, trackScreen) - // } else { - // alert("video conversion failed"); - // } - - // // delete all files in MEMFS - // inputPaths.forEach(path => ffmpeg.FS('unlink', path)); - // ffmpeg.FS('unlink', 'order.txt'); - // ffmpeg.FS('unlink', 'output.mp4'); + const result: Upload.AccessPathInfo | Error = await Networking.PostToServer('/concatVideos', serverPaths) + console.log(result) + if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox + props.setResult(result, trackScreen) + } else { + alert("video conversion failed"); + } + })(); } @@ -162,8 +91,6 @@ export function RecordingView(props: IRecordingViewProps) { }, [videos]) useEffect(() => { - - console.log('in finish useEffect') if (finished) { setOrderVideos(true); @@ -177,7 +104,7 @@ export function RecordingView(props: IRecordingViewProps) { if (!navigator.mediaDevices) { console.log('This browser does not support getUserMedia.') } - console.log('This device has the correct media devices.') + // console.log('This device has the correct media devices.') }, []) useEffect(() => { @@ -244,7 +171,6 @@ export function RecordingView(props: IRecordingViewProps) { // reset the temporary chunks videoChunks = [] setRecording(false); - setFinished(true); trackScreen && RecordingApi.Instance.pause(); } @@ -269,8 +195,10 @@ export function RecordingView(props: IRecordingViewProps) { } - const stop = () => { - if (videoRecorder.current) { + const stop = (e: React.MouseEvent) => { + e.stopPropagation() + if (videoRecorder.current) { + setFinished(true); if (videoRecorder.current.state !== "inactive") { videoRecorder.current.stop(); // recorder.current.stream.getTracks().forEach((track: any) => track.stop()) @@ -278,19 +206,21 @@ export function RecordingView(props: IRecordingViewProps) { } } - const pause = () => { + const pause = (e: React.MouseEvent) => { + e.stopPropagation() if (videoRecorder.current) { if (videoRecorder.current.state === "recording") { - videoRecorder.current.pause(); + videoRecorder.current.stop(); } } } - const startOrResume = () => { + const startOrResume = (e: React.MouseEvent) => { + e.stopPropagation() if (!videoRecorder.current || videoRecorder.current.state === "inactive") { record(); } else if (videoRecorder.current.state === "paused") { - videoRecorder.current.resume(); + videoRecorder.current.start(); } } @@ -315,16 +245,6 @@ export function RecordingView(props: IRecordingViewProps) { const seconds = Math.floor((milliseconds % (1000 * 60)) / 1000); return toTwoDigit(minutes) + " : " + toTwoDigit(seconds); } - - const doTranscode = async () => { - - // console.log('Start transcoding'); - // ffmpeg.FS('writeFile', 'test.avi', await fetchFile('/flame.avi')); - // await ffmpeg.run('-i', 'test.avi', 'test.mp4'); - // console.log('Complete transcoding'); - // const data = ffmpeg.FS('readFile', 'test.mp4'); - // console.log(URL.createObjectURL(new Blob([data.buffer], { type: 'video/mp4' }))); - }; return (
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 398b007b5..0ee0a34df 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -44,25 +44,10 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, - subscription: "/uploadVideosandConcatenate", - secureHandler: async ({ req, res }) => { - const form = new formidable.IncomingForm(); - form.keepExtensions = true; - form.uploadDir = pathToDirectory(Directory.parsed_files); - return new Promise(resolve => { - form.parse(req, async (_err, _fields, files) => { - const results: Upload.FileResponse[] = []; - - // create an array of all the file paths - const filePaths: string[] = Object.keys(files).map(key => files[key].path); - console.log("uploading files", Array.isArray(filePaths)); - const result = await DashUploadUtils.concatenateVideos(filePaths); - console.log('concatenated', result); - result && !(result.result instanceof Error) && results.push(result); - _success(res, results) - resolve(); - }); - }); + subscription: "/concatVideos", + secureHandler: async ({ req, res }) => { + // req.body contains the array of server paths to the videos + _success(res, await DashUploadUtils.concatVideos(req.body)); } }); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 148b0df65..be30c115d 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -63,21 +63,20 @@ export namespace DashUploadUtils { const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia; //TODO:glr - export async function concatenateVideos(filePaths: string[]): Promise { + export async function concatVideos(filePaths: string[]): Promise { // make a list of paths to create the ordered text file for ffmpeg - //const filePaths = videoFiles.map(file => file.path); - // write the text file to the file system const inputListName = 'concat.txt'; const textFilePath = path.join(filesDirectory, inputListName); // make a list of paths to create the ordered text file for ffmpeg - const filePathsText = filePaths.map(filePath => `file '${filePath}'`).join('\n'); + const filePathsText = filePaths.map(filePath => `file '${filePath}'`).join('\n'); + // write the text file to the file system writeFile(textFilePath, filePathsText, (err) => console.log(err)); - console.log(filePathsText) + console.log('fileTextPaths', filePathsText) // make output file name based on timestamp - const outputFileName = `output-${Utils.GenerateGuid()}.mkv`; - // create the output file path in the parsed_file directory - const outputFilePath = path.join(filesDirectory, outputFileName); + const outputFileName = `output-${Utils.GenerateGuid()}.mp4`; + // create the output file path in the videos directory + const outputFilePath = path.join(pathToDirectory(Directory.videos), outputFileName); // concatenate the videos await new Promise((resolve, reject) => { @@ -92,45 +91,15 @@ export namespace DashUploadUtils { .on("end", resolve); }) - // delete concat.txt from the file system - unlinkSync(textFilePath); - - // read the output file from the file system - // const outputFile = fs.readFileSync(outputFilePath); - - // make a new blob object with the output file buffer - // const outputFileBlob = new Blob([outputFile.buffer], { type: 'x-matroska/mkv' }); - - // TODO: make with toJSON() + // delete concat.txt from the file system + unlinkSync(textFilePath); + // delete the old segment videos from the server + filePaths.forEach(filePath => unlinkSync(filePath)); - // make a data object - const outputFileObject: formidable.File = { - size: 0, - name: outputFileName, - path: outputFilePath, - // size: outputFileBlob.size, - type: 'video/x-matroska;codecs=avc1,opus', - lastModifiedDate: new Date(), - - toJSON: () => ({ ...outputFileObject, filename: outputFilePath.replace(/.*\//, ""), mtime: null, length: 0, mime: "", toJson: () => undefined as any }) + // return the path(s) to the output file + return { + accessPaths: getAccessPaths(Directory.videos, outputFileName) } - - // const file = { ...outputFileObject, toJSON: () => ({ ...outputFileObject, filename: outputFilePath.replace(/.*\//, ""), mtime: null, length: 0, mime: "", toJson: () => undefined as any }) }; - - // this will convert it to mp4 and save it to the server - //return await MoveParsedFile(outputFileObject, Directory.videos); - - return await upload(outputFileObject); - - // // return only the output (first) file to the videos directory - // return { - // source: file, result: { - // accessPaths: { - // agnostic: getAccessPaths(Directory.videos, outputFileName) - // }, - // rawText: undefined - // } - // } } export function uploadYoutube(videoId: string): Promise { -- cgit v1.2.3-70-g09d2 From 1688af3de54419029773fb85b78bca4500f7f0de Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 10 Jun 2022 15:41:38 -0400 Subject: big bug problems with recording. some reason interacting with the canvas is making the recording inactive - i think :/ --- src/client/util/RecordingApi.ts | 153 +++++++++++++-------- .../views/nodes/RecordingBox/ProgressBar.tsx | 8 +- .../views/nodes/RecordingBox/RecordingBox.tsx | 12 +- .../views/nodes/RecordingBox/RecordingView.tsx | 99 ++++++++----- 4 files changed, 166 insertions(+), 106 deletions(-) (limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx') diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts index 021feee9a..ab6935e3b 100644 --- a/src/client/util/RecordingApi.ts +++ b/src/client/util/RecordingApi.ts @@ -3,8 +3,6 @@ import { IReactionDisposer, observable, reaction } from "mobx"; import { NumCast } from "../../fields/Types"; import { Doc } from "../../fields/Doc"; import { VideoBox } from "../views/nodes/VideoBox"; -import { scaleDiverging } from "d3-scale"; -import { Transform } from "./Transform"; type Movement = { time: number, @@ -13,17 +11,17 @@ type Movement = { scale: number, } -type Presentation = { - movements: Array | null - meta: Object, +export type Presentation = { + movements: Movement[] | null, + totalTime: number, + meta: any, } export class RecordingApi { - private static NULL_PRESENTATION: Presentation = { - movements: null, - meta: {}, - } + private static get NULL_PRESENTATION(): Presentation { + return { movements: null, meta: {}, totalTime: -1,} + } // instance variables private currentPresentation: Presentation; @@ -50,6 +48,8 @@ export class RecordingApi { // for now, set playFFView this.playFFView = null; this.timers = null; + + // put a pointerdown event on the doucment to see what the target } // little helper :) @@ -59,8 +59,9 @@ export class RecordingApi { public start = (meta?: Object): Error | undefined => { // check if already init a presentation - if (!this.isInitPresenation) { - console.error('[recordingApi.ts] start() failed: current presentation data exists. please call clear() first.') + if (!this.isInitPresenation) { + console.log(this.currentPresentation) + console.trace('[recordingApi.ts] start() failed: current presentation data exists. please call clear() first.') return new Error('[recordingApi.ts] start()') } @@ -80,50 +81,61 @@ export class RecordingApi { this.isRecording = true } - public clear = (): Error | Presentation => { + /* stops the video and returns the presentatation; if no presentation, returns undefined */ + public getPresentation = (): undefined | Presentation => { // TODO: maybe archive the data? - if (this.isRecording) { - console.error('[recordingApi.ts] clear() failed: currently recording presentation. call pause() first') - return new Error('[recordingApi.ts] clear()') - } - - // update the presentation mode - Doc.UserDoc().presentationMode = 'none' - // set the previus recording view to the play view - this.playFFView = this.recordingFFView - - const presCopy = { ...this.currentPresentation } - - // clear presenation data - this.currentPresentation = RecordingApi.NULL_PRESENTATION - // clear isRecording - this.isRecording = false - // clear absoluteStart - this.absoluteStart = -1 - // clear the disposeFunc - this.removeRecordingFFView() - - return presCopy; + if (this.isRecording) console.warn('[recordingApi.ts] getPresentation() : currently recording presentation.'); + + // update the presentation mode + Doc.UserDoc().presentationMode = 'none'; + // set the previus recording view to the play view + this.playFFView = this.recordingFFView; + + // ensure we add the endTime now that they are done recording + return { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart }; + } + + public stop = (): void => { + // make is recording false + this.isRecording = false + } + + public clear = (): void => { + // clear presenation data + this.currentPresentation = RecordingApi.NULL_PRESENTATION + // clear isRecording + this.isRecording = false + // clear absoluteStart + this.absoluteStart = -1 + // clear the disposeFunc + this.removeRecordingFFView() + } + + + // call on dispose function to stop tracking movements + public removeRecordingFFView = (): void => { + this.disposeFunc?.(); + this.disposeFunc = null; } - public pause = (): Error | undefined => { - if (this.isInitPresenation) { - console.error('[recordingApi.ts] pause() failed: no presentation started. try calling init() first') - return new Error('[recordingApi.ts] pause(): no presentation') - } - // don't allow track movments - this.isRecording = false - - // set adjust absoluteStart to add the time difference - const timestamp = new Date().getTime() - this.absoluteStart = timestamp - this.absoluteStart - } + // public pause = (): Error | undefined => { + // if (this.isInitPresenation) { + // console.error('[recordingApi.ts] pause() failed: no presentation started. try calling init() first') + // return new Error('[recordingApi.ts] pause(): no presentation') + // } + // // don't allow track movments + // this.isRecording = false + + // // set adjust absoluteStart to add the time difference + // const timestamp = new Date().getTime() + // this.absoluteStart = timestamp - this.absoluteStart + // } - public resume = () => { - this.isRecording = true - // set absoluteStart to the difference in time - this.absoluteStart = new Date().getTime() - this.absoluteStart - } + // public resume = () => { + // this.isRecording = true + // // set absoluteStart to the difference in time + // this.absoluteStart = new Date().getTime() - this.absoluteStart + // } private trackMovements = (panX: number, panY: number, scale: number = 0): Error | undefined => { // ensure we are recording @@ -134,6 +146,8 @@ export class RecordingApi { if (this.isInitPresenation) { return new Error('[recordingApi.ts] trackMovements(): no presentation') } + + console.log('track movment') // get the time const time = new Date().getTime() - this.absoluteStart @@ -164,12 +178,6 @@ export class RecordingApi { this.recordingFFView = view; } - // call on dispose function to stop tracking movements - public removeRecordingFFView = (): void => { - this.disposeFunc?.(); - this.disposeFunc = null; - } - // TODO: extract this into different class with pause and resume recording // TODO: store the FFview with the movements private playFFView: CollectionFreeFormView | null; @@ -250,6 +258,37 @@ export class RecordingApi { }, timeDiff) }) } + + // make a public method that concatenates the movements of the an array of presentations into one array + // TODO: consider the meta data of the presentations + public concatPresentations = (presentations: Presentation[]): Presentation => { + console.info(presentations); + if (presentations.length === 0) return RecordingApi.NULL_PRESENTATION; + const firstPresentation = presentations[0]; + + let sumTime = presentations[0].totalTime; + let combinedPresentations = { ...firstPresentation } + presentations.forEach((presentation, i) => { + // already consider the first presentation + if (i === 0) return; + + const { movements, totalTime } = presentation; + if (movements === null) return; + + // add the summed time to the movements + const addedTimeMovements = movements.map(move => { return { ...move, time: move.time + sumTime } }); + // concat the movements already in the combined presentation with these new ones + const newMovements = [...combinedPresentations.movements || [], ...addedTimeMovements]; + + combinedPresentations = { ...combinedPresentations, movements: newMovements } + + // update the totalTime + sumTime += totalTime; + }); + + // return the combined presentation with the updated total summed time + return { ...combinedPresentations, totalTime: sumTime }; + } // Unfinished code for tracing multiple free form views // export let pres: Map = new Map() diff --git a/src/client/views/nodes/RecordingBox/ProgressBar.tsx b/src/client/views/nodes/RecordingBox/ProgressBar.tsx index 493069394..1bb2b7c84 100644 --- a/src/client/views/nodes/RecordingBox/ProgressBar.tsx +++ b/src/client/views/nodes/RecordingBox/ProgressBar.tsx @@ -161,12 +161,12 @@ export function ProgressBar(props: ProgressBarProps) { } // pointerdown event for the progress bar - const onPointerDown = (e: React.PointerEvent) => { - // don't move the videobox element - e.stopPropagation() + const onPointerDown = (e: React.PointerEvent) => { + // don't move the videobox element + e.stopPropagation(); // if recording, do nothing - if (props.recording) return; + if (props.recording) return; // get the segment the user clicked on to be dragged const clickedSegment = e.target as HTMLDivElement & EventTarget diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx index 5e97e3eb5..6fe67b6db 100644 --- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx @@ -8,8 +8,8 @@ import { FieldView } from "../FieldView"; import { VideoBox } from "../VideoBox"; import { RecordingView } from './RecordingView'; import { DocumentType } from "../../../documents/DocumentTypes"; -import { RecordingApi } from "../../../util/RecordingApi"; -import { Doc, FieldsSym } from "../../../../fields/Doc"; +import { Presentation } from "../../../util/RecordingApi"; +import { Doc } from "../../../../fields/Doc"; import { Id } from "../../../../fields/FieldSymbols"; @@ -21,7 +21,7 @@ export class RecordingBox extends ViewBoxBaseComponent() { private _ref: React.RefObject = React.createRef(); constructor(props: any) { - super(props); + super(props); } componentDidMount() { @@ -38,7 +38,7 @@ export class RecordingBox extends ViewBoxBaseComponent() { } @action - setResult = (info: Upload.AccessPathInfo, trackScreen: boolean) => { + setResult = (info: Upload.AccessPathInfo, presentation?: Presentation) => { this.result = info this.dataDoc.type = DocumentType.VID; this.dataDoc[this.fieldKey + "-duration"] = this.videoDuration; @@ -47,9 +47,7 @@ export class RecordingBox extends ViewBoxBaseComponent() { this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client); this.dataDoc[this.fieldKey + "-recorded"] = true; // stringify the presenation and store it - if (trackScreen) { - this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(RecordingApi.Instance.clear()); - } + presentation?.movements && (this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(presentation)); } render() { diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx index 138e72274..208eaf45a 100644 --- a/src/client/views/nodes/RecordingBox/RecordingView.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx @@ -8,16 +8,17 @@ import { IconContext } from "react-icons"; import { Networking } from '../../../Network'; import { Upload } from '../../../../server/SharedMediaTypes'; import { returnFalse, returnTrue, setupMoveUpEvents } from '../../../../Utils'; -import { RecordingApi } from '../../../util/RecordingApi'; +import { Presentation, RecordingApi } from '../../../util/RecordingApi'; export interface MediaSegment { videoChunks: any[], endTime: number, - startTime: number + startTime: number, + presenation?: Presentation, } interface IRecordingViewProps { - setResult: (info: Upload.AccessPathInfo, trackScreen: boolean) => void + setResult: (info: Upload.AccessPathInfo, presentation?: Presentation) => void setDuration: (seconds: number) => void id: string } @@ -58,13 +59,18 @@ export function RecordingView(props: IRecordingViewProps) { sampleRate: 44100 } } + + useEffect(() => console.info('progess', progress), [progress]) useEffect(() => { - if (finished) { + if (finished) { + // make the total presentation that'll match the concatted video + const concatPres = trackScreen ? RecordingApi.Instance.concatPresentations(videos.map(v => v.presenation as Presentation)) : undefined; + // this async function uses the server to create the concatted video and then sets the result to it's accessPaths (async () => { - const videoFiles = videos.map((vid, i) => new File(vid.videoChunks, `segvideo${i}.mkv`, { type: vid.videoChunks[0].type, lastModified: Date.now() })); + const videoFiles = videos.map((vid, i) => new File(vid.videoChunks, `segvideo${i}.mkv`, { type: vid.videoChunks[0].type, lastModified: Date.now() })); // upload the segments to the server and get their server access paths const serverPaths: string[] = (await Networking.UploadFilesToServer(videoFiles)) @@ -72,7 +78,7 @@ export function RecordingView(props: IRecordingViewProps) { // concat the segments together using post call const result: Upload.AccessPathInfo | Error = await Networking.PostToServer('/concatVideos', serverPaths); - !(result instanceof Error) ? props.setResult(result, trackScreen) : console.error("video conversion failed"); + !(result instanceof Error) ? props.setResult(result, concatPres) : console.error("video conversion failed"); })(); } }, [videos]) @@ -133,53 +139,70 @@ export function RecordingView(props: IRecordingViewProps) { trackScreen && RecordingApi.Instance.start(); } - videoRecorder.current.onstop = () => { + videoRecorder.current.onstop = () => { + RecordingApi.Instance.stop(); // if we have a last portion - if (videoChunks.length > 1) { + if (videoChunks.length > 1) { // append the current portion to the video pieces - setVideos(videos => [...videos, { videoChunks: videoChunks, endTime: recordingTimerRef.current, startTime: videos?.lastElement()?.endTime || 0 }]) + setVideos(videos => [...videos, { + videoChunks: videoChunks, + endTime: recordingTimerRef.current, + startTime: videos?.lastElement()?.endTime || 0, + // RecordingApi.stop() will return undefined if no track screen + presenation: RecordingApi.Instance.getPresentation() + }]) + // now that we got the presentation data, we can clear for the next segment to be recorded + RecordingApi.Instance.clear(); } - // reset the temporary chunks - videoChunks = [] - setRecording(false); - trackScreen && RecordingApi.Instance.pause(); + // reset the temporary chunks + videoChunks = [] + setRecording(false); } videoRecorder.current.start(200) } - const stop = (e: React.PointerEvent) => { - e.stopPropagation(); - if (videoRecorder.current) { - if (videoRecorder.current.state !== "inactive") { - videoRecorder.current.stop(); - } - - // this will call upon progessbar to update videos to be in the correct order - setFinished(true); - - // end the streams (audio/video) to remove recording icon - const stream = videoElementRef.current!.srcObject; - stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop()); - } + const finish = (e: React.PointerEvent) => { + e.stopPropagation(); + console.log('finish', videoRecorder.current) + // if inactive, then we're done recording all the segments + if (videoRecorder.current && videoRecorder.current.state !== "inactive") { + console.log('stopping recorder', videoRecorder.current?.state) + + + + // call stop on the video recorder + videoRecorder.current?.stop(); + + // end the streams (audio/video) to remove recording icon + const stream = videoElementRef.current!.srcObject; + stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop()); + + // clear the recoringApi - this is done in the stop method + // RecordingApi.Instance.clear(); + + // this will call upon progessbar to update videos to be in the correct order + console.log('setFinished to true', finished) + setFinished(true); + } } const pause = (e: React.PointerEvent) => { - e.stopPropagation() - if (videoRecorder.current) { - if (videoRecorder.current.state === "recording") { - videoRecorder.current.stop(); - } - } + e.stopPropagation() + // if recording, then this is just a new segment + videoRecorder.current?.state === "recording" && videoRecorder.current.stop(); } - const startOrResume = (e: React.PointerEvent) => { + const start = (e: React.PointerEvent) => { // the code to start or resume does not get triggered if we start dragging the button setupMoveUpEvents({}, e, returnTrue, returnFalse, e => { - (!videoRecorder.current || videoRecorder.current.state === "inactive") && record(); - return true; // cancels propagation to documentView to avoid selecting it. + if (!videoRecorder.current || videoRecorder.current.state === "inactive") { + record(); + // trackScreen && RecordingApi.Instance.start(); + } + return true; // cancels propagation to documentView to avoid selecting it. }, false, false); } @@ -218,7 +241,7 @@ export function RecordingView(props: IRecordingViewProps) {
{recording ?
@@ -229,7 +252,7 @@ export function RecordingView(props: IRecordingViewProps) { - +
-- cgit v1.2.3-70-g09d2 From d91ff1f937e9588b88258b07738eea56bac5fcca Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 13 Jun 2022 13:21:38 -0400 Subject: fix bugs with selecting tracking. very stable now. --- src/client/util/RecordingApi.ts | 408 ++++++++++----------- .../views/nodes/RecordingBox/RecordingBox.tsx | 2 +- .../views/nodes/RecordingBox/RecordingView.tsx | 18 +- 3 files changed, 208 insertions(+), 220 deletions(-) (limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx') diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts index 12c1654a2..d56093cee 100644 --- a/src/client/util/RecordingApi.ts +++ b/src/client/util/RecordingApi.ts @@ -5,10 +5,10 @@ import { Doc } from "../../fields/Doc"; import { VideoBox } from "../views/nodes/VideoBox"; type Movement = { - time: number, - panX: number, - panY: number, - scale: number, + time: number, + panX: number, + panY: number, + scale: number, } export type Presentation = { @@ -20,78 +20,82 @@ export type Presentation = { export class RecordingApi { private static get NULL_PRESENTATION(): Presentation { - return { movements: null, meta: {}, totalTime: -1,} + return { movements: null, meta: {}, totalTime: -1, } } - // instance variables - private currentPresentation: Presentation; - private tracking: boolean; - private absoluteStart: number; - - - // create static instance and getter for global use - @observable static _instance: RecordingApi; - public static get Instance(): RecordingApi { return RecordingApi._instance } - public constructor() { - // init the global instance - RecordingApi._instance = this; - - // init the instance variables - this.currentPresentation = RecordingApi.NULL_PRESENTATION - this.tracking = false; - this.absoluteStart = -1; - - // used for tracking movements in the view frame - this.disposeFunc = null; - this.recordingFFView = null; - - // for now, set playFFView - this.playFFView = null; - this.timers = null; - - // put a pointerdown event on the doucment to see what the target - } + // instance variables + private currentPresentation: Presentation; + private tracking: boolean; + private absoluteStart: number; - // little helper :) - private get isInitPresenation(): boolean { - return this.currentPresentation.movements === null - } - public start = (meta?: Object) => { - // update the presentation mode - Doc.UserDoc().presentationMode = 'recording'; - - // (1a) get start date for presenation - const startDate = new Date(); - // (1b) set start timestamp to absolute timestamp - this.absoluteStart = startDate.getTime(); - - // (2) assign meta content if it exists - this.currentPresentation.meta = meta || {} - // (3) assign start date to currentPresenation - this.currentPresentation.movements = [] - // (4) set tracking true to allow trackMovements - this.tracking = true - } + // create static instance and getter for global use + @observable static _instance: RecordingApi; + public static get Instance(): RecordingApi { return RecordingApi._instance } + public constructor() { + // init the global instance + RecordingApi._instance = this; + + // init the instance variables + this.currentPresentation = RecordingApi.NULL_PRESENTATION + this.tracking = false; + this.absoluteStart = -1; + + // used for tracking movements in the view frame + this.disposeFunc = null; + this.recordingFFView = null; - /* stops the video and returns the presentatation; if no presentation, returns undefined */ - public * yieldPresentation(clearData: boolean = true): Generator { + // for now, set playFFView + this.playFFView = null; + this.timers = null; + + // put a pointerdown event on the doucment to see what the target + } + + // little helper :) + private get nullPresentation(): boolean { + return this.currentPresentation.movements === null + } + + public start = (meta?: Object) => { + // update the presentation mode + Doc.UserDoc().presentationMode = 'recording'; + + // (1a) get start date for presenation + const startDate = new Date(); + // (1b) set start timestamp to absolute timestamp + this.absoluteStart = startDate.getTime(); + + // (2) assign meta content if it exists + this.currentPresentation.meta = meta || {} + // (3) assign start date to currentPresenation + this.currentPresentation.movements = [] + // (4) set tracking true to allow trackMovements + this.tracking = true + } + + /* stops the video and returns the presentatation; if no presentation, returns undefined */ + public yieldPresentation(clearData: boolean = true): Presentation | null { // if no presentation or done tracking, return null - if (!this.isInitPresenation || !this.tracking) return null; + if (this.nullPresentation || !this.tracking) return null; // set the previus recording view to the play view this.playFFView = this.recordingFFView; // ensure we add the endTime now that they are done recording - yield { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart }; + const cpy = { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart }; // reset the current presentation clearData && this.clear(); + + return cpy; } - public stop = (): void => { + public finish = (): void => { // make is tracking false this.tracking = false + // reset the RecordingApi instance + this.clear(); } public clear = (): void => { @@ -108,155 +112,138 @@ export class RecordingApi { // clear absoluteStart this.absoluteStart = -1 } - - // call on dispose function to stop tracking movements - public removeRecordingFFView = (): void => { - this.disposeFunc?.(); - this.disposeFunc = null; - } - // public pause = (): Error | undefined => { - // if (this.isInitPresenation) { - // console.error('[recordingApi.ts] pause() failed: no presentation started. try calling init() first') - // return new Error('[recordingApi.ts] pause(): no presentation') - // } - // // don't allow track movments - // this.tracking = false - - // // set adjust absoluteStart to add the time difference - // const timestamp = new Date().getTime() - // this.absoluteStart = timestamp - this.absoluteStart - // } - - // public resume = () => { - // this.tracking = true - // // set absoluteStart to the difference in time - // this.absoluteStart = new Date().getTime() - this.absoluteStart - // } - - private trackMovements = (panX: number, panY: number, scale: number = 0): Error | undefined => { - // ensure we are recording - if (!this.tracking) { - return new Error('[recordingApi.ts] trackMovements()') - } - // check to see if the presetation is init - if (this.isInitPresenation) { - return new Error('[recordingApi.ts] trackMovements(): no presentation') - } - - // TO FIX: bob - // console.debug('track movment') - - // get the time - const time = new Date().getTime() - this.absoluteStart - // make new movement object - const movement: Movement = { time, panX, panY, scale } - - // add that movement to the current presentation data's movement array - this.currentPresentation.movements && this.currentPresentation.movements.push(movement) - } + // call on dispose function to stop tracking movements + public removeRecordingFFView = (): void => { + this.disposeFunc?.(); + this.disposeFunc = null; + } - // instance variable for the FFView - private disposeFunc: IReactionDisposer | null; - private recordingFFView: CollectionFreeFormView | null; - - // set the FFView that will be used in a reaction to track the movements - public setRecordingFFView = (view: CollectionFreeFormView): void => { - // set the view to the current view - if (view === this.recordingFFView || view == null) return; - - // this.recordingFFView = view; - // set the reaction to track the movements - this.disposeFunc = reaction( - () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }), - (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale) - ) - - // for now, set the most recent recordingFFView to the playFFView - this.recordingFFView = view; + private trackMovements = (panX: number, panY: number, scale: number = 0) => { + // ensure we are recording + if (!this.tracking) { + console.error('[recordingApi.ts] trackMovements(): tracking is false') + return; } + // check to see if the presetation is init + if (this.nullPresentation) { + console.error('[recordingApi.ts] trackMovements(): no presentation') + return; + } + + // TO FIX: bob + // console.debug('track movment') + + // get the time + const time = new Date().getTime() - this.absoluteStart + // make new movement object + const movement: Movement = { time, panX, panY, scale } + + // add that movement to the current presentation data's movement array + this.currentPresentation.movements && this.currentPresentation.movements.push(movement) + } - // TODO: extract this into different class with pause and resume recording - // TODO: store the FFview with the movements - private playFFView: CollectionFreeFormView | null; - private timers: NodeJS.Timeout[] | null; + // instance variable for the FFView + private disposeFunc: IReactionDisposer | null; + private recordingFFView: CollectionFreeFormView | null; - public setPlayFFView = (view: CollectionFreeFormView): void => { - this.playFFView = view + // set the FFView that will be used in a reaction to track the movements + public setRecordingFFView = (view: CollectionFreeFormView): void => { + // set the view to the current view + if (view === this.recordingFFView || view == null) return; + + // this.recordingFFView = view; + // set the reaction to track the movements + this.disposeFunc = reaction( + () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }), + (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale) + ) + + // for now, set the most recent recordingFFView to the playFFView + this.recordingFFView = view; + } + + // TODO: extract this into different class with pause and resume recording + // TODO: store the FFview with the movements + private playFFView: CollectionFreeFormView | null; + private timers: NodeJS.Timeout[] | null; + + public setPlayFFView = (view: CollectionFreeFormView): void => { + this.playFFView = view + } + + // pausing movements will dispose all timers that are planned to replay the movements + // play movemvents will recreate them when the user resumes the presentation + public pauseMovements = (): undefined | Error => { + if (this.playFFView === null) { + return new Error('[recordingApi.ts] pauseMovements() failed: no view') } - // pausing movements will dispose all timers that are planned to replay the movements - // play movemvents will recreate them when the user resumes the presentation - public pauseMovements = (): undefined | Error => { - if (this.playFFView === null) { - return new Error('[recordingApi.ts] pauseMovements() failed: no view') - } - - if (!this._isPlaying) { - //return new Error('[recordingApi.ts] pauseMovements() failed: not playing') - return - } - this._isPlaying = false - // TODO: set userdoc presentMode to browsing - this.timers?.map(timer => clearTimeout(timer)) - - // this.videoBox = null; + if (!this._isPlaying) { + //return new Error('[recordingApi.ts] pauseMovements() failed: not playing') + return } - - private videoBox: VideoBox | null = null; + this._isPlaying = false + // TODO: set userdoc presentMode to browsing + this.timers?.map(timer => clearTimeout(timer)) + + // this.videoBox = null; + } - // by calling pause on the VideoBox, the pauseMovements will be called - public pauseVideoAndMovements = (): boolean => { - this.videoBox?.Pause() + private videoBox: VideoBox | null = null; + + // by calling pause on the VideoBox, the pauseMovements will be called + public pauseVideoAndMovements = (): boolean => { + this.videoBox?.Pause() + + this.pauseMovements() + return this.videoBox == null + } - this.pauseMovements() - return this.videoBox == null + public _isPlaying = false; + + public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => { + if (presentation.movements === null || this.playFFView === null) { + return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view') } + if (this._isPlaying) return; + + this._isPlaying = true; + Doc.UserDoc().presentationMode = 'watching'; + + // TODO: consider this bug at the end of the clip on seek + this.videoBox = videoBox || null; - public _isPlaying = false; - - public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => { - if (presentation.movements === null || this.playFFView === null) { - return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view') - } - if(this._isPlaying) return; - - this._isPlaying = true; - Doc.UserDoc().presentationMode = 'watching'; - - // TODO: consider this bug at the end of the clip on seek - this.videoBox = videoBox || null; - - // only get the movements that are remaining in the video time left - const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000) - - // helper to replay a movement - const document = this.playFFView - let preScale = -1; - const zoomAndPan = (movement: Movement) => { - const { panX, panY, scale } = movement; - (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0); - document.Document._panX = panX; - document.Document._panY = panY; - - preScale = scale; - } - - // set the first frame to be at the start of the pres - zoomAndPan(filteredMovements[0]); - - // make timers that will execute each movement at the correct replay time - this.timers = filteredMovements.map(movement => { - const timeDiff = movement.time - timeViewed*1000 - return setTimeout(() => { - // replay the movement - zoomAndPan(movement) - // if last movement, presentation is done -> set the instance var - if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false; - }, timeDiff) - }) + // only get the movements that are remaining in the video time left + const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000) + + // helper to replay a movement + const document = this.playFFView + let preScale = -1; + const zoomAndPan = (movement: Movement) => { + const { panX, panY, scale } = movement; + (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0); + document.Document._panX = panX; + document.Document._panY = panY; + + preScale = scale; } - + + // set the first frame to be at the start of the pres + zoomAndPan(filteredMovements[0]); + + // make timers that will execute each movement at the correct replay time + this.timers = filteredMovements.map(movement => { + const timeDiff = movement.time - timeViewed * 1000 + return setTimeout(() => { + // replay the movement + zoomAndPan(movement) + // if last movement, presentation is done -> set the instance var + if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false; + }, timeDiff) + }) + } + // method that concatenates an array of presentatations into one public concatPresentations = (presentations: Presentation[]): Presentation => { // these three will lead to the combined presentation @@ -266,6 +253,7 @@ export class RecordingApi { presentations.forEach((presentation) => { const { movements, totalTime, meta } = presentation; + // update movements if they had one if (movements) { // add the summed time to the movements @@ -273,8 +261,10 @@ export class RecordingApi { // concat the movements already in the combined presentation with these new ones combinedMovements.push(...addedTimeMovements); } + // update the totalTime sumTime += totalTime; + // concatenate the metas combinedMetas.push(...meta); }); @@ -283,19 +273,19 @@ export class RecordingApi { return { movements: combinedMovements, totalTime: sumTime, meta: combinedMetas }; } - // Unfinished code for tracing multiple free form views - // export let pres: Map = new Map() - - // export function AddRecordingFFView(ffView: CollectionFreeFormView): void { - // pres.set(ffView, - // reaction(() => ({ x: ffView.panX, y: ffView.panY }), - // (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y))) - // ) - // } - - // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void { - // const disposer = pres.get(ffView); - // disposer?.(); - // pres.delete(ffView) - // } + // Unfinished code for tracing multiple free form views + // export let pres: Map = new Map() + + // export function AddRecordingFFView(ffView: CollectionFreeFormView): void { + // pres.set(ffView, + // reaction(() => ({ x: ffView.panX, y: ffView.panY }), + // (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y))) + // ) + // } + + // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void { + // const disposer = pres.get(ffView); + // disposer?.(); + // pres.delete(ffView) + // } } diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx index 6fe67b6db..a28677525 100644 --- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx @@ -46,7 +46,7 @@ export class RecordingBox extends ViewBoxBaseComponent() { this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey); this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client); this.dataDoc[this.fieldKey + "-recorded"] = true; - // stringify the presenation and store it + // stringify the presentation and store it presentation?.movements && (this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(presentation)); } diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx index 35a6aa07e..15883632a 100644 --- a/src/client/views/nodes/RecordingBox/RecordingView.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx @@ -140,7 +140,6 @@ export function RecordingView(props: IRecordingViewProps) { } videoRecorder.current.onstop = () => { - // RecordingApi.Instance.stop(); // if we have a last portion if (videoChunks.length > 1) { // append the current portion to the video pieces @@ -149,9 +148,10 @@ export function RecordingView(props: IRecordingViewProps) { endTime: recordingTimerRef.current, startTime: videos?.lastElement()?.endTime || 0 }; + // depending on if a presenation exists, add it to the video - const { done: presError, value: presentation } = RecordingApi.Instance.yieldPresentation().next(); - setVideos(videos => [...videos, (!presError && trackScreen) ? { ...nextVideo, presentation } : nextVideo]); + const presentation = RecordingApi.Instance.yieldPresentation(); + setVideos(videos => [...videos, (presentation != null && trackScreen) ? { ...nextVideo, presentation } : nextVideo]); } // reset the temporary chunks @@ -174,8 +174,8 @@ export function RecordingView(props: IRecordingViewProps) { const stream = videoElementRef.current!.srcObject; stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop()); - // clear the recoringApi - RecordingApi.Instance.clear(); + // finish/clear the recoringApi + RecordingApi.Instance.finish(); // this will call upon progessbar to update videos to be in the correct order setFinished(true); @@ -188,12 +188,10 @@ export function RecordingView(props: IRecordingViewProps) { } const start = (e: React.PointerEvent) => { - // the code to start or resume does not get triggered if we start dragging the button setupMoveUpEvents({}, e, returnTrue, returnFalse, e => { - if (!videoRecorder.current || videoRecorder.current.state === "inactive") { - record(); - // trackScreen && - } + // start recording if not already recording + if (!videoRecorder.current || videoRecorder.current.state === "inactive") record(); + return true; // cancels propagation to documentView to avoid selecting it. }, false, false); } -- cgit v1.2.3-70-g09d2 From f8bfda0b0139db5f8bf1de602ce211a091d5d9ef Mon Sep 17 00:00:00 2001 From: Michael Foiani Date: Mon, 13 Jun 2022 14:56:03 -0400 Subject: revert to previous stable commit. lmao get ratioed by a mouse --- src/client/util/RecordingApi.ts | 383 ++++++++++++--------- .../views/nodes/RecordingBox/RecordingBox.tsx | 2 +- .../views/nodes/RecordingBox/RecordingView.tsx | 18 +- 3 files changed, 223 insertions(+), 180 deletions(-) (limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx') diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts index bcd21a48a..4c8f55ce5 100644 --- a/src/client/util/RecordingApi.ts +++ b/src/client/util/RecordingApi.ts @@ -5,10 +5,10 @@ import { Doc } from "../../fields/Doc"; import { VideoBox } from "../views/nodes/VideoBox"; type Movement = { - time: number, - panX: number, - panY: number, - scale: number, + time: number, + panX: number, + panY: number, + scale: number, } export type Presentation = { @@ -20,9 +20,10 @@ export type Presentation = { export class RecordingApi { private static get NULL_PRESENTATION(): Presentation { - return { movements: null, meta: {}, totalTime: -1, } + return { movements: null, meta: {}, totalTime: -1,} } +<<<<<<< HEAD // instance variables private currentPresentation: Presentation; private tracking: boolean; @@ -49,51 +50,77 @@ export class RecordingApi { this.playFFView = null; this.timers = null; } +======= + // instance variables + private currentPresentation: Presentation; + private tracking: boolean; + private absoluteStart: number; + + + // create static instance and getter for global use + @observable static _instance: RecordingApi; + public static get Instance(): RecordingApi { return RecordingApi._instance } + public constructor() { + // init the global instance + RecordingApi._instance = this; + + // init the instance variables + this.currentPresentation = RecordingApi.NULL_PRESENTATION + this.tracking = false; + this.absoluteStart = -1; + + // used for tracking movements in the view frame + this.disposeFunc = null; + this.recordingFFView = null; + + // for now, set playFFView + this.playFFView = null; + this.timers = null; + + // put a pointerdown event on the doucment to see what the target + } +>>>>>>> parent of d91ff1f93 (fix bugs with selecting tracking. very stable now.) - // little helper :) - private get nullPresentation(): boolean { - return this.currentPresentation.movements === null - } + // little helper :) + private get isInitPresenation(): boolean { + return this.currentPresentation.movements === null + } - public start = (meta?: Object) => { - // update the presentation mode - Doc.UserDoc().presentationMode = 'recording'; - - // (1a) get start date for presenation - const startDate = new Date(); - // (1b) set start timestamp to absolute timestamp - this.absoluteStart = startDate.getTime(); - - // (2) assign meta content if it exists - this.currentPresentation.meta = meta || {} - // (3) assign start date to currentPresenation - this.currentPresentation.movements = [] - // (4) set tracking true to allow trackMovements - this.tracking = true - } + public start = (meta?: Object) => { + // update the presentation mode + Doc.UserDoc().presentationMode = 'recording'; + + // (1a) get start date for presenation + const startDate = new Date(); + // (1b) set start timestamp to absolute timestamp + this.absoluteStart = startDate.getTime(); + + // (2) assign meta content if it exists + this.currentPresentation.meta = meta || {} + // (3) assign start date to currentPresenation + this.currentPresentation.movements = [] + // (4) set tracking true to allow trackMovements + this.tracking = true + } - /* stops the video and returns the presentatation; if no presentation, returns undefined */ - public yieldPresentation(clearData: boolean = true): Presentation | null { + /* stops the video and returns the presentatation; if no presentation, returns undefined */ + public * yieldPresentation(clearData: boolean = true): Generator { // if no presentation or done tracking, return null - if (this.nullPresentation || !this.tracking) return null; + if (!this.isInitPresenation || !this.tracking) return null; // set the previus recording view to the play view this.playFFView = this.recordingFFView; // ensure we add the endTime now that they are done recording - const cpy = { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart }; + yield { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart }; // reset the current presentation clearData && this.clear(); - - return cpy; } - public finish = (): void => { + public stop = (): void => { // make is tracking false this.tracking = false - // reset the RecordingApi instance - this.clear(); } public clear = (): void => { @@ -110,138 +137,155 @@ export class RecordingApi { // clear absoluteStart this.absoluteStart = -1 } - - // call on dispose function to stop tracking movements - public removeRecordingFFView = (): void => { - this.disposeFunc?.(); - this.disposeFunc = null; - } - - private trackMovements = (panX: number, panY: number, scale: number = 0) => { - // ensure we are recording - if (!this.tracking) { - console.error('[recordingApi.ts] trackMovements(): tracking is false') - return; - } - // check to see if the presetation is init - if (this.nullPresentation) { - console.error('[recordingApi.ts] trackMovements(): no presentation') - return; + + // call on dispose function to stop tracking movements + public removeRecordingFFView = (): void => { + this.disposeFunc?.(); + this.disposeFunc = null; } - // TO FIX: bob - // console.debug('track movment') - - // get the time - const time = new Date().getTime() - this.absoluteStart - // make new movement object - const movement: Movement = { time, panX, panY, scale } - - // add that movement to the current presentation data's movement array - this.currentPresentation.movements && this.currentPresentation.movements.push(movement) - } - - // instance variable for the FFView - private disposeFunc: IReactionDisposer | null; - private recordingFFView: CollectionFreeFormView | null; - - // set the FFView that will be used in a reaction to track the movements - public setRecordingFFView = (view: CollectionFreeFormView): void => { - // set the view to the current view - if (view === this.recordingFFView || view == null) return; - - // this.recordingFFView = view; - // set the reaction to track the movements - this.disposeFunc = reaction( - () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }), - (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale) - ) - - // for now, set the most recent recordingFFView to the playFFView - this.recordingFFView = view; - } - - // TODO: extract this into different class with pause and resume recording - // TODO: store the FFview with the movements - private playFFView: CollectionFreeFormView | null; - private timers: NodeJS.Timeout[] | null; - - public setPlayFFView = (view: CollectionFreeFormView): void => { - this.playFFView = view - } - - // pausing movements will dispose all timers that are planned to replay the movements - // play movemvents will recreate them when the user resumes the presentation - public pauseMovements = (): undefined | Error => { - if (this.playFFView === null) { - return new Error('[recordingApi.ts] pauseMovements() failed: no view') + // public pause = (): Error | undefined => { + // if (this.isInitPresenation) { + // console.error('[recordingApi.ts] pause() failed: no presentation started. try calling init() first') + // return new Error('[recordingApi.ts] pause(): no presentation') + // } + // // don't allow track movments + // this.tracking = false + + // // set adjust absoluteStart to add the time difference + // const timestamp = new Date().getTime() + // this.absoluteStart = timestamp - this.absoluteStart + // } + + // public resume = () => { + // this.tracking = true + // // set absoluteStart to the difference in time + // this.absoluteStart = new Date().getTime() - this.absoluteStart + // } + + private trackMovements = (panX: number, panY: number, scale: number = 0): Error | undefined => { + // ensure we are recording + if (!this.tracking) { + return new Error('[recordingApi.ts] trackMovements()') + } + // check to see if the presetation is init + if (this.isInitPresenation) { + return new Error('[recordingApi.ts] trackMovements(): no presentation') + } + + // TO FIX: bob + // console.debug('track movment') + + // get the time + const time = new Date().getTime() - this.absoluteStart + // make new movement object + const movement: Movement = { time, panX, panY, scale } + + // add that movement to the current presentation data's movement array + this.currentPresentation.movements && this.currentPresentation.movements.push(movement) } - if (!this._isPlaying) { - //return new Error('[recordingApi.ts] pauseMovements() failed: not playing') - return + // instance variable for the FFView + private disposeFunc: IReactionDisposer | null; + private recordingFFView: CollectionFreeFormView | null; + + // set the FFView that will be used in a reaction to track the movements + public setRecordingFFView = (view: CollectionFreeFormView): void => { + // set the view to the current view + if (view === this.recordingFFView || view == null) return; + + // this.recordingFFView = view; + // set the reaction to track the movements + this.disposeFunc = reaction( + () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }), + (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale) + ) + + // for now, set the most recent recordingFFView to the playFFView + this.recordingFFView = view; } - this._isPlaying = false - // TODO: set userdoc presentMode to browsing - this.timers?.map(timer => clearTimeout(timer)) - - // this.videoBox = null; - } - - private videoBox: VideoBox | null = null; - - // by calling pause on the VideoBox, the pauseMovements will be called - public pauseVideoAndMovements = (): boolean => { - this.videoBox?.Pause() - - this.pauseMovements() - return this.videoBox == null - } - public _isPlaying = false; + // TODO: extract this into different class with pause and resume recording + // TODO: store the FFview with the movements + private playFFView: CollectionFreeFormView | null; + private timers: NodeJS.Timeout[] | null; - public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => { - if (presentation.movements === null || this.playFFView === null) { - return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view') + public setPlayFFView = (view: CollectionFreeFormView): void => { + this.playFFView = view } - if (this._isPlaying) return; - this._isPlaying = true; - Doc.UserDoc().presentationMode = 'watching'; - - // TODO: consider this bug at the end of the clip on seek - this.videoBox = videoBox || null; - - // only get the movements that are remaining in the video time left - const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000) + // pausing movements will dispose all timers that are planned to replay the movements + // play movemvents will recreate them when the user resumes the presentation + public pauseMovements = (): undefined | Error => { + if (this.playFFView === null) { + return new Error('[recordingApi.ts] pauseMovements() failed: no view') + } + + if (!this._isPlaying) { + //return new Error('[recordingApi.ts] pauseMovements() failed: not playing') + return + } + this._isPlaying = false + // TODO: set userdoc presentMode to browsing + this.timers?.map(timer => clearTimeout(timer)) + + // this.videoBox = null; + } + + private videoBox: VideoBox | null = null; - // helper to replay a movement - const document = this.playFFView - let preScale = -1; - const zoomAndPan = (movement: Movement) => { - const { panX, panY, scale } = movement; - (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0); - document.Document._panX = panX; - document.Document._panY = panY; + // by calling pause on the VideoBox, the pauseMovements will be called + public pauseVideoAndMovements = (): boolean => { + this.videoBox?.Pause() - preScale = scale; + this.pauseMovements() + return this.videoBox == null } - // set the first frame to be at the start of the pres - zoomAndPan(filteredMovements[0]); - - // make timers that will execute each movement at the correct replay time - this.timers = filteredMovements.map(movement => { - const timeDiff = movement.time - timeViewed * 1000 - return setTimeout(() => { - // replay the movement - zoomAndPan(movement) - // if last movement, presentation is done -> set the instance var - if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false; - }, timeDiff) - }) - } - + public _isPlaying = false; + + public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => { + if (presentation.movements === null || this.playFFView === null) { + return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view') + } + if(this._isPlaying) return; + + this._isPlaying = true; + Doc.UserDoc().presentationMode = 'watching'; + + // TODO: consider this bug at the end of the clip on seek + this.videoBox = videoBox || null; + + // only get the movements that are remaining in the video time left + const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000) + + // helper to replay a movement + const document = this.playFFView + let preScale = -1; + const zoomAndPan = (movement: Movement) => { + const { panX, panY, scale } = movement; + (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0); + document.Document._panX = panX; + document.Document._panY = panY; + + preScale = scale; + } + + // set the first frame to be at the start of the pres + zoomAndPan(filteredMovements[0]); + + // make timers that will execute each movement at the correct replay time + this.timers = filteredMovements.map(movement => { + const timeDiff = movement.time - timeViewed*1000 + return setTimeout(() => { + // replay the movement + zoomAndPan(movement) + // if last movement, presentation is done -> set the instance var + if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false; + }, timeDiff) + }) + } + // method that concatenates an array of presentatations into one public concatPresentations = (presentations: Presentation[]): Presentation => { // these three will lead to the combined presentation @@ -251,7 +295,6 @@ export class RecordingApi { presentations.forEach((presentation) => { const { movements, totalTime, meta } = presentation; - // update movements if they had one if (movements) { // add the summed time to the movements @@ -259,10 +302,8 @@ export class RecordingApi { // concat the movements already in the combined presentation with these new ones combinedMovements.push(...addedTimeMovements); } - // update the totalTime sumTime += totalTime; - // concatenate the metas combinedMetas.push(...meta); }); @@ -271,19 +312,19 @@ export class RecordingApi { return { movements: combinedMovements, totalTime: sumTime, meta: combinedMetas }; } - // Unfinished code for tracing multiple free form views - // export let pres: Map = new Map() - - // export function AddRecordingFFView(ffView: CollectionFreeFormView): void { - // pres.set(ffView, - // reaction(() => ({ x: ffView.panX, y: ffView.panY }), - // (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y))) - // ) - // } - - // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void { - // const disposer = pres.get(ffView); - // disposer?.(); - // pres.delete(ffView) - // } + // Unfinished code for tracing multiple free form views + // export let pres: Map = new Map() + + // export function AddRecordingFFView(ffView: CollectionFreeFormView): void { + // pres.set(ffView, + // reaction(() => ({ x: ffView.panX, y: ffView.panY }), + // (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y))) + // ) + // } + + // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void { + // const disposer = pres.get(ffView); + // disposer?.(); + // pres.delete(ffView) + // } } diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx index a28677525..6fe67b6db 100644 --- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx @@ -46,7 +46,7 @@ export class RecordingBox extends ViewBoxBaseComponent() { this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey); this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client); this.dataDoc[this.fieldKey + "-recorded"] = true; - // stringify the presentation and store it + // stringify the presenation and store it presentation?.movements && (this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(presentation)); } diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx index d4d19f3d8..ba9479f41 100644 --- a/src/client/views/nodes/RecordingBox/RecordingView.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx @@ -141,6 +141,7 @@ export function RecordingView(props: IRecordingViewProps) { } videoRecorder.current.onstop = () => { + // RecordingApi.Instance.stop(); // if we have a last portion if (videoChunks.length > 1) { // append the current portion to the video pieces @@ -149,10 +150,9 @@ export function RecordingView(props: IRecordingViewProps) { endTime: recordingTimerRef.current, startTime: videos?.lastElement()?.endTime || 0 }; - // depending on if a presenation exists, add it to the video - const presentation = RecordingApi.Instance.yieldPresentation(); - setVideos(videos => [...videos, (presentation != null && trackScreen) ? { ...nextVideo, presentation } : nextVideo]); + const { done: presError, value: presentation } = RecordingApi.Instance.yieldPresentation().next(); + setVideos(videos => [...videos, (!presError && trackScreen) ? { ...nextVideo, presentation } : nextVideo]); } // reset the temporary chunks @@ -175,8 +175,8 @@ export function RecordingView(props: IRecordingViewProps) { const stream = videoElementRef.current!.srcObject; stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop()); - // finish/clear the recoringApi - RecordingApi.Instance.finish(); + // clear the recoringApi + RecordingApi.Instance.clear(); // this will call upon progessbar to update videos to be in the correct order setFinished(true); @@ -189,10 +189,12 @@ export function RecordingView(props: IRecordingViewProps) { } const start = (e: React.PointerEvent) => { + // the code to start or resume does not get triggered if we start dragging the button setupMoveUpEvents({}, e, returnTrue, returnFalse, e => { - // start recording if not already recording - if (!videoRecorder.current || videoRecorder.current.state === "inactive") record(); - + if (!videoRecorder.current || videoRecorder.current.state === "inactive") { + record(); + // trackScreen && + } return true; // cancels propagation to documentView to avoid selecting it. }, false, false); } -- cgit v1.2.3-70-g09d2 From c7e4bf8c4e19669666cce610020969cf0d865e49 Mon Sep 17 00:00:00 2001 From: Michael Foiani Date: Mon, 13 Jun 2022 14:57:34 -0400 Subject: Revert "debug w bob" This reverts commit 1dfb694ffa8f34a834387eb073ffb9bd3a678039. return to stable version without printing the logs. --- src/client/util/RecordingApi.ts | 385 +++++++++------------ .../views/nodes/RecordingBox/RecordingBox.tsx | 2 +- .../views/nodes/RecordingBox/RecordingView.tsx | 29 +- 3 files changed, 186 insertions(+), 230 deletions(-) (limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx') diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts index 4c8f55ce5..d56093cee 100644 --- a/src/client/util/RecordingApi.ts +++ b/src/client/util/RecordingApi.ts @@ -5,10 +5,10 @@ import { Doc } from "../../fields/Doc"; import { VideoBox } from "../views/nodes/VideoBox"; type Movement = { - time: number, - panX: number, - panY: number, - scale: number, + time: number, + panX: number, + panY: number, + scale: number, } export type Presentation = { @@ -20,10 +20,9 @@ export type Presentation = { export class RecordingApi { private static get NULL_PRESENTATION(): Presentation { - return { movements: null, meta: {}, totalTime: -1,} + return { movements: null, meta: {}, totalTime: -1, } } -<<<<<<< HEAD // instance variables private currentPresentation: Presentation; private tracking: boolean; @@ -49,78 +48,54 @@ export class RecordingApi { // for now, set playFFView this.playFFView = null; this.timers = null; + + // put a pointerdown event on the doucment to see what the target } -======= - // instance variables - private currentPresentation: Presentation; - private tracking: boolean; - private absoluteStart: number; - - - // create static instance and getter for global use - @observable static _instance: RecordingApi; - public static get Instance(): RecordingApi { return RecordingApi._instance } - public constructor() { - // init the global instance - RecordingApi._instance = this; - - // init the instance variables - this.currentPresentation = RecordingApi.NULL_PRESENTATION - this.tracking = false; - this.absoluteStart = -1; - - // used for tracking movements in the view frame - this.disposeFunc = null; - this.recordingFFView = null; - - // for now, set playFFView - this.playFFView = null; - this.timers = null; - - // put a pointerdown event on the doucment to see what the target - } ->>>>>>> parent of d91ff1f93 (fix bugs with selecting tracking. very stable now.) - // little helper :) - private get isInitPresenation(): boolean { - return this.currentPresentation.movements === null - } + // little helper :) + private get nullPresentation(): boolean { + return this.currentPresentation.movements === null + } - public start = (meta?: Object) => { - // update the presentation mode - Doc.UserDoc().presentationMode = 'recording'; - - // (1a) get start date for presenation - const startDate = new Date(); - // (1b) set start timestamp to absolute timestamp - this.absoluteStart = startDate.getTime(); - - // (2) assign meta content if it exists - this.currentPresentation.meta = meta || {} - // (3) assign start date to currentPresenation - this.currentPresentation.movements = [] - // (4) set tracking true to allow trackMovements - this.tracking = true - } + public start = (meta?: Object) => { + // update the presentation mode + Doc.UserDoc().presentationMode = 'recording'; + + // (1a) get start date for presenation + const startDate = new Date(); + // (1b) set start timestamp to absolute timestamp + this.absoluteStart = startDate.getTime(); + + // (2) assign meta content if it exists + this.currentPresentation.meta = meta || {} + // (3) assign start date to currentPresenation + this.currentPresentation.movements = [] + // (4) set tracking true to allow trackMovements + this.tracking = true + } - /* stops the video and returns the presentatation; if no presentation, returns undefined */ - public * yieldPresentation(clearData: boolean = true): Generator { + /* stops the video and returns the presentatation; if no presentation, returns undefined */ + public yieldPresentation(clearData: boolean = true): Presentation | null { // if no presentation or done tracking, return null - if (!this.isInitPresenation || !this.tracking) return null; + if (this.nullPresentation || !this.tracking) return null; // set the previus recording view to the play view this.playFFView = this.recordingFFView; // ensure we add the endTime now that they are done recording - yield { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart }; + const cpy = { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart }; // reset the current presentation clearData && this.clear(); + + return cpy; } - public stop = (): void => { + public finish = (): void => { // make is tracking false this.tracking = false + // reset the RecordingApi instance + this.clear(); } public clear = (): void => { @@ -137,155 +112,138 @@ export class RecordingApi { // clear absoluteStart this.absoluteStart = -1 } - - // call on dispose function to stop tracking movements - public removeRecordingFFView = (): void => { - this.disposeFunc?.(); - this.disposeFunc = null; - } - // public pause = (): Error | undefined => { - // if (this.isInitPresenation) { - // console.error('[recordingApi.ts] pause() failed: no presentation started. try calling init() first') - // return new Error('[recordingApi.ts] pause(): no presentation') - // } - // // don't allow track movments - // this.tracking = false - - // // set adjust absoluteStart to add the time difference - // const timestamp = new Date().getTime() - // this.absoluteStart = timestamp - this.absoluteStart - // } - - // public resume = () => { - // this.tracking = true - // // set absoluteStart to the difference in time - // this.absoluteStart = new Date().getTime() - this.absoluteStart - // } - - private trackMovements = (panX: number, panY: number, scale: number = 0): Error | undefined => { - // ensure we are recording - if (!this.tracking) { - return new Error('[recordingApi.ts] trackMovements()') - } - // check to see if the presetation is init - if (this.isInitPresenation) { - return new Error('[recordingApi.ts] trackMovements(): no presentation') - } - - // TO FIX: bob - // console.debug('track movment') - - // get the time - const time = new Date().getTime() - this.absoluteStart - // make new movement object - const movement: Movement = { time, panX, panY, scale } - - // add that movement to the current presentation data's movement array - this.currentPresentation.movements && this.currentPresentation.movements.push(movement) - } + // call on dispose function to stop tracking movements + public removeRecordingFFView = (): void => { + this.disposeFunc?.(); + this.disposeFunc = null; + } - // instance variable for the FFView - private disposeFunc: IReactionDisposer | null; - private recordingFFView: CollectionFreeFormView | null; - - // set the FFView that will be used in a reaction to track the movements - public setRecordingFFView = (view: CollectionFreeFormView): void => { - // set the view to the current view - if (view === this.recordingFFView || view == null) return; - - // this.recordingFFView = view; - // set the reaction to track the movements - this.disposeFunc = reaction( - () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }), - (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale) - ) - - // for now, set the most recent recordingFFView to the playFFView - this.recordingFFView = view; + private trackMovements = (panX: number, panY: number, scale: number = 0) => { + // ensure we are recording + if (!this.tracking) { + console.error('[recordingApi.ts] trackMovements(): tracking is false') + return; + } + // check to see if the presetation is init + if (this.nullPresentation) { + console.error('[recordingApi.ts] trackMovements(): no presentation') + return; } - // TODO: extract this into different class with pause and resume recording - // TODO: store the FFview with the movements - private playFFView: CollectionFreeFormView | null; - private timers: NodeJS.Timeout[] | null; + // TO FIX: bob + // console.debug('track movment') + + // get the time + const time = new Date().getTime() - this.absoluteStart + // make new movement object + const movement: Movement = { time, panX, panY, scale } + + // add that movement to the current presentation data's movement array + this.currentPresentation.movements && this.currentPresentation.movements.push(movement) + } + + // instance variable for the FFView + private disposeFunc: IReactionDisposer | null; + private recordingFFView: CollectionFreeFormView | null; + + // set the FFView that will be used in a reaction to track the movements + public setRecordingFFView = (view: CollectionFreeFormView): void => { + // set the view to the current view + if (view === this.recordingFFView || view == null) return; + + // this.recordingFFView = view; + // set the reaction to track the movements + this.disposeFunc = reaction( + () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }), + (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale) + ) + + // for now, set the most recent recordingFFView to the playFFView + this.recordingFFView = view; + } + + // TODO: extract this into different class with pause and resume recording + // TODO: store the FFview with the movements + private playFFView: CollectionFreeFormView | null; + private timers: NodeJS.Timeout[] | null; - public setPlayFFView = (view: CollectionFreeFormView): void => { - this.playFFView = view + public setPlayFFView = (view: CollectionFreeFormView): void => { + this.playFFView = view + } + + // pausing movements will dispose all timers that are planned to replay the movements + // play movemvents will recreate them when the user resumes the presentation + public pauseMovements = (): undefined | Error => { + if (this.playFFView === null) { + return new Error('[recordingApi.ts] pauseMovements() failed: no view') } - // pausing movements will dispose all timers that are planned to replay the movements - // play movemvents will recreate them when the user resumes the presentation - public pauseMovements = (): undefined | Error => { - if (this.playFFView === null) { - return new Error('[recordingApi.ts] pauseMovements() failed: no view') - } - - if (!this._isPlaying) { - //return new Error('[recordingApi.ts] pauseMovements() failed: not playing') - return - } - this._isPlaying = false - // TODO: set userdoc presentMode to browsing - this.timers?.map(timer => clearTimeout(timer)) - - // this.videoBox = null; + if (!this._isPlaying) { + //return new Error('[recordingApi.ts] pauseMovements() failed: not playing') + return } - - private videoBox: VideoBox | null = null; + this._isPlaying = false + // TODO: set userdoc presentMode to browsing + this.timers?.map(timer => clearTimeout(timer)) + + // this.videoBox = null; + } + + private videoBox: VideoBox | null = null; + + // by calling pause on the VideoBox, the pauseMovements will be called + public pauseVideoAndMovements = (): boolean => { + this.videoBox?.Pause() + + this.pauseMovements() + return this.videoBox == null + } - // by calling pause on the VideoBox, the pauseMovements will be called - public pauseVideoAndMovements = (): boolean => { - this.videoBox?.Pause() + public _isPlaying = false; - this.pauseMovements() - return this.videoBox == null + public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => { + if (presentation.movements === null || this.playFFView === null) { + return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view') } + if (this._isPlaying) return; - public _isPlaying = false; - - public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => { - if (presentation.movements === null || this.playFFView === null) { - return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view') - } - if(this._isPlaying) return; - - this._isPlaying = true; - Doc.UserDoc().presentationMode = 'watching'; - - // TODO: consider this bug at the end of the clip on seek - this.videoBox = videoBox || null; - - // only get the movements that are remaining in the video time left - const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000) - - // helper to replay a movement - const document = this.playFFView - let preScale = -1; - const zoomAndPan = (movement: Movement) => { - const { panX, panY, scale } = movement; - (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0); - document.Document._panX = panX; - document.Document._panY = panY; - - preScale = scale; - } - - // set the first frame to be at the start of the pres - zoomAndPan(filteredMovements[0]); - - // make timers that will execute each movement at the correct replay time - this.timers = filteredMovements.map(movement => { - const timeDiff = movement.time - timeViewed*1000 - return setTimeout(() => { - // replay the movement - zoomAndPan(movement) - // if last movement, presentation is done -> set the instance var - if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false; - }, timeDiff) - }) + this._isPlaying = true; + Doc.UserDoc().presentationMode = 'watching'; + + // TODO: consider this bug at the end of the clip on seek + this.videoBox = videoBox || null; + + // only get the movements that are remaining in the video time left + const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000) + + // helper to replay a movement + const document = this.playFFView + let preScale = -1; + const zoomAndPan = (movement: Movement) => { + const { panX, panY, scale } = movement; + (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0); + document.Document._panX = panX; + document.Document._panY = panY; + + preScale = scale; } - + + // set the first frame to be at the start of the pres + zoomAndPan(filteredMovements[0]); + + // make timers that will execute each movement at the correct replay time + this.timers = filteredMovements.map(movement => { + const timeDiff = movement.time - timeViewed * 1000 + return setTimeout(() => { + // replay the movement + zoomAndPan(movement) + // if last movement, presentation is done -> set the instance var + if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false; + }, timeDiff) + }) + } + // method that concatenates an array of presentatations into one public concatPresentations = (presentations: Presentation[]): Presentation => { // these three will lead to the combined presentation @@ -295,6 +253,7 @@ export class RecordingApi { presentations.forEach((presentation) => { const { movements, totalTime, meta } = presentation; + // update movements if they had one if (movements) { // add the summed time to the movements @@ -302,8 +261,10 @@ export class RecordingApi { // concat the movements already in the combined presentation with these new ones combinedMovements.push(...addedTimeMovements); } + // update the totalTime sumTime += totalTime; + // concatenate the metas combinedMetas.push(...meta); }); @@ -312,19 +273,19 @@ export class RecordingApi { return { movements: combinedMovements, totalTime: sumTime, meta: combinedMetas }; } - // Unfinished code for tracing multiple free form views - // export let pres: Map = new Map() - - // export function AddRecordingFFView(ffView: CollectionFreeFormView): void { - // pres.set(ffView, - // reaction(() => ({ x: ffView.panX, y: ffView.panY }), - // (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y))) - // ) - // } - - // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void { - // const disposer = pres.get(ffView); - // disposer?.(); - // pres.delete(ffView) - // } + // Unfinished code for tracing multiple free form views + // export let pres: Map = new Map() + + // export function AddRecordingFFView(ffView: CollectionFreeFormView): void { + // pres.set(ffView, + // reaction(() => ({ x: ffView.panX, y: ffView.panY }), + // (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y))) + // ) + // } + + // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void { + // const disposer = pres.get(ffView); + // disposer?.(); + // pres.delete(ffView) + // } } diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx index 6fe67b6db..a28677525 100644 --- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx @@ -46,7 +46,7 @@ export class RecordingBox extends ViewBoxBaseComponent() { this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey); this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client); this.dataDoc[this.fieldKey + "-recorded"] = true; - // stringify the presenation and store it + // stringify the presentation and store it presentation?.movements && (this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(presentation)); } diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx index ba9479f41..15883632a 100644 --- a/src/client/views/nodes/RecordingBox/RecordingView.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx @@ -60,7 +60,7 @@ export function RecordingView(props: IRecordingViewProps) { } } - useEffect(() => console.log('progress', progress), [progress]) + // useEffect(() => console.debug('progress', progress), [progress]) useEffect(() => { if (finished) { @@ -93,8 +93,7 @@ export function RecordingView(props: IRecordingViewProps) { useEffect(() => { let interval: any = null; if (recording) { - interval = setTimeout(() => { - console.log('update interval') + interval = setInterval(() => { setRecordingTimer(unit => unit + 1); }, 10); } else if (!recording && recordingTimer !== 0) { @@ -141,7 +140,6 @@ export function RecordingView(props: IRecordingViewProps) { } videoRecorder.current.onstop = () => { - // RecordingApi.Instance.stop(); // if we have a last portion if (videoChunks.length > 1) { // append the current portion to the video pieces @@ -150,9 +148,10 @@ export function RecordingView(props: IRecordingViewProps) { endTime: recordingTimerRef.current, startTime: videos?.lastElement()?.endTime || 0 }; + // depending on if a presenation exists, add it to the video - const { done: presError, value: presentation } = RecordingApi.Instance.yieldPresentation().next(); - setVideos(videos => [...videos, (!presError && trackScreen) ? { ...nextVideo, presentation } : nextVideo]); + const presentation = RecordingApi.Instance.yieldPresentation(); + setVideos(videos => [...videos, (presentation != null && trackScreen) ? { ...nextVideo, presentation } : nextVideo]); } // reset the temporary chunks @@ -175,8 +174,8 @@ export function RecordingView(props: IRecordingViewProps) { const stream = videoElementRef.current!.srcObject; stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop()); - // clear the recoringApi - RecordingApi.Instance.clear(); + // finish/clear the recoringApi + RecordingApi.Instance.finish(); // this will call upon progessbar to update videos to be in the correct order setFinished(true); @@ -189,12 +188,10 @@ export function RecordingView(props: IRecordingViewProps) { } const start = (e: React.PointerEvent) => { - // the code to start or resume does not get triggered if we start dragging the button setupMoveUpEvents({}, e, returnTrue, returnFalse, e => { - if (!videoRecorder.current || videoRecorder.current.state === "inactive") { - record(); - // trackScreen && - } + // start recording if not already recording + if (!videoRecorder.current || videoRecorder.current.state === "inactive") record(); + return true; // cancels propagation to documentView to avoid selecting it. }, false, false); } @@ -204,9 +201,7 @@ export function RecordingView(props: IRecordingViewProps) { setDoUndo(prev => !prev); } - const handleOnTimeUpdate = () => { - playing && setVideoProgressHelper(videoElementRef.current!.currentTime); - }; + const handleOnTimeUpdate = () => { playing && setVideoProgressHelper(videoElementRef.current!.currentTime); }; const millisecondToMinuteSecond = (milliseconds: number) => { const toTwoDigit = (digit: number) => { @@ -223,7 +218,7 @@ export function RecordingView(props: IRecordingViewProps) {