diff options
Diffstat (limited to 'src')
30 files changed, 2680 insertions, 1538 deletions
diff --git a/src/client/Network.ts b/src/client/Network.ts index bf2918734..3597e7b2b 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -4,46 +4,46 @@ import { Upload } from "../server/SharedMediaTypes"; export namespace Networking { - export async function FetchFromServer(relativeRoute: string) { - return (await fetch(relativeRoute)).text(); - } + export async function FetchFromServer(relativeRoute: string) { + return (await fetch(relativeRoute)).text(); + } - export async function PostToServer(relativeRoute: string, body?: any) { - const options = { - uri: Utils.prepend(relativeRoute), - method: "POST", - body, - json: true - }; - return requestPromise.post(options); - } + export async function PostToServer(relativeRoute: string, body?: any) { + const options = { + uri: Utils.prepend(relativeRoute), + method: "POST", + body, + json: true + }; + return requestPromise.post(options); + } - export async function UploadFilesToServer<T extends Upload.FileInformation = Upload.FileInformation>(files: File | File[]): Promise<Upload.FileResponse<T>[]> { - const formData = new FormData(); - if (Array.isArray(files)) { - if (!files.length) { - return []; + export async function UploadFilesToServer<T extends Upload.FileInformation = Upload.FileInformation>(files: File | File[]): Promise<Upload.FileResponse<T>[]> { + const formData = new FormData(); + if (Array.isArray(files)) { + if (!files.length) { + return []; + } + files.forEach(file => formData.append(Utils.GenerateGuid(), file)); + } else { + formData.append(Utils.GenerateGuid(), files); } - files.forEach(file => formData.append(Utils.GenerateGuid(), file)); - } else { - formData.append(Utils.GenerateGuid(), files); - } - const parameters = { - method: 'POST', - body: formData - }; - const response = await fetch("/uploadFormData", parameters); - return response.json(); - } + const parameters = { + method: 'POST', + body: formData + }; + const response = await fetch("/uploadFormData", parameters); + return response.json(); + } - export async function UploadYoutubeToServer<T extends Upload.FileInformation = Upload.FileInformation>(videoId: string): Promise<Upload.FileResponse<T>[]> { - const parameters = { - method: 'POST', - body: JSON.stringify({ videoId }), - json: true - }; - const response = await fetch("/uploadYoutubeVideo", parameters); - return response.json(); - } + export async function UploadYoutubeToServer<T extends Upload.FileInformation = Upload.FileInformation>(videoId: string): Promise<Upload.FileResponse<T>[]> { + const parameters = { + method: 'POST', + body: JSON.stringify({ videoId }), + json: true + }; + const response = await fetch("/uploadYoutubeVideo", parameters); + return response.json(); + } }
\ No newline at end of file diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index 161dff6e0..ca942a38a 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -9,6 +9,7 @@ export enum DocumentType { KVP = "kvp", VID = "video", AUDIO = "audio", + REC = "recording", PDF = "pdf", INK = "inks", SCREENSHOT = "screenshot", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 144a3d9cf..088593871 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -11,7 +11,7 @@ import { RichTextField } from "../../fields/RichTextField"; import { SchemaHeaderField } from "../../fields/SchemaHeaderField"; import { ComputedField, ScriptField } from "../../fields/ScriptField"; import { Cast, NumCast, StrCast } from "../../fields/Types"; -import { AudioField, ImageField, MapField, PdfField, VideoField, WebField, YoutubeField } from "../../fields/URLField"; +import { AudioField, ImageField, MapField, PdfField, RecordingField, VideoField, WebField, YoutubeField } from "../../fields/URLField"; import { SharingPermissions } from "../../fields/util"; import { Upload } from "../../server/SharedMediaTypes"; import { OmitKeys, Utils, aggregateBounds } from "../../Utils"; @@ -61,6 +61,7 @@ import { DashWebRTCVideo } from "../views/webcam/DashWebRTCVideo"; import { DocumentType } from "./DocumentTypes"; import { IconProp } from "@fortawesome/fontawesome-svg-core"; import { MapBox } from "../views/nodes/MapBox/MapBox"; +import { RecordingBox } from "../views/nodes/RecordingBox/RecordingBox"; const defaultNativeImageDim = Number(DFLT_IMAGE_NATIVE_DIM.replace("px", "")); class EmptyBox { @@ -410,6 +411,10 @@ export namespace Docs { layout: { view: AudioBox, dataField: defaultDataKey }, options: { _height: 100, backgroundColor: "lightGray", links: "@links(self)" } }], + [DocumentType.REC, { + layout: { view: VideoBox, dataField: defaultDataKey }, + options: { _height: 100, backgroundColor: "pink", links: "@links(self)" } + }], [DocumentType.PDF, { layout: { view: PDFBox, dataField: defaultDataKey }, options: { _curPage: 1, _fitWidth: true, nativeDimModifiable: true, nativeHeightUnfrozen: true, links: "@links(self)" } @@ -477,7 +482,7 @@ export namespace Docs { options: { hideLinkButton: true, _width: 40, _height: 40, borderRounding: "100%", links: "@links(self)" }, }], [DocumentType.WEBCAM, { - layout: { view: DashWebRTCVideo, dataField: defaultDataKey }, + layout: { view: RecordingBox, dataField: defaultDataKey }, options: { links: "@links(self)" } }], [DocumentType.PRESELEMENT, { @@ -711,6 +716,10 @@ export namespace Docs { { ...options, backgroundColor: ComputedField.MakeFunction("this._mediaState === 'playing' ? 'green':'gray'") as any }); } + export function RecordingDocument(url: string, options: DocumentOptions = {}) { + return InstanceFromProto(Prototypes.get(DocumentType.REC), "", options); + } + export function SearchDocument(options: DocumentOptions = {}) { return InstanceFromProto(Prototypes.get(DocumentType.SEARCH), new List<Doc>([]), options); } @@ -1174,6 +1183,9 @@ export namespace DocUtils { } else if (field instanceof AudioField) { created = Docs.Create.AudioDocument((field).url.href, resolved); layout = AudioBox.LayoutString; + } else if (field instanceof RecordingField) { + created = Docs.Create.RecordingDocument((field).url.href, resolved); + layout = RecordingBox.LayoutString; } else if (field instanceof InkField) { created = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), (field).inkData, resolved); layout = InkingStroke.LayoutString; diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index a325122bc..b1329e349 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -271,8 +271,8 @@ export class CurrentUserUtils { doc.emptyScreenshot = Docs.Create.ScreenshotDocument({ ...standardOps(), title: "empty screenshot", _width: 400, _height: 200 }); } if (doc.emptyWall === undefined) { - doc.emptyWall = Docs.Create.ScreenshotDocument({ ...standardOps(), title: "screen snapshot", _width: 400, _height: 200, }); - (doc.emptyWall as Doc).videoWall = true; + doc.emptyWall = Docs.Create.WebCamDocument("", { _width: 400, _height: 200, title: "recording", system: true, cloneFieldFilter: new List<string>(["system"]) }); + (doc.emptyWall as Doc).recording = true; } if (doc.emptyAudio === undefined) { doc.emptyAudio = Docs.Create.AudioDocument(nullAudio, { ...standardOps(), title: "audio recording", x: 200, y: 200, _width: 200, _height: 100, }); diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts new file mode 100644 index 000000000..021feee9a --- /dev/null +++ b/src/client/util/RecordingApi.ts @@ -0,0 +1,269 @@ +import { CollectionFreeFormView } from "../views/collections/collectionFreeForm"; +import { IReactionDisposer, observable, reaction } from "mobx"; +import { NumCast } from "../../fields/Types"; +import { Doc } from "../../fields/Doc"; +import { VideoBox } from "../views/nodes/VideoBox"; +import { scaleDiverging } from "d3-scale"; +import { Transform } from "./Transform"; + +type Movement = { + time: number, + panX: number, + panY: number, + scale: number, +} + +type Presentation = { + movements: Array<Movement> | null + meta: Object, +} + +export class RecordingApi { + + private static NULL_PRESENTATION: Presentation = { + movements: null, + meta: {}, + } + + // instance variables + private currentPresentation: Presentation; + private isRecording: boolean; + private absoluteStart: number; + + + // create static instance and getter for global use + @observable static _instance: RecordingApi; + public static get Instance(): RecordingApi { return RecordingApi._instance } + public constructor() { + // init the global instance + RecordingApi._instance = this; + + // init the instance variables + this.currentPresentation = RecordingApi.NULL_PRESENTATION + this.isRecording = false; + this.absoluteStart = -1; + + // used for tracking movements in the view frame + this.disposeFunc = null; + this.recordingFFView = null; + + // for now, set playFFView + this.playFFView = null; + this.timers = null; + } + + // little helper :) + private get isInitPresenation(): boolean { + return this.currentPresentation.movements === null + } + + public start = (meta?: Object): Error | undefined => { + // check if already init a presentation + if (!this.isInitPresenation) { + console.error('[recordingApi.ts] start() failed: current presentation data exists. please call clear() first.') + return new Error('[recordingApi.ts] start()') + } + + // update the presentation mode + Doc.UserDoc().presentationMode = 'recording' + + // (1a) get start date for presenation + const startDate = new Date() + // (1b) set start timestamp to absolute timestamp + this.absoluteStart = startDate.getTime() + + // (2) assign meta content if it exists + this.currentPresentation.meta = meta || {} + // (3) assign start date to currentPresenation + this.currentPresentation.movements = [] + // (4) set isRecording true to allow trackMovements + this.isRecording = true + } + + public clear = (): Error | Presentation => { + // TODO: maybe archive the data? + if (this.isRecording) { + console.error('[recordingApi.ts] clear() failed: currently recording presentation. call pause() first') + return new Error('[recordingApi.ts] clear()') + } + + // update the presentation mode + Doc.UserDoc().presentationMode = 'none' + // set the previus recording view to the play view + this.playFFView = this.recordingFFView + + const presCopy = { ...this.currentPresentation } + + // clear presenation data + this.currentPresentation = RecordingApi.NULL_PRESENTATION + // clear isRecording + this.isRecording = false + // clear absoluteStart + this.absoluteStart = -1 + // clear the disposeFunc + this.removeRecordingFFView() + + return presCopy; + } + + public pause = (): Error | undefined => { + if (this.isInitPresenation) { + console.error('[recordingApi.ts] pause() failed: no presentation started. try calling init() first') + return new Error('[recordingApi.ts] pause(): no presentation') + } + // don't allow track movments + this.isRecording = false + + // set adjust absoluteStart to add the time difference + const timestamp = new Date().getTime() + this.absoluteStart = timestamp - this.absoluteStart + } + + public resume = () => { + this.isRecording = true + // set absoluteStart to the difference in time + this.absoluteStart = new Date().getTime() - this.absoluteStart + } + + private trackMovements = (panX: number, panY: number, scale: number = 0): Error | undefined => { + // ensure we are recording + if (!this.isRecording) { + return new Error('[recordingApi.ts] trackMovements()') + } + // check to see if the presetation is init + if (this.isInitPresenation) { + return new Error('[recordingApi.ts] trackMovements(): no presentation') + } + + // get the time + const time = new Date().getTime() - this.absoluteStart + // make new movement object + const movement: Movement = { time, panX, panY, scale } + + // add that movement to the current presentation data's movement array + this.currentPresentation.movements && this.currentPresentation.movements.push(movement) + } + + // instance variable for the FFView + private disposeFunc: IReactionDisposer | null; + private recordingFFView: CollectionFreeFormView | null; + + // set the FFView that will be used in a reaction to track the movements + public setRecordingFFView = (view: CollectionFreeFormView): void => { + // set the view to the current view + if (view === this.recordingFFView || view == null) return; + + // this.recordingFFView = view; + // set the reaction to track the movements + this.disposeFunc = reaction( + () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }), + (res) => (res.x !== -1 && res.y !== -1 && this.isRecording) && this.trackMovements(res.x, res.y, res.scale) + ) + + // for now, set the most recent recordingFFView to the playFFView + this.recordingFFView = view; + } + + // call on dispose function to stop tracking movements + public removeRecordingFFView = (): void => { + this.disposeFunc?.(); + this.disposeFunc = null; + } + + // TODO: extract this into different class with pause and resume recording + // TODO: store the FFview with the movements + private playFFView: CollectionFreeFormView | null; + private timers: NodeJS.Timeout[] | null; + + public setPlayFFView = (view: CollectionFreeFormView): void => { + this.playFFView = view + } + + // pausing movements will dispose all timers that are planned to replay the movements + // play movemvents will recreate them when the user resumes the presentation + public pauseMovements = (): undefined | Error => { + if (this.playFFView === null) { + return new Error('[recordingApi.ts] pauseMovements() failed: no view') + } + + if (!this._isPlaying) { + //return new Error('[recordingApi.ts] pauseMovements() failed: not playing') + return + } + this._isPlaying = false + // TODO: set userdoc presentMode to browsing + this.timers?.map(timer => clearTimeout(timer)) + + // this.videoBox = null; + } + + private videoBox: VideoBox | null = null; + + // by calling pause on the VideoBox, the pauseMovements will be called + public pauseVideoAndMovements = (): boolean => { + this.videoBox?.Pause() + + this.pauseMovements() + return this.videoBox == null + } + + public _isPlaying = false; + + public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => { + if (presentation.movements === null || this.playFFView === null) { + return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view') + } + if(this._isPlaying) return; + + this._isPlaying = true; + Doc.UserDoc().presentationMode = 'watching'; + + // TODO: consider this bug at the end of the clip on seek + this.videoBox = videoBox || null; + + // only get the movements that are remaining in the video time left + const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000) + + // helper to replay a movement + const document = this.playFFView + let preScale = -1; + const zoomAndPan = (movement: Movement) => { + const { panX, panY, scale } = movement; + (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0); + document.Document._panX = panX; + document.Document._panY = panY; + + preScale = scale; + } + + // set the first frame to be at the start of the pres + zoomAndPan(filteredMovements[0]); + + // make timers that will execute each movement at the correct replay time + this.timers = filteredMovements.map(movement => { + const timeDiff = movement.time - timeViewed*1000 + return setTimeout(() => { + // replay the movement + zoomAndPan(movement) + // if last movement, presentation is done -> set the instance var + if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false; + }, timeDiff) + }) + } + + // Unfinished code for tracing multiple free form views + // export let pres: Map<CollectionFreeFormView, IReactionDisposer> = new Map() + + // export function AddRecordingFFView(ffView: CollectionFreeFormView): void { + // pres.set(ffView, + // reaction(() => ({ x: ffView.panX, y: ffView.panY }), + // (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y))) + // ) + // } + + // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void { + // const disposer = pres.get(ffView); + // disposer?.(); + // pres.delete(ffView) + // } +} diff --git a/src/client/views/DocComponent.tsx b/src/client/views/DocComponent.tsx index 67f8c8585..80835447d 100644 --- a/src/client/views/DocComponent.tsx +++ b/src/client/views/DocComponent.tsx @@ -92,7 +92,6 @@ export interface ViewBoxAnnotatableProps { export function ViewBoxAnnotatableComponent<P extends ViewBoxAnnotatableProps>() { class Component extends Touchable<P> { @observable _annotationKeySuffix = () => "annotations"; - @observable _isAnyChildContentActive = false; //TODO This might be pretty inefficient if doc isn't observed, because computed doesn't cache then @computed get Document() { return this.props.Document; } diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index a118cc144..317f5f5d7 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -25,6 +25,7 @@ import { RadialMenu } from "./nodes/RadialMenu"; import HorizontalPalette from "./Palette"; import { Touchable } from "./Touchable"; import TouchScrollableMenu, { TouchScrollableMenuItem } from "./TouchScrollableMenu"; +import { InkTranscription } from "./InkTranscription"; @observer export class GestureOverlay extends Touchable { @@ -127,7 +128,7 @@ export class GestureOverlay extends Touchable { // pen is also a touch, but with a radius of 0.5 (at least with the surface pens) // and this seems to be the only way of differentiating pen and touch on touch events if (pt.radiusX > 1 && pt.radiusY > 1) { - createInkGroup(); + InkTranscription.Instance.createInkGroup(); Doc.UserDoc().activeInkTool = InkTool.None; this.prevPoints.set(pt.identifier, pt); } @@ -496,7 +497,7 @@ export class GestureOverlay extends Touchable { if (InteractionUtils.IsType(e, InteractionUtils.TOUCHTYPE)) { setupMoveUpEvents(this, e, returnFalse, returnFalse, action((e: PointerEvent, doubleTap?: boolean) => { if (doubleTap) { - createInkGroup(); + InkTranscription.Instance.createInkGroup(); CurrentUserUtils.SelectedTool = InkTool.None; return; } @@ -644,7 +645,7 @@ export class GestureOverlay extends Touchable { (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y)); if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0]; this._points = controlPoints; - this.dispatchGesture(GestureUtils.Gestures.Stroke); + this.dispatchGesture(GestureUtils.Gestures.Stroke); // TODO: nda - check inks to group here checkInksToGroup(); } diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index 0ac1770cc..ab189c0d6 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -2,14 +2,14 @@ import * as iink from 'iink-js'; import { action, observable } from 'mobx'; import * as React from 'react'; import { Doc, DocListCast, HeightSym, WidthSym } from '../../fields/Doc'; -import { InkData, InkField } from "../../fields/InkField"; +import { InkData, InkField, InkTool } from "../../fields/InkField"; import { Cast, DateCast, NumCast } from '../../fields/Types'; import { aggregateBounds } from '../../Utils'; import { DocumentType } from "../documents/DocumentTypes"; import { DocumentManager } from "../util/DocumentManager"; import { CollectionFreeFormView } from './collections/collectionFreeForm'; import { InkingStroke } from './InkingStroke'; -import './InkTranscription.scss'; +import { CurrentUserUtils } from '../util/CurrentUserUtils'; export class InkTranscription extends React.Component { @@ -42,8 +42,8 @@ export class InkTranscription extends React.Component { protocol: 'WEBSOCKET', server: { host: 'cloud.myscript.com', - applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', - hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + applicationKey: process.env.IINKJS_APP, + hmacKey: process.env.IINKJS_HMAC, websocket: { pingEnabled: false, autoReconnect: true @@ -171,53 +171,7 @@ export class InkTranscription extends React.Component { if (!docView) return; const marqViewRef = docView._marqueeViewRef.current; if (!marqViewRef) return; - // loop through selected an get the bound - const bounds: { x: number, y: number, width?: number, height?: number }[] = [] - - selected.map(action(d => { - const x = NumCast(d.x); - const y = NumCast(d.y); - const width = d[WidthSym](); - const height = d[HeightSym](); - bounds.push({ x, y, width, height }); - })) - - const aggregBounds = aggregateBounds(bounds, 0, 0); - - if (marqViewRef) { - marqViewRef._downX = aggregBounds.x; - marqViewRef._downY = aggregBounds.y; - marqViewRef._lastX = aggregBounds.r; - marqViewRef._lastY = aggregBounds.b; - } - - // set the vals for bounds in marqueeView - - selected.map(action(d => { - const dx = NumCast(d.x); - const dy = NumCast(d.y); - delete d.x; - delete d.y; - delete d.activeFrame; - delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection - delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection - // calculate pos based on bounds - if (marqViewRef?.Bounds) { - d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2; - d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2; - } - return d; - })); - - docView.props.removeDocument?.(selected); - const newCollection = marqViewRef?.getCollection(selected, undefined, true); - if (newCollection) { - newCollection.height = newCollection[HeightSym](); - newCollection.width = newCollection[WidthSym](); - newCollection.title = word; - } - // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs - newCollection && docView.props.addDocument?.(newCollection); + this.groupInkDocs(selected, docView, word); }); } @@ -280,6 +234,88 @@ export class InkTranscription extends React.Component { } } } + + /** + * Creates the ink grouping once the user leaves the writing mode + */ +createInkGroup() { + // TODO nda - if document being added to is a inkGrouping then we can just add to that group + if (CurrentUserUtils.SelectedTool === InkTool.Write) { + CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => { + // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those + const selected = ffView.unprocessedDocs; + const newCollection = this.groupInkDocs(selected, ffView); + ffView.unprocessedDocs = []; + + InkTranscription.Instance.transcribeInk(newCollection, ffView.layoutDoc, selected, false, ffView); + }); + } + CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); +} + +/** + * Creates the groupings for a given list of ink docs on a specific doc view + * @param selected: the list of ink docs to create a grouping of + * @param docView: the view in which we want the grouping to be created + * @param word: optional param if the group we are creating is a word (subgrouping individual words) + * @returns a new collection Doc or undefined if the grouping fails + */ + groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined { + const bounds: { x: number, y: number, width?: number, height?: number }[] = [] + + // calculate the necessary bounds from the selected ink docs + selected.map(action(d => { + const x = NumCast(d.x); + const y = NumCast(d.y); + const width = d[WidthSym](); + const height = d[HeightSym](); + bounds.push({ x, y, width, height }); + })) + + // calculate the aggregated bounds + const aggregBounds = aggregateBounds(bounds, 0, 0); + const marqViewRef = docView._marqueeViewRef.current; + + // set the vals for bounds in marqueeView + if (marqViewRef) { + marqViewRef._downX = aggregBounds.x; + marqViewRef._downY = aggregBounds.y; + marqViewRef._lastX = aggregBounds.r; + marqViewRef._lastY = aggregBounds.b; + } + + // map through all the selected ink strokes and create the groupings + selected.map(action(d => { + const dx = NumCast(d.x); + const dy = NumCast(d.y); + delete d.x; + delete d.y; + delete d.activeFrame; + delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection + delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection + // calculate pos based on bounds + if (marqViewRef?.Bounds) { + d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2; + d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2; + } + return d; + })); + docView.props.removeDocument?.(selected); + // Gets a collection based on the selected nodes using a marquee view ref + const newCollection = marqViewRef?.getCollection(selected, undefined, true); + if (newCollection) { + newCollection.height = newCollection[HeightSym](); + newCollection.width = newCollection[WidthSym](); + // if the grouping we are creating is an individual word + if (word) { + newCollection.title = word; + } + } + + // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs + newCollection && docView.props.addDocument?.(newCollection); + return newCollection; +} render() { return ( diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 8560ccb29..517fe097c 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -8,6 +8,7 @@ import { AssignAllExtensions } from "../../extensions/General/Extensions"; import { Docs } from "../documents/Documents"; import { CurrentUserUtils } from "../util/CurrentUserUtils"; import { LinkManager } from "../util/LinkManager"; +import { RecordingApi } from "../util/RecordingApi"; import { CollectionView } from "./collections/CollectionView"; import { MainView } from "./MainView"; @@ -36,5 +37,6 @@ AssignAllExtensions(); const expires = "expires=" + d.toUTCString(); document.cookie = `loadtime=${loading};${expires};path=/`; new LinkManager(); + new RecordingApi; ReactDOM.render(<MainView />, document.getElementById('root')); })();
\ No newline at end of file diff --git a/src/client/views/MainView.scss b/src/client/views/MainView.scss index 001c91b40..a695577d0 100644 --- a/src/client/views/MainView.scss +++ b/src/client/views/MainView.scss @@ -31,6 +31,11 @@ left: calc(100% + 5px); z-index: 1; pointer-events: none; + display: flex; + flex-direction: row; + align-items: center; + justify-content: space-between; + gap: 10px; } .mainView-snapLines { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 84a131a57..adcfd82dc 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -179,7 +179,7 @@ export class MainView extends React.Component { fa.faClone, fa.faCloudUploadAlt, fa.faCommentAlt, fa.faCompressArrowsAlt, fa.faCut, fa.faEllipsisV, fa.faEraser, fa.faExclamation, fa.faFileAlt, fa.faFileAudio, fa.faFileVideo, fa.faFilePdf, fa.faFilm, fa.faFilter, fa.faFont, fa.faGlobeAmericas, fa.faGlobeAsia, fa.faHighlighter, fa.faLongArrowAltRight, fa.faMousePointer, fa.faMusic, fa.faObjectGroup, fa.faPause, fa.faPen, fa.faPenNib, fa.faPhone, fa.faPlay, fa.faPortrait, fa.faRedoAlt, fa.faStamp, fa.faStickyNote, fa.faArrowsAltV, - fa.faTimesCircle, fa.faThumbtack, fa.faTree, fa.faTv, fa.faUndoAlt, fa.faVideo, fa.faAsterisk, fa.faBrain, fa.faImage, fa.faPaintBrush, fa.faTimes, fa.faFlag, + fa.faTimesCircle, fa.faThumbtack, fa.faTree, fa.faTv, fa.faUndoAlt, fa.faVideoSlash, fa.faVideo, fa.faAsterisk, fa.faBrain, fa.faImage, fa.faPaintBrush, fa.faTimes, fa.faFlag, fa.faEye, fa.faArrowsAlt, fa.faQuoteLeft, fa.faSortAmountDown, fa.faAlignLeft, fa.faAlignCenter, fa.faAlignRight, fa.faHeading, fa.faRulerCombined, fa.faFillDrip, fa.faLink, fa.faUnlink, fa.faBold, fa.faItalic, fa.faClipboard, fa.faUnderline, fa.faStrikethrough, fa.faSuperscript, fa.faSubscript, fa.faIndent, fa.faEyeDropper, fa.faPaintRoller, fa.faBars, fa.faBrush, fa.faShapes, fa.faEllipsisH, fa.faHandPaper, fa.faMap, fa.faUser, faHireAHelper as any, @@ -573,6 +573,7 @@ export class MainView extends React.Component { searchFilterDocs={returnEmptyDoclist} ContainingCollectionView={undefined} ContainingCollectionDoc={undefined} /> + {['watching', 'recording'].includes(String(this.userDoc?.presentationMode) ?? '') ? <div style={{ border: '.5rem solid green', padding: '5px' }}>{this.userDoc?.presentationMode}</div> : <></>} </div>; } @computed get snapLines() { diff --git a/src/client/views/OverlayView.scss b/src/client/views/OverlayView.scss index f22e68ff5..302e7a5e3 100644 --- a/src/client/views/OverlayView.scss +++ b/src/client/views/OverlayView.scss @@ -1,3 +1,13 @@ +.overlayView { + position: absolute; + pointer-events: none; + top: 0; + width: 100vw; + height: 100vh; + /* background-color: pink; */ + z-index: 100000; +} + .overlayWindow-outerDiv { border-radius: 5px; overflow: hidden; diff --git a/src/client/views/OverlayView.tsx b/src/client/views/OverlayView.tsx index f5e686473..18abcda72 100644 --- a/src/client/views/OverlayView.tsx +++ b/src/client/views/OverlayView.tsx @@ -3,7 +3,7 @@ import { observer } from "mobx-react"; import { computedFn } from "mobx-utils"; import * as React from "react"; import ReactLoading from 'react-loading'; -import { Doc, HeightSym, WidthSym } from "../../fields/Doc"; +import { Doc, WidthSym, HeightSym } from "../../fields/Doc"; import { Id } from "../../fields/FieldSymbols"; import { Cast, NumCast } from "../../fields/Types"; import { emptyFunction, returnEmptyDoclist, returnEmptyFilter, returnFalse, returnOne, returnTrue, returnZero, setupMoveUpEvents, Utils } from "../../Utils"; @@ -188,6 +188,7 @@ export class OverlayView extends React.Component { offsetx = NumCast(d.x) - e.clientX; offsety = NumCast(d.y) - e.clientY; }; + return <div className="overlayView-doc" ref={dref} key={d[Id]} onPointerDown={onPointerDown} style={{ top: d.type === 'presentation' ? 0 : undefined, width: NumCast(d._width), height: NumCast(d._height), transform: `translate(${d.x}px, ${d.y}px)` }}> <DocumentView Document={d} diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index ec3cf1fe8..f4d2d55d5 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -26,6 +26,7 @@ import { DragManager, dropActionType } from "../../../util/DragManager"; import { HistoryUtil } from "../../../util/History"; import { InteractionUtils } from "../../../util/InteractionUtils"; import { LinkManager } from "../../../util/LinkManager"; +import { RecordingApi } from "../../../util/RecordingApi"; import { ScriptingGlobals } from "../../../util/ScriptingGlobals"; import { SearchUtil } from "../../../util/SearchUtil"; import { SelectionManager } from "../../../util/SelectionManager"; @@ -57,6 +58,7 @@ import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCurso import "./CollectionFreeFormView.scss"; import { MarqueeView } from "./MarqueeView"; import React = require("react"); +import e = require("connect-flash"); export type collectionFreeformViewProps = { @@ -994,6 +996,16 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection @action setPan(panX: number, panY: number, panTime: number = 0, clamp: boolean = false) { + // set the current respective FFview to the tab being panned. + (Doc.UserDoc()?.presentationMode === 'recording') && RecordingApi.Instance.setRecordingFFView(this); + // TODO: make this based off the specific recording FFView + (Doc.UserDoc()?.presentationMode === 'none') && RecordingApi.Instance.setPlayFFView(this); + if (Doc.UserDoc()?.presentationMode === 'watching') { + RecordingApi.Instance.pauseVideoAndMovements(); + Doc.UserDoc().presentationMode = 'none'; + // RecordingApi.Instance.pauseMovements() + } + if (!this.isAnnotationOverlay && clamp) { // this section wraps the pan position, horizontally and/or vertically whenever the content is panned out of the viewing bounds const docs = this.childLayoutPairs.map(pair => pair.layout).filter(doc => doc instanceof Doc); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index 6df2bb102..051da795f 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -434,6 +434,7 @@ export class MarqueeView extends React.Component<SubCollectionViewProps & Marque })); this.props.removeDocument?.(selected); } + // TODO: nda - this is the code to actually get a new grouped collection const newCollection = this.getCollection(selected, (e as KeyboardEvent)?.key === "t" ? Docs.Create.StackingDocument : undefined, group); this.props.addDocument?.(newCollection); this.props.selectDocuments([newCollection]); diff --git a/src/client/views/nodes/AudioBox.tsx b/src/client/views/nodes/AudioBox.tsx index ae04f0f00..669622455 100644 --- a/src/client/views/nodes/AudioBox.tsx +++ b/src/client/views/nodes/AudioBox.tsx @@ -4,8 +4,6 @@ import { action, computed, IReactionDisposer, observable, runInAction } from "mo import { observer } from "mobx-react"; import { DateField } from "../../../fields/DateField"; import { Doc, DocListCast } from "../../../fields/Doc"; -import { documentSchema } from "../../../fields/documentSchemas"; -import { makeInterface } from "../../../fields/Schema"; import { ComputedField } from "../../../fields/ScriptField"; import { Cast, DateCast, NumCast } from "../../../fields/Types"; import { AudioField, nullAudio } from "../../../fields/URLField"; diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index d4e8ffc7f..70732e74c 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -41,6 +41,7 @@ import { WebBox } from "./WebBox"; import React = require("react"); import XRegExp = require("xregexp"); import { MapBox } from "./MapBox/MapBox"; +import { RecordingBox } from "./RecordingBox"; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? @@ -225,7 +226,7 @@ export class DocumentContentsView extends React.Component<DocumentViewProps & Fo components={{ FormattedTextBox, ImageBox, DirectoryImportBox, FontIconBox, LabelBox, EquationBox, SliderBox, FieldView, CollectionFreeFormView, CollectionDockingView, CollectionSchemaView, CollectionView, WebBox, KeyValueBox, - PDFBox, VideoBox, AudioBox, PresBox, YoutubeBox, PresElementBox, SearchBox, FilterBox, FunctionPlotBox, + PDFBox, VideoBox, AudioBox, RecordingBox, PresBox, YoutubeBox, PresElementBox, SearchBox, FilterBox, FunctionPlotBox, ColorBox, DashWebRTCVideo, LinkAnchorBox, InkingStroke, LinkBox, ScriptingBox, MapBox, ScreenshotBox, HTMLtag, ComparisonBox diff --git a/src/client/views/nodes/FieldView.tsx b/src/client/views/nodes/FieldView.tsx index 79c1f1c40..67cf18d8b 100644 --- a/src/client/views/nodes/FieldView.tsx +++ b/src/client/views/nodes/FieldView.tsx @@ -7,6 +7,7 @@ import { List } from "../../../fields/List"; import { WebField } from "../../../fields/URLField"; import { DocumentView, DocumentViewSharedProps } from "./DocumentView"; import { ScriptField } from "../../../fields/ScriptField"; +import { RecordingBox } from "./RecordingBox"; // // these properties get assigned through the render() method of the DocumentView when it creates this node. diff --git a/src/client/views/nodes/RecordingBox/ProgressBar.scss b/src/client/views/nodes/RecordingBox/ProgressBar.scss new file mode 100644 index 000000000..a493b0b89 --- /dev/null +++ b/src/client/views/nodes/RecordingBox/ProgressBar.scss @@ -0,0 +1,26 @@ + +.progressbar { + position: absolute; + display: flex; + justify-content: flex-start; + bottom: 10px; + width: 80%; + height: 5px; + background-color: gray; + + &.done { + top: 0; + width: 0px; + height: 5px; + background-color: red; + z-index: 2; + } + + &.mark { + top: 0; + background-color: transparent; + border-right: 2px solid white; + z-index: 3; + pointer-events: none; + } +}
\ No newline at end of file diff --git a/src/client/views/nodes/RecordingBox/ProgressBar.tsx b/src/client/views/nodes/RecordingBox/ProgressBar.tsx new file mode 100644 index 000000000..82d5e1f04 --- /dev/null +++ b/src/client/views/nodes/RecordingBox/ProgressBar.tsx @@ -0,0 +1,45 @@ +import * as React from 'react'; +import { useEffect } from "react" +import "./ProgressBar.scss" + +interface ProgressBarProps { + progress: number, + marks: number[], +} + +export function ProgressBar(props: ProgressBarProps) { + + // const handleClick = (e: React.MouseEvent) => { + // let progressbar = document.getElementById('progressbar')! + // let bounds = progressbar!.getBoundingClientRect(); + // let x = e.clientX - bounds.left; + // let percent = x / progressbar.clientWidth * 100 + + // for (let i = 0; i < props.marks.length; i++) { + // let start = i == 0 ? 0 : props.marks[i-1]; + // if (percent > start && percent < props.marks[i]) { + // props.playSegment(i) + // // console.log(i) + // // console.log(percent) + // // console.log(props.marks[i]) + // break + // } + // } + // } + + return ( + <div className="progressbar" id="progressbar"> + <div + className="progressbar done" + style={{ width: `${props.progress}%` }} + // onClick={handleClick} + ></div> + {props.marks.map((mark) => { + return <div + className="progressbar mark" + style={{ width: `${mark}%` }} + ></div> + })} + </div> + ) +}
\ No newline at end of file diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx new file mode 100644 index 000000000..159271223 --- /dev/null +++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx @@ -0,0 +1,62 @@ +import { action, observable } from "mobx"; +import { observer } from "mobx-react"; +import * as React from "react"; +import { VideoField } from "../../../../fields/URLField"; +import { Upload } from "../../../../server/SharedMediaTypes"; +import { ViewBoxBaseComponent } from "../../DocComponent"; +import { FieldView } from "../FieldView"; +import { VideoBox } from "../VideoBox"; +import { RecordingView } from './RecordingView'; +import { DocumentType } from "../../../documents/DocumentTypes"; +import { RecordingApi } from "../../../util/RecordingApi"; +import { Doc, FieldsSym } from "../../../../fields/Doc"; +import { Id } from "../../../../fields/FieldSymbols"; + + +@observer +export class RecordingBox extends ViewBoxBaseComponent() { + + public static LayoutString(fieldKey: string) { return FieldView.LayoutString(RecordingBox, fieldKey); } + + private _ref: React.RefObject<HTMLDivElement> = React.createRef(); + + constructor(props: any) { + super(props); + } + + componentDidMount() { + console.log("set native width and height") + Doc.SetNativeWidth(this.dataDoc, 1280); + Doc.SetNativeHeight(this.dataDoc, 720); + } + + @observable result: Upload.FileInformation | undefined = undefined + @observable videoDuration: number | undefined = undefined + + @action + setVideoDuration = (duration: number) => { + this.videoDuration = duration + } + + @action + setResult = (info: Upload.FileInformation, trackScreen: boolean) => { + this.result = info + this.dataDoc.type = DocumentType.VID; + this.dataDoc[this.fieldKey + "-duration"] = this.videoDuration; + + this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey); + this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.agnostic.client); + this.dataDoc[this.fieldKey + "-recorded"] = true; + // stringify the presenation and store it + if (trackScreen) { + this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(RecordingApi.Instance.clear()); + } + } + + render() { + // console.log("Proto[Is]: ", this.rootDoc.proto?.[Id]) + return <div className="recordingBox" ref={this._ref}> + {!this.result && <RecordingView setResult={this.setResult} setDuration={this.setVideoDuration} id={this.rootDoc.proto?.[Id]} />} + </div>; + } +} diff --git a/src/client/views/nodes/RecordingBox/RecordingView.scss b/src/client/views/nodes/RecordingBox/RecordingView.scss new file mode 100644 index 000000000..9b2f6d070 --- /dev/null +++ b/src/client/views/nodes/RecordingBox/RecordingView.scss @@ -0,0 +1,207 @@ +video { + // flex: 100%; + width: 100%; + // min-height: 400px; + //height: auto; + height: 100%; + //display: block; + object-fit: cover; + background-color: black; +} + +button { + margin: 0 .5rem +} + +.recording-container { + height: 100%; + width: 100%; + // display: flex; + pointer-events: all; + background-color: grey; +} + +.video-wrapper { + // max-width: 600px; + // max-width: 700px; + position: relative; + display: flex; + justify-content: center; + // overflow: hidden; + border-radius: 10px; + margin: 0; +} + +.video-wrapper:hover .controls { + bottom: 30px; + transform: translateY(0%); + opacity: 100%; +} + +.controls { + display: flex; + align-items: center; + justify-content: space-evenly; + position: absolute; + padding: 14px; + width: 100%; + max-width: 500px; + // max-height: 20%; + flex-wrap: wrap; + background: rgba(255, 255, 255, 0.25); + box-shadow: 0 8px 32px 0 rgba(255, 255, 255, 0.1); + backdrop-filter: blur(4px); + border-radius: 10px; + border: 1px solid rgba(255, 255, 255, 0.18); + // transform: translateY(150%); + transition: all 0.3s ease-in-out; + // opacity: 0%; + bottom: 30px; + // bottom: -150px; +} + +.actions button { + background: none; + border: none; + outline: none; + cursor: pointer; +} + +.actions button i { + background-color: none; + color: white; + font-size: 30px; +} + + +.velocity { + appearance: none; + background: none; + color: white; + outline: none; + border: none; + text-align: center; + font-size: 16px; +} + +.mute-btn { + background: none; + border: none; + outline: none; + cursor: pointer; +} + +.mute-btn i { + background-color: none; + color: white; + font-size: 20px; +} + +.recording-sign { + height: 20px; + width: auto; + display: flex; + flex-direction: row; + position: absolute; + top: 10px; + right: 15px; + align-items: center; + justify-content: center; + + .timer { + font-size: 15px; + color: white; + margin: 0; + } + + .dot { + height: 15px; + width: 15px; + margin: 5px; + background-color: red; + border-radius: 50%; + display: inline-block; + } +} + +.controls-inner-container { + display: flex; + flex-direction: row; + justify-content: center; + width: 100%; + +} + +.record-button-wrapper { + width: 35px; + height: 35px; + font-size: 0; + background-color: grey; + border: 0px; + border-radius: 35px; + margin: 10px; + display: flex; + justify-content: center; + + .record-button { + background-color: red; + border: 0px; + border-radius: 50%; + height: 80%; + width: 80%; + align-self: center; + margin: 0; + + &:hover { + height: 85%; + width: 85%; + } + } + + .stop-button { + background-color: red; + border: 0px; + border-radius: 10%; + height: 70%; + width: 70%; + align-self: center; + margin: 0; + + + // &:hover { + // width: 40px; + // height: 40px + // } + } + +} + +.options-wrapper { + height: 100%; + display: flex; + flex-direction: row; + align-items: center; + position: absolute; + top: 0; + bottom: 0; + + &.video-edit-wrapper { + + right: 50% - 15; + + .track-screen { + font-weight: 200; + } + + } + + &.track-screen-wrapper { + + right: 50% - 30; + + .track-screen { + font-weight: 200; + } + + } +}
\ No newline at end of file diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx new file mode 100644 index 000000000..87716e9cc --- /dev/null +++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx @@ -0,0 +1,277 @@ +import * as React from 'react'; +import "./RecordingView.scss"; +import { ReactElement, useCallback, useEffect, useRef, useState } from "react"; +import { ProgressBar } from "./ProgressBar" +import { MdBackspace } from 'react-icons/md'; +import { FaCheckCircle } from 'react-icons/fa'; +import { IconContext } from "react-icons"; +import { Networking } from '../../../Network'; +import { Upload } from '../../../../server/SharedMediaTypes'; + +import { RecordingApi } from '../../../util/RecordingApi'; + +interface MediaSegment { + videoChunks: any[], + endTime: number +} + +interface IRecordingViewProps { + setResult: (info: Upload.FileInformation, trackScreen: boolean) => void + setDuration: (seconds: number) => void + id: string +} + +const MAXTIME = 100000; + +export function RecordingView(props: IRecordingViewProps) { + + const [recording, setRecording] = useState(false); + const recordingTimerRef = useRef<number>(0); + const [recordingTimer, setRecordingTimer] = useState(0); // unit is 0.01 second + const [playing, setPlaying] = useState(false); + const [progress, setProgress] = useState(0); + + const [videos, setVideos] = useState<MediaSegment[]>([]); + const videoRecorder = useRef<MediaRecorder | null>(null); + const videoElementRef = useRef<HTMLVideoElement | null>(null); + + const [finished, setFinished] = useState<boolean>(false) + const [trackScreen, setTrackScreen] = useState<boolean>(true) + + + + const DEFAULT_MEDIA_CONSTRAINTS = { + video: { + width: 1280, + height: 720, + }, + audio: { + echoCancellation: true, + noiseSuppression: true, + sampleRate: 44100 + } + } + + useEffect(() => { + + if (finished) { + props.setDuration(recordingTimer * 100) + let allVideoChunks: any = [] + videos.forEach((vid) => { + console.log(vid.videoChunks) + allVideoChunks = allVideoChunks.concat(vid.videoChunks) + }) + + const videoFile = new File(allVideoChunks, "video.mkv", { type: allVideoChunks[0].type, lastModified: Date.now() }); + + Networking.UploadFilesToServer(videoFile) + .then((data) => { + const result = data[0].result + if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox + props.setResult(result, trackScreen) + } else { + alert("video conversion failed"); + } + }) + + } + + + }, [finished]) + + useEffect(() => { + // check if the browser supports media devices on first load + if (!navigator.mediaDevices) { + console.log('This browser does not support getUserMedia.') + } + console.log('This device has the correct media devices.') + }, []) + + useEffect(() => { + // get access to the video element on every render + videoElementRef.current = document.getElementById(`video-${props.id}`) as HTMLVideoElement; + }) + + useEffect(() => { + let interval: any = null; + if (recording) { + interval = setInterval(() => { + setRecordingTimer(unit => unit + 1); + }, 10); + } else if (!recording && recordingTimer !== 0) { + clearInterval(interval); + } + return () => clearInterval(interval); + }, [recording]) + + useEffect(() => { + setVideoProgressHelper(recordingTimer) + recordingTimerRef.current = recordingTimer; + }, [recordingTimer]) + + const setVideoProgressHelper = (progress: number) => { + const newProgress = (progress / MAXTIME) * 100; + setProgress(newProgress) + } + const startShowingStream = async (mediaConstraints = DEFAULT_MEDIA_CONSTRAINTS) => { + const stream = await navigator.mediaDevices.getUserMedia(mediaConstraints) + + videoElementRef.current!.src = "" + videoElementRef.current!.srcObject = stream + videoElementRef.current!.muted = true + + return stream + } + + const record = async () => { + const stream = await startShowingStream(); + videoRecorder.current = new MediaRecorder(stream) + + // temporary chunks of video + let videoChunks: any = [] + + videoRecorder.current.ondataavailable = (event: any) => { + if (event.data.size > 0) { + videoChunks.push(event.data) + } + } + + videoRecorder.current.onstart = (event: any) => { + setRecording(true); + trackScreen && RecordingApi.Instance.start(); + } + + videoRecorder.current.onstop = () => { + // if we have a last portion + if (videoChunks.length > 1) { + // append the current portion to the video pieces + setVideos(videos => [...videos, { videoChunks: videoChunks, endTime: recordingTimerRef.current }]) + } + + // reset the temporary chunks + videoChunks = [] + setRecording(false); + setFinished(true); + trackScreen && RecordingApi.Instance.pause(); + } + + // recording paused + videoRecorder.current.onpause = (event: any) => { + // append the current portion to the video pieces + setVideos(videos => [...videos, { videoChunks: videoChunks, endTime: recordingTimerRef.current }]) + + // reset the temporary chunks + videoChunks = [] + setRecording(false); + trackScreen && RecordingApi.Instance.pause(); + } + + videoRecorder.current.onresume = async (event: any) => { + await startShowingStream(); + setRecording(true); + trackScreen && RecordingApi.Instance.resume(); + } + + videoRecorder.current.start(200) + } + + + const stop = () => { + if (videoRecorder.current) { + if (videoRecorder.current.state !== "inactive") { + videoRecorder.current.stop(); + // recorder.current.stream.getTracks().forEach((track: any) => track.stop()) + } + } + } + + const pause = () => { + if (videoRecorder.current) { + if (videoRecorder.current.state === "recording") { + videoRecorder.current.pause(); + } + } + } + + const startOrResume = () => { + if (!videoRecorder.current || videoRecorder.current.state === "inactive") { + record(); + } else if (videoRecorder.current.state === "paused") { + videoRecorder.current.resume(); + } + } + + const clearPrevious = () => { + const numVideos = videos.length + setRecordingTimer(numVideos == 1 ? 0 : videos[numVideos - 2].endTime) + setVideoProgressHelper(numVideos == 1 ? 0 : videos[numVideos - 2].endTime) + setVideos(videos.filter((_, idx) => idx !== numVideos - 1)); + } + + const handleOnTimeUpdate = () => { + if (playing) { + setVideoProgressHelper(videoElementRef.current!.currentTime) + } + }; + + const millisecondToMinuteSecond = (milliseconds: number) => { + const toTwoDigit = (digit: number) => { + return String(digit).length == 1 ? "0" + digit : digit + } + const minutes = Math.floor((milliseconds % (1000 * 60 * 60)) / (1000 * 60)); + const seconds = Math.floor((milliseconds % (1000 * 60)) / 1000); + return toTwoDigit(minutes) + " : " + toTwoDigit(seconds); + } + + return ( + <div className="recording-container"> + <div className="video-wrapper"> + <video id={`video-${props.id}`} + autoPlay + muted + onTimeUpdate={handleOnTimeUpdate} + /> + <div className="recording-sign"> + <span className="dot" /> + <p className="timer">{millisecondToMinuteSecond(recordingTimer * 10)}</p> + </div> + <div className="controls"> + + <div className="controls-inner-container"> + <div className="record-button-wrapper"> + {recording ? + <button className="stop-button" onClick={pause} /> : + <button className="record-button" onClick={startOrResume} /> + } + </div> + + {!recording && (videos.length > 0 ? + + <div className="options-wrapper video-edit-wrapper"> + {/* <IconContext.Provider value={{ color: "grey", className: "video-edit-buttons" }}> + <MdBackspace onClick={clearPrevious} /> + </IconContext.Provider> */} + <IconContext.Provider value={{ color: "#cc1c08", className: "video-edit-buttons" }}> + <FaCheckCircle onClick={stop} /> + </IconContext.Provider> + </div> + + : <div className="options-wrapper track-screen-wrapper"> + <label className="track-screen"> + <input type="checkbox" checked={trackScreen} onChange={(e) => { setTrackScreen(e.target.checked) }} /> + <span className="checkmark"></span> + Track Screen + </label> + </div>)} + + </div> + + <ProgressBar + progress={progress} + marks={videos.map((elt) => elt.endTime / MAXTIME * 100)} + // playSegment={playSegment} + /> + </div> + </div> + </div>) +}
\ No newline at end of file diff --git a/src/client/views/nodes/RecordingBox/index.ts b/src/client/views/nodes/RecordingBox/index.ts new file mode 100644 index 000000000..ff21eaed6 --- /dev/null +++ b/src/client/views/nodes/RecordingBox/index.ts @@ -0,0 +1,2 @@ +export * from './RecordingView' +export * from './RecordingBox'
\ No newline at end of file diff --git a/src/client/views/nodes/ScreenshotBox.tsx b/src/client/views/nodes/ScreenshotBox.tsx index dbb567d3a..0c7d7dc31 100644 --- a/src/client/views/nodes/ScreenshotBox.tsx +++ b/src/client/views/nodes/ScreenshotBox.tsx @@ -26,7 +26,7 @@ import { FormattedTextBox } from "./formattedText/FormattedTextBox"; import "./ScreenshotBox.scss"; import { VideoBox } from "./VideoBox"; declare class MediaRecorder { - constructor(e: any, options?: any); // whatever MediaRecorder has + constructor(e: any, options?: any); // whatever MediaRecorder has } // interface VideoTileProps { @@ -107,225 +107,226 @@ declare class MediaRecorder { @observer export class ScreenshotBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps & FieldViewProps>() { - public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ScreenshotBox, fieldKey); } - private _audioRec: any; - private _videoRec: any; - @observable private _videoRef: HTMLVideoElement | null = null; - @observable _screenCapture = false; - @computed get recordingStart() { return Cast(this.dataDoc[this.props.fieldKey + "-recordingStart"], DateField)?.date.getTime(); } + public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ScreenshotBox, fieldKey); } + private _audioRec: any; + private _videoRec: any; + @observable private _videoRef: HTMLVideoElement | null = null; + @observable _screenCapture = false; + @computed get recordingStart() { return Cast(this.dataDoc[this.props.fieldKey + "-recordingStart"], DateField)?.date.getTime(); } - constructor(props: any) { - super(props); - if (this.rootDoc.videoWall) { - this.rootDoc.nativeWidth = undefined; - this.rootDoc.nativeHeight = undefined; - this.layoutDoc.popOff = 0; - this.layoutDoc.popOut = 1; - } else { - this.setupDictation(); - } - } - getAnchor = () => { - const startTime = Cast(this.layoutDoc._currentTimecode, "number", null) || (this._videoRec ? (Date.now() - (this.recordingStart || 0)) / 1000 : undefined); - return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow", "_timecodeToHide", - startTime, startTime === undefined ? undefined : startTime + 3) - || this.rootDoc; - } + constructor(props: any) { + super(props); + if (this.rootDoc.videoWall) { + this.rootDoc.nativeWidth = undefined; + this.rootDoc.nativeHeight = undefined; + this.layoutDoc.popOff = 0; + this.layoutDoc.popOut = 1; + } else { + this.setupDictation(); + } + } + getAnchor = () => { + const startTime = Cast(this.layoutDoc._currentTimecode, "number", null) || (this._videoRec ? (Date.now() - (this.recordingStart || 0)) / 1000 : undefined); + return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow", "_timecodeToHide", + startTime, startTime === undefined ? undefined : startTime + 3) + || this.rootDoc; + } - videoLoad = () => { - const aspect = this._videoRef!.videoWidth / this._videoRef!.videoHeight; - const nativeWidth = Doc.NativeWidth(this.layoutDoc); - const nativeHeight = Doc.NativeHeight(this.layoutDoc); - if (!nativeWidth || !nativeHeight) { - if (!nativeWidth) Doc.SetNativeWidth(this.dataDoc, 1200); - Doc.SetNativeHeight(this.dataDoc, (nativeWidth || 1200) / aspect); - this.layoutDoc._height = (this.layoutDoc[WidthSym]() || 0) / aspect; - } - } + videoLoad = () => { + const aspect = this._videoRef!.videoWidth / this._videoRef!.videoHeight; + const nativeWidth = Doc.NativeWidth(this.layoutDoc); + const nativeHeight = Doc.NativeHeight(this.layoutDoc); + if (!nativeWidth || !nativeHeight) { + if (!nativeWidth) Doc.SetNativeWidth(this.dataDoc, 1200); + Doc.SetNativeHeight(this.dataDoc, (nativeWidth || 1200) / aspect); + this.layoutDoc._height = (this.layoutDoc[WidthSym]() || 0) / aspect; + } + } - componentDidMount() { - this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = 0; - this.props.setContentView?.(this); // this tells the DocumentView that this ScreenshotBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. - // this.rootDoc.videoWall && reaction(() => ({ width: this.props.PanelWidth(), height: this.props.PanelHeight() }), - // ({ width, height }) => { - // if (this._camera) { - // const angle = -Math.abs(1 - width / height); - // const xz = [0, (this._numScreens - 2) / Math.abs(1 + angle)]; - // this._camera.position.set(this._numScreens / 2 + xz[1] * Math.sin(angle), this._numScreens / 2, xz[1] * Math.cos(angle)); - // this._camera.lookAt(this._numScreens / 2, this._numScreens / 2, 0); - // (this._camera as any).updateProjectionMatrix(); - // } - // }); - } - componentWillUnmount() { - const ind = DocUtils.ActiveRecordings.indexOf(this); - ind !== -1 && (DocUtils.ActiveRecordings.splice(ind, 1)); - } + componentDidMount() { + this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = 0; + this.props.setContentView?.(this); // this tells the DocumentView that this ScreenshotBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. + // this.rootDoc.videoWall && reaction(() => ({ width: this.props.PanelWidth(), height: this.props.PanelHeight() }), + // ({ width, height }) => { + // if (this._camera) { + // const angle = -Math.abs(1 - width / height); + // const xz = [0, (this._numScreens - 2) / Math.abs(1 + angle)]; + // this._camera.position.set(this._numScreens / 2 + xz[1] * Math.sin(angle), this._numScreens / 2, xz[1] * Math.cos(angle)); + // this._camera.lookAt(this._numScreens / 2, this._numScreens / 2, 0); + // (this._camera as any).updateProjectionMatrix(); + // } + // }); + } + componentWillUnmount() { + const ind = DocUtils.ActiveRecordings.indexOf(this); + ind !== -1 && (DocUtils.ActiveRecordings.splice(ind, 1)); + } - specificContextMenu = (e: React.MouseEvent): void => { - const subitems = [{ description: "Screen Capture", event: this.toggleRecording, icon: "expand-arrows-alt" as any }]; - ContextMenu.Instance.addItem({ description: "Options...", subitems, icon: "video" }); - } + specificContextMenu = (e: React.MouseEvent): void => { + const subitems = [{ description: "Screen Capture", event: this.toggleRecording, icon: "expand-arrows-alt" as any }]; + ContextMenu.Instance.addItem({ description: "Options...", subitems, icon: "video" }); + } - @computed get content() { - if (this.rootDoc.videoWall) return (null); - return <video className={"videoBox-content"} key="video" - ref={r => { - this._videoRef = r; - setTimeout(() => { - if (this.rootDoc.mediaState === "pendingRecording" && this._videoRef) { - this.toggleRecording(); - } - }, 100); - }} - autoPlay={this._screenCapture} - style={{ width: this._screenCapture ? "100%" : undefined, height: this._screenCapture ? "100%" : undefined }} - onCanPlay={this.videoLoad} - controls={true} - onClick={e => e.preventDefault()}> - <source type="video/mp4" /> - Not supported. - </video>; - } + @computed get content() { + if (this.rootDoc.videoWall) return (null); + return <video className={"videoBox-content"} key="video" + ref={r => { + this._videoRef = r; + setTimeout(() => { + if (this.rootDoc.mediaState === "pendingRecording" && this._videoRef) { + this.toggleRecording(); + } + }, 100); + }} + autoPlay={this._screenCapture} + style={{ width: this._screenCapture ? "100%" : undefined, height: this._screenCapture ? "100%" : undefined }} + onCanPlay={this.videoLoad} + controls={true} + onClick={e => e.preventDefault()}> + <source type="video/mp4" /> + Not supported. + </video>; + } - // _numScreens = 5; - // _camera: Camera | undefined; - // @observable _raised = [] as { coord: Vector2, off: Vector3 }[]; - // @action setRaised = (r: { coord: Vector2, off: Vector3 }[]) => this._raised = r; - @computed get threed() { - // if (this.rootDoc.videoWall) { - // const screens: any[] = []; - // const colors = ["yellow", "red", "orange", "brown", "maroon", "gray"]; - // let count = 0; - // numberRange(this._numScreens).forEach(x => numberRange(this._numScreens).forEach(y => screens.push( - // <VideoTile rootDoc={this.rootDoc} color={colors[count++ % colors.length]} x={x} y={y} raised={this._raised} setRaised={this.setRaised} />))); - // return <Canvas key="canvas" id="CANCAN" style={{ width: this.props.PanelWidth(), height: this.props.PanelHeight() }} gl={{ antialias: false }} colorManagement={false} onCreated={props => { - // this._camera = props.camera; - // props.camera.position.set(this._numScreens / 2, this._numScreens / 2, this._numScreens - 2); - // props.camera.lookAt(this._numScreens / 2, this._numScreens / 2, 0); - // }}> - // {/* <ambientLight />*/} - // <pointLight position={[10, 10, 10]} intensity={1} /> - // {screens} - // </ Canvas>; - // } - return (null); - } - toggleRecording = async () => { - if (!this._screenCapture) { - this._audioRec = new MediaRecorder(await navigator.mediaDevices.getUserMedia({ audio: true })); - const aud_chunks: any = []; - this._audioRec.ondataavailable = (e: any) => aud_chunks.push(e.data); - this._audioRec.onstop = async (e: any) => { - const [{ result }] = await Networking.UploadFilesToServer(aud_chunks); - if (!(result instanceof Error)) { - this.dataDoc[this.props.fieldKey + "-audio"] = new AudioField(result.accessPaths.agnostic.client); - } - }; - this._videoRef!.srcObject = await (navigator.mediaDevices as any).getDisplayMedia({ video: true }); - this._videoRec = new MediaRecorder(this._videoRef!.srcObject); - const vid_chunks: any = []; - this._videoRec.onstart = () => this.dataDoc[this.props.fieldKey + "-recordingStart"] = new DateField(new Date()); - this._videoRec.ondataavailable = (e: any) => vid_chunks.push(e.data); - this._videoRec.onstop = async (e: any) => { - const file = new File(vid_chunks, `${this.rootDoc[Id]}.mkv`, { type: vid_chunks[0].type, lastModified: Date.now() }); - const [{ result }] = await Networking.UploadFilesToServer(file); - this.dataDoc[this.fieldKey + "-duration"] = (new Date().getTime() - this.recordingStart!) / 1000; - if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox - this.dataDoc.type = DocumentType.VID; - this.layoutDoc.layout = VideoBox.LayoutString(this.fieldKey); - this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = undefined; - this.layoutDoc._fitWidth = undefined; - this.dataDoc[this.props.fieldKey] = new VideoField(result.accessPaths.agnostic.client); - } else alert("video conversion failed"); - }; - this._audioRec.start(); - this._videoRec.start(); - runInAction(() => { - this._screenCapture = true; - this.dataDoc.mediaState = "recording"; - }); - DocUtils.ActiveRecordings.push(this); - } else { - this._audioRec?.stop(); - this._videoRec?.stop(); - runInAction(() => { - this._screenCapture = false; - this.dataDoc.mediaState = "paused"; - }); - const ind = DocUtils.ActiveRecordings.indexOf(this); - ind !== -1 && (DocUtils.ActiveRecordings.splice(ind, 1)); + // _numScreens = 5; + // _camera: Camera | undefined; + // @observable _raised = [] as { coord: Vector2, off: Vector3 }[]; + // @action setRaised = (r: { coord: Vector2, off: Vector3 }[]) => this._raised = r; + @computed get threed() { + // if (this.rootDoc.videoWall) { + // const screens: any[] = []; + // const colors = ["yellow", "red", "orange", "brown", "maroon", "gray"]; + // let count = 0; + // numberRange(this._numScreens).forEach(x => numberRange(this._numScreens).forEach(y => screens.push( + // <VideoTile rootDoc={this.rootDoc} color={colors[count++ % colors.length]} x={x} y={y} raised={this._raised} setRaised={this.setRaised} />))); + // return <Canvas key="canvas" id="CANCAN" style={{ width: this.props.PanelWidth(), height: this.props.PanelHeight() }} gl={{ antialias: false }} colorManagement={false} onCreated={props => { + // this._camera = props.camera; + // props.camera.position.set(this._numScreens / 2, this._numScreens / 2, this._numScreens - 2); + // props.camera.lookAt(this._numScreens / 2, this._numScreens / 2, 0); + // }}> + // {/* <ambientLight />*/} + // <pointLight position={[10, 10, 10]} intensity={1} /> + // {screens} + // </ Canvas>; + // } + return (null); + } + toggleRecording = async () => { + if (!this._screenCapture) { + this._audioRec = new MediaRecorder(await navigator.mediaDevices.getUserMedia({ audio: true })); + const aud_chunks: any = []; + this._audioRec.ondataavailable = (e: any) => aud_chunks.push(e.data); + this._audioRec.onstop = async (e: any) => { + const [{ result }] = await Networking.UploadFilesToServer(aud_chunks); + if (!(result instanceof Error)) { + this.dataDoc[this.props.fieldKey + "-audio"] = new AudioField(result.accessPaths.agnostic.client); + } + }; + this._videoRef!.srcObject = await (navigator.mediaDevices as any).getDisplayMedia({ video: true }); + this._videoRec = new MediaRecorder(this._videoRef!.srcObject); + const vid_chunks: any = []; + this._videoRec.onstart = () => this.dataDoc[this.props.fieldKey + "-recordingStart"] = new DateField(new Date()); + this._videoRec.ondataavailable = (e: any) => vid_chunks.push(e.data); + this._videoRec.onstop = async (e: any) => { + console.log("screenshotbox: upload") + const file = new File(vid_chunks, `${this.rootDoc[Id]}.mkv`, { type: vid_chunks[0].type, lastModified: Date.now() }); + const [{ result }] = await Networking.UploadFilesToServer(file); + this.dataDoc[this.fieldKey + "-duration"] = (new Date().getTime() - this.recordingStart!) / 1000; + if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox + this.dataDoc.type = DocumentType.VID; + this.layoutDoc.layout = VideoBox.LayoutString(this.fieldKey); + this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = undefined; + this.layoutDoc._fitWidth = undefined; + this.dataDoc[this.props.fieldKey] = new VideoField(result.accessPaths.agnostic.client); + } else alert("video conversion failed"); + }; + this._audioRec.start(); + this._videoRec.start(); + runInAction(() => { + this._screenCapture = true; + this.dataDoc.mediaState = "recording"; + }); + DocUtils.ActiveRecordings.push(this); + } else { + this._audioRec?.stop(); + this._videoRec?.stop(); + runInAction(() => { + this._screenCapture = false; + this.dataDoc.mediaState = "paused"; + }); + const ind = DocUtils.ActiveRecordings.indexOf(this); + ind !== -1 && (DocUtils.ActiveRecordings.splice(ind, 1)); - CaptureManager.Instance.open(this.rootDoc); - } - } + CaptureManager.Instance.open(this.rootDoc); + } + } - setupDictation = () => { - if (this.dataDoc[this.fieldKey + "-dictation"]) return; - const dictationText = CurrentUserUtils.GetNewTextDoc("dictation", - NumCast(this.rootDoc.x), NumCast(this.rootDoc.y) + NumCast(this.layoutDoc._height) + 10, - NumCast(this.layoutDoc._width), 2 * NumCast(this.layoutDoc._height)); - dictationText._autoHeight = false; - const dictationTextProto = Doc.GetProto(dictationText); - dictationTextProto.recordingSource = this.dataDoc; - dictationTextProto.recordingStart = ComputedField.MakeFunction(`self.recordingSource["${this.props.fieldKey}-recordingStart"]`); - dictationTextProto.mediaState = ComputedField.MakeFunction("self.recordingSource.mediaState"); - this.dataDoc[this.fieldKey + "-dictation"] = dictationText; - } - contentFunc = () => [this.threed, this.content]; - videoPanelHeight = () => NumCast(this.dataDoc[this.fieldKey + "-nativeHeight"], this.layoutDoc[HeightSym]()) / NumCast(this.dataDoc[this.fieldKey + "-nativeWidth"], this.layoutDoc[WidthSym]()) * this.props.PanelWidth(); - formattedPanelHeight = () => Math.max(0, this.props.PanelHeight() - this.videoPanelHeight()); - render() { - TraceMobx(); - return <div className="videoBox" onContextMenu={this.specificContextMenu} style={{ width: "100%", height: "100%" }} > - <div className="videoBox-viewer" > - <div style={{ position: "relative", height: this.videoPanelHeight() }}> - <CollectionFreeFormView {...OmitKeys(this.props, ["NativeWidth", "NativeHeight"]).omit} - PanelHeight={this.videoPanelHeight} - PanelWidth={this.props.PanelWidth} - focus={this.props.focus} - isSelected={this.props.isSelected} - isAnnotationOverlay={true} - select={emptyFunction} - isContentActive={returnFalse} - scaling={returnOne} - whenChildContentsActiveChanged={emptyFunction} - removeDocument={returnFalse} - moveDocument={returnFalse} - addDocument={returnFalse} - CollectionView={undefined} - ScreenToLocalTransform={this.props.ScreenToLocalTransform} - renderDepth={this.props.renderDepth + 1} - ContainingCollectionDoc={this.props.ContainingCollectionDoc}> - {this.contentFunc} - </CollectionFreeFormView></div> - <div style={{ position: "relative", height: this.formattedPanelHeight() }}> - {!(this.dataDoc[this.fieldKey + "-dictation"] instanceof Doc) ? (null) : - <FormattedTextBox {...OmitKeys(this.props, ["NativeWidth", "NativeHeight"]).omit} - Document={this.dataDoc[this.fieldKey + "-dictation"]} - fieldKey={"text"} - PanelHeight={this.formattedPanelHeight} - isAnnotationOverlay={true} - select={emptyFunction} - isContentActive={emptyFunction} - scaling={returnOne} - xPadding={25} - yPadding={10} - whenChildContentsActiveChanged={emptyFunction} - removeDocument={returnFalse} - moveDocument={returnFalse} - addDocument={returnFalse} - CollectionView={undefined} - renderDepth={this.props.renderDepth + 1} - ContainingCollectionDoc={this.props.ContainingCollectionDoc}> - </FormattedTextBox>} - </div> - </div> - {!this.props.isSelected() ? (null) : <div className="screenshotBox-uiButtons"> - <div className="screenshotBox-recorder" key="snap" onPointerDown={this.toggleRecording} > - <FontAwesomeIcon icon="file" size="lg" /> - </div> - </div>} - </div >; - } + setupDictation = () => { + if (this.dataDoc[this.fieldKey + "-dictation"]) return; + const dictationText = CurrentUserUtils.GetNewTextDoc("dictation", + NumCast(this.rootDoc.x), NumCast(this.rootDoc.y) + NumCast(this.layoutDoc._height) + 10, + NumCast(this.layoutDoc._width), 2 * NumCast(this.layoutDoc._height)); + dictationText._autoHeight = false; + const dictationTextProto = Doc.GetProto(dictationText); + dictationTextProto.recordingSource = this.dataDoc; + dictationTextProto.recordingStart = ComputedField.MakeFunction(`self.recordingSource["${this.props.fieldKey}-recordingStart"]`); + dictationTextProto.mediaState = ComputedField.MakeFunction("self.recordingSource.mediaState"); + this.dataDoc[this.fieldKey + "-dictation"] = dictationText; + } + contentFunc = () => [this.threed, this.content]; + videoPanelHeight = () => NumCast(this.dataDoc[this.fieldKey + "-nativeHeight"], this.layoutDoc[HeightSym]()) / NumCast(this.dataDoc[this.fieldKey + "-nativeWidth"], this.layoutDoc[WidthSym]()) * this.props.PanelWidth(); + formattedPanelHeight = () => Math.max(0, this.props.PanelHeight() - this.videoPanelHeight()); + render() { + TraceMobx(); + return <div className="videoBox" onContextMenu={this.specificContextMenu} style={{ width: "100%", height: "100%" }} > + <div className="videoBox-viewer" > + <div style={{ position: "relative", height: this.videoPanelHeight() }}> + <CollectionFreeFormView {...OmitKeys(this.props, ["NativeWidth", "NativeHeight"]).omit} + PanelHeight={this.videoPanelHeight} + PanelWidth={this.props.PanelWidth} + focus={this.props.focus} + isSelected={this.props.isSelected} + isAnnotationOverlay={true} + select={emptyFunction} + isContentActive={returnFalse} + scaling={returnOne} + whenChildContentsActiveChanged={emptyFunction} + removeDocument={returnFalse} + moveDocument={returnFalse} + addDocument={returnFalse} + CollectionView={undefined} + ScreenToLocalTransform={this.props.ScreenToLocalTransform} + renderDepth={this.props.renderDepth + 1} + ContainingCollectionDoc={this.props.ContainingCollectionDoc}> + {this.contentFunc} + </CollectionFreeFormView></div> + <div style={{ position: "relative", height: this.formattedPanelHeight() }}> + {!(this.dataDoc[this.fieldKey + "-dictation"] instanceof Doc) ? (null) : + <FormattedTextBox {...OmitKeys(this.props, ["NativeWidth", "NativeHeight"]).omit} + Document={this.dataDoc[this.fieldKey + "-dictation"]} + fieldKey={"text"} + PanelHeight={this.formattedPanelHeight} + isAnnotationOverlay={true} + select={emptyFunction} + isContentActive={emptyFunction} + scaling={returnOne} + xPadding={25} + yPadding={10} + whenChildContentsActiveChanged={emptyFunction} + removeDocument={returnFalse} + moveDocument={returnFalse} + addDocument={returnFalse} + CollectionView={undefined} + renderDepth={this.props.renderDepth + 1} + ContainingCollectionDoc={this.props.ContainingCollectionDoc}> + </FormattedTextBox>} + </div> + </div> + {!this.props.isSelected() ? (null) : <div className="screenshotBox-uiButtons"> + <div className="screenshotBox-recorder" key="snap" onPointerDown={this.toggleRecording} > + <FontAwesomeIcon icon="file" size="lg" /> + </div> + </div>} + </div >; + } }
\ No newline at end of file diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx index 0017e5f5e..b14a1f0f6 100644 --- a/src/client/views/nodes/VideoBox.tsx +++ b/src/client/views/nodes/VideoBox.tsx @@ -7,7 +7,7 @@ import * as rp from 'request-promise'; import { Doc, DocListCast, HeightSym, WidthSym } from "../../../fields/Doc"; import { InkTool } from "../../../fields/InkField"; import { Cast, NumCast, StrCast } from "../../../fields/Types"; -import { AudioField, ImageField, VideoField } from "../../../fields/URLField"; +import { AudioField, ImageField, RecordingField, VideoField } from "../../../fields/URLField"; import { emptyFunction, formatTime, OmitKeys, returnFalse, returnOne, setupMoveUpEvents, Utils } from "../../../Utils"; import { Docs, DocUtils } from "../../documents/Documents"; import { DocumentType } from "../../documents/DocumentTypes"; @@ -28,6 +28,9 @@ import { AnchorMenu } from "../pdf/AnchorMenu"; import { StyleProp } from "../StyleProvider"; import { FieldView, FieldViewProps } from './FieldView'; import "./VideoBox.scss"; +import { RecordingApi } from "../../util/RecordingApi"; +import { List } from "../../../fields/List"; +import { RecordingBox } from "./RecordingBox"; const path = require('path'); @@ -45,825 +48,874 @@ const path = require('path'); @observer export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps & FieldViewProps>() { - public static LayoutString(fieldKey: string) { return FieldView.LayoutString(VideoBox, fieldKey); } - /** - * Uploads an image buffer to the server and stores with specified filename. by default the image - * is stored at multiple resolutions each retrieved by using the filename appended with _o, _s, _m, _l (indicating original, small, medium, or large) - * @param imageUri the bytes of the image - * @param returnedFilename the base filename to store the image on the server - * @param nosuffix optionally suppress creating multiple resolution images - */ - public static async convertDataUri(imageUri: string, returnedFilename: string, nosuffix = false, replaceRootFilename?: string) { - try { - const posting = Utils.prepend("/uploadURI"); - const returnedUri = await rp.post(posting, { - body: { - uri: imageUri, - name: returnedFilename, - nosuffix, - replaceRootFilename - }, - json: true, - }); - return returnedUri; - - } catch (e) { - console.log("VideoBox :" + e); - } - } - - static _youtubeIframeCounter: number = 0; - static heightPercent = 80; // height of video relative to videoBox when timeline is open - private _disposers: { [name: string]: IReactionDisposer } = {}; - private _youtubePlayer: YT.Player | undefined = undefined; - private _videoRef: HTMLVideoElement | null = null; // <video> ref - private _contentRef: HTMLDivElement | null = null; // ref to div that wraps video and controls for full screen - private _youtubeIframeId: number = -1; - private _youtubeContentCreated = false; - private _audioPlayer: HTMLAudioElement | null = null; - private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); // outermost div - private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef(); - private _playRegionTimer: any = null; // timeout for playback - @observable _stackedTimeline: any; // CollectionStackedTimeline ref - @observable static _nativeControls: boolean; // default html controls - @observable _marqueeing: number[] | undefined; // coords for marquee selection - @observable _savedAnnotations = new ObservableMap<number, HTMLDivElement[]>(); - @observable _screenCapture = false; - @observable _clicking = false; // used for transition between showing/hiding timeline - @observable _forceCreateYouTubeIFrame = false; - @observable _playTimer?: NodeJS.Timeout = undefined; - @observable _fullScreen = false; - @observable _playing = false; - @observable _finished: boolean = false; // has playback reached end of clip - @observable _volume: number = 1; - @observable _muted: boolean = false; - - @computed get links() { return DocListCast(this.dataDoc.links); } - @computed get heightPercent() { return NumCast(this.layoutDoc._timelineHeightPercent, 100); } // current percent of video relative to VideoBox height - // @computed get rawDuration() { return NumCast(this.dataDoc[this.fieldKey + "-duration"]); } - @observable rawDuration: number = 0; - - - @computed get youtubeVideoId() { - const field = Cast(this.dataDoc[this.props.fieldKey], VideoField); - return field && field.url.href.indexOf("youtube") !== -1 ? ((arr: string[]) => arr[arr.length - 1])(field.url.href.split("/")) : ""; - } - - - // returns the path of the audio file - @computed get audiopath() { - const field = Cast(this.props.Document[this.props.fieldKey + '-audio'], AudioField, null); - const vfield = Cast(this.dataDoc[this.fieldKey], VideoField, null); - return field?.url.href ?? vfield?.url.href ?? ""; - } - - - @computed private get timeline() { return this._stackedTimeline; } - private get transition() { return this._clicking ? "left 0.5s, width 0.5s, height 0.5s" : ""; } // css transition for hiding/showing timeline - public get player(): HTMLVideoElement | null { return this._videoRef; } - - - componentDidMount() { - this.props.setContentView?.(this); // this tells the DocumentView that this VideoBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the VideoBox when making a link. - if (this.youtubeVideoId) { - const youtubeaspect = 400 / 315; - const nativeWidth = Doc.NativeWidth(this.layoutDoc); - const nativeHeight = Doc.NativeHeight(this.layoutDoc); - if (!nativeWidth || !nativeHeight) { - if (!nativeWidth) Doc.SetNativeWidth(this.dataDoc, 600); - Doc.SetNativeHeight(this.dataDoc, (nativeWidth || 600) / youtubeaspect); - this.layoutDoc._height = NumCast(this.layoutDoc._width) / youtubeaspect; + public static LayoutString(fieldKey: string) { return FieldView.LayoutString(VideoBox, fieldKey); } + /** + * Uploads an image buffer to the server and stores with specified filename. by default the image + * is stored at multiple resolutions each retrieved by using the filename appended with _o, _s, _m, _l (indicating original, small, medium, or large) + * @param imageUri the bytes of the image + * @param returnedFilename the base filename to store the image on the server + * @param nosuffix optionally suppress creating multiple resolution images + */ + public static async convertDataUri(imageUri: string, returnedFilename: string, nosuffix = false, replaceRootFilename?: string) { + try { + const posting = Utils.prepend("/uploadURI"); + const returnedUri = await rp.post(posting, { + body: { + uri: imageUri, + name: returnedFilename, + nosuffix, + replaceRootFilename + }, + json: true, + }); + return returnedUri; + + } catch (e) { + console.log("VideoBox :" + e); } - } - this.player && this.setPlayheadTime(0); - } - - componentWillUnmount() { - this.removeCurrentlyPlaying(); - this.Pause(); - Object.keys(this._disposers).forEach(d => this._disposers[d]?.()); - } - - - // plays video - @action public Play = (update: boolean = true) => { - if (this._playRegionTimer) return; - this._playing = true; - const eleTime = this.player?.currentTime || 0; - if (this.timeline) { - let start = eleTime >= this.timeline.trimEnd || eleTime <= this.timeline.trimStart ? this.timeline.trimStart : eleTime; - - if (this._finished) { - // restarts video if reached end on previous play - this._finished = false; - start = this.timeline.trimStart; + } + + static _youtubeIframeCounter: number = 0; + static heightPercent = 80; // height of video relative to videoBox when timeline is open + private _disposers: { [name: string]: IReactionDisposer } = {}; + private _youtubePlayer: YT.Player | undefined = undefined; + private _videoRef: HTMLVideoElement | null = null; // <video> ref + private _contentRef: HTMLDivElement | null = null; // ref to div that wraps video and controls for full screen + private _youtubeIframeId: number = -1; + private _youtubeContentCreated = false; + private _audioPlayer: HTMLAudioElement | null = null; + private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); // outermost div + private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef(); + private _playRegionTimer: any = null; // timeout for playback + @observable _stackedTimeline: any; // CollectionStackedTimeline ref + @observable static _nativeControls: boolean; // default html controls + @observable _marqueeing: number[] | undefined; // coords for marquee selection + @observable _savedAnnotations = new ObservableMap<number, HTMLDivElement[]>(); + @observable _screenCapture = false; + @observable _clicking = false; // used for transition between showing/hiding timeline + @observable _forceCreateYouTubeIFrame = false; + @observable _playTimer?: NodeJS.Timeout = undefined; + @observable _fullScreen = false; + @observable _playing = false; + @observable _finished: boolean = false; // has playback reached end of clip + @observable _volume: number = 1; + @observable _muted: boolean = false; + + @computed get links() { return DocListCast(this.dataDoc.links); } + @computed get heightPercent() { return NumCast(this.layoutDoc._timelineHeightPercent, 100); } // current percent of video relative to VideoBox height + // @computed get rawDuration() { return NumCast(this.dataDoc[this.fieldKey + "-duration"]); } + @observable rawDuration: number = 0; + + + @computed get youtubeVideoId() { + const field = Cast(this.dataDoc[this.props.fieldKey], VideoField); + return field && field.url.href.indexOf("youtube") !== -1 ? ((arr: string[]) => arr[arr.length - 1])(field.url.href.split("/")) : ""; + } + + + // returns the path of the audio file + @computed get audiopath() { + const field = Cast(this.props.Document[this.props.fieldKey + '-audio'], AudioField, null); + const vfield = Cast(this.dataDoc[this.fieldKey], VideoField, null); + return field?.url.href ?? vfield?.url.href ?? ""; + } + + // returns the presentation data if it exists, null otherwise + @computed get presentation() { + const data = this.dataDoc[this.fieldKey + '-presentation']; + return data ? JSON.parse(data) : null; + } + + + @computed private get timeline() { return this._stackedTimeline; } + private get transition() { return this._clicking ? "left 0.5s, width 0.5s, height 0.5s" : ""; } // css transition for hiding/showing timeline + public get player(): HTMLVideoElement | null { return this._videoRef; } + + + componentDidMount() { + this.props.setContentView?.(this); // this tells the DocumentView that this VideoBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the VideoBox when making a link. + if (this.youtubeVideoId) { + const youtubeaspect = 400 / 315; + const nativeWidth = Doc.NativeWidth(this.layoutDoc); + const nativeHeight = Doc.NativeHeight(this.layoutDoc); + if (!nativeWidth || !nativeHeight) { + if (!nativeWidth) Doc.SetNativeWidth(this.dataDoc, 600); + Doc.SetNativeHeight(this.dataDoc, (nativeWidth || 600) / youtubeaspect); + this.layoutDoc._height = NumCast(this.layoutDoc._width) / youtubeaspect; + } + } + this.player && this.setPlayheadTime(0); + } + + componentWillUnmount() { + this.removeCurrentlyPlaying(); + this.Pause(); + Object.keys(this._disposers).forEach(d => this._disposers[d]?.()); + } + + + // plays video + @action public Play = (update: boolean = true) => { + // if (Doc.UserDoc().presentationMode === 'watching' && !this._playing) { + // console.log('VideoBox : Play : presentation mode', this._playing); + // return; + // } + + // if presentation isn't null, call followmovements on the recording api + if (this.presentation) { + console.log("presentation isn't null") + const err = RecordingApi.Instance.playMovements(this.presentation, this.player?.currentTime || 0, this); + err && console.log(err) + } else { + console.log("presentation is null") + } + + this._playing = true; + const eleTime = this.player?.currentTime || 0; + if (this.timeline) { + let start = eleTime >= this.timeline.trimEnd || eleTime <= this.timeline.trimStart ? this.timeline.trimStart : eleTime; + + if (this._finished) { + // restarts video if reached end on previous play + this._finished = false; + start = this.timeline.trimStart; + } + + try { + this._audioPlayer && this.player && (this._audioPlayer.currentTime = this.player?.currentTime); + update && this.player && this.playFrom(start, undefined, true); + update && this._audioPlayer?.play(); + update && this._youtubePlayer?.playVideo(); + this._youtubePlayer && !this._playTimer && (this._playTimer = setInterval(this.updateTimecode, 5)); + } catch (e) { + console.log("Video Play Exception:", e); + } } + this.updateTimecode(); + } + // goes to time + @action public Seek(time: number) { try { - this._audioPlayer && this.player && (this._audioPlayer.currentTime = this.player?.currentTime); - update && this.player && this.playFrom(start, undefined, true); - update && this._audioPlayer?.play(); - update && this._youtubePlayer?.playVideo(); - this._youtubePlayer && !this._playTimer && (this._playTimer = setInterval(this.updateTimecode, 5)); + this._youtubePlayer?.seekTo(Math.round(time), true); } catch (e) { - console.log("Video Play Exception:", e); + console.log("Video Seek Exception:", e); } - } - this.updateTimecode(); - } - - // goes to time - @action public Seek(time: number) { - try { - this._youtubePlayer?.seekTo(Math.round(time), true); - } catch (e) { - console.log("Video Seek Exception:", e); - } - this.player && (this.player.currentTime = time); - this._audioPlayer && (this._audioPlayer.currentTime = time); - } - - // pauses video - @action public Pause = (update: boolean = true) => { - this._playing = false; - this.removeCurrentlyPlaying(); - try { - update && this.player?.pause(); - update && this._audioPlayer?.pause(); - update && this._youtubePlayer?.pauseVideo(); - this._youtubePlayer && this._playTimer && clearInterval(this._playTimer); - this._youtubePlayer?.seekTo(this._youtubePlayer?.getCurrentTime(), true); - } catch (e) { - console.log("Video Pause Exception:", e); - } - this._youtubePlayer && SelectionManager.DeselectAll(); // if we don't deselect the player, then we get an annoying YouTube spinner I guess telling us we're paused. - this._playTimer = undefined; - this.updateTimecode(); - if (!this._finished) { - clearTimeout(this._playRegionTimer); // if paused in the middle of playback, prevents restart on next play - } - this._playRegionTimer = undefined; - } - - // toggles video full screen - @action public FullScreen = () => { - if (document.fullscreenElement === this._contentRef) { - this._fullScreen = false; - this.player && this._contentRef && document.exitFullscreen(); - } - else { - this._fullScreen = true; - this.player && this._contentRef && this._contentRef.requestFullscreen(); - - } - try { - this._youtubePlayer && this.props.addDocTab(this.rootDoc, "add"); - } catch (e) { - console.log("Video FullScreen Exception:", e); - } - } - - - // creates and links snapshot photo of current video frame - @action public Snapshot = (downX?: number, downY?: number, cb?: (filename: string, x: number | undefined, y: number | undefined) => void) => { - const width = NumCast(this.layoutDoc._width); - const canvas = document.createElement('canvas'); - canvas.width = 640; - canvas.height = 640 * Doc.NativeHeight(this.layoutDoc) / (Doc.NativeWidth(this.layoutDoc) || 1); - const ctx = canvas.getContext('2d');//draw image to canvas. scale to target dimensions - if (ctx) { - this._videoRef && ctx.drawImage(this._videoRef, 0, 0, canvas.width, canvas.height); - } - - if (!this._videoRef) { - const b = Docs.Create.LabelDocument({ - x: NumCast(this.layoutDoc.x) + width, y: NumCast(this.layoutDoc.y, 1), - _width: 150, _height: 50, title: (this.layoutDoc._currentTimecode || 0).toString(), - _isLinkButton: true - }); - this.props.addDocument?.(b); - DocUtils.MakeLink({ doc: b }, { doc: this.rootDoc }, "video snapshot"); - Networking.PostToServer("/youtubeScreenshot", { - id: this.youtubeVideoId, - timecode: this.layoutDoc._currentTimecode - }).then(response => { - const resolved = response?.accessPaths?.agnostic?.client; - if (resolved) { - this.props.removeDocument?.(b); - this.createRealSummaryLink(resolved); - } - }); - } else { - //convert to desired file format - const dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png' - // if you want to preview the captured image, - const retitled = StrCast(this.rootDoc.title).replace(/[ -\.]/g, ""); - const encodedFilename = encodeURIComponent("snapshot" + retitled + "_" + (this.layoutDoc._currentTimecode || 0).toString().replace(/\./, "_")); - const filename = basename(encodedFilename); - VideoBox.convertDataUri(dataUrl, filename).then((returnedFilename: string) => - returnedFilename && (cb ?? this.createRealSummaryLink)(returnedFilename, downX, downY)); - } - } - - updateIcon = () => { - const makeIcon = (returnedfilename: string) => { - this.dataDoc.icon = new ImageField(returnedfilename); - this.dataDoc["icon-nativeWidth"] = this.layoutDoc[WidthSym](); - this.dataDoc["icon-nativeHeight"] = this.layoutDoc[HeightSym](); - }; - this.Snapshot(undefined, undefined, makeIcon); - } - - // creates link for snapshot - createRealSummaryLink = (imagePath: string, downX?: number, downY?: number) => { - const url = !imagePath.startsWith("/") ? Utils.CorsProxy(imagePath) : imagePath; - const width = NumCast(this.layoutDoc._width) || 1; - const height = NumCast(this.layoutDoc._height); - const imageSummary = Docs.Create.ImageDocument(url, { - _nativeWidth: Doc.NativeWidth(this.layoutDoc), _nativeHeight: Doc.NativeHeight(this.layoutDoc), - x: NumCast(this.layoutDoc.x) + width, y: NumCast(this.layoutDoc.y), _isLinkButton: true, - _width: 150, _height: height / width * 150, title: "--snapshot" + NumCast(this.layoutDoc._currentTimecode) + " image-" - }); - Doc.SetNativeWidth(Doc.GetProto(imageSummary), Doc.NativeWidth(this.layoutDoc)); - Doc.SetNativeHeight(Doc.GetProto(imageSummary), Doc.NativeHeight(this.layoutDoc)); - this.props.addDocument?.(imageSummary); - const link = DocUtils.MakeLink({ doc: imageSummary }, { doc: this.getAnchor() }, "video snapshot"); - link && (Doc.GetProto(link.anchor2 as Doc).timecodeToHide = NumCast((link.anchor2 as Doc).timecodeToShow) + 3); - setTimeout(() => - (downX !== undefined && downY !== undefined) && DocumentManager.Instance.getFirstDocumentView(imageSummary)?.startDragging(downX, downY, "move", true)); - } - - - getAnchor = () => { - const timecode = Cast(this.layoutDoc._currentTimecode, "number", null); - const marquee = AnchorMenu.Instance.GetAnchor?.(); - return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow"/* videoStart */, "_timecodeToHide" /* videoEnd */, timecode ? timecode : undefined, undefined, marquee) || this.rootDoc; - } - - - // sets video info on load - videoLoad = action(() => { - const aspect = this.player!.videoWidth / this.player!.videoHeight; - Doc.SetNativeWidth(this.dataDoc, this.player!.videoWidth); - Doc.SetNativeHeight(this.dataDoc, this.player!.videoHeight); - this.layoutDoc._height = NumCast(this.layoutDoc._width) / aspect; - if (Number.isFinite(this.player!.duration)) { - this.rawDuration = this.player!.duration; - } else this.rawDuration = NumCast(this.dataDoc[this.fieldKey + "-duration"]); - }); - - - // updates video time - @action - updateTimecode = () => { - this.player && (this.layoutDoc._currentTimecode = this.player.currentTime); - try { - this._youtubePlayer && (this.layoutDoc._currentTimecode = this._youtubePlayer.getCurrentTime?.()); - } catch (e) { - console.log("Video Timecode Exception:", e); - } - } - - - // sets video element ref - @action - setVideoRef = (vref: HTMLVideoElement | null) => { - this._videoRef = vref; - if (vref) { - this._videoRef!.ontimeupdate = this.updateTimecode; - // @ts-ignore - // vref.onfullscreenchange = action((e) => this._fullScreen = vref.webkitDisplayingFullscreen); - this._disposers.reactionDisposer?.(); - this._disposers.reactionDisposer = reaction(() => NumCast(this.layoutDoc._currentTimecode), - time => !this._playing && (vref.currentTime = time), { fireImmediately: true }); - } - } - - // set ref for div that wraps video and controls for fullscreen - @action - setContentRef = (cref: HTMLDivElement | null) => { - this._contentRef = cref; - if (cref) { - cref.onfullscreenchange = action((e) => this._fullScreen = (document.fullscreenElement === cref)); - } - } - - - // context menu - specificContextMenu = (e: React.MouseEvent): void => { - const field = Cast(this.dataDoc[this.props.fieldKey], VideoField); - if (field) { - const url = field.url.href; - const subitems: ContextMenuProps[] = []; - subitems.push({ description: "Full Screen", event: this.FullScreen, icon: "expand" }); - subitems.push({ description: "Take Snapshot", event: this.Snapshot, icon: "expand-arrows-alt" }); - this.rootDoc.type === DocumentType.SCREENSHOT && subitems.push({ - description: "Screen Capture", event: (async () => { - runInAction(() => this._screenCapture = !this._screenCapture); - this._videoRef!.srcObject = !this._screenCapture ? undefined : await (navigator.mediaDevices as any).getDisplayMedia({ video: true }); - }), icon: "expand-arrows-alt" + this.player && (this.player.currentTime = time); + this._audioPlayer && (this._audioPlayer.currentTime = time); + // TODO: revisit this and clean it + if ((this.player?.currentTime || -1) < this.rawDuration) { + this._finished = false; + } + } + + // pauses video + @action public Pause = (update: boolean = true) => { + if (this.presentation) { + console.log('VideoBox : Pause'); + const err = RecordingApi.Instance.pauseMovements(); + err && console.log(err); + } + + this._playing = false; + this.removeCurrentlyPlaying(); + try { + update && this.player?.pause(); + update && this._audioPlayer?.pause(); + update && this._youtubePlayer?.pauseVideo(); + this._youtubePlayer && this._playTimer && clearInterval(this._playTimer); + this._youtubePlayer?.seekTo(this._youtubePlayer?.getCurrentTime(), true); + } catch (e) { + console.log("Video Pause Exception:", e); + } + this._youtubePlayer && SelectionManager.DeselectAll(); // if we don't deselect the player, then we get an annoying YouTube spinner I guess telling us we're paused. + this._playTimer = undefined; + this.updateTimecode(); + if (!this._finished) clearTimeout(this._playRegionTimer); // if paused in the middle of playback, prevents restart on next play + } + + // toggles video full screen + @action public FullScreen = () => { + if (document.fullscreenElement === this._contentRef) { + this._fullScreen = false; + this.player && this._contentRef && document.exitFullscreen(); + } + else { + this._fullScreen = true; + this.player && this._contentRef && this._contentRef.requestFullscreen(); + + } + try { + this._youtubePlayer && this.props.addDocTab(this.rootDoc, "add"); + } catch (e) { + console.log("Video FullScreen Exception:", e); + } + } + + + // creates and links snapshot photo of current video frame + @action public Snapshot = (downX?: number, downY?: number, cb?: (filename: string, x: number | undefined, y: number | undefined) => void) => { + const width = NumCast(this.layoutDoc._width); + const canvas = document.createElement('canvas'); + canvas.width = 640; + canvas.height = 640 * Doc.NativeHeight(this.layoutDoc) / (Doc.NativeWidth(this.layoutDoc) || 1); + const ctx = canvas.getContext('2d');//draw image to canvas. scale to target dimensions + if (ctx) { + this._videoRef && ctx.drawImage(this._videoRef, 0, 0, canvas.width, canvas.height); + } + + if (!this._videoRef) { + const b = Docs.Create.LabelDocument({ + x: NumCast(this.layoutDoc.x) + width, y: NumCast(this.layoutDoc.y, 1), + _width: 150, _height: 50, title: (this.layoutDoc._currentTimecode || 0).toString(), + _isLinkButton: true + }); + this.props.addDocument?.(b); + DocUtils.MakeLink({ doc: b }, { doc: this.rootDoc }, "video snapshot"); + Networking.PostToServer("/youtubeScreenshot", { + id: this.youtubeVideoId, + timecode: this.layoutDoc._currentTimecode + }).then(response => { + const resolved = response?.accessPaths?.agnostic?.client; + if (resolved) { + this.props.removeDocument?.(b); + this.createRealSummaryLink(resolved); + } + }); + } else { + //convert to desired file format + const dataUrl = canvas.toDataURL('image/png'); // can also use 'image/png' + // if you want to preview the captured image, + const retitled = StrCast(this.rootDoc.title).replace(/[ -\.]/g, ""); + const encodedFilename = encodeURIComponent("snapshot" + retitled + "_" + (this.layoutDoc._currentTimecode || 0).toString().replace(/\./, "_")); + const filename = basename(encodedFilename); + VideoBox.convertDataUri(dataUrl, filename).then((returnedFilename: string) => + returnedFilename && (cb ?? this.createRealSummaryLink)(returnedFilename, downX, downY)); + } + } + + updateIcon = () => { + const makeIcon = (returnedfilename: string) => { + this.dataDoc.icon = new ImageField(returnedfilename); + this.dataDoc["icon-nativeWidth"] = this.layoutDoc[WidthSym](); + this.dataDoc["icon-nativeHeight"] = this.layoutDoc[HeightSym](); + }; + this.Snapshot(undefined, undefined, makeIcon); + } + + // creates link for snapshot + createRealSummaryLink = (imagePath: string, downX?: number, downY?: number) => { + const url = !imagePath.startsWith("/") ? Utils.CorsProxy(imagePath) : imagePath; + const width = NumCast(this.layoutDoc._width) || 1; + const height = NumCast(this.layoutDoc._height); + const imageSummary = Docs.Create.ImageDocument(url, { + _nativeWidth: Doc.NativeWidth(this.layoutDoc), _nativeHeight: Doc.NativeHeight(this.layoutDoc), + x: NumCast(this.layoutDoc.x) + width, y: NumCast(this.layoutDoc.y), _isLinkButton: true, + _width: 150, _height: height / width * 150, title: "--snapshot" + NumCast(this.layoutDoc._currentTimecode) + " image-" }); - subitems.push({ description: (this.layoutDoc.dontAutoFollowLinks ? "" : "Don't") + " follow links when encountered", event: () => this.layoutDoc.dontAutoFollowLinks = !this.layoutDoc.dontAutoFollowLinks, icon: "expand-arrows-alt" }); - subitems.push({ description: (this.layoutDoc.dontAutoPlayFollowedLinks ? "" : "Don't") + " play when link is selected", event: () => this.layoutDoc.dontAutoPlayFollowedLinks = !this.layoutDoc.dontAutoPlayFollowedLinks, icon: "expand-arrows-alt" }); - subitems.push({ description: (this.layoutDoc.autoPlayAnchors ? "Don't auto play" : "Auto play") + " anchors onClick", event: () => this.layoutDoc.autoPlayAnchors = !this.layoutDoc.autoPlayAnchors, icon: "expand-arrows-alt" }); - // subitems.push({ description: "Toggle Native Controls", event: action(() => VideoBox._nativeControls = !VideoBox._nativeControls), icon: "expand-arrows-alt" }); - // subitems.push({ description: "Start Trim All", event: () => this.startTrim(TrimScope.All), icon: "expand-arrows-alt" }); - // subitems.push({ description: "Start Trim Clip", event: () => this.startTrim(TrimScope.Clip), icon: "expand-arrows-alt" }); - // subitems.push({ description: "Stop Trim", event: () => this.finishTrim(), icon: "expand-arrows-alt" }); - subitems.push({ description: "Copy path", event: () => { Utils.CopyText(url); }, icon: "expand-arrows-alt" }); - ContextMenu.Instance.addItem({ description: "Options...", subitems: subitems, icon: "video" }); - } - } - - - // ref for updating time - setAudioRef = (e: HTMLAudioElement | null) => this._audioPlayer = e; - - // renders the video and audio - @computed get content() { - const field = Cast(this.dataDoc[this.fieldKey], VideoField); - const interactive = CurrentUserUtils.SelectedTool !== InkTool.None || !this.props.isSelected() ? "" : "-interactive"; - const classname = "videoBox-content" + (this._fullScreen ? "-fullScreen" : "") + interactive; - return !field ? <div key="loading">Loading</div> : - <div className="videoBox-contentContainer" key="container" style={{ mixBlendMode: "multiply" }}> - <div className={classname} ref={this.setContentRef} onPointerDown={(e) => this._fullScreen && e.stopPropagation()}> - {this.uIButtons} - <video key="video" autoPlay={this._screenCapture} ref={this.setVideoRef} style={this._fullScreen ? this.fullScreenSize() : {}} - onCanPlay={this.videoLoad} - controls={VideoBox._nativeControls} - onPlay={() => this.Play()} - onSeeked={this.updateTimecode} - onPause={() => this.Pause()} - onClick={e => e.preventDefault()}> - <source src={field.url.href} type="video/mp4" /> - Not supported. - </video> - {!this.audiopath || this.audiopath === field.url.href ? (null) : - <audio ref={this.setAudioRef} className={`audiobox-control${this.props.isContentActive() ? "-interactive" : ""}`}> - <source src={this.audiopath} type="audio/mpeg" /> - Not supported. - </audio>} - </div> - </div>; - } - - - @action youtubeIframeLoaded = (e: any) => { - if (!this._youtubeContentCreated) { - this._forceCreateYouTubeIFrame = !this._forceCreateYouTubeIFrame; - return; - } - else this._youtubeContentCreated = false; - - this.loadYouTube(e.target); - } - - loadYouTube = (iframe: any) => { - let started = true; - const onYoutubePlayerStateChange = (event: any) => runInAction(() => { - if (started && event.data === YT.PlayerState.PLAYING) { - started = false; - this._youtubePlayer?.unMute(); - //this.Pause(); - return; + Doc.SetNativeWidth(Doc.GetProto(imageSummary), Doc.NativeWidth(this.layoutDoc)); + Doc.SetNativeHeight(Doc.GetProto(imageSummary), Doc.NativeHeight(this.layoutDoc)); + this.props.addDocument?.(imageSummary); + const link = DocUtils.MakeLink({ doc: imageSummary }, { doc: this.getAnchor() }, "video snapshot"); + link && (Doc.GetProto(link.anchor2 as Doc).timecodeToHide = NumCast((link.anchor2 as Doc).timecodeToShow) + 3); + setTimeout(() => + (downX !== undefined && downY !== undefined) && DocumentManager.Instance.getFirstDocumentView(imageSummary)?.startDragging(downX, downY, "move", true)); + } + + + getAnchor = () => { + const timecode = Cast(this.layoutDoc._currentTimecode, "number", null); + const marquee = AnchorMenu.Instance.GetAnchor?.(); + return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow"/* videoStart */, "_timecodeToHide" /* videoEnd */, timecode ? timecode : undefined, undefined, marquee) || this.rootDoc; + } + + + // sets video info on load + videoLoad = action(() => { + const aspect = this.player!.videoWidth / this.player!.videoHeight; + Doc.SetNativeWidth(this.dataDoc, this.player!.videoWidth); + Doc.SetNativeHeight(this.dataDoc, this.player!.videoHeight); + this.layoutDoc._height = NumCast(this.layoutDoc._width) / aspect; + if (Number.isFinite(this.player!.duration)) { + this.rawDuration = this.player!.duration; + } else this.rawDuration = NumCast(this.dataDoc[this.fieldKey + "-duration"]); + }); + + + // updates video time + @action + updateTimecode = () => { + this.player && (this.layoutDoc._currentTimecode = this.player.currentTime); + try { + this._youtubePlayer && (this.layoutDoc._currentTimecode = this._youtubePlayer.getCurrentTime?.()); + } catch (e) { + console.log("Video Timecode Exception:", e); + } + } + + + // sets video element ref + @action + setVideoRef = (vref: HTMLVideoElement | null) => { + this._videoRef = vref; + if (vref) { + this._videoRef!.ontimeupdate = this.updateTimecode; + // @ts-ignore + // vref.onfullscreenchange = action((e) => this._fullScreen = vref.webkitDisplayingFullscreen); + this._disposers.reactionDisposer?.(); + this._disposers.reactionDisposer = reaction(() => NumCast(this.layoutDoc._currentTimecode), + time => { + !this._playing && (vref.currentTime = time); + console.log("vref time = " + vref.currentTime) + }, { fireImmediately: true }); } - if (event.data === YT.PlayerState.PLAYING && !this._playing) this.Play(false); - if (event.data === YT.PlayerState.PAUSED && this._playing) this.Pause(false); - }); - const onYoutubePlayerReady = (event: any) => { - this._disposers.reactionDisposer?.(); - this._disposers.youtubeReactionDisposer?.(); - this._disposers.reactionDisposer = reaction(() => this.layoutDoc._currentTimecode, () => !this._playing && this.Seek(NumCast(this.layoutDoc._currentTimecode))); - this._disposers.youtubeReactionDisposer = reaction( - () => CurrentUserUtils.SelectedTool === InkTool.None && this.props.isSelected(true) && !SnappingManager.GetIsDragging() && !DocumentDecorations.Instance.Interacting, - (interactive) => iframe.style.pointerEvents = interactive ? "all" : "none", { fireImmediately: true }); - }; - if (typeof (YT) === undefined) setTimeout(() => this.loadYouTube(iframe), 100); - else { - (YT as any)?.ready(() => { - this._youtubePlayer = new YT.Player(`${this.youtubeVideoId + this._youtubeIframeId}-player`, { - events: { - 'onReady': this.props.dontRegisterView ? undefined : onYoutubePlayerReady, - 'onStateChange': this.props.dontRegisterView ? undefined : onYoutubePlayerStateChange, - } - }); + } + + // set ref for div that wraps video and controls for fullscreen + @action + setContentRef = (cref: HTMLDivElement | null) => { + this._contentRef = cref; + if (cref) { + cref.onfullscreenchange = action((e) => this._fullScreen = (document.fullscreenElement === cref)); + } + } + + + // context menu + specificContextMenu = (e: React.MouseEvent): void => { + const field = Cast(this.dataDoc[this.props.fieldKey], VideoField); + if (field) { + const url = field.url.href; + const subitems: ContextMenuProps[] = []; + subitems.push({ description: "Full Screen", event: this.FullScreen, icon: "expand" }); + subitems.push({ description: "Take Snapshot", event: this.Snapshot, icon: "expand-arrows-alt" }); + this.rootDoc.type === DocumentType.SCREENSHOT && subitems.push({ + description: "Screen Capture", event: (async () => { + runInAction(() => this._screenCapture = !this._screenCapture); + this._videoRef!.srcObject = !this._screenCapture ? undefined : await (navigator.mediaDevices as any).getDisplayMedia({ video: true }); + }), icon: "expand-arrows-alt" + }); + subitems.push({ description: (this.layoutDoc.dontAutoFollowLinks ? "" : "Don't") + " follow links when encountered", event: () => this.layoutDoc.dontAutoFollowLinks = !this.layoutDoc.dontAutoFollowLinks, icon: "expand-arrows-alt" }); + subitems.push({ description: (this.layoutDoc.dontAutoPlayFollowedLinks ? "" : "Don't") + " play when link is selected", event: () => this.layoutDoc.dontAutoPlayFollowedLinks = !this.layoutDoc.dontAutoPlayFollowedLinks, icon: "expand-arrows-alt" }); + subitems.push({ description: (this.layoutDoc.autoPlayAnchors ? "Don't auto play" : "Auto play") + " anchors onClick", event: () => this.layoutDoc.autoPlayAnchors = !this.layoutDoc.autoPlayAnchors, icon: "expand-arrows-alt" }); + // subitems.push({ description: "Toggle Native Controls", event: action(() => VideoBox._nativeControls = !VideoBox._nativeControls), icon: "expand-arrows-alt" }); + // subitems.push({ description: "Start Trim All", event: () => this.startTrim(TrimScope.All), icon: "expand-arrows-alt" }); + // subitems.push({ description: "Start Trim Clip", event: () => this.startTrim(TrimScope.Clip), icon: "expand-arrows-alt" }); + // subitems.push({ description: "Stop Trim", event: () => this.finishTrim(), icon: "expand-arrows-alt" }); + subitems.push({ description: "Copy path", event: () => { Utils.CopyText(url); }, icon: "expand-arrows-alt" }); + // if the videobox was turned from a recording box + if (this.dataDoc[this.fieldKey + "-recorded"] === true) { + subitems.push({ + description: "Recreate recording", event: () => { + this.dataDoc.layout = RecordingBox.LayoutString(this.fieldKey); + // delete assoicated video data + this.dataDoc[this.props.fieldKey] = ""; + this.dataDoc[this.fieldKey + "-duration"] = ""; + // delete assoicated presentation data + this.dataDoc[this.fieldKey + "-presentation"] = ""; + }, icon: "expand-arrows-alt" + }); + + } + ContextMenu.Instance.addItem({ description: "Options...", subitems: subitems, icon: "video" }); + } + } + + + // ref for updating time + setAudioRef = (e: HTMLAudioElement | null) => this._audioPlayer = e; + + // renders the video and audio + @computed get content() { + const field = Cast(this.dataDoc[this.fieldKey], VideoField); + const interactive = CurrentUserUtils.SelectedTool !== InkTool.None || !this.props.isSelected() ? "" : "-interactive"; + const classname = "videoBox-content" + (this._fullScreen ? "-fullScreen" : "") + interactive; + return !field ? <div key="loading">Loading</div> : + <div className="videoBox-contentContainer" key="container" style={{ mixBlendMode: "multiply" }}> + <div className={classname} ref={this.setContentRef} onPointerDown={(e) => this._fullScreen && e.stopPropagation()}> + {this.uIButtons} + <video key="video" autoPlay={this._screenCapture} ref={this.setVideoRef} style={this._fullScreen ? this.fullScreenSize() : {}} + onCanPlay={this.videoLoad} + controls={VideoBox._nativeControls} + onPlay={() => { + // console.log("PLAY from CONTENT") + //this.Play() + }} + onSeeked={this.updateTimecode} + // onPause={() => this.Pause() } + onClick={e => e.preventDefault()}> + <source src={field.url.href} type="video/mp4" /> + Not supported. + </video> + {!this.audiopath || this.audiopath === field.url.href ? (null) : + <audio ref={this.setAudioRef} className={`audiobox-control${this.props.isContentActive() ? "-interactive" : ""}`}> + <source src={this.audiopath} type="audio/mpeg" /> + Not supported. + </audio>} + </div> + </div>; + } + + + @action youtubeIframeLoaded = (e: any) => { + if (!this._youtubeContentCreated) { + this._forceCreateYouTubeIFrame = !this._forceCreateYouTubeIFrame; + return; + } + else this._youtubeContentCreated = false; + + this.loadYouTube(e.target); + } + + loadYouTube = (iframe: any) => { + let started = true; + const onYoutubePlayerStateChange = (event: any) => runInAction(() => { + if (started && event.data === YT.PlayerState.PLAYING) { + started = false; + this._youtubePlayer?.unMute(); + //this.Pause(); + return; + } + if (event.data === YT.PlayerState.PLAYING && !this._playing) this.Play(false); + if (event.data === YT.PlayerState.PAUSED && this._playing) this.Pause(false); }); - } - } - - - // for play button - onPlayDown = () => this._playing ? this.Pause() : this.Play(); - - // for fullscreen button - onFullDown = (e: React.PointerEvent) => { - this.FullScreen(); - e.stopPropagation(); - e.preventDefault(); - } - - // for snapshot button - onSnapshotDown = (e: React.PointerEvent) => { - setupMoveUpEvents(this, e, (e) => { - this.Snapshot(e.clientX, e.clientY); - return true; - }, emptyFunction, () => this.Snapshot()); - } - - // for show/hide timeline button, transitions between show/hide - @action - onTimelineHdlDown = (e: React.PointerEvent) => { - this._clicking = true; - setupMoveUpEvents(this, e, - action(encodeURIComponent => { - this._clicking = false; - if (this.props.isContentActive()) { - // const local = this.props.ScreenToLocalTransform().scale(this.props.scaling?.() || 1).transformPoint(e.clientX, e.clientY); - // this.layoutDoc._timelineHeightPercent = Math.max(0, Math.min(100, local[1] / this.props.PanelHeight() * 100)); - - this.layoutDoc._timelineHeightPercent = 80; - } - return false; - }), emptyFunction, - () => { - this.layoutDoc._timelineHeightPercent = this.heightPercent !== 100 ? 100 : VideoBox.heightPercent; - setTimeout(action(() => this._clicking = false), 500); - }, this.props.isContentActive(), this.props.isContentActive()); - } - - - // removes video from currently playing display - @action - removeCurrentlyPlaying = () => { - if (CollectionStackedTimeline.CurrentlyPlaying) { - const index = CollectionStackedTimeline.CurrentlyPlaying.indexOf(this.layoutDoc); - index !== -1 && CollectionStackedTimeline.CurrentlyPlaying.splice(index, 1); - } - } - - // adds video to currently playing display - @action - addCurrentlyPlaying = () => { - if (!CollectionStackedTimeline.CurrentlyPlaying) { - CollectionStackedTimeline.CurrentlyPlaying = []; - } - if (CollectionStackedTimeline.CurrentlyPlaying.indexOf(this.layoutDoc) === -1) { - CollectionStackedTimeline.CurrentlyPlaying.push(this.layoutDoc); - } - } - - - @computed get youtubeContent() { - this._youtubeIframeId = VideoBox._youtubeIframeCounter++; - this._youtubeContentCreated = this._forceCreateYouTubeIFrame ? true : true; - const classname = "videoBox-content-YouTube" + (this._fullScreen ? "-fullScreen" : ""); - const start = untracked(() => Math.round(NumCast(this.layoutDoc._currentTimecode))); - return <iframe key={this._youtubeIframeId} id={`${this.youtubeVideoId + this._youtubeIframeId}-player`} - onPointerLeave={this.updateTimecode} - onLoad={this.youtubeIframeLoaded} className={classname} width={Doc.NativeWidth(this.layoutDoc) || 640} height={Doc.NativeHeight(this.layoutDoc) || 390} - src={`https://www.youtube.com/embed/${this.youtubeVideoId}?enablejsapi=1&rel=0&showinfo=1&autoplay=0&mute=1&start=${start}&modestbranding=1&controls=${VideoBox._nativeControls ? 1 : 0}`} />; - } - - - // for annotating, adds doc with time info - @action.bound - addDocWithTimecode(doc: Doc | Doc[]): boolean { - const docs = doc instanceof Doc ? [doc] : doc; - const curTime = NumCast(this.layoutDoc._currentTimecode); - docs.forEach(doc => doc._timecodeToHide = (doc._timecodeToShow = curTime) + 1); - return this.addDocument(doc); - } - - - // play back the audio from seekTimeInSeconds, fullPlay tells whether clip is being played to end vs link range - @action - playFrom = (seekTimeInSeconds: number, endTime?: number, fullPlay: boolean = false) => { - clearTimeout(this._playRegionTimer); - this._playRegionTimer = undefined; - if (Number.isNaN(this.player?.duration)) { - setTimeout(() => this.playFrom(seekTimeInSeconds, endTime), 500); - } - else if (this.player) { - // trimBounds override requested playback bounds - const end = Math.min(this.timeline?.trimEnd ?? this.rawDuration, endTime ?? this.timeline?.trimEnd ?? this.rawDuration); - const start = Math.max(this.timeline?.trimStart ?? 0, seekTimeInSeconds); - const playRegionDuration = end - start; - // checks if times are within clip range - if (seekTimeInSeconds >= 0 && (this.timeline?.trimStart || 0) <= end && seekTimeInSeconds <= (this.timeline?.trimEnd || this.rawDuration)) { - this.player.currentTime = start; - this._audioPlayer && (this._audioPlayer.currentTime = seekTimeInSeconds); - this.player.play(); - this._audioPlayer?.play(); - this._playing = true; - this.addCurrentlyPlaying(); - this._playRegionTimer = setTimeout( - () => { - // need to keep track of if end of clip is reached so on next play, clip restarts - if (fullPlay) this._finished = true; - // removes from currently playing if playback has reached end of range marker - else this.removeCurrentlyPlaying(); + const onYoutubePlayerReady = (event: any) => { + this._disposers.reactionDisposer?.(); + this._disposers.youtubeReactionDisposer?.(); + this._disposers.reactionDisposer = reaction(() => this.layoutDoc._currentTimecode, () => !this._playing && this.Seek(NumCast(this.layoutDoc._currentTimecode))); + this._disposers.youtubeReactionDisposer = reaction( + () => CurrentUserUtils.SelectedTool === InkTool.None && this.props.isSelected(true) && !SnappingManager.GetIsDragging() && !DocumentDecorations.Instance.Interacting, + (interactive) => iframe.style.pointerEvents = interactive ? "all" : "none", { fireImmediately: true }); + }; + if (typeof (YT) === undefined) setTimeout(() => this.loadYouTube(iframe), 100); + else { + (YT as any)?.ready(() => { + this._youtubePlayer = new YT.Player(`${this.youtubeVideoId + this._youtubeIframeId}-player`, { + events: { + 'onReady': this.props.dontRegisterView ? undefined : onYoutubePlayerReady, + 'onStateChange': this.props.dontRegisterView ? undefined : onYoutubePlayerStateChange, + } + }); + }); + } + } + + + // for play button + + onPlayDown = () => { + console.log("PLAY DOWN"); + this._playing ? this.Pause() : this.Play(); + } + + // for fullscreen button + onFullDown = (e: React.PointerEvent) => { + this.FullScreen(); + e.stopPropagation(); + e.preventDefault(); + } + + // for snapshot button + onSnapshotDown = (e: React.PointerEvent) => { + setupMoveUpEvents(this, e, (e) => { + this.Snapshot(e.clientX, e.clientY); + return true; + }, emptyFunction, () => this.Snapshot()); + } + + // for show/hide timeline button, transitions between show/hide + @action + onTimelineHdlDown = (e: React.PointerEvent) => { + this._clicking = true; + setupMoveUpEvents(this, e, + action(encodeURIComponent => { + this._clicking = false; + if (this.props.isContentActive()) { + // const local = this.props.ScreenToLocalTransform().scale(this.props.scaling?.() || 1).transformPoint(e.clientX, e.clientY); + // this.layoutDoc._timelineHeightPercent = Math.max(0, Math.min(100, local[1] / this.props.PanelHeight() * 100)); + + this.layoutDoc._timelineHeightPercent = 80; + } + return false; + }), emptyFunction, + () => { + this.layoutDoc._timelineHeightPercent = this.heightPercent !== 100 ? 100 : VideoBox.heightPercent; + setTimeout(action(() => this._clicking = false), 500); + }, this.props.isContentActive(), this.props.isContentActive()); + } + + + // removes video from currently playing display + @action + removeCurrentlyPlaying = () => { + if (CollectionStackedTimeline.CurrentlyPlaying) { + const index = CollectionStackedTimeline.CurrentlyPlaying.indexOf(this.layoutDoc); + index !== -1 && CollectionStackedTimeline.CurrentlyPlaying.splice(index, 1); + } + } + + // adds video to currently playing display + @action + addCurrentlyPlaying = () => { + if (!CollectionStackedTimeline.CurrentlyPlaying) { + CollectionStackedTimeline.CurrentlyPlaying = []; + } + if (CollectionStackedTimeline.CurrentlyPlaying.indexOf(this.layoutDoc) === -1) { + CollectionStackedTimeline.CurrentlyPlaying.push(this.layoutDoc); + } + } + + + @computed get youtubeContent() { + this._youtubeIframeId = VideoBox._youtubeIframeCounter++; + this._youtubeContentCreated = this._forceCreateYouTubeIFrame ? true : true; + const classname = "videoBox-content-YouTube" + (this._fullScreen ? "-fullScreen" : ""); + const start = untracked(() => Math.round(NumCast(this.layoutDoc._currentTimecode))); + return <iframe key={this._youtubeIframeId} id={`${this.youtubeVideoId + this._youtubeIframeId}-player`} + onPointerLeave={this.updateTimecode} + onLoad={this.youtubeIframeLoaded} className={classname} width={Doc.NativeWidth(this.layoutDoc) || 640} height={Doc.NativeHeight(this.layoutDoc) || 390} + src={`https://www.youtube.com/embed/${this.youtubeVideoId}?enablejsapi=1&rel=0&showinfo=1&autoplay=0&mute=1&start=${start}&modestbranding=1&controls=${VideoBox._nativeControls ? 1 : 0}`} />; + } + + + // for annotating, adds doc with time info + @action.bound + addDocWithTimecode(doc: Doc | Doc[]): boolean { + const docs = doc instanceof Doc ? [doc] : doc; + const curTime = NumCast(this.layoutDoc._currentTimecode); + docs.forEach(doc => doc._timecodeToHide = (doc._timecodeToShow = curTime) + 1); + return this.addDocument(doc); + } + + + // play back the audio from seekTimeInSeconds, fullPlay tells whether clip is being played to end vs link range + @action + playFrom = (seekTimeInSeconds: number, endTime?: number, fullPlay: boolean = false) => { + clearTimeout(this._playRegionTimer); + if (Number.isNaN(this.player?.duration)) { + setTimeout(() => this.playFrom(seekTimeInSeconds, endTime), 500); + } + else if (this.player) { + // trimBounds override requested playback bounds + const end = Math.min(this.timeline?.trimEnd ?? this.rawDuration, endTime ?? this.timeline?.trimEnd ?? this.rawDuration); + const start = Math.max(this.timeline?.trimStart ?? 0, seekTimeInSeconds); + const playRegionDuration = end - start; + // checks if times are within clip range + if (seekTimeInSeconds >= 0 && (this.timeline?.trimStart || 0) <= end && seekTimeInSeconds <= (this.timeline?.trimEnd || this.rawDuration)) { + this.player.currentTime = start; + this._audioPlayer && (this._audioPlayer.currentTime = seekTimeInSeconds); + this.player.play(); + this._audioPlayer?.play(); + this._playing = true; + this.addCurrentlyPlaying(); + this._playRegionTimer = setTimeout( + () => { + // need to keep track of if end of clip is reached so on next play, clip restarts + if (fullPlay) { + Doc.UserDoc().presentationMode = 'none'; + this._finished = true; + } + // removes from currently playing if playback has reached end of range marker + else this.removeCurrentlyPlaying(); + this.Pause(); + }, playRegionDuration * 1000); + } else { this.Pause(); - }, playRegionDuration * 1000); - } else { - this.Pause(); + } } - } - } - - - // ends trim, hides trim controls and displays new clip - @undoBatch - finishTrim = action(() => { - this.Pause(); - this.setPlayheadTime(Math.max(Math.min(this.timeline?.trimEnd || 0, this.player!.currentTime), this.timeline?.trimStart || 0)); - this.timeline?.StopTrimming(); - }); - - // displays trim controls to start trimming clip - startTrim = (scope: TrimScope) => { - this.Pause(); - this.timeline?.StartTrimming(scope); - } - - // for trim button, double click displays full clip, single displays curr trim bounds - onClipPointerDown = (e: React.PointerEvent) => { - // if timeline isn't shown, show first then trim - this.heightPercent >= 100 && this.onTimelineHdlDown(e); - this.timeline && setupMoveUpEvents(this, e, returnFalse, returnFalse, action((e: PointerEvent, doubleTap?: boolean) => { - if (doubleTap) { - this.startTrim(TrimScope.All); - } else if (this.timeline) { - this.Pause(); - this.timeline.IsTrimming !== TrimScope.None ? this.finishTrim() : this.startTrim(TrimScope.Clip); + } + + + // ends trim, hides trim controls and displays new clip + @undoBatch + finishTrim = action(() => { + this.Pause(); + this.setPlayheadTime(Math.max(Math.min(this.timeline?.trimEnd || 0, this.player!.currentTime), this.timeline?.trimStart || 0)); + this.timeline?.StopTrimming(); + }); + + // displays trim controls to start trimming clip + startTrim = (scope: TrimScope) => { + this.Pause(); + this.timeline?.StartTrimming(scope); + } + + // for trim button, double click displays full clip, single displays curr trim bounds + onClipPointerDown = (e: React.PointerEvent) => { + // if timeline isn't shown, show first then trim + this.heightPercent >= 100 && this.onTimelineHdlDown(e); + this.timeline && setupMoveUpEvents(this, e, returnFalse, returnFalse, action((e: PointerEvent, doubleTap?: boolean) => { + if (doubleTap) { + this.startTrim(TrimScope.All); + } else if (this.timeline) { + this.Pause(); + this.timeline.IsTrimming !== TrimScope.None ? this.finishTrim() : this.startTrim(TrimScope.Clip); + } + })); + } + + + // for volume slider sets volume + @action + setVolume = (volume: number) => { + if (this.player) { + this._volume = volume; + this.player.volume = volume; + if (this._muted) { + this.toggleMute(); + } } - })); - } - - - // for volume slider sets volume - @action - setVolume = (volume: number) => { - if (this.player) { - this._volume = volume; - this.player.volume = volume; - if (this._muted) { - this.toggleMute(); + } + + // toggles video mute + @action + toggleMute = () => { + if (this.player) { + this._muted = !this._muted; + this.player.muted = this._muted; } - } - } - - // toggles video mute - @action - toggleMute = () => { - if (this.player) { - this._muted = !this._muted; - this.player.muted = this._muted; - } - } - - - // stretches vertically or horizontally depending on video orientation so video fits full screen - fullScreenSize() { - if (this._videoRef && this._videoRef.videoHeight / this._videoRef.videoWidth > 1) { - return { height: "100%" }; - } - else { - return { width: "100%" }; - } - } - - - // for zoom slider, sets timeline waveform zoom - zoom = (zoom: number) => { - this.timeline?.setZoom(zoom); - } - - - // plays link - playLink = (doc: Doc) => { - const startTime = Math.max(0, (this._stackedTimeline?.anchorStart(doc) || 0)); - const endTime = this.timeline?.anchorEnd(doc); - if (startTime !== undefined) { - if (!this.layoutDoc.dontAutoPlayFollowedLinks) endTime ? this.playFrom(startTime, endTime) : this.playFrom(startTime); - else this.Seek(startTime); - } - } - - - // starts marquee selection - marqueeDown = (e: React.PointerEvent) => { - if (!e.altKey && e.button === 0 && this.layoutDoc._viewScale === 1 && this.props.isContentActive(true) && ![InkTool.Highlighter, InkTool.Pen].includes(CurrentUserUtils.SelectedTool)) { - setupMoveUpEvents(this, e, action(e => { - MarqueeAnnotator.clearAnnotations(this._savedAnnotations); - this._marqueeing = [e.clientX, e.clientY]; - return true; - }), returnFalse, () => MarqueeAnnotator.clearAnnotations(this._savedAnnotations), false); - } - } - - // ends marquee selection - @action - finishMarquee = () => { - this._marqueeing = undefined; - this.props.select(true); - } - - timelineWhenChildContentsActiveChanged = action((isActive: boolean) => this.props.whenChildContentsActiveChanged(this._isAnyChildContentActive = isActive)); - - timelineScreenToLocal = () => this.props.ScreenToLocalTransform().scale(this.scaling()).translate(0, -this.heightPercent / 100 * this.props.PanelHeight()); - - setPlayheadTime = (time: number) => this.player!.currentTime = this.layoutDoc._currentTimecode = time; - - timelineHeight = () => this.props.PanelHeight() * (100 - this.heightPercent) / 100; - - playing = () => this._playing; - - contentFunc = () => [this.youtubeVideoId ? this.youtubeContent : this.content]; - - scaling = () => this.props.scaling?.() || 1; - - panelWidth = () => this.props.PanelWidth() * this.heightPercent / 100; - panelHeight = () => this.layoutDoc._fitWidth ? this.panelWidth() / (Doc.NativeAspect(this.rootDoc) || 1) : this.props.PanelHeight() * this.heightPercent / 100; - - screenToLocalTransform = () => { - const offset = (this.props.PanelWidth() - this.panelWidth()) / 2 / this.scaling(); - return this.props.ScreenToLocalTransform().translate(-offset, 0).scale(100 / this.heightPercent); - } - - marqueeFitScaling = () => (this.props.scaling?.() || 1) * this.heightPercent / 100; - marqueeOffset = () => [this.panelWidth() / 2 * (1 - this.heightPercent / 100) / (this.heightPercent / 100), 0]; - - timelineDocFilter = () => [`_timelineLabel:true,${Utils.noRecursionHack}:x`]; - - - // renders video controls - @computed get uIButtons() { - const curTime = NumCast(this.layoutDoc._currentTimecode) - (this.timeline?.clipStart || 0); - const scaling = (this.props.scaling?.() || 1); - return <div className="videoBox-ui" style={{ - transform: `scale(${1 / scaling})`, width: `${100 * scaling}%`, bottom: 20 / scaling - }}> - <div className="videobox-button" - title={this._playing ? "play" : "pause"} - onPointerDown={this.onPlayDown}> - <FontAwesomeIcon icon={this._playing ? "pause" : "play"} /> - </div> - - {this.timeline && <div className="timecode-controls"> - <div className="timecode-current"> - {formatTime(curTime)} - </div> - - {this._fullScreen || this.heightPercent === 100 ? - <div className="timeline-slider"> - <input type="range" step="0.1" min={this.timeline.clipStart} max={this.timeline.clipEnd} value={curTime} - className="toolbar-slider time-progress" - onPointerDown={(e: React.PointerEvent) => { e.stopPropagation(); }} - onChange={(e: React.ChangeEvent<HTMLInputElement>) => this.setPlayheadTime(Number(e.target.value))} - /> - </div> - : - <div>/</div>} - - <div className="timecode-end"> - {formatTime(this.timeline.clipDuration)} - </div> - </div>} - - <div className="videobox-button" - title={"full screen"} - onPointerDown={this.onFullDown}> - <FontAwesomeIcon icon="expand" /> - </div> - - {!this._fullScreen && <div className="videobox-button" - title={"show timeline"} - onPointerDown={this.onTimelineHdlDown}> - <FontAwesomeIcon icon="eye" /> - </div>} - - {!this._fullScreen && <div className="videobox-button" - title={this.timeline?.IsTrimming !== TrimScope.None ? "finish trimming" : "start trim"} - onPointerDown={this.onClipPointerDown}> - <FontAwesomeIcon icon={this.timeline?.IsTrimming !== TrimScope.None ? "check" : "cut"} /> - </div>} - - <div className="videobox-button show-slider" - title={this._muted ? "unmute" : "mute"} - onPointerDown={(e) => { e.stopPropagation(); this.toggleMute(); }}> - <FontAwesomeIcon icon={this._muted ? "volume-mute" : "volume-up"} /> - </div> - <input type="range" style={{ width: `min(25%, 50px)` }} step="0.1" min="0" max="1" value={this._muted ? 0 : this._volume} - className="toolbar-slider volume" - onPointerDown={(e: React.PointerEvent) => e.stopPropagation()} - onChange={(e: React.ChangeEvent<HTMLInputElement>) => this.setVolume(Number(e.target.value))} - /> - - {!this._fullScreen && this.heightPercent !== 100 && - <> - <div className="videobox-button" title="zoom"> - <FontAwesomeIcon icon="search-plus" /> - </div> - <input type="range" step="0.1" min="1" max="5" value={this.timeline?._zoomFactor} - className="toolbar-slider zoom" - onPointerDown={(e: React.PointerEvent) => { e.stopPropagation(); }} - onChange={(e: React.ChangeEvent<HTMLInputElement>) => { this.zoom(Number(e.target.value)); }} - /> - </>} - </div>; - } - - // renders CollectionStackedTimeline - @computed get renderTimeline() { - return <div className="videoBox-stackPanel" style={{ transition: this.transition, height: `${100 - this.heightPercent}%` }}> - <CollectionStackedTimeline ref={action((r: any) => this._stackedTimeline = r)} {...this.props} - fieldKey={this.annotationKey} - dictationKey={this.fieldKey + "-dictation"} - mediaPath={this.audiopath} - renderDepth={this.props.renderDepth + 1} - startTag={"_timecodeToShow" /* videoStart */} - endTag={"_timecodeToHide" /* videoEnd */} - bringToFront={emptyFunction} - CollectionView={undefined} - playFrom={this.playFrom} - setTime={this.setPlayheadTime} - playing={this.playing} - isAnyChildContentActive={this.isAnyChildContentActive} - whenChildContentsActiveChanged={this.timelineWhenChildContentsActiveChanged} - moveDocument={this.moveDocument} - addDocument={this.addDocument} - removeDocument={this.removeDocument} - ScreenToLocalTransform={this.timelineScreenToLocal} - Play={this.Play} - Pause={this.Pause} - playLink={this.playLink} - PanelHeight={this.timelineHeight} - rawDuration={this.rawDuration} - /> - </div>; - } - - // renders annotation layer - @computed get annotationLayer() { - return <div className="videoBox-annotationLayer" style={{ transition: this.transition, height: `${this.heightPercent}%` }} ref={this._annotationLayer} />; - } - - savedAnnotations = () => this._savedAnnotations; - render() { - const borderRad = this.props.styleProvider?.(this.layoutDoc, this.props, StyleProp.BorderRounding); - const borderRadius = borderRad?.includes("px") ? `${Number(borderRad.split("px")[0]) / this.scaling()}px` : borderRad; - return (<div className="videoBox" onContextMenu={this.specificContextMenu} ref={this._mainCont} - style={{ - pointerEvents: this.layoutDoc._lockedPosition ? "none" : undefined, - borderRadius, - overflow: this.props.docViewPath?.().slice(-1)[0].fitWidth ? "auto" : undefined - }} onWheel={e => { e.stopPropagation(); e.preventDefault(); }}> - <div className="videoBox-viewer" onPointerDown={this.marqueeDown} > - <div style={{ - position: "absolute", transition: this.transition, - width: this.panelWidth(), - height: this.panelHeight(), - top: 0, - left: (this.props.PanelWidth() - this.panelWidth()) / 2 - }}> - <CollectionFreeFormView {...OmitKeys(this.props, ["NativeWidth", "NativeHeight", "setContentView"]).omit} - renderDepth={this.props.renderDepth + 1} + } + + + // stretches vertically or horizontally depending on video orientation so video fits full screen + fullScreenSize() { + if (this._videoRef && this._videoRef.videoHeight / this._videoRef.videoWidth > 1) { + return { height: "100%" }; + } + else { + return { width: "100%" }; + } + } + + + // for zoom slider, sets timeline waveform zoom + zoom = (zoom: number) => { + this.timeline?.setZoom(zoom); + } + + + // plays link + playLink = (doc: Doc) => { + const startTime = Math.max(0, (this._stackedTimeline?.anchorStart(doc) || 0)); + const endTime = this.timeline?.anchorEnd(doc); + if (startTime !== undefined) { + if (!this.layoutDoc.dontAutoPlayFollowedLinks) endTime ? this.playFrom(startTime, endTime) : this.playFrom(startTime); + else this.Seek(startTime); + } + } + + + // starts marquee selection + marqueeDown = (e: React.PointerEvent) => { + if (!e.altKey && e.button === 0 && this.layoutDoc._viewScale === 1 && this.props.isContentActive(true) && ![InkTool.Highlighter, InkTool.Pen].includes(CurrentUserUtils.SelectedTool)) { + setupMoveUpEvents(this, e, action(e => { + MarqueeAnnotator.clearAnnotations(this._savedAnnotations); + this._marqueeing = [e.clientX, e.clientY]; + return true; + }), returnFalse, () => MarqueeAnnotator.clearAnnotations(this._savedAnnotations), false); + } + } + + // ends marquee selection + @action + finishMarquee = () => { + this._marqueeing = undefined; + this.props.select(true); + } + + timelineWhenChildContentsActiveChanged = action((isActive: boolean) => this.props.whenChildContentsActiveChanged(this._isAnyChildContentActive = isActive)); + + timelineScreenToLocal = () => this.props.ScreenToLocalTransform().scale(this.scaling()).translate(0, -this.heightPercent / 100 * this.props.PanelHeight()); + + setPlayheadTime = (time: number) => this.player!.currentTime = this.layoutDoc._currentTimecode = time; + + timelineHeight = () => this.props.PanelHeight() * (100 - this.heightPercent) / 100; + + playing = () => this._playing; + + contentFunc = () => [this.youtubeVideoId ? this.youtubeContent : this.content]; + + scaling = () => this.props.scaling?.() || 1; + + panelWidth = () => this.props.PanelWidth() * this.heightPercent / 100; + panelHeight = () => this.layoutDoc._fitWidth ? this.panelWidth() / (Doc.NativeAspect(this.rootDoc) || 1) : this.props.PanelHeight() * this.heightPercent / 100; + + screenToLocalTransform = () => { + const offset = (this.props.PanelWidth() - this.panelWidth()) / 2 / this.scaling(); + return this.props.ScreenToLocalTransform().translate(-offset, 0).scale(100 / this.heightPercent); + } + + marqueeFitScaling = () => (this.props.scaling?.() || 1) * this.heightPercent / 100; + marqueeOffset = () => [this.panelWidth() / 2 * (1 - this.heightPercent / 100) / (this.heightPercent / 100), 0]; + + timelineDocFilter = () => [`_timelineLabel:true,${Utils.noRecursionHack}:x`]; + + + // renders video controls + @computed get uIButtons() { + const curTime = NumCast(this.layoutDoc._currentTimecode) - (this.timeline?.clipStart || 0); + return <div className="videoBox-ui" style={this._fullScreen || this.heightPercent === 100 ? { fontSize: "40px", minWidth: "80%" } : {}}> + <div className="videobox-button" + title={this._playing ? "play" : "pause"} + onPointerDown={this.onPlayDown}> + <FontAwesomeIcon icon={this._playing ? "pause" : "play"} /> + </div> + + {this.timeline && <div className="timecode-controls"> + <div className="timecode-current"> + {formatTime(curTime)} + </div> + + {this._fullScreen || this.heightPercent === 100 ? + <div className="timeline-slider"> + <input type="range" step="0.1" min={this.timeline.clipStart} max={this.timeline.clipEnd} value={curTime} + className="toolbar-slider time-progress" + onPointerDown={(e: React.PointerEvent) => { e.stopPropagation(); }} + onChange={(e: React.ChangeEvent<HTMLInputElement>) => this.setPlayheadTime(Number(e.target.value))} + /> + </div> + : + <div>/</div>} + + <div className="timecode-end"> + {formatTime(this.timeline.clipDuration)} + </div> + </div>} + + <div className="videobox-button" + title={"full screen"} + onPointerDown={this.onFullDown}> + <FontAwesomeIcon icon="expand" /> + </div> + + {!this._fullScreen && <div className="videobox-button" + title={"show timeline"} + onPointerDown={this.onTimelineHdlDown}> + <FontAwesomeIcon icon="eye" /> + </div>} + + {!this._fullScreen && <div className="videobox-button" + title={this.timeline?.IsTrimming !== TrimScope.None ? "finish trimming" : "start trim"} + onPointerDown={this.onClipPointerDown}> + <FontAwesomeIcon icon={this.timeline?.IsTrimming !== TrimScope.None ? "check" : "cut"} /> + </div>} + + <div className="videobox-button show-slider" + title={this._muted ? "unmute" : "mute"} + onPointerDown={(e) => { e.stopPropagation(); this.toggleMute(); }}> + <FontAwesomeIcon icon={this._muted ? "volume-mute" : "volume-up"} /> + </div> + <input type="range" step="0.1" min="0" max="1" value={this._muted ? 0 : this._volume} + className="toolbar-slider volume" + onPointerDown={(e: React.PointerEvent) => e.stopPropagation()} + onChange={(e: React.ChangeEvent<HTMLInputElement>) => this.setVolume(Number(e.target.value))} + /> + + {!this._fullScreen && this.heightPercent !== 100 && + <> + <div className="videobox-button" title="zoom"> + <FontAwesomeIcon icon="search-plus" /> + </div> + <input type="range" step="0.1" min="1" max="5" value={this.timeline?._zoomFactor} + className="toolbar-slider zoom" + onPointerDown={(e: React.PointerEvent) => { e.stopPropagation(); }} + onChange={(e: React.ChangeEvent<HTMLInputElement>) => { this.zoom(Number(e.target.value)); }} + /> + </>} + </div>; + } + + // renders CollectionStackedTimeline + @computed get renderTimeline() { + return <div className="videoBox-stackPanel" style={{ transition: this.transition, height: `${100 - this.heightPercent}%` }}> + <CollectionStackedTimeline ref={action((r: any) => this._stackedTimeline = r)} {...this.props} fieldKey={this.annotationKey} + dictationKey={this.fieldKey + "-dictation"} + mediaPath={this.audiopath} + renderDepth={this.props.renderDepth + 1} + startTag={"_timecodeToShow" /* videoStart */} + endTag={"_timecodeToHide" /* videoEnd */} + bringToFront={emptyFunction} CollectionView={undefined} - isAnnotationOverlay={true} - annotationLayerHostsContent={true} - PanelWidth={this.panelWidth} - PanelHeight={this.panelHeight} - ScreenToLocalTransform={this.screenToLocalTransform} - docFilters={this.timelineDocFilter} - select={emptyFunction} - scaling={returnOne} - whenChildContentsActiveChanged={this.whenChildContentsActiveChanged} - removeDocument={this.removeDocument} + playFrom={this.playFrom} + setTime={this.setPlayheadTime} + playing={this.playing} + isAnyChildContentActive={this.isAnyChildContentActive} + whenChildContentsActiveChanged={this.timelineWhenChildContentsActiveChanged} moveDocument={this.moveDocument} - addDocument={this.addDocWithTimecode}> - {this.contentFunc} - </CollectionFreeFormView> - </div> - {this.annotationLayer} - {!this._marqueeing || !this._mainCont.current || !this._annotationLayer.current ? (null) : - <MarqueeAnnotator - rootDoc={this.rootDoc} - scrollTop={0} - down={this._marqueeing} - scaling={this.marqueeFitScaling} - docView={this.props.docViewPath().slice(-1)[0]} - containerOffset={this.marqueeOffset} - addDocument={this.addDocWithTimecode} - finishMarquee={this.finishMarquee} - savedAnnotations={this.savedAnnotations} - annotationLayer={this._annotationLayer.current} - mainCont={this._mainCont.current} - />} - {this.renderTimeline} - </div> - </div >); - } + addDocument={this.addDocument} + removeDocument={this.removeDocument} + ScreenToLocalTransform={this.timelineScreenToLocal} + Play={this.Play} + Pause={this.Pause} + playLink={this.playLink} + PanelHeight={this.timelineHeight} + rawDuration={this.rawDuration} + /> + </div>; + } + + // renders annotation layer + @computed get annotationLayer() { + return <div className="videoBox-annotationLayer" style={{ transition: this.transition, height: `${this.heightPercent}%` }} ref={this._annotationLayer} />; + } + + savedAnnotations = () => this._savedAnnotations; + render() { + const borderRad = this.props.styleProvider?.(this.layoutDoc, this.props, StyleProp.BorderRounding); + const borderRadius = borderRad?.includes("px") ? `${Number(borderRad.split("px")[0]) / this.scaling()}px` : borderRad; + return (<div className="videoBox" onContextMenu={this.specificContextMenu} ref={this._mainCont} + style={{ + pointerEvents: this.layoutDoc._lockedPosition ? "none" : undefined, + borderRadius, + overflow: this.props.docViewPath?.().slice(-1)[0].fitWidth ? "auto" : undefined + }} onWheel={e => { e.stopPropagation(); e.preventDefault(); }}> + <div className="videoBox-viewer" onPointerDown={this.marqueeDown} > + <div style={{ + position: "absolute", transition: this.transition, + width: this.panelWidth(), + height: this.panelHeight(), + top: 0, + left: (this.props.PanelWidth() - this.panelWidth()) / 2 + }}> + <CollectionFreeFormView {...OmitKeys(this.props, ["NativeWidth", "NativeHeight", "setContentView"]).omit} + renderDepth={this.props.renderDepth + 1} + fieldKey={this.annotationKey} + CollectionView={undefined} + isAnnotationOverlay={true} + annotationLayerHostsContent={true} + PanelWidth={this.panelWidth} + PanelHeight={this.panelHeight} + ScreenToLocalTransform={this.screenToLocalTransform} + docFilters={this.timelineDocFilter} + select={emptyFunction} + scaling={returnOne} + whenChildContentsActiveChanged={this.whenChildContentsActiveChanged} + removeDocument={this.removeDocument} + moveDocument={this.moveDocument} + addDocument={this.addDocWithTimecode}> + {this.contentFunc} + </CollectionFreeFormView> + </div> + {this.annotationLayer} + {!this._marqueeing || !this._mainCont.current || !this._annotationLayer.current ? (null) : + <MarqueeAnnotator + rootDoc={this.rootDoc} + scrollTop={0} + down={this._marqueeing} + scaling={this.marqueeFitScaling} + docView={this.props.docViewPath().slice(-1)[0]} + containerOffset={this.marqueeOffset} + addDocument={this.addDocWithTimecode} + finishMarquee={this.finishMarquee} + savedAnnotations={this.savedAnnotations} + annotationLayer={this._annotationLayer.current} + mainCont={this._mainCont.current} + />} + {this.renderTimeline} + </div> + </div >); + } } VideoBox._nativeControls = false;
\ No newline at end of file diff --git a/src/client/views/nodes/button/FontIconBox.tsx b/src/client/views/nodes/button/FontIconBox.tsx index 42c650a09..a1b9023f3 100644 --- a/src/client/views/nodes/button/FontIconBox.tsx +++ b/src/client/views/nodes/button/FontIconBox.tsx @@ -708,8 +708,6 @@ ScriptingGlobals.add(function toggleItalic(checkResult?: boolean) { }); - - export function checkInksToGroup() { // console.log("getting here to inks group"); if (CurrentUserUtils.SelectedTool === InkTool.Write) { @@ -793,8 +791,7 @@ export function createInkGroup(inksToGroup?: Doc[], isSubGroup?: boolean) { **/ ScriptingGlobals.add(function setActiveInkTool(tool: string, checkResult?: boolean) { - createInkGroup(); - + InkTranscription.Instance?.createInkGroup(); if (checkResult) { return ((Doc.UserDoc().activeInkTool === tool && !GestureOverlay.Instance?.InkShape) || GestureOverlay.Instance?.InkShape === tool) ? Colors.MEDIUM_BLUE : "transparent"; diff --git a/src/client/views/nodes/trails/PresElementBox.tsx b/src/client/views/nodes/trails/PresElementBox.tsx index ba0193e4b..ef70f8e36 100644 --- a/src/client/views/nodes/trails/PresElementBox.tsx +++ b/src/client/views/nodes/trails/PresElementBox.tsx @@ -4,9 +4,10 @@ import { action, computed, IReactionDisposer, observable, reaction } from "mobx" import { observer } from "mobx-react"; import { Doc, DocListCast, Opt } from "../../../../fields/Doc"; import { Id } from "../../../../fields/FieldSymbols"; -import { BoolCast, Cast, NumCast, StrCast } from "../../../../fields/Types"; +import { List } from "../../../../fields/List"; +import { Cast, NumCast, StrCast } from "../../../../fields/Types"; import { emptyFunction, returnEmptyDoclist, returnFalse, returnTrue, setupMoveUpEvents } from "../../../../Utils"; -import { DocUtils } from "../../../documents/Documents"; +import { Docs, DocUtils } from "../../../documents/Documents"; import { DocumentType } from "../../../documents/DocumentTypes"; import { CurrentUserUtils } from "../../../util/CurrentUserUtils"; import { DocumentManager } from "../../../util/DocumentManager"; @@ -30,8 +31,8 @@ import React = require("react"); */ @observer export class PresElementBox extends ViewBoxBaseComponent<FieldViewProps>() { - public static LayoutString(fieldKey: string) { return FieldView.LayoutString(PresElementBox, fieldKey); } - _heightDisposer: IReactionDisposer | undefined; + public static LayoutString(fieldKey: string) { return FieldView.LayoutString(PresElementBox, fieldKey); } + _heightDisposer: IReactionDisposer | undefined; @observable _dragging = false; @computed get indexInPres() { return DocListCast(this.presBox[StrCast(this.presBox.presFieldKey, "data")]).indexOf(this.rootDoc); } // the index field is where this document is in the presBox display list (since this value is different for each presentation element, the value can't be stored on the layout template which is used by all display elements) @@ -181,10 +182,11 @@ export class PresElementBox extends ViewBoxBaseComponent<FieldViewProps>() { e.preventDefault(); } - /** - * Function to drag and drop the pres element to a diferent location - */ - startDrag = (e: PointerEvent) => { + + /** + * Function to drag and drop the pres element to a diferent location + */ + startDrag = (e: PointerEvent) => { const miniView: boolean = this.toolbarWidth <= 100; const activeItem = this.rootDoc; const dragArray = PresBox.Instance._dragArray; @@ -195,27 +197,32 @@ export class PresElementBox extends ViewBoxBaseComponent<FieldViewProps>() { dragData.moveDocument = this.props.docViewPath().lastElement()?.props.moveDocument; const dragItem: HTMLElement[] = []; if (dragArray.length === 1) { - const doc = this._itemRef.current || dragArray[0]; - doc.className = miniView ? "presItem-miniSlide" : "presItem-slide"; - dragItem.push(doc); + const doc = this._itemRef.current || dragArray[0]; + doc.className = miniView ? "presItem-miniSlide" : "presItem-slide"; + dragItem.push(doc); } else if (dragArray.length >= 1) { - const doc = document.createElement('div'); - doc.className = "presItem-multiDrag"; - doc.innerText = "Move " + PresBox.Instance._selectedArray.size + " slides"; - doc.style.position = 'absolute'; - doc.style.top = (e.clientY) + 'px'; - doc.style.left = (e.clientX - 50) + 'px'; - dragItem.push(doc); + const doc = document.createElement('div'); + doc.className = "presItem-multiDrag"; + doc.innerText = "Move " + PresBox.Instance._selectedArray.size + " slides"; + doc.style.position = 'absolute'; + doc.style.top = (e.clientY) + 'px'; + doc.style.left = (e.clientX - 50) + 'px'; + dragItem.push(doc); } // const dropEvent = () => runInAction(() => this._dragging = false); if (activeItem) { - DragManager.StartDocumentDrag(dragItem.map(ele => ele), dragData, e.clientX, e.clientY, undefined); - // runInAction(() => this._dragging = true); - return true; + DragManager.StartDocumentDrag(dragItem.map(ele => ele), dragData, e.clientX, e.clientY, undefined); + // runInAction(() => this._dragging = true); + return true; } return false; - } + } + + /** + * Returns a local transformed coordinate array for given coordinates. + */ + ScreenToLocalListTransform = (xCord: number, yCord: number) => [xCord, yCord]; onPointerOver = (e: any) => { document.removeEventListener("pointermove", this.onPointerMove); @@ -314,168 +321,283 @@ export class PresElementBox extends ViewBoxBaseComponent<FieldViewProps>() { return width; } - @computed get mainItem() { - const isSelected: boolean = PresBox.Instance._selectedArray.has(this.rootDoc); - const toolbarWidth: number = this.toolbarWidth; - const showMore: boolean = this.toolbarWidth >= 300; - const miniView: boolean = this.toolbarWidth <= 110; - const presBox: Doc = this.presBox; //presBox - const presBoxColor: string = StrCast(presBox._backgroundColor); - const presColorBool: boolean = presBoxColor ? (presBoxColor !== Colors.WHITE && presBoxColor !== "transparent") : false; - const targetDoc: Doc = this.targetDoc; - const activeItem: Doc = this.rootDoc; - const isGroup: boolean = BoolCast(targetDoc._isGroup); - return ( - <div className={`presItem-container`} - key={this.props.Document[Id] + this.indexInPres} - ref={this._itemRef} - style={{ - opacity: this._dragging ? 0.3 : 1 - }} - onClick={e => { - e.stopPropagation(); - e.preventDefault(); - PresBox.Instance.modifierSelect(this.rootDoc, this._itemRef.current!, this._dragRef.current!, !e.shiftKey && !e.ctrlKey && !e.metaKey, e.ctrlKey || e.metaKey, e.shiftKey); - }} - onDoubleClick={action(e => { - if (isGroup) { - this.rootDoc.presExpandGroup = !this.rootDoc.presExpandGroup; - } else { - this.toggleProperties(); - PresBox.Instance.regularSelect(this.rootDoc, this._itemRef.current!, this._dragRef.current!, true); - } - })} - onPointerOver={this.onPointerOver} - onPointerLeave={this.onPointerLeave} - onPointerDown={this.headerDown} - onPointerUp={this.headerUp} - > - {miniView ? - // when width is LESS than 110 px - <div className={`presItem-miniSlide ${isSelected ? "active" : ""}`} ref={miniView ? this._dragRef : null}> - {`${this.indexInPres + 1}.`} - </div> - : - // when width is MORE than 110 px - <div className="presItem-number"> - {`${this.indexInPres + 1}.`} - </div>} - {isGroup ? - <div ref={miniView ? null : this._dragRef} className={`presItem-slide ${isSelected ? "activeGroup" : "group"}`} + onPointerOver = (e: any) => { + document.removeEventListener("pointermove", this.onPointerMove); + document.addEventListener("pointermove", this.onPointerMove); + } + + onPointerMove = (e: PointerEvent) => { + const slide = this._itemRef.current!; + let dragIsPresItem: boolean = DragManager.docsBeingDragged.length > 0 ? true : false; + for (const doc of DragManager.docsBeingDragged) { + if (!doc.presentationTargetDoc) dragIsPresItem = false; + } + if (slide && dragIsPresItem) { + const rect = slide.getBoundingClientRect(); + const y = e.clientY - rect.top; //y position within the element. + const height = slide.clientHeight; + const halfLine = height / 2; + if (y <= halfLine) { + slide.style.borderTop = `solid 2px ${Colors.MEDIUM_BLUE}`; + slide.style.borderBottom = "0px"; + } else if (y > halfLine) { + slide.style.borderTop = "0px"; + slide.style.borderBottom = `solid 2px ${Colors.MEDIUM_BLUE}`; + } + } + document.removeEventListener("pointermove", this.onPointerMove); + } + + onPointerLeave = (e: any) => { + this._itemRef.current!.style.borderTop = "0px"; + this._itemRef.current!.style.borderBottom = "0px"; + document.removeEventListener("pointermove", this.onPointerMove); + } + + @action + toggleProperties = () => { + if (CurrentUserUtils.propertiesWidth < 5) { + action(() => (CurrentUserUtils.propertiesWidth = 250)); + } + } + + @undoBatch + removeItem = action((e: React.MouseEvent) => { + this.props.removeDocument?.(this.rootDoc); + if (PresBox.Instance._selectedArray.has(this.rootDoc)) { + PresBox.Instance._selectedArray.delete(this.rootDoc); + } + this.hideRecording(); + e.stopPropagation(); + }); + + // set the value/title of the individual pres element + @undoBatch + @action + onSetValue = (value: string) => { + this.rootDoc.title = !value.trim().length ? "-untitled-" : value; + return true; + } + + /** + * Method called for updating the view of the currently selected document + * + * @param targetDoc + * @param activeItem + */ + @undoBatch + @action + updateView = (targetDoc: Doc, activeItem: Doc) => { + if (targetDoc.type === DocumentType.PDF || targetDoc.type === DocumentType.WEB || targetDoc.type === DocumentType.RTF) { + const scroll = targetDoc._scrollTop; + activeItem.presPinViewScroll = scroll; + } else if (targetDoc.type === DocumentType.VID) { + activeItem.presPinTimecode = targetDoc._currentTimecode; + } else if (targetDoc.type === DocumentType.COMPARISON) { + const clipWidth = targetDoc._clipWidth; + activeItem.presPinClipWidth = clipWidth; + } else { + const x = targetDoc._panX; + const y = targetDoc._panY; + const scale = targetDoc._viewScale; + activeItem.presPinViewX = x; + activeItem.presPinViewY = y; + activeItem.presPinViewScale = scale; + } + } + + @computed get recordingIsInOverlay() { + let isInOverlay = false + DocListCast((Doc.UserDoc().myOverlayDocs as Doc).data).forEach((doc) => { + if (doc.slides === this.rootDoc) { + isInOverlay = true + return + } + }) + return isInOverlay + } + + @undoBatch + @action + hideRecording = () => { + DocListCast((Doc.UserDoc().myOverlayDocs as Doc).data).forEach((doc) => { + if (doc.slides === this.rootDoc) { + Doc.RemoveDocFromList((Doc.UserDoc().myOverlayDocs as Doc), undefined, doc); + } + }) + } + + @undoBatch + @action + showRecording = (activeItem: Doc) => { + // Remove every recording that already exists in overlay view + DocListCast((Doc.UserDoc().myOverlayDocs as Doc).data).forEach((doc) => { + if (doc.slides !== null) { + Doc.RemoveDocFromList((Doc.UserDoc().myOverlayDocs as Doc), undefined, doc); + } + }) + if (activeItem.recording) { + // if we already have an existing recording + Doc.AddDocToList((Doc.UserDoc().myOverlayDocs as Doc), undefined, Cast(activeItem.recording, Doc, null)); + + } + } + + @undoBatch + @action + startRecording = (activeItem: Doc) => { + // Remove every recording that already exists in overlay view + DocListCast((Doc.UserDoc().myOverlayDocs as Doc).data).forEach((doc) => { + if (doc.slides !== null) { + Doc.RemoveDocFromList((Doc.UserDoc().myOverlayDocs as Doc), undefined, doc); + } + }) + + if (activeItem.recording) { + // if we already have an existing recording + Doc.AddDocToList((Doc.UserDoc().myOverlayDocs as Doc), undefined, Cast(activeItem.recording, Doc, null)); + + } else { + // if we dont have any recording + const recording = Docs.Create.WebCamDocument("", { _width: 400, _height: 200, title: "recording", cloneFieldFilter: new List<string>(["system"]) }); + + // attach the recording to the slide, and attach the slide to the recording + recording.slides = activeItem + activeItem.recording = recording + + // TODO: Figure out exactly where we want the video to appear + const pt = this.props.ScreenToLocalTransform().inverse().transformPoint(0, 0); + recording.x = pt[0]; + recording.y = pt[1]; + Doc.AddDocToList((Doc.UserDoc().myOverlayDocs as Doc), undefined, recording); + } + } + + @computed + get toolbarWidth(): number { + const presBoxDocView = DocumentManager.Instance.getDocumentView(this.presBox); + let width: number = NumCast(this.presBox._width); + if (presBoxDocView) width = presBoxDocView.props.PanelWidth(); + if (width === 0) width = 300; + return width; + } + + @computed get mainItem() { + const isSelected: boolean = PresBox.Instance?._selectedArray.has(this.rootDoc); + const toolbarWidth: number = this.toolbarWidth; + const showMore: boolean = this.toolbarWidth >= 300; + const miniView: boolean = this.toolbarWidth <= 110; + const presBox: Doc = this.presBox; //presBox + const presBoxColor: string = StrCast(presBox._backgroundColor); + const presColorBool: boolean = presBoxColor ? (presBoxColor !== Colors.WHITE && presBoxColor !== "transparent") : false; + const targetDoc: Doc = this.targetDoc; + const activeItem: Doc = this.rootDoc; + return ( + <div className={`presItem-container`} + key={this.props.Document[Id] + this.indexInPres} + ref={this._itemRef} style={{ - backgroundColor: this.props.styleProvider?.(this.layoutDoc, this.props, StyleProp.BackgroundColor), - boxShadow: presBoxColor && presBoxColor !== "white" && presBoxColor !== "transparent" ? isSelected ? "0 0 0px 1.5px" + presBoxColor : undefined : undefined - }}> - <div className="presItem-name" style={{ maxWidth: showMore ? (toolbarWidth - 195) : toolbarWidth - 105, cursor: isSelected ? 'text' : 'grab' }}> - <EditableView - ref={this._titleRef} - editing={!isSelected ? false : undefined} - contents={activeItem.title} - overflow={'ellipsis'} - GetValue={() => StrCast(activeItem.title)} - SetValue={this.onSetValue} - /> - </div> - <div className={"presItem-slideButtons"}> - <Tooltip title={<><div className="dash-tooltip">{"Update view"}</div></>}> - <div className="slideButton" - onClick={() => this.updateView(targetDoc, activeItem)} - style={{ fontWeight: 700, display: activeItem.presPinView ? "flex" : "none" }}>V</div> - </Tooltip> - {this.indexInPres === 0 ? (null) : <Tooltip title={<><div className="dash-tooltip">{activeItem.groupWithUp ? "Ungroup" : "Group with up"}</div></>}> - <div className="slideButton" - onClick={() => activeItem.groupWithUp = !activeItem.groupWithUp} - style={{ - zIndex: 1000 - this.indexInPres, - fontWeight: 700, - backgroundColor: activeItem.groupWithUp ? presColorBool ? presBoxColor : Colors.MEDIUM_BLUE : undefined, - height: activeItem.groupWithUp ? 53 : 18, - transform: activeItem.groupWithUp ? "translate(0, -17px)" : undefined - }}> - <div style={{ transform: activeItem.groupWithUp ? "rotate(180deg) translate(0, -17.5px)" : "rotate(0deg)" }}> - <FontAwesomeIcon icon={"arrow-up"} onPointerDown={e => e.stopPropagation()} /> - </div> - </div> - </Tooltip>} - <Tooltip title={<><div className="dash-tooltip">{this.rootDoc.presExpandInlineButton ? "Minimize" : "Expand"}</div></>}><div className={"slideButton"} onClick={e => { e.stopPropagation(); this.presExpandDocumentClick(); }}> - <FontAwesomeIcon icon={this.rootDoc.presExpandInlineButton ? "eye-slash" : "eye"} onPointerDown={e => e.stopPropagation()} /> - </div></Tooltip> - <Tooltip title={<><div className="dash-tooltip">{"Remove from presentation"}</div></>}><div - className={"slideButton"} - onClick={this.removeItem}> - <FontAwesomeIcon icon={"trash"} onPointerDown={e => e.stopPropagation()} /> - </div></Tooltip> - <div className="group"></div> - </div> - <div className="presItem-groupSlideContainer" style={{ top: 28, height: 'calc(100% - 28px)' }}> - {this.rootDoc.presExpandGroup ? this.renderGroupSlides : (null)} - </div> - <div className="presItem-docName" style={{ maxWidth: showMore ? (toolbarWidth - 195) : toolbarWidth - 105 }}>{activeItem.presPinView ? (<><i>View of </i> {targetDoc.title}</>) : targetDoc.title}</div> - <div className="expandButton" - onClick={e => { - if (isGroup) { - this.rootDoc.presExpandGroup = !this.rootDoc.presExpandGroup; - } - }} - > - <FontAwesomeIcon icon={"caret-down"} style={{ transform: this.rootDoc.presExpandGroup ? "rotate(180deg)" : "rotate(0deg)" }} /> - </div> - </div> - : (null)} - {miniView || isGroup ? (null) : <div ref={miniView ? null : this._dragRef} className={`presItem-slide ${isSelected ? "active" : ""}`} - style={{ - backgroundColor: this.props.styleProvider?.(this.layoutDoc, this.props, StyleProp.BackgroundColor), - boxShadow: presBoxColor && presBoxColor !== "white" && presBoxColor !== "transparent" ? isSelected ? "0 0 0px 1.5px" + presBoxColor : undefined : undefined - }}> - <div className="presItem-name" style={{ maxWidth: showMore ? (toolbarWidth - (this.presBox._viewType === CollectionViewType.Stacking ? 195 : 220)) : toolbarWidth - (this.presBox._viewType === CollectionViewType.Stacking ? 105 : 145), cursor: isSelected ? 'text' : 'grab' }}> - <EditableView - ref={this._titleRef} - editing={!isSelected ? false : undefined} - contents={activeItem.title} - overflow={'ellipsis'} - GetValue={() => StrCast(activeItem.title)} - SetValue={this.onSetValue} - /> - </div> - <Tooltip title={<><div className="dash-tooltip">{"Movement speed"}</div></>}><div className="presItem-time" style={{ display: showMore ? "block" : "none" }}>{this.transition}</div></Tooltip> - <Tooltip title={<><div className="dash-tooltip">{"Duration"}</div></>}><div className="presItem-time" style={{ display: showMore ? "block" : "none" }}>{this.duration}</div></Tooltip> - <div className={"presItem-slideButtons"}> - <Tooltip title={<><div className="dash-tooltip">{"Update view"}</div></>}> - <div className="slideButton" - onClick={() => this.updateView(targetDoc, activeItem)} - style={{ fontWeight: 700, display: activeItem.presPinView ? "flex" : "none" }}>V</div> - </Tooltip> - {this.indexInPres === 0 ? (null) : <Tooltip title={<><div className="dash-tooltip">{activeItem.groupWithUp ? "Ungroup" : "Group with up"}</div></>}> - <div className="slideButton" - onClick={() => activeItem.groupWithUp = !activeItem.groupWithUp} - style={{ - zIndex: 1000 - this.indexInPres, - fontWeight: 700, - backgroundColor: activeItem.groupWithUp ? presColorBool ? presBoxColor : Colors.MEDIUM_BLUE : undefined, - height: activeItem.groupWithUp ? 53 : 18, - transform: activeItem.groupWithUp ? "translate(0, -17px)" : undefined - }}> - <div style={{ transform: activeItem.groupWithUp ? "rotate(180deg) translate(0, -17.5px)" : "rotate(0deg)" }}> - <FontAwesomeIcon icon={"arrow-up"} onPointerDown={e => e.stopPropagation()} /> - </div> - </div> - </Tooltip>} - <Tooltip title={<><div className="dash-tooltip">{this.rootDoc.presExpandInlineButton ? "Minimize" : "Expand"}</div></>}><div className={"slideButton"} onClick={e => { e.stopPropagation(); this.presExpandDocumentClick(); }}> - <FontAwesomeIcon icon={this.rootDoc.presExpandInlineButton ? "eye-slash" : "eye"} onPointerDown={e => e.stopPropagation()} /> - </div></Tooltip> - <Tooltip title={<><div className="dash-tooltip">{"Remove from presentation"}</div></>}><div - className={"slideButton"} - onClick={this.removeItem}> - <FontAwesomeIcon icon={"trash"} onPointerDown={e => e.stopPropagation()} /> - </div></Tooltip> - </div> - <div className="presItem-docName" style={{ maxWidth: showMore ? (toolbarWidth - 195) : toolbarWidth - 105 }}>{activeItem.presPinView ? (<><i>View of </i> {targetDoc.title}</>) : targetDoc.title}</div> - {this.renderEmbeddedInline} - </div>} - </div >); - } + backgroundColor: presColorBool ? isSelected ? "rgba(250,250,250,0.3)" : "transparent" : isSelected ? Colors.LIGHT_BLUE : "transparent", + opacity: this._dragging ? 0.3 : 1 + }} + onClick={e => { + e.stopPropagation(); + e.preventDefault(); + PresBox.Instance.modifierSelect(this.rootDoc, this._itemRef.current!, this._dragRef.current!, !e.shiftKey && !e.ctrlKey && !e.metaKey, e.ctrlKey || e.metaKey, e.shiftKey); + this.showRecording(activeItem); + }} + onDoubleClick={action(e => { + this.toggleProperties(); + PresBox.Instance.regularSelect(this.rootDoc, this._itemRef.current!, this._dragRef.current!, true); + })} + onPointerOver={this.onPointerOver} + onPointerLeave={this.onPointerLeave} + onPointerDown={this.headerDown} + onPointerUp={this.headerUp} + > + {miniView ? + // when width is LESS than 110 px + <div className={`presItem-miniSlide ${isSelected ? "active" : ""}`} ref={miniView ? this._dragRef : null}> + {`${this.indexInPres + 1}.`} + </div> + : + // when width is MORE than 110 px + <div className="presItem-number"> + {`${this.indexInPres + 1}.`} + </div>} + {miniView ? (null) : <div ref={miniView ? null : this._dragRef} className={`presItem-slide ${isSelected ? "active" : ""}`} + style={{ + backgroundColor: this.props.styleProvider?.(this.layoutDoc, this.props, StyleProp.BackgroundColor), + boxShadow: presBoxColor && presBoxColor !== "white" && presBoxColor !== "transparent" ? isSelected ? "0 0 0px 1.5px" + presBoxColor : undefined : undefined + }}> + <div className="presItem-name" style={{ maxWidth: showMore ? (toolbarWidth - 195) : toolbarWidth - 105, cursor: isSelected ? 'text' : 'grab' }}> + <EditableView + ref={this._titleRef} + editing={!isSelected ? false : undefined} + contents={activeItem.title} + overflow={'ellipsis'} + GetValue={() => StrCast(activeItem.title)} + SetValue={this.onSetValue} + /> + </div> + <Tooltip title={<><div className="dash-tooltip">{"Movement speed"}</div></>}><div className="presItem-time" style={{ display: showMore ? "block" : "none" }}>{this.transition}</div></Tooltip> + <Tooltip title={<><div className="dash-tooltip">{"Duration"}</div></>}><div className="presItem-time" style={{ display: showMore ? "block" : "none" }}>{this.duration}</div></Tooltip> + <div className={"presItem-slideButtons"}> + <Tooltip title={<><div className="dash-tooltip">{"Update view"}</div></>}> + <div className="slideButton" + onClick={() => this.updateView(targetDoc, activeItem)} + style={{ fontWeight: 700, display: activeItem.presPinView ? "flex" : "none" }}>V</div> + </Tooltip> - render() { - return !(this.rootDoc instanceof Doc) || this.targetDoc instanceof Promise ? (null) : this.mainItem; - } + {this.recordingIsInOverlay ? + <Tooltip title={<><div className="dash-tooltip">{"Hide Recording"}</div></>}> + <div className="slideButton" + onClick={this.hideRecording} + style={{ fontWeight: 700 }}> + <FontAwesomeIcon icon={"video-slash"} onPointerDown={e => e.stopPropagation()} /> + </div> + </Tooltip> : + <Tooltip title={<><div className="dash-tooltip">{"Start recording"}</div></>}> + <div className="slideButton" + onClick={() => this.startRecording(activeItem)} + style={{ fontWeight: 700 }}> + <FontAwesomeIcon icon={"video"} onPointerDown={e => e.stopPropagation()} /> + </div> + </Tooltip> + } + + + {this.indexInPres === 0 ? (null) : <Tooltip title={<><div className="dash-tooltip">{activeItem.groupWithUp ? "Ungroup" : "Group with up"}</div></>}> + <div className="slideButton" + onClick={() => activeItem.groupWithUp = !activeItem.groupWithUp} + style={{ + zIndex: 1000 - this.indexInPres, + fontWeight: 700, + backgroundColor: activeItem.groupWithUp ? presColorBool ? presBoxColor : Colors.MEDIUM_BLUE : undefined, + height: activeItem.groupWithUp ? 53 : 18, + transform: activeItem.groupWithUp ? "translate(0, -17px)" : undefined + }}> + <div style={{ transform: activeItem.groupWithUp ? "rotate(180deg) translate(0, -17.5px)" : "rotate(0deg)" }}> + <FontAwesomeIcon icon={"arrow-up"} onPointerDown={e => e.stopPropagation()} /> + </div> + </div> + </Tooltip>} + {/* <Tooltip title={<><div className="dash-tooltip">{this.rootDoc.presExpandInlineButton ? "Minimize" : "Expand"}</div></>}><div className={"slideButton"} onClick={e => { e.stopPropagation(); this.presExpandDocumentClick(); }}> + <FontAwesomeIcon icon={this.rootDoc.presExpandInlineButton ? "eye-slash" : "eye"} onPointerDown={e => e.stopPropagation()} /> + </div></Tooltip> */} + <Tooltip title={<><div className="dash-tooltip">{"Remove from presentation"}</div></>}><div + className={"slideButton"} + onClick={this.removeItem}> + <FontAwesomeIcon icon={"trash"} onPointerDown={e => e.stopPropagation()} /> + </div></Tooltip> + </div> + <div className="presItem-docName" style={{ maxWidth: showMore ? (toolbarWidth - 195) : toolbarWidth - 105 }}>{activeItem.presPinView ? (<><i>View of </i> {targetDoc.title}</>) : targetDoc.title}</div> + {this.renderEmbeddedInline} + </div>} + </div >); + } + + render() { + return !(this.rootDoc instanceof Doc) || this.targetDoc instanceof Promise ? (null) : this.mainItem; + } }
\ No newline at end of file diff --git a/src/fields/URLField.ts b/src/fields/URLField.ts index 1d4bbaed0..3e7542e46 100644 --- a/src/fields/URLField.ts +++ b/src/fields/URLField.ts @@ -53,6 +53,7 @@ export abstract class URLField extends ObjectField { export const nullAudio = "https://actions.google.com/sounds/v1/alarms/beep_short.ogg"; @scriptingGlobal @Deserializable("audio") export class AudioField extends URLField { } +@scriptingGlobal @Deserializable("recording") export class RecordingField extends URLField { } @scriptingGlobal @Deserializable("image") export class ImageField extends URLField { } @scriptingGlobal @Deserializable("video") export class VideoField extends URLField { } @scriptingGlobal @Deserializable("pdf") export class PdfField extends URLField { } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2c667ba3e..e7b7056a1 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -17,239 +17,239 @@ import { SolrManager } from "./SearchManager"; const fs = require('fs'); export enum Directory { - parsed_files = "parsed_files", - images = "images", - videos = "videos", - pdfs = "pdfs", - text = "text", - pdf_thumbnails = "pdf_thumbnails", - audio = "audio", + parsed_files = "parsed_files", + images = "images", + videos = "videos", + pdfs = "pdfs", + text = "text", + pdf_thumbnails = "pdf_thumbnails", + audio = "audio", } export function serverPathToFile(directory: Directory, filename: string) { - return normalize(`${filesDirectory}/${directory}/${filename}`); + return normalize(`${filesDirectory}/${directory}/${filename}`); } export function pathToDirectory(directory: Directory) { - return normalize(`${filesDirectory}/${directory}`); + return normalize(`${filesDirectory}/${directory}`); } export function clientPathToFile(directory: Directory, filename: string) { - return `/files/${directory}/${filename}`; + return `/files/${directory}/${filename}`; } export default class UploadManager extends ApiManager { - protected initialize(register: Registration): void { + protected initialize(register: Registration): void { - register({ - method: Method.POST, - subscription: "/uploadFormData", - secureHandler: async ({ req, res }) => { - const form = new formidable.IncomingForm(); - form.keepExtensions = true; - form.uploadDir = pathToDirectory(Directory.parsed_files); - return new Promise<void>(resolve => { - form.parse(req, async (_err, _fields, files) => { - const results: Upload.FileResponse[] = []; - for (const key in files) { - const f = files[key]; - if (!Array.isArray(f)) { - const result = await DashUploadUtils.upload(f); - result && !(result.result instanceof Error) && results.push(result); - } - } - _success(res, results); - resolve(); - }); - }); - } - }); + register({ + method: Method.POST, + subscription: "/uploadFormData", + secureHandler: async ({ req, res }) => { + const form = new formidable.IncomingForm(); + form.keepExtensions = true; + form.uploadDir = pathToDirectory(Directory.parsed_files); + return new Promise<void>(resolve => { + form.parse(req, async (_err, _fields, files) => { + const results: Upload.FileResponse[] = []; + for (const key in files) { + const f = files[key]; + if (!Array.isArray(f)) { + const result = await DashUploadUtils.upload(f); + result && !(result.result instanceof Error) && results.push(result); + } + } + _success(res, results); + resolve(); + }); + }); + } + }); - register({ - method: Method.POST, - subscription: "/uploadYoutubeVideo", - secureHandler: async ({ req, res }) => { - //req.readableBuffer.head.data - return new Promise<void>(async resolve => { - req.addListener("data", async (args) => { - console.log(args); - const payload = String.fromCharCode.apply(String, args); - const videoId = JSON.parse(payload).videoId; - const results: Upload.FileResponse[] = []; - const result = await DashUploadUtils.uploadYoutube(videoId); - result && !(result.result instanceof Error) && results.push(result); - _success(res, results); - resolve(); - }); - }); - } - }); + register({ + method: Method.POST, + subscription: "/uploadYoutubeVideo", + secureHandler: async ({ req, res }) => { + //req.readableBuffer.head.data + return new Promise<void>(async resolve => { + req.addListener("data", async (args) => { + console.log(args); + const payload = String.fromCharCode.apply(String, args); + const videoId = JSON.parse(payload).videoId; + const results: Upload.FileResponse[] = []; + const result = await DashUploadUtils.uploadYoutube(videoId); + result && !(result.result instanceof Error) && results.push(result); + _success(res, results); + resolve(); + }); + }); + } + }); - register({ - method: Method.POST, - subscription: new RouteSubscriber("youtubeScreenshot"), - secureHandler: async ({ req, res }) => { - const { id, timecode } = req.body; - const convert = (raw: string) => { - const number = Math.floor(Number(raw)); - const seconds = number % 60; - const minutes = (number - seconds) / 60; - return `${minutes}m${seconds}s`; - }; - const suffix = timecode ? `&t=${convert(timecode)}` : ``; - const targetUrl = `https://www.youtube.com/watch?v=${id}${suffix}`; - const buffer = await captureYoutubeScreenshot(targetUrl); - if (!buffer) { - return res.send(); - } - const resolvedName = `youtube_capture_${id}_${suffix}.png`; - const resolvedPath = serverPathToFile(Directory.images, resolvedName); - return new Promise<void>(resolve => { - writeFile(resolvedPath, buffer, async error => { - if (error) { - return res.send(); + register({ + method: Method.POST, + subscription: new RouteSubscriber("youtubeScreenshot"), + secureHandler: async ({ req, res }) => { + const { id, timecode } = req.body; + const convert = (raw: string) => { + const number = Math.floor(Number(raw)); + const seconds = number % 60; + const minutes = (number - seconds) / 60; + return `${minutes}m${seconds}s`; + }; + const suffix = timecode ? `&t=${convert(timecode)}` : ``; + const targetUrl = `https://www.youtube.com/watch?v=${id}${suffix}`; + const buffer = await captureYoutubeScreenshot(targetUrl); + if (!buffer) { + return res.send(); } - await DashUploadUtils.outputResizedImages(() => createReadStream(resolvedPath), resolvedName, pathToDirectory(Directory.images)); - res.send({ - accessPaths: { - agnostic: DashUploadUtils.getAccessPaths(Directory.images, resolvedName) - } - } as Upload.FileInformation); - resolve(); - }); - }); - } - }); + const resolvedName = `youtube_capture_${id}_${suffix}.png`; + const resolvedPath = serverPathToFile(Directory.images, resolvedName); + return new Promise<void>(resolve => { + writeFile(resolvedPath, buffer, async error => { + if (error) { + return res.send(); + } + await DashUploadUtils.outputResizedImages(() => createReadStream(resolvedPath), resolvedName, pathToDirectory(Directory.images)); + res.send({ + accessPaths: { + agnostic: DashUploadUtils.getAccessPaths(Directory.images, resolvedName) + } + } as Upload.FileInformation); + resolve(); + }); + }); + } + }); - register({ - method: Method.POST, - subscription: "/uploadRemoteImage", - secureHandler: async ({ req, res }) => { + register({ + method: Method.POST, + subscription: "/uploadRemoteImage", + secureHandler: async ({ req, res }) => { - const { sources } = req.body; - if (Array.isArray(sources)) { - const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source))); - return res.send(results); - } - res.send(); - } - }); + const { sources } = req.body; + if (Array.isArray(sources)) { + const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source))); + return res.send(results); + } + res.send(); + } + }); - register({ - method: Method.POST, - subscription: "/uploadDoc", - secureHandler: ({ req, res }) => { + register({ + method: Method.POST, + subscription: "/uploadDoc", + secureHandler: ({ req, res }) => { - const form = new formidable.IncomingForm(); - form.keepExtensions = true; - // let path = req.body.path; - const ids: { [id: string]: string } = {}; - let remap = true; - const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith("Proto")) return id; - if (id in ids) { - return ids[id]; - } else { - return ids[id] = v4(); - } - }; - const mapFn = (doc: any) => { - if (doc.id) { - doc.id = getId(doc.id); - } - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { continue; } - const field = doc.fields[key]; - if (field === undefined || field === null) { continue; } + const form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); + } + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); + } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { continue; } + const field = doc.fields[key]; + if (field === undefined || field === null) { continue; } - if (field.__type === "Doc") { - mapFn(field); - } else if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - field.fieldId = getId(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - field.captures.fieldId = getId(field.captures.fieldId); - } - } else if (field.__type === "list") { - mapFn(field); - } else if (typeof field === "string") { - const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; - doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } else if (field.__type === "RichTextField") { - const re = /("href"\s*:\s*")(.*?)"/g; - field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } - } - }; - return new Promise<void>(resolve => { - form.parse(req, async (_err, fields, files) => { - remap = fields.remap !== "false"; - let id: string = ""; - try { - for (const name in files) { - const f = files[name]; - const path_2 = Array.isArray(f) ? "" : f.path; - const zip = new AdmZip(path_2); - zip.getEntries().forEach((entry: any) => { - if (!entry.entryName.startsWith("files/")) return; - let directory = dirname(entry.entryName) + "/"; - const extension = extname(entry.entryName); - const base = basename(entry.entryName).split(".")[0]; + if (field.__type === "Doc") { + mapFn(field); + } else if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); + } + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } + } + }; + return new Promise<void>(resolve => { + form.parse(req, async (_err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; try { - zip.extractEntryTo(entry.entryName, publicDirectory, true, false); - directory = "/" + directory; + for (const name in files) { + const f = files[name]; + const path_2 = Array.isArray(f) ? "" : f.path; + const zip = new AdmZip(path_2); + zip.getEntries().forEach((entry: any) => { + if (!entry.entryName.startsWith("files/")) return; + let directory = dirname(entry.entryName) + "/"; + const extension = extname(entry.entryName); + const base = basename(entry.entryName).split(".")[0]; + try { + zip.extractEntryTo(entry.entryName, publicDirectory, true, false); + directory = "/" + directory; - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_o" + extension)); - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_s" + extension)); - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_m" + extension)); - createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_l" + extension)); - } catch (e) { - console.log(e); - } - }); - const json = zip.getEntry("doc.json"); - try { - const data = JSON.parse(json.getData().toString("utf8")); - const datadocs = data.docs; - id = getId(data.id); - const docs = Object.keys(datadocs).map(key => datadocs[key]); - docs.forEach(mapFn); - await Promise.all(docs.map((doc: any) => new Promise<void>(res => { - Database.Instance.replace(doc.id, doc, (err, r) => { - err && console.log(err); - res(); - }, true); - }))); - } catch (e) { console.log(e); } - unlink(path_2, () => { }); - } - SolrManager.update(); - res.send(JSON.stringify(id || "error")); - } catch (e) { console.log(e); } - resolve(); - }); - }); - } - }); + createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_o" + extension)); + createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_s" + extension)); + createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_m" + extension)); + createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_l" + extension)); + } catch (e) { + console.log(e); + } + }); + const json = zip.getEntry("doc.json"); + try { + const data = JSON.parse(json.getData().toString("utf8")); + const datadocs = data.docs; + id = getId(data.id); + const docs = Object.keys(datadocs).map(key => datadocs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise<void>(res => { + Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true); + }))); + } catch (e) { console.log(e); } + unlink(path_2, () => { }); + } + SolrManager.update(); + res.send(JSON.stringify(id || "error")); + } catch (e) { console.log(e); } + resolve(); + }); + }); + } + }); - register({ - method: Method.POST, - subscription: "/inspectImage", - secureHandler: async ({ req, res }) => { + register({ + method: Method.POST, + subscription: "/inspectImage", + secureHandler: async ({ req, res }) => { - const { source } = req.body; - if (typeof source === "string") { - return res.send(await DashUploadUtils.InspectImage(source)); - } - res.send({}); - } - }); + const { source } = req.body; + if (typeof source === "string") { + return res.send(await DashUploadUtils.InspectImage(source)); + } + res.send({}); + } + }); register({ method: Method.POST, @@ -296,17 +296,17 @@ export default class UploadManager extends ApiManager { .pipe(createWriteStream(path).on("error", e => console.log("Resizing write: " + e))); }); - } - res.send(clientPathToFile(Directory.images, filename + ext)); - }); - } - }); + } + res.send(clientPathToFile(Directory.images, filename + ext)); + }); + } + }); - } + } } function delay(ms: number) { - return new Promise(resolve => setTimeout(resolve, ms)); + return new Promise(resolve => setTimeout(resolve, ms)); } /** * On success, returns a buffer containing the bytes of a screenshot @@ -315,25 +315,25 @@ function delay(ms: number) { * On failure, returns undefined. */ async function captureYoutubeScreenshot(targetUrl: string) { - // const browser = await launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] }); - // const page = await browser.newPage(); - // // await page.setViewport({ width: 1920, height: 1080 }); + // const browser = await launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] }); + // const page = await browser.newPage(); + // // await page.setViewport({ width: 1920, height: 1080 }); - // // await page.goto(targetUrl, { waitUntil: 'domcontentloaded' as any }); + // // await page.goto(targetUrl, { waitUntil: 'domcontentloaded' as any }); - // const videoPlayer = await page.$('.html5-video-player'); - // videoPlayer && await page.focus("video"); - // await delay(7000); - // const ad = await page.$('.ytp-ad-skip-button-text'); - // await ad?.click(); - // await videoPlayer?.click(); - // await delay(1000); - // // hide youtube player controls. - // await page.evaluate(() => (document.querySelector('.ytp-chrome-bottom') as HTMLElement).style.display = 'none'); + // const videoPlayer = await page.$('.html5-video-player'); + // videoPlayer && await page.focus("video"); + // await delay(7000); + // const ad = await page.$('.ytp-ad-skip-button-text'); + // await ad?.click(); + // await videoPlayer?.click(); + // await delay(1000); + // // hide youtube player controls. + // await page.evaluate(() => (document.querySelector('.ytp-chrome-bottom') as HTMLElement).style.display = 'none'); - // const buffer = await videoPlayer?.screenshot({ encoding: "binary" }); - // await browser.close(); + // const buffer = await videoPlayer?.screenshot({ encoding: "binary" }); + // await browser.close(); - // return buffer; - return null; + // return buffer; + return null; }
\ No newline at end of file |