From ee03fa6e04dd9dba3099f75154de6ffab566ff5c Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Fri, 13 Sep 2019 16:43:09 -0400 Subject: Trial 5(Most succesful yet) --- src/client/documents/DocumentTypes.ts | 1 + src/client/documents/Documents.ts | 10 +- src/client/views/MainView.tsx | 5 +- src/client/views/nodes/DocumentContentsView.tsx | 4 +- src/client/views/webcam/DashWebCam.tsx | 302 ++++++++++++++++++++++++ src/new_fields/URLField.ts | 2 + 6 files changed, 321 insertions(+), 3 deletions(-) create mode 100644 src/client/views/webcam/DashWebCam.tsx (limited to 'src') diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index 381981e1b..4430ffd62 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -17,6 +17,7 @@ export enum DocumentType { TEMPLATE = "template", EXTENSION = "extension", YOUTUBE = "youtube", + WEBCAM = "webcam", DRAGBOX = "dragbox", PRES = "presentation", LINKFOLLOW = "linkfollow", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 602a7f9ad..036cc75a0 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -22,7 +22,7 @@ import { MINIMIZED_ICON_SIZE } from "../views/globalCssVariables.scss"; import { IconBox } from "../views/nodes/IconBox"; import { Field, Doc, Opt } from "../../new_fields/Doc"; import { OmitKeys, JSONUtils } from "../../Utils"; -import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField } from "../../new_fields/URLField"; +import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField, WebCamField } from "../../new_fields/URLField"; import { HtmlField } from "../../new_fields/HtmlField"; import { List } from "../../new_fields/List"; import { Cast, NumCast } from "../../new_fields/Types"; @@ -46,6 +46,7 @@ import { ComputedField } from "../../new_fields/ScriptField"; import { ProxyField } from "../../new_fields/Proxy"; import { DocumentType } from "./DocumentTypes"; import { LinkFollowBox } from "../views/linking/LinkFollowBox"; +import { DashWebCam } from "../views/webcam/DashWebCam"; //import { PresBox } from "../views/nodes/PresBox"; //import { PresField } from "../../new_fields/PresField"; var requestImageSize = require('../util/request-image-size'); @@ -176,6 +177,9 @@ export namespace Docs { }], [DocumentType.LINKFOLLOW, { layout: { view: LinkFollowBox } + }], + [DocumentType.WEBCAM, { + layout: { view: DashWebCam } }] ]); @@ -357,6 +361,10 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.YOUTUBE), new YoutubeField(new URL(url)), options); } + export function WebCamDocument(url: string, options: DocumentOptions = {}) { + return InstanceFromProto(Prototypes.get(DocumentType.WEBCAM), "", options); + } + export function AudioDocument(url: string, options: DocumentOptions = {}) { return InstanceFromProto(Prototypes.get(DocumentType.AUDIO), new AudioField(new URL(url)), options); } diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index b64986084..62d4fb1b8 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -1,5 +1,5 @@ import { IconName, library } from '@fortawesome/fontawesome-svg-core'; -import { faLink, faArrowDown, faArrowUp, faBolt, faCaretUp, faCat, faCheck, faClone, faCloudUploadAlt, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faLongArrowAltRight, faMusic, faObjectGroup, faPause, faPenNib, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faUndoAlt, faTv } from '@fortawesome/free-solid-svg-icons'; +import { faLink, faArrowDown, faArrowUp, faBolt, faCaretUp, faCat, faCheck, faClone, faCloudUploadAlt, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faLongArrowAltRight, faMusic, faObjectGroup, faPause, faPenNib, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faUndoAlt, faTv, faVideo } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, computed, configure, observable, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; @@ -195,6 +195,7 @@ export class MainView extends React.Component { library.add(faArrowUp); library.add(faCloudUploadAlt); library.add(faBolt); + library.add(faVideo); this.initEventListeners(); this.initAuthenticationRouters(); } @@ -456,6 +457,7 @@ export class MainView extends React.Component { let addImportCollectionNode = action(() => Docs.Create.DirectoryImportDocument({ title: "Directory Import", width: 400, height: 400 })); // let youtubeurl = "https://www.youtube.com/embed/TqcApsGRzWw"; // let addYoutubeSearcher = action(() => Docs.Create.YoutubeDocument(youtubeurl, { width: 600, height: 600, title: "youtube search" })); + let addWebCam = action(() => Docs.Create.WebCamDocument("", {})); let btns: [React.RefObject, IconName, string, () => Doc][] = [ [React.createRef(), "object-group", "Add Collection", addColNode], @@ -464,6 +466,7 @@ export class MainView extends React.Component { [React.createRef(), "bolt", "Add Button", addButtonDocument], [React.createRef(), "file", "Add Document Dragger", addDragboxNode], [React.createRef(), "cloud-upload-alt", "Import Directory", addImportCollectionNode], //remove at some point in favor of addImportCollectionNode + [React.createRef(), "video", "Add WebCam", addWebCam] //[React.createRef(), "play", "Add Youtube Searcher", addYoutubeSearcher], ]; if (!ClientUtils.RELEASE) btns.unshift([React.createRef(), "cat", "Add Cat Image", addImageNode]); diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index d0e117fe4..df5ff04dd 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -32,6 +32,8 @@ import { Doc } from "../../../new_fields/Doc"; import DirectoryImportBox from "../../util/Import & Export/DirectoryImportBox"; import { ScriptField } from "../../../new_fields/ScriptField"; import { fromPromise } from "mobx-utils"; +import { DashWebCam } from "../../views/webcam/DashWebCam"; + const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? type BindingProps = Without; @@ -110,7 +112,7 @@ export class DocumentContentsView extends React.Component void; + onUserMediaError: (error: string) => void; + screenshotFormat: "image/webp" | "image/png" | "image/jpeg"; + screenshotQuality: number; + videoConstraints?: MediaStreamConstraints["video"]; +} + +@observer +export class DashWebCam extends React.Component, WebcamState> { + static defaultProps = { + audio: true, + imageSmoothing: true, + onUserMedia: () => { }, + onUserMediaError: () => { }, + screenshotFormat: "image/webp", + screenshotQuality: 0.92 + }; + + private static mountedInstances: DashWebCam[] = []; + private static userMediaRequested = false; + private canvas: HTMLCanvasElement | undefined; + private ctx: CanvasRenderingContext2D | null = null; + private stream: MediaStream | undefined; + private video: HTMLVideoElement | null | undefined; + + @observable private hasUserMedia: boolean | undefined; + @observable private src: string | undefined; + + // constructor(props: any) { + // super(props); + // this.state = { + // hasUserMedia: false + // }; + // } + + componentDidMount() { + if (!hasGetUserMedia()) return; + + const { state } = this; + + DashWebCam.mountedInstances.push(this); + + if (!state.hasUserMedia && !DashWebCam.userMediaRequested) { + this.requestUserMedia(); + } + } + + componentDidUpdate(nextProps: WebcamProps) { + const { props } = this; + if ( + JSON.stringify(nextProps.audioConstraints) !== + JSON.stringify(props.audioConstraints) || + JSON.stringify(nextProps.videoConstraints) !== + JSON.stringify(props.videoConstraints) + ) { + this.requestUserMedia(); + } + } + + componentWillUnmount() { + //const { state } = this; + const index = DashWebCam.mountedInstances.indexOf(this); + DashWebCam.mountedInstances.splice(index, 1); + + DashWebCam.userMediaRequested = false; + if (DashWebCam.mountedInstances.length === 0 && this.hasUserMedia) { + if (this.stream!.getVideoTracks && this.stream!.getAudioTracks) { + this.stream!.getVideoTracks().map(track => track.stop()); + this.stream!.getAudioTracks().map(track => track.stop()); + } else { + ((this.stream as unknown) as MediaStreamTrack).stop(); + } + + if (this.src) { + window.URL.revokeObjectURL(this.src); + } + } + } + + getScreenshot() { + const { props } = this; + + if (!this.hasUserMedia) return null; + + const canvas = this.getCanvas(); + return ( + canvas && + canvas.toDataURL(props.screenshotFormat, props.screenshotQuality) + ); + } + + getCanvas() { + const { props } = this; + + if (!this.video) { + return null; + } + + if (!this.hasUserMedia || !this.video.videoHeight) return null; + + if (!this.ctx) { + const canvas = document.createElement("canvas"); + const aspectRatio = this.video.videoWidth / this.video.videoHeight; + + let canvasWidth = props.minScreenshotWidth || this.video.clientWidth; + let canvasHeight = canvasWidth / aspectRatio; + + if ( + props.minScreenshotHeight && + canvasHeight < props.minScreenshotHeight + ) { + canvasHeight = props.minScreenshotHeight; + canvasWidth = canvasHeight * aspectRatio; + } + + canvas.width = canvasWidth; + canvas.height = canvasHeight; + + this.canvas = canvas; + this.ctx = canvas.getContext("2d"); + } + + const { ctx, canvas } = this; + + if (ctx) { + ctx.imageSmoothingEnabled = props.imageSmoothing; + ctx.drawImage(this.video, 0, 0, canvas!.width, canvas!.height); + } + + return canvas; + } + + requestUserMedia() { + const { props } = this; + + navigator.getUserMedia = + navigator.mediaDevices.getUserMedia; + + const sourceSelected = (audioConstraints, videoConstraints) => { + const constraints: MediaStreamConstraints = { + video: typeof videoConstraints !== "undefined" ? videoConstraints : true + }; + + if (props.audio) { + constraints.audio = + typeof audioConstraints !== "undefined" ? audioConstraints : true; + } + + navigator.mediaDevices + .getUserMedia(constraints) + .then(stream => { + DashWebCam.mountedInstances.forEach(instance => + instance.handleUserMedia(null, stream) + ); + }) + .catch(e => { + DashWebCam.mountedInstances.forEach(instance => + instance.handleUserMedia(e) + ); + }); + }; + + if ("mediaDevices" in navigator) { + sourceSelected(props.audioConstraints, props.videoConstraints); + } else { + const optionalSource = id => ({ optional: [{ sourceId: id }] }); + + const constraintToSourceId = constraint => { + const { deviceId } = constraint; + + if (typeof deviceId === "string") { + return deviceId; + } + + if (Array.isArray(deviceId) && deviceId.length > 0) { + return deviceId[0]; + } + + if (typeof deviceId === "object" && deviceId.ideal) { + return deviceId.ideal; + } + + return null; + }; + + // @ts-ignore: deprecated api + MediaStreamTrack.getSources(sources => { + let audioSource = null; + let videoSource = null; + + sources.forEach(source => { + if (source.kind === "audio") { + audioSource = source.id; + } else if (source.kind === "video") { + videoSource = source.id; + } + }); + + const audioSourceId = constraintToSourceId(props.audioConstraints); + if (audioSourceId) { + audioSource = audioSourceId; + } + + const videoSourceId = constraintToSourceId(props.videoConstraints); + if (videoSourceId) { + videoSource = videoSourceId; + } + + sourceSelected( + optionalSource(audioSource), + optionalSource(videoSource) + ); + }); + } + + DashWebCam.userMediaRequested = true; + } + + handleUserMedia(err, stream?: MediaStream) { + const { props } = this; + + if (err || !stream) { + this.setState({ hasUserMedia: false }); + props.onUserMediaError(err); + + return; + } + + this.stream = stream; + + try { + if (this.video) { + this.video.srcObject = stream; + } + this.setState({ hasUserMedia: true }); + } catch (error) { + this.setState({ + hasUserMedia: true, + src: window.URL.createObjectURL(stream) + }); + } + + props.onUserMedia(); + } + + + + + + public static LayoutString() { return FieldView.LayoutString(DashWebCam); } + + + render() { + const { props } = this; + + const { + audio, + onUserMedia, + onUserMediaError, + screenshotFormat, + screenshotQuality, + minScreenshotWidth, + minScreenshotHeight, + audioConstraints, + videoConstraints, + imageSmoothing, + ...rest + } = props; + + + return ( +