diff options
| -rw-r--r-- | src/client/documents/Documents.ts | 16 | ||||
| -rw-r--r-- | src/client/views/MainView.tsx | 4 | ||||
| -rw-r--r-- | src/client/views/nodes/DocumentContentsView.tsx | 16 | ||||
| -rw-r--r-- | src/client/views/webcam/DashWebRTC.tsx | 225 |
4 files changed, 214 insertions, 47 deletions
diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 3482033d5..6bd91c3b9 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -20,12 +20,8 @@ import { AggregateFunction } from "../northstar/model/idea/idea"; import { MINIMIZED_ICON_SIZE } from "../views/globalCssVariables.scss"; import { IconBox } from "../views/nodes/IconBox"; import { OmitKeys, JSONUtils } from "../../Utils"; -<<<<<<< HEAD -import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField, WebCamField } from "../../new_fields/URLField"; -======= import { Field, Doc, Opt, DocListCastAsync } from "../../new_fields/Doc"; -import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField } from "../../new_fields/URLField"; ->>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 +import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField, WebCamField } from "../../new_fields/URLField"; import { HtmlField } from "../../new_fields/HtmlField"; import { List } from "../../new_fields/List"; import { Cast, NumCast } from "../../new_fields/Types"; @@ -49,14 +45,9 @@ import { ComputedField } from "../../new_fields/ScriptField"; import { ProxyField } from "../../new_fields/Proxy"; import { DocumentType } from "./DocumentTypes"; import { LinkFollowBox } from "../views/linking/LinkFollowBox"; -<<<<<<< HEAD import { DashWebCam } from "../views/webcam/DashWebCam"; import { DashWebRTC } from "../views/webcam/DashWebRTC"; -//import { PresBox } from "../views/nodes/PresBox"; -//import { PresField } from "../../new_fields/PresField"; -======= import { PresElementBox } from "../views/presentationview/PresElementBox"; ->>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 var requestImageSize = require('../util/request-image-size'); var path = require('path'); @@ -186,15 +177,12 @@ export namespace Docs { [DocumentType.LINKFOLLOW, { layout: { view: LinkFollowBox } }], -<<<<<<< HEAD [DocumentType.WEBCAM, { layout: { view: DashWebRTC } - }] -======= + }], [DocumentType.PRESELEMENT, { layout: { view: PresElementBox } }], ->>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 ]); // All document prototypes are initialized with at least these values diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 22d8171c0..7e2b4fa3a 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -1,9 +1,5 @@ import { IconName, library } from '@fortawesome/fontawesome-svg-core'; -<<<<<<< HEAD import { faLink, faArrowDown, faArrowUp, faBolt, faCaretUp, faCat, faCheck, faClone, faCloudUploadAlt, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faLongArrowAltRight, faMusic, faObjectGroup, faPause, faPenNib, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faUndoAlt, faTv, faVideo } from '@fortawesome/free-solid-svg-icons'; -======= -import { faArrowDown, faArrowUp, faBolt, faCaretUp, faCat, faCheck, faClone, faCloudUploadAlt, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faLongArrowAltRight, faMusic, faObjectGroup, faPause, faPenNib, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faTv, faUndoAlt } from '@fortawesome/free-solid-svg-icons'; ->>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, computed, configure, observable, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index f1c10f2f2..f1678d4f9 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -30,20 +30,12 @@ import { PresElementBox } from "../presentationview/PresElementBox"; import { VideoBox } from "./VideoBox"; import { WebBox } from "./WebBox"; import React = require("react"); -<<<<<<< HEAD -import { FieldViewProps } from "./FieldView"; -import { Without, OmitKeys } from "../../../Utils"; -import { Cast, StrCast, NumCast } from "../../../new_fields/Types"; +import { StrCast, NumCast } from "../../../new_fields/Types"; import { List } from "../../../new_fields/List"; -import { Doc } from "../../../new_fields/Doc"; -import DirectoryImportBox from "../../util/Import & Export/DirectoryImportBox"; -import { ScriptField } from "../../../new_fields/ScriptField"; import { fromPromise } from "mobx-utils"; import { DashWebCam } from "../../views/webcam/DashWebCam"; import { DashWebRTC } from "../webcam/DashWebRTC"; -======= ->>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? type BindingProps = Without<FieldViewProps, 'fieldKey'>; @@ -115,11 +107,7 @@ export class DocumentContentsView extends React.Component<DocumentViewProps & { if (!this.layout && this.props.layoutKey !== "overlayLayout") return (null); return <ObserverJsxParser blacklistedAttrs={[]} -<<<<<<< HEAD - components={{ FormattedTextBox, ImageBox, IconBox, DirectoryImportBox, DragBox, ButtonBox, FieldView, CollectionFreeFormView, CollectionDockingView, CollectionSchemaView, CollectionView, CollectionPDFView, CollectionVideoView, WebBox, KeyValueBox, PDFBox, VideoBox, AudioBox, HistogramBox, PresBox, YoutubeBox, LinkFollowBox, DashWebCam, DashWebRTC }} -======= - components={{ FormattedTextBox, ImageBox, IconBox, DirectoryImportBox, DragBox, ButtonBox, FieldView, CollectionFreeFormView, CollectionDockingView, CollectionSchemaView, CollectionView, CollectionPDFView, CollectionVideoView, WebBox, KeyValueBox, PDFBox, VideoBox, AudioBox, HistogramBox, PresBox, YoutubeBox, LinkFollowBox, PresElementBox }} ->>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 + components={{ FormattedTextBox, ImageBox, IconBox, DirectoryImportBox, DragBox, ButtonBox, FieldView, CollectionFreeFormView, CollectionDockingView, CollectionSchemaView, CollectionView, CollectionPDFView, CollectionVideoView, WebBox, KeyValueBox, PDFBox, VideoBox, AudioBox, HistogramBox, PresBox, YoutubeBox, LinkFollowBox, PresElementBox, DashWebCam, DashWebRTC }} bindings={this.CreateBindings()} jsx={this.finalLayout} showWarnings={true} diff --git a/src/client/views/webcam/DashWebRTC.tsx b/src/client/views/webcam/DashWebRTC.tsx index 9c289b40f..64ddb318f 100644 --- a/src/client/views/webcam/DashWebRTC.tsx +++ b/src/client/views/webcam/DashWebRTC.tsx @@ -2,7 +2,7 @@ import { observer } from "mobx-react"; import React = require("react"); import { CollectionFreeFormDocumentViewProps } from "../nodes/CollectionFreeFormDocumentView"; import { FieldViewProps, FieldView } from "../nodes/FieldView"; -import { observable, trace } from "mobx"; +import { observable } from "mobx"; import { DocumentDecorations } from "../DocumentDecorations"; import { InkingControl } from "../InkingControl"; import "../../views/nodes/WebBox.scss"; @@ -11,7 +11,7 @@ import adapter from 'webrtc-adapter'; -const mediaStreamConstaints = { +const mediaStreamConstraints = { video: true, }; @@ -26,7 +26,7 @@ export class DashWebRTC extends React.Component<CollectionFreeFormDocumentViewPr @observable private localVideoEl: HTMLVideoElement | undefined; @observable private peerVideoEl: HTMLVideoElement | undefined; @observable private localStream: MediaStream | undefined; - @observable private startTime = null; + @observable private startTime: any = null; @observable private remoteStream: MediaStream | undefined; @observable private localPeerConnection: any; @observable private remotePeerConnection: any; @@ -38,7 +38,7 @@ export class DashWebRTC extends React.Component<CollectionFreeFormDocumentViewPr componentDidMount() { this.callButton!.disabled = true; this.hangupButton!.disabled = true; - navigator.mediaDevices.getUserMedia(mediaStreamConstaints).then(this.gotLocalMediaStream).catch(this.handleLocalMediaStreamError); + navigator.mediaDevices.getUserMedia(mediaStreamConstraints).then(this.gotLocalMediaStream).catch(this.handleLocalMediaStreamError); this.localVideoEl!.addEventListener('loadedmetadata', this.logVideoLoaded); this.peerVideoEl!.addEventListener('loadedmetadata', this.logVideoLoaded); this.peerVideoEl!.addEventListener('onresize', this.logResizedVideo); @@ -50,7 +50,7 @@ export class DashWebRTC extends React.Component<CollectionFreeFormDocumentViewPr if (this.localVideoEl) { this.localVideoEl.srcObject = mediaStream; } - trace('Received local stream.'); + this.trace('Received local stream.'); this.callButton!.disabled = false; } @@ -64,27 +64,222 @@ export class DashWebRTC extends React.Component<CollectionFreeFormDocumentViewPr handleLocalMediaStreamError = (error: string) => { //console.log("navigator.getUserMedia error: ", error); - trace(`navigator.getUserMedia error: ${error.toString()}.`); + this.trace(`navigator.getUserMedia error: ${error.toString()}.`); } - logVideoLoaded(event: any) { + logVideoLoaded = (event: any) => { let video = event.target; - trace(`${video!.id} videoWidth: ${video!.videoWidth}px, ` + - `videoHeight: ${video!.videoHeight}px.`); + this.trace(`${video.id} videoWidth: ${video.videoWidth}px, ` + + `videoHeight: ${video.videoHeight}px.`); } - logResizedVideo(event: any) { + logResizedVideo = (event: any) => { this.logVideoLoaded(event); if (this.startTime) { - let elapsedTime = window.performance.now() - this.startTime!; + let elapsedTime = window.performance.now() - this.startTime; this.startTime = null; - trace(`Setup time: ${elapsedTime.toFixed(3)}ms.`); + this.trace(`Setup time: ${elapsedTime.toFixed(3)}ms.`); } } + handleConnection = (event: any) => { + let peerConnection = event.target; + let iceCandidate = event.candidate; + + if (iceCandidate) { + let newIceCandidate: RTCIceCandidate = new RTCIceCandidate(iceCandidate); + let otherPeer: any = this.getOtherPeer(peerConnection); + + otherPeer.addIceCandidate(newIceCandidate).then(() => { + this.handleConnectionSuccess(peerConnection); + }).catch((error: any) => { + this.handleConnectionFailure(peerConnection, error); + }); + + this.trace(`${this.getPeerName(peerConnection)} ICE candidate:\n` + + `${event.candidate.candidate}.`); + + } + } + + // Logs that the connection succeeded. + handleConnectionSuccess = (peerConnection: any) => { + this.trace(`${this.getPeerName(peerConnection)} addIceCandidate success.`); + } + + handleConnectionFailure = (peerConnection: any, error: any) => { + this.trace(`${this.getPeerName(peerConnection)} failed to add ICE Candidate:\n` + + `${error.toString()}.`); + } + + // Logs changes to the connection state. + handleConnectionChange = (event: any) => { + let peerConnection = event.target; + console.log('ICE state change event: ', event); + this.trace(`${this.getPeerName(peerConnection)} ICE state: ` + + `${peerConnection.iceConnectionState}.`); + } + + // Logs error when setting session description fails. + setSessionDescriptionError = (error: any) => { + this.trace(`Failed to create session description: ${error.toString()}.`); + } + + // Logs success when setting session description. + setDescriptionSuccess = (peerConnection: any, functionName: any) => { + let peerName = this.getPeerName(peerConnection); + this.trace(`${peerName} ${functionName} complete.`); + } + + + // Logs success when localDescription is set. + setLocalDescriptionSuccess = (peerConnection: any) => { + this.setDescriptionSuccess(peerConnection, 'setLocalDescription'); + } + + // Logs success when remoteDescription is set. + setRemoteDescriptionSuccess = (peerConnection: any) => { + this.setDescriptionSuccess(peerConnection, 'setRemoteDescription'); + } + + createdOffer = (description: any) => { + this.trace(`Offer from localPeerConnection:\n${description.sdp}`); + this.trace('localPeerConnection setLocalDescription start.'); + + this.localPeerConnection.setLocalDescription(description).then(() => { + this.setLocalDescriptionSuccess(this.localPeerConnection); + }).catch(this.setSessionDescriptionError); + + + this.trace('remotePeerConnection setRemoteDescription start.'); + this.remotePeerConnection.setRemoteDescription(description) + .then(() => { + this.setRemoteDescriptionSuccess(this.remotePeerConnection); + }).catch(this.setSessionDescriptionError); + + this.trace('remotePeerConnection createAnswer start.'); + this.remotePeerConnection.createAnswer() + .then(this.createdAnswer) + .catch(this.setSessionDescriptionError); + + } + + createdAnswer = (description: any) => { + this.trace(`Answer from remotePeerConnection:\n${description.sdp}.`); + + this.trace('remotePeerConnection setLocalDescription start.'); + this.remotePeerConnection.setLocalDescription(description) + .then(() => { + this.setLocalDescriptionSuccess(this.remotePeerConnection); + }).catch(this.setSessionDescriptionError); + + this.trace('localPeerConnection setRemoteDescription start.'); + this.localPeerConnection.setRemoteDescription(description) + .then(() => { + this.setRemoteDescriptionSuccess(this.localPeerConnection); + }).catch(this.setSessionDescriptionError); + } + + + startAction = () => { + this.startButton!.disabled = true; + navigator.mediaDevices.getUserMedia(mediaStreamConstraints) + .then(this.gotLocalMediaStream).catch(this.handleLocalMediaStreamError); + this.trace('Requesting local stream.'); + } + + + // Handles call button action: creates peer connection. + callAction = () => { + this.callButton!.disabled = true; + this.hangupButton!.disabled = false; + + this.trace('Starting call.'); + this.startTime = window.performance.now(); + + // Get local media stream tracks. + const videoTracks = this.localStream!.getVideoTracks(); + const audioTracks = this.localStream!.getAudioTracks(); + if (videoTracks.length > 0) { + this.trace(`Using video device: ${videoTracks[0].label}.`); + } + if (audioTracks.length > 0) { + this.trace(`Using audio device: ${audioTracks[0].label}.`); + } + + let servers: RTCConfiguration | undefined = undefined; // Allows for RTC server configuration. + + // Create peer connections and add behavior. + this.localPeerConnection = new RTCPeerConnection(servers); + this.trace('Created local peer connection object localPeerConnection.'); + + this.localPeerConnection.addEventListener('icecandidate', this.handleConnection); + this.localPeerConnection.addEventListener( + 'iceconnectionstatechange', this.handleConnectionChange); + + this.remotePeerConnection = new RTCPeerConnection(servers); + this.trace('Created remote peer connection object remotePeerConnection.'); + + this.remotePeerConnection.addEventListener('icecandidate', this.handleConnection); + this.remotePeerConnection.addEventListener( + 'iceconnectionstatechange', this.handleConnectionChange); + this.remotePeerConnection.addEventListener('addstream', this.gotRemoteMediaStream); + + // Add local stream to connection and create offer to connect. + this.localPeerConnection.addStream(this.localStream); + this.trace('Added local stream to localPeerConnection.'); + + this.trace('localPeerConnection createOffer start.'); + this.localPeerConnection.createOffer(offerOptions) + .then(this.createdOffer).catch(this.setSessionDescriptionError); + } + + + // Handles hangup action: ends up call, closes connections and resets peers. + hangupAction = () => { + this.localPeerConnection.close(); + this.remotePeerConnection.close(); + this.localPeerConnection = null; + this.remotePeerConnection = null; + this.hangupButton!.disabled = true; + this.callButton!.disabled = false; + this.trace('Ending call.'); + } + + // Gets the "other" peer connection. + getOtherPeer = (peerConnection: any) => { + return (peerConnection === this.localPeerConnection) ? + this.remotePeerConnection : this.localPeerConnection; + } + + // Gets the name of a certain peer connection. + getPeerName = (peerConnection: any) => { + return (peerConnection === this.localPeerConnection) ? + 'localPeerConnection' : 'remotePeerConnection'; + } + + // Logs an action (text) and the time when it happened on the console. + trace = (text: string) => { + text = text.trim(); + const now = (window.performance.now() / 1000).toFixed(3); + + console.log(now, text); + } + + + + + + + + + + + + @@ -116,9 +311,9 @@ export class DashWebRTC extends React.Component<CollectionFreeFormDocumentViewPr <div className="webcam-cont" style={{ width: "100%", height: "100%" }} onWheel={this.onPostWheel} onPointerDown={this.onPostPointer} onPointerMove={this.onPostPointer} onPointerUp={this.onPostPointer}> <video id="localVideo" autoPlay playsInline ref={(e) => this.localVideoEl = e!}></video> <video id="remoteVideo" autoPlay playsInline ref={(e) => this.peerVideoEl = e!}></video> - <button id="startButton" ref={(e) => this.startButton = e!}>Start</button> - <button id="callButton" ref={(e) => this.callButton = e!}>Call</button> - <button id="hangupButton" ref={(e) => this.hangupButton = e!}>Hang Up</button> + <button id="startButton" ref={(e) => this.startButton = e!} onClick={this.startAction}>Start</button> + <button id="callButton" ref={(e) => this.callButton = e!} onClick={this.callAction}>Call</button> + <button id="hangupButton" ref={(e) => this.hangupButton = e!} onClick={this.hangupAction}>Hang Up</button> </div>; let frozen = !this.props.isSelected() || DocumentDecorations.Instance.Interacting; |