From ee03fa6e04dd9dba3099f75154de6ffab566ff5c Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Fri, 13 Sep 2019 16:43:09 -0400 Subject: Trial 5(Most succesful yet) --- src/client/views/nodes/DocumentContentsView.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/client/views/nodes') diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index d0e117fe4..df5ff04dd 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -32,6 +32,8 @@ import { Doc } from "../../../new_fields/Doc"; import DirectoryImportBox from "../../util/Import & Export/DirectoryImportBox"; import { ScriptField } from "../../../new_fields/ScriptField"; import { fromPromise } from "mobx-utils"; +import { DashWebCam } from "../../views/webcam/DashWebCam"; + const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? type BindingProps = Without; @@ -110,7 +112,7 @@ export class DocumentContentsView extends React.Component Date: Tue, 1 Oct 2019 18:14:04 -0400 Subject: trials 2 --- package.json | 1 + src/client/documents/Documents.ts | 3 ++- src/client/views/nodes/DocumentContentsView.tsx | 3 ++- src/client/views/webcam/DashWebRTC.tsx | 9 +++++++-- 4 files changed, 12 insertions(+), 4 deletions(-) (limited to 'src/client/views/nodes') diff --git a/package.json b/package.json index d0718345f..7ae8c3399 100644 --- a/package.json +++ b/package.json @@ -216,6 +216,7 @@ "typescript-collections": "^1.3.2", "url-loader": "^1.1.2", "uuid": "^3.3.2", + "webrtc-adapter": "^7.3.0", "words-to-numbers": "^1.5.1", "xoauth2": "^1.2.0", "youtube": "^0.1.0" diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 036cc75a0..1f9b575a4 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -47,6 +47,7 @@ import { ProxyField } from "../../new_fields/Proxy"; import { DocumentType } from "./DocumentTypes"; import { LinkFollowBox } from "../views/linking/LinkFollowBox"; import { DashWebCam } from "../views/webcam/DashWebCam"; +import { DashWebRTC } from "../views/webcam/DashWebRTC"; //import { PresBox } from "../views/nodes/PresBox"; //import { PresField } from "../../new_fields/PresField"; var requestImageSize = require('../util/request-image-size'); @@ -179,7 +180,7 @@ export namespace Docs { layout: { view: LinkFollowBox } }], [DocumentType.WEBCAM, { - layout: { view: DashWebCam } + layout: { view: DashWebRTC } }] ]); diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index df5ff04dd..2b797eeca 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -33,6 +33,7 @@ import DirectoryImportBox from "../../util/Import & Export/DirectoryImportBox"; import { ScriptField } from "../../../new_fields/ScriptField"; import { fromPromise } from "mobx-utils"; import { DashWebCam } from "../../views/webcam/DashWebCam"; +import { DashWebRTC } from "../webcam/DashWebRTC"; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? @@ -112,7 +113,7 @@ export class DocumentContentsView extends React.Component - + + + + + ; let frozen = !this.props.isSelected() || DocumentDecorations.Instance.Interacting; @@ -77,7 +83,6 @@ export class DashWebRTC extends React.Component {!frozen ? (null) :
} ); - ); } -- cgit v1.2.3-70-g09d2 From a8f14c501cf676f6a2697b73d7f2a162d4100a9e Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Wed, 2 Oct 2019 16:52:57 -0400 Subject: Merge conflicts fixed --- src/client/documents/Documents.ts | 16 +- src/client/views/MainView.tsx | 4 - src/client/views/nodes/DocumentContentsView.tsx | 16 +- src/client/views/webcam/DashWebRTC.tsx | 225 ++++++++++++++++++++++-- 4 files changed, 214 insertions(+), 47 deletions(-) (limited to 'src/client/views/nodes') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 3482033d5..6bd91c3b9 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -20,12 +20,8 @@ import { AggregateFunction } from "../northstar/model/idea/idea"; import { MINIMIZED_ICON_SIZE } from "../views/globalCssVariables.scss"; import { IconBox } from "../views/nodes/IconBox"; import { OmitKeys, JSONUtils } from "../../Utils"; -<<<<<<< HEAD -import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField, WebCamField } from "../../new_fields/URLField"; -======= import { Field, Doc, Opt, DocListCastAsync } from "../../new_fields/Doc"; -import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField } from "../../new_fields/URLField"; ->>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 +import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField, WebCamField } from "../../new_fields/URLField"; import { HtmlField } from "../../new_fields/HtmlField"; import { List } from "../../new_fields/List"; import { Cast, NumCast } from "../../new_fields/Types"; @@ -49,14 +45,9 @@ import { ComputedField } from "../../new_fields/ScriptField"; import { ProxyField } from "../../new_fields/Proxy"; import { DocumentType } from "./DocumentTypes"; import { LinkFollowBox } from "../views/linking/LinkFollowBox"; -<<<<<<< HEAD import { DashWebCam } from "../views/webcam/DashWebCam"; import { DashWebRTC } from "../views/webcam/DashWebRTC"; -//import { PresBox } from "../views/nodes/PresBox"; -//import { PresField } from "../../new_fields/PresField"; -======= import { PresElementBox } from "../views/presentationview/PresElementBox"; ->>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 var requestImageSize = require('../util/request-image-size'); var path = require('path'); @@ -186,15 +177,12 @@ export namespace Docs { [DocumentType.LINKFOLLOW, { layout: { view: LinkFollowBox } }], -<<<<<<< HEAD [DocumentType.WEBCAM, { layout: { view: DashWebRTC } - }] -======= + }], [DocumentType.PRESELEMENT, { layout: { view: PresElementBox } }], ->>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 ]); // All document prototypes are initialized with at least these values diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 22d8171c0..7e2b4fa3a 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -1,9 +1,5 @@ import { IconName, library } from '@fortawesome/fontawesome-svg-core'; -<<<<<<< HEAD import { faLink, faArrowDown, faArrowUp, faBolt, faCaretUp, faCat, faCheck, faClone, faCloudUploadAlt, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faLongArrowAltRight, faMusic, faObjectGroup, faPause, faPenNib, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faUndoAlt, faTv, faVideo } from '@fortawesome/free-solid-svg-icons'; -======= -import { faArrowDown, faArrowUp, faBolt, faCaretUp, faCat, faCheck, faClone, faCloudUploadAlt, faCommentAlt, faCut, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faLongArrowAltRight, faMusic, faObjectGroup, faPause, faPenNib, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faTv, faUndoAlt } from '@fortawesome/free-solid-svg-icons'; ->>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, computed, configure, observable, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index f1c10f2f2..f1678d4f9 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -30,20 +30,12 @@ import { PresElementBox } from "../presentationview/PresElementBox"; import { VideoBox } from "./VideoBox"; import { WebBox } from "./WebBox"; import React = require("react"); -<<<<<<< HEAD -import { FieldViewProps } from "./FieldView"; -import { Without, OmitKeys } from "../../../Utils"; -import { Cast, StrCast, NumCast } from "../../../new_fields/Types"; +import { StrCast, NumCast } from "../../../new_fields/Types"; import { List } from "../../../new_fields/List"; -import { Doc } from "../../../new_fields/Doc"; -import DirectoryImportBox from "../../util/Import & Export/DirectoryImportBox"; -import { ScriptField } from "../../../new_fields/ScriptField"; import { fromPromise } from "mobx-utils"; import { DashWebCam } from "../../views/webcam/DashWebCam"; import { DashWebRTC } from "../webcam/DashWebRTC"; -======= ->>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? type BindingProps = Without; @@ -115,11 +107,7 @@ export class DocumentContentsView extends React.Component>>>>>> 69e4a936c4eb0cc2e35e4e7f3258aed1f72b8da7 + components={{ FormattedTextBox, ImageBox, IconBox, DirectoryImportBox, DragBox, ButtonBox, FieldView, CollectionFreeFormView, CollectionDockingView, CollectionSchemaView, CollectionView, CollectionPDFView, CollectionVideoView, WebBox, KeyValueBox, PDFBox, VideoBox, AudioBox, HistogramBox, PresBox, YoutubeBox, LinkFollowBox, PresElementBox, DashWebCam, DashWebRTC }} bindings={this.CreateBindings()} jsx={this.finalLayout} showWarnings={true} diff --git a/src/client/views/webcam/DashWebRTC.tsx b/src/client/views/webcam/DashWebRTC.tsx index 9c289b40f..64ddb318f 100644 --- a/src/client/views/webcam/DashWebRTC.tsx +++ b/src/client/views/webcam/DashWebRTC.tsx @@ -2,7 +2,7 @@ import { observer } from "mobx-react"; import React = require("react"); import { CollectionFreeFormDocumentViewProps } from "../nodes/CollectionFreeFormDocumentView"; import { FieldViewProps, FieldView } from "../nodes/FieldView"; -import { observable, trace } from "mobx"; +import { observable } from "mobx"; import { DocumentDecorations } from "../DocumentDecorations"; import { InkingControl } from "../InkingControl"; import "../../views/nodes/WebBox.scss"; @@ -11,7 +11,7 @@ import adapter from 'webrtc-adapter'; -const mediaStreamConstaints = { +const mediaStreamConstraints = { video: true, }; @@ -26,7 +26,7 @@ export class DashWebRTC extends React.Component { //console.log("navigator.getUserMedia error: ", error); - trace(`navigator.getUserMedia error: ${error.toString()}.`); + this.trace(`navigator.getUserMedia error: ${error.toString()}.`); } - logVideoLoaded(event: any) { + logVideoLoaded = (event: any) => { let video = event.target; - trace(`${video!.id} videoWidth: ${video!.videoWidth}px, ` + - `videoHeight: ${video!.videoHeight}px.`); + this.trace(`${video.id} videoWidth: ${video.videoWidth}px, ` + + `videoHeight: ${video.videoHeight}px.`); } - logResizedVideo(event: any) { + logResizedVideo = (event: any) => { this.logVideoLoaded(event); if (this.startTime) { - let elapsedTime = window.performance.now() - this.startTime!; + let elapsedTime = window.performance.now() - this.startTime; this.startTime = null; - trace(`Setup time: ${elapsedTime.toFixed(3)}ms.`); + this.trace(`Setup time: ${elapsedTime.toFixed(3)}ms.`); } } + handleConnection = (event: any) => { + let peerConnection = event.target; + let iceCandidate = event.candidate; + + if (iceCandidate) { + let newIceCandidate: RTCIceCandidate = new RTCIceCandidate(iceCandidate); + let otherPeer: any = this.getOtherPeer(peerConnection); + + otherPeer.addIceCandidate(newIceCandidate).then(() => { + this.handleConnectionSuccess(peerConnection); + }).catch((error: any) => { + this.handleConnectionFailure(peerConnection, error); + }); + + this.trace(`${this.getPeerName(peerConnection)} ICE candidate:\n` + + `${event.candidate.candidate}.`); + + } + } + + // Logs that the connection succeeded. + handleConnectionSuccess = (peerConnection: any) => { + this.trace(`${this.getPeerName(peerConnection)} addIceCandidate success.`); + } + + handleConnectionFailure = (peerConnection: any, error: any) => { + this.trace(`${this.getPeerName(peerConnection)} failed to add ICE Candidate:\n` + + `${error.toString()}.`); + } + + // Logs changes to the connection state. + handleConnectionChange = (event: any) => { + let peerConnection = event.target; + console.log('ICE state change event: ', event); + this.trace(`${this.getPeerName(peerConnection)} ICE state: ` + + `${peerConnection.iceConnectionState}.`); + } + + // Logs error when setting session description fails. + setSessionDescriptionError = (error: any) => { + this.trace(`Failed to create session description: ${error.toString()}.`); + } + + // Logs success when setting session description. + setDescriptionSuccess = (peerConnection: any, functionName: any) => { + let peerName = this.getPeerName(peerConnection); + this.trace(`${peerName} ${functionName} complete.`); + } + + + // Logs success when localDescription is set. + setLocalDescriptionSuccess = (peerConnection: any) => { + this.setDescriptionSuccess(peerConnection, 'setLocalDescription'); + } + + // Logs success when remoteDescription is set. + setRemoteDescriptionSuccess = (peerConnection: any) => { + this.setDescriptionSuccess(peerConnection, 'setRemoteDescription'); + } + + createdOffer = (description: any) => { + this.trace(`Offer from localPeerConnection:\n${description.sdp}`); + this.trace('localPeerConnection setLocalDescription start.'); + + this.localPeerConnection.setLocalDescription(description).then(() => { + this.setLocalDescriptionSuccess(this.localPeerConnection); + }).catch(this.setSessionDescriptionError); + + + this.trace('remotePeerConnection setRemoteDescription start.'); + this.remotePeerConnection.setRemoteDescription(description) + .then(() => { + this.setRemoteDescriptionSuccess(this.remotePeerConnection); + }).catch(this.setSessionDescriptionError); + + this.trace('remotePeerConnection createAnswer start.'); + this.remotePeerConnection.createAnswer() + .then(this.createdAnswer) + .catch(this.setSessionDescriptionError); + + } + + createdAnswer = (description: any) => { + this.trace(`Answer from remotePeerConnection:\n${description.sdp}.`); + + this.trace('remotePeerConnection setLocalDescription start.'); + this.remotePeerConnection.setLocalDescription(description) + .then(() => { + this.setLocalDescriptionSuccess(this.remotePeerConnection); + }).catch(this.setSessionDescriptionError); + + this.trace('localPeerConnection setRemoteDescription start.'); + this.localPeerConnection.setRemoteDescription(description) + .then(() => { + this.setRemoteDescriptionSuccess(this.localPeerConnection); + }).catch(this.setSessionDescriptionError); + } + + + startAction = () => { + this.startButton!.disabled = true; + navigator.mediaDevices.getUserMedia(mediaStreamConstraints) + .then(this.gotLocalMediaStream).catch(this.handleLocalMediaStreamError); + this.trace('Requesting local stream.'); + } + + + // Handles call button action: creates peer connection. + callAction = () => { + this.callButton!.disabled = true; + this.hangupButton!.disabled = false; + + this.trace('Starting call.'); + this.startTime = window.performance.now(); + + // Get local media stream tracks. + const videoTracks = this.localStream!.getVideoTracks(); + const audioTracks = this.localStream!.getAudioTracks(); + if (videoTracks.length > 0) { + this.trace(`Using video device: ${videoTracks[0].label}.`); + } + if (audioTracks.length > 0) { + this.trace(`Using audio device: ${audioTracks[0].label}.`); + } + + let servers: RTCConfiguration | undefined = undefined; // Allows for RTC server configuration. + + // Create peer connections and add behavior. + this.localPeerConnection = new RTCPeerConnection(servers); + this.trace('Created local peer connection object localPeerConnection.'); + + this.localPeerConnection.addEventListener('icecandidate', this.handleConnection); + this.localPeerConnection.addEventListener( + 'iceconnectionstatechange', this.handleConnectionChange); + + this.remotePeerConnection = new RTCPeerConnection(servers); + this.trace('Created remote peer connection object remotePeerConnection.'); + + this.remotePeerConnection.addEventListener('icecandidate', this.handleConnection); + this.remotePeerConnection.addEventListener( + 'iceconnectionstatechange', this.handleConnectionChange); + this.remotePeerConnection.addEventListener('addstream', this.gotRemoteMediaStream); + + // Add local stream to connection and create offer to connect. + this.localPeerConnection.addStream(this.localStream); + this.trace('Added local stream to localPeerConnection.'); + + this.trace('localPeerConnection createOffer start.'); + this.localPeerConnection.createOffer(offerOptions) + .then(this.createdOffer).catch(this.setSessionDescriptionError); + } + + + // Handles hangup action: ends up call, closes connections and resets peers. + hangupAction = () => { + this.localPeerConnection.close(); + this.remotePeerConnection.close(); + this.localPeerConnection = null; + this.remotePeerConnection = null; + this.hangupButton!.disabled = true; + this.callButton!.disabled = false; + this.trace('Ending call.'); + } + + // Gets the "other" peer connection. + getOtherPeer = (peerConnection: any) => { + return (peerConnection === this.localPeerConnection) ? + this.remotePeerConnection : this.localPeerConnection; + } + + // Gets the name of a certain peer connection. + getPeerName = (peerConnection: any) => { + return (peerConnection === this.localPeerConnection) ? + 'localPeerConnection' : 'remotePeerConnection'; + } + + // Logs an action (text) and the time when it happened on the console. + trace = (text: string) => { + text = text.trim(); + const now = (window.performance.now() / 1000).toFixed(3); + + console.log(now, text); + } + + + + + + + + + + + + @@ -116,9 +311,9 @@ export class DashWebRTC extends React.Component - - - + + +
; let frozen = !this.props.isSelected() || DocumentDecorations.Instance.Interacting; -- cgit v1.2.3-70-g09d2 From d97091421468238d32edf55c240bfb3b021e9e05 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 29 Oct 2019 18:52:26 -0400 Subject: couple fixes --- src/client/views/nodes/DocumentContentsView.tsx | 4 ++-- src/client/views/webcam/DashWebRTCVideo.tsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src/client/views/nodes') diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index f1678d4f9..ee58e4152 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -34,7 +34,7 @@ import { StrCast, NumCast } from "../../../new_fields/Types"; import { List } from "../../../new_fields/List"; import { fromPromise } from "mobx-utils"; import { DashWebCam } from "../../views/webcam/DashWebCam"; -import { DashWebRTC } from "../webcam/DashWebRTC"; +import { DashWebRTCVideo } from "../webcam/DashWebRTCVideo"; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? @@ -107,7 +107,7 @@ export class DocumentContentsView extends React.Component Date: Sun, 10 Nov 2019 16:23:00 -0500 Subject: messages changed to only room instead of server broadcast! --- src/client/views/nodes/CollectionFreeFormDocumentView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 2 +- src/client/views/webcam/DashWebRTC.ts | 8 +++++--- src/server/index.ts | 5 +++-- 4 files changed, 10 insertions(+), 7 deletions(-) (limited to 'src/client/views/nodes') diff --git a/src/client/views/nodes/CollectionFreeFormDocumentView.tsx b/src/client/views/nodes/CollectionFreeFormDocumentView.tsx index a035bdc3d..c2c402819 100644 --- a/src/client/views/nodes/CollectionFreeFormDocumentView.tsx +++ b/src/client/views/nodes/CollectionFreeFormDocumentView.tsx @@ -84,7 +84,7 @@ export class CollectionFreeFormDocumentView extends DocComponent this.dataProvider ? this.dataProvider.height : this.panelHeight(); render() { - trace(); + //trace(); return
(Docu render() { if (!this.props.Document) return (null); - trace(); + // trace(); const animDims = this.Document.animateToDimensions ? Array.from(this.Document.animateToDimensions) : undefined; const ruleColor = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleColor_" + this.Document.heading]) : undefined; const ruleRounding = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleRounding_" + this.Document.heading]) : undefined; diff --git a/src/client/views/webcam/DashWebRTC.ts b/src/client/views/webcam/DashWebRTC.ts index 846de4115..c61469cb8 100644 --- a/src/client/views/webcam/DashWebRTC.ts +++ b/src/client/views/webcam/DashWebRTC.ts @@ -19,6 +19,7 @@ export namespace DashWebRTC { let turnReady; let localVideo: HTMLVideoElement; let remoteVideo: HTMLVideoElement; + let curRoom: string = ""; let pcConfig = { @@ -35,6 +36,8 @@ export namespace DashWebRTC { export function init(room: string) { + curRoom = room; + if (room !== '') { DocServer._socket.emit('create or join', room); console.log('Attempted to create or join room', room); @@ -73,7 +76,6 @@ export namespace DashWebRTC { if (message.message === 'got user media') { maybeStart(); } else if (message.message.type === 'offer') { - console.log("I have entered here bro!!!"); if (!isInitiator && !isStarted) { maybeStart(); } @@ -82,7 +84,7 @@ export namespace DashWebRTC { } else if (message.message.type === 'answer' && isStarted) { pc.setRemoteDescription(new RTCSessionDescription(message.message)); } else if (message.message.type === 'candidate' && isStarted) { - var candidate = new RTCIceCandidate({ + let candidate = new RTCIceCandidate({ sdpMLineIndex: message.message.label, candidate: message.message.candidate }); @@ -123,7 +125,7 @@ export namespace DashWebRTC { function sendMessage(message: any) { console.log('Client sending message: ', message); - Utils.Emit(DocServer._socket, MessageStore.NotifyRoommates, { message: message, room: "" }); + Utils.Emit(DocServer._socket, MessageStore.NotifyRoommates, { message: message, room: curRoom }); //DocServer._socket.emit('message', message); } diff --git a/src/server/index.ts b/src/server/index.ts index 77f048a1e..93d257721 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -983,8 +983,9 @@ function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any } function HandleRoommateNotification(socket: Socket, message: RoomMessage) { - socket.broadcast.emit('message', message); - //server.sockets.in(message.room).emit('message', message.message); + //socket.broadcast.emit('message', message); + console.log("The room that sent this: ", message.room, " and message is : ", message.message); + server.sockets.in(message.room).emit('message', message); } -- cgit v1.2.3-70-g09d2