diff options
| -rw-r--r-- | package.json | 3 | ||||
| -rw-r--r-- | src/Utils.ts | 8 | ||||
| -rw-r--r-- | src/client/DocServer.ts | 2 | ||||
| -rw-r--r-- | src/client/documents/DocumentTypes.ts | 2 | ||||
| -rw-r--r-- | src/client/documents/Documents.ts | 9 | ||||
| -rw-r--r-- | src/client/views/MainView.tsx | 3 | ||||
| -rw-r--r-- | src/client/views/nodes/DocumentContentsView.tsx | 8 | ||||
| -rw-r--r-- | src/client/views/webcam/DashWebCam.tsx | 396 | ||||
| -rw-r--r-- | src/client/views/webcam/DashWebRTC.scss | 23 | ||||
| -rw-r--r-- | src/client/views/webcam/DashWebRTC.ts | 321 | ||||
| -rw-r--r-- | src/client/views/webcam/DashWebRTCVideo.tsx | 341 | ||||
| -rw-r--r-- | src/new_fields/URLField.ts | 2 | ||||
| -rw-r--r-- | src/server/Message.ts | 9 | ||||
| -rw-r--r-- | src/server/authentication/models/current_user_utils.ts | 1 | ||||
| -rw-r--r-- | src/server/index.ts | 50 | ||||
| -rw-r--r-- | src/typings/index.d.ts | 3 |
16 files changed, 1176 insertions, 5 deletions
diff --git a/package.json b/package.json index 8ee080933..9ed0a4f85 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,7 @@ "@types/react-color": "^2.14.1", "@types/react-measure": "^2.0.4", "@types/react-table": "^6.7.22", + "@types/react-webcam": "^3.0.0", "@types/request": "^2.48.1", "@types/request-promise": "^4.1.42", "@types/sharp": "^0.22.2", @@ -207,6 +208,7 @@ "react-mosaic": "0.0.20", "react-simple-dropdown": "^3.2.3", "react-split-pane": "^0.1.85", + "react-webcam": "^3.0.1", "react-table": "^6.10.3", "readline": "^1.3.0", "request": "^2.88.0", @@ -221,6 +223,7 @@ "typescript-collections": "^1.3.2", "url-loader": "^1.1.2", "uuid": "^3.3.2", + "webrtc-adapter": "^7.3.0", "words-to-numbers": "^1.5.1", "xoauth2": "^1.2.0", "youtube": "^0.1.0" diff --git a/src/Utils.ts b/src/Utils.ts index 7bb025e49..2dd4eace4 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -1,6 +1,6 @@ import v4 = require('uuid/v4'); import v5 = require("uuid/v5"); -import { Socket } from 'socket.io'; +import { Socket, Room } from 'socket.io'; import { Message } from './server/Message'; import { RouteStore } from './server/RouteStore'; @@ -234,6 +234,12 @@ export namespace Utils { handler([arg, loggingCallback('S sending', fn, message.Name)]); }); } + export type RoomHandler = (socket: Socket, room: string) => any; + export type UsedSockets = Socket | SocketIOClient.Socket; + export type RoomMessage = "create or join" | "created" | "joined"; + export function AddRoomHandler(socket: Socket, message: RoomMessage, handler: RoomHandler) { + socket.on(message, room => handler(socket, room)); + } } export function OmitKeys(obj: any, keys: string[], addKeyFunc?: (dup: any) => void): { omit: any, extract: any } { diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index 2cec1046b..7ffb43684 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -21,7 +21,7 @@ import { Id, HandleUpdate } from '../new_fields/FieldSymbols'; */ export namespace DocServer { let _cache: { [id: string]: RefField | Promise<Opt<RefField>> } = {}; - let _socket: SocketIOClient.Socket; + export let _socket: SocketIOClient.Socket; // this client's distinct GUID created at initialization let GUID: string; // indicates whether or not a document is currently being udpated, and, if so, its id diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index 12501065a..957836a7a 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -17,6 +17,8 @@ export enum DocumentType { TEMPLATE = "template", EXTENSION = "extension", YOUTUBE = "youtube", + WEBCAM = "webcam", + DRAGBOX = "dragbox", FONTICON = "fonticonbox", PRES = "presentation", LINKFOLLOW = "linkfollow", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index d1fcabc4a..8ba24133a 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -44,7 +44,9 @@ import { ComputedField, ScriptField } from "../../new_fields/ScriptField"; import { ProxyField } from "../../new_fields/Proxy"; import { DocumentType } from "./DocumentTypes"; import { LinkFollowBox } from "../views/linking/LinkFollowBox"; +import { DashWebCam } from "../views/webcam/DashWebCam"; import { PresElementBox } from "../views/presentationview/PresElementBox"; +import { DashWebRTCVideo } from "../views/webcam/DashWebRTCVideo"; import { QueryBox } from "../views/nodes/QueryBox"; import { ColorBox } from "../views/nodes/ColorBox"; import { DocuLinkBox } from "../views/nodes/DocuLinkBox"; @@ -206,6 +208,9 @@ export namespace Docs { [DocumentType.LINKFOLLOW, { layout: { view: LinkFollowBox, dataField: data } }], + [DocumentType.WEBCAM, { + layout: { view: DashWebRTCVideo, dataField: data } + }], [DocumentType.PRESELEMENT, { layout: { view: PresElementBox, dataField: data } }], @@ -391,6 +396,10 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.YOUTUBE), new YoutubeField(new URL(url)), options); } + export function WebCamDocument(url: string, options: DocumentOptions = {}) { + return InstanceFromProto(Prototypes.get(DocumentType.WEBCAM), "", options); + } + export function AudioDocument(url: string, options: DocumentOptions = {}) { return InstanceFromProto(Prototypes.get(DocumentType.AUDIO), new AudioField(new URL(url)), options); } diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 383300b22..68f5d5446 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -1,7 +1,7 @@ import { library } from '@fortawesome/fontawesome-svg-core'; import { faArrowDown, faArrowUp, faBolt, faCaretUp, faCat, faCheck, faChevronRight, faClone, faCloudUploadAlt, faCommentAlt, faCut, faEllipsisV, faExclamation, faFilePdf, faFilm, faFont, faGlobeAsia, faLongArrowAltRight, - faMusic, faObjectGroup, faPause, faMousePointer, faPenNib, faFileAudio, faPen, faEraser, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faTv, faUndoAlt, faHighlighter, faMicrophone + faMusic, faObjectGroup, faPause, faMousePointer, faPenNib, faFileAudio, faPen, faEraser, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faTv, faUndoAlt, faHighlighter, faMicrophone, faVideo } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, computed, configure, observable, reaction, runInAction } from 'mobx'; @@ -129,6 +129,7 @@ export class MainView extends React.Component { library.add(faArrowUp); library.add(faCloudUploadAlt); library.add(faBolt); + library.add(faVideo); library.add(faChevronRight); library.add(faEllipsisV); this.initEventListeners(); diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index 779d25cdd..60d5a16d7 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -31,6 +31,12 @@ import { PresElementBox } from "../presentationview/PresElementBox"; import { VideoBox } from "./VideoBox"; import { WebBox } from "./WebBox"; import React = require("react"); +import { NumCast } from "../../../new_fields/Types"; +import { List } from "../../../new_fields/List"; +import { fromPromise } from "mobx-utils"; +import { DashWebCam } from "../../views/webcam/DashWebCam"; +import { DashWebRTCVideo } from "../webcam/DashWebRTCVideo"; + const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? type BindingProps = Without<FieldViewProps, 'fieldKey'>; @@ -97,7 +103,7 @@ export class DocumentContentsView extends React.Component<DocumentViewProps & { components={{ FormattedTextBox, ImageBox, IconBox, DirectoryImportBox, FontIconBox: FontIconBox, ButtonBox, FieldView, CollectionFreeFormView, CollectionDockingView, CollectionSchemaView, CollectionView, WebBox, KeyValueBox, - PDFBox, VideoBox, AudioBox, HistogramBox, PresBox, YoutubeBox, LinkFollowBox, PresElementBox, QueryBox, ColorBox, DocuLinkBox + PDFBox, VideoBox, AudioBox, HistogramBox, PresBox, YoutubeBox, LinkFollowBox, PresElementBox, QueryBox, ColorBox, DocuLinkBox, DashWebRTCVideo }} bindings={this.CreateBindings()} jsx={this.layout} diff --git a/src/client/views/webcam/DashWebCam.tsx b/src/client/views/webcam/DashWebCam.tsx new file mode 100644 index 000000000..a9669750f --- /dev/null +++ b/src/client/views/webcam/DashWebCam.tsx @@ -0,0 +1,396 @@ +import React = require("react"); +import { observer } from "mobx-react"; +import { FieldViewProps, FieldView } from "../nodes/FieldView"; +import { observable, action } from "mobx"; +import { DocumentDecorations } from "../DocumentDecorations"; +import { InkingControl } from "../InkingControl"; +import { CollectionFreeFormDocumentViewProps } from "../nodes/CollectionFreeFormDocumentView"; +import "../../views/nodes/WebBox.scss"; + + +//https://github.com/mozmorris/react-webcam is the source code used for the bigger part of this implementation. It's only modified to fit our current system. + + +function hasGetUserMedia() { + return !!( + (navigator.mediaDevices && navigator.mediaDevices.getUserMedia)); +} + +interface WebcamProps { + audio: boolean; + audioConstraints?: MediaStreamConstraints["audio"]; + imageSmoothing: boolean; + minScreenshotHeight?: number; + minScreenshotWidth?: number; + onUserMedia: () => void; + onUserMediaError: (error: string) => void; + screenshotFormat: "image/webp" | "image/png" | "image/jpeg"; + screenshotQuality: number; + videoConstraints?: MediaStreamConstraints["video"]; +} + +interface WebcamState { + hasUserMedia: boolean; + src?: string; +} + +@observer +export class DashWebCam extends React.Component<CollectionFreeFormDocumentViewProps & FieldViewProps & WebcamProps & React.HTMLAttributes<HTMLVideoElement> & { + layoutKey: string, +}, WebcamState> { + static defaultProps = { + audio: true, + imageSmoothing: true, + onUserMedia: () => { }, + onUserMediaError: () => { }, + screenshotFormat: "image/webp", + screenshotQuality: 0.92 + }; + + private static mountedInstances: DashWebCam[] = []; + private static userMediaRequested = false; + private canvas: HTMLCanvasElement | undefined; + private ctx: CanvasRenderingContext2D | null = null; + private stream: MediaStream | undefined; + private video: HTMLVideoElement | null | undefined; + + // @observable private hasUserMedia: boolean | undefined; + // @observable private src: string | undefined; + + constructor(props: any) { + super(props); + this.state = { + hasUserMedia: false + }; + } + + componentDidMount() { + if (!hasGetUserMedia()) return; + + const { state } = this; + + DashWebCam.mountedInstances.push(this); + + if (!state.hasUserMedia && !DashWebCam.userMediaRequested) { + this.requestUserMedia(); + } + } + + componentDidUpdate(nextProps: WebcamProps) { + const { props } = this; + if ( + JSON.stringify(nextProps.audioConstraints) !== + JSON.stringify(props.audioConstraints) || + JSON.stringify(nextProps.videoConstraints) !== + JSON.stringify(props.videoConstraints) + ) { + this.requestUserMedia(); + } + } + + componentWillUnmount() { + const { state } = this; + const index = DashWebCam.mountedInstances.indexOf(this); + DashWebCam.mountedInstances.splice(index, 1); + + DashWebCam.userMediaRequested = false; + if (DashWebCam.mountedInstances.length === 0 && state.hasUserMedia) { + if (this.stream!.getVideoTracks && this.stream!.getAudioTracks) { + this.stream!.getVideoTracks().map(track => track.stop()); + this.stream!.getAudioTracks().map(track => track.stop()); + } else { + ((this.stream as unknown) as MediaStreamTrack).stop(); + } + + if (state.src) { + window.URL.revokeObjectURL(state.src); + } + } + } + + //These are for screenshot if wanted. + + // getScreenshot() { + // const { state, props } = this; + + // if (!state.hasUserMedia) return null; + + // const canvas = this.getCanvas(); + // return ( + // canvas && + // canvas.toDataURL(props.screenshotFormat, props.screenshotQuality) + // ); + // } + + // getCanvas() { + // const { state, props } = this; + + // if (!this.video) { + // return null; + // } + + // if (!state.hasUserMedia || !this.video.videoHeight) return null; + + // if (!this.ctx) { + // const canvas = document.createElement("canvas"); + // const aspectRatio = this.video.videoWidth / this.video.videoHeight; + + // let canvasWidth = props.minScreenshotWidth || this.video.clientWidth; + // let canvasHeight = canvasWidth / aspectRatio; + + // if ( + // props.minScreenshotHeight && + // canvasHeight < props.minScreenshotHeight + // ) { + // canvasHeight = props.minScreenshotHeight; + // canvasWidth = canvasHeight * aspectRatio; + // } + + // canvas.width = canvasWidth; + // canvas.height = canvasHeight; + + // this.canvas = canvas; + // this.ctx = canvas.getContext("2d"); + // } + + // const { ctx, canvas } = this; + + // if (ctx) { + // ctx.imageSmoothingEnabled = props.imageSmoothing; + // ctx.drawImage(this.video, 0, 0, canvas!.width, canvas!.height); + // } + + // return canvas; + // } + + requestUserMedia() { + const { props } = this; + + navigator.getUserMedia = + navigator.mediaDevices.getUserMedia; + + const sourceSelected = (audioConstraints: any, videoConstraints: any) => { + const constraints: MediaStreamConstraints = { + video: typeof videoConstraints !== "undefined" ? videoConstraints : true + }; + + if (props.audio) { + constraints.audio = + typeof audioConstraints !== "undefined" ? audioConstraints : true; + } + + navigator.mediaDevices + .getUserMedia(constraints) + .then(stream => { + DashWebCam.mountedInstances.forEach(instance => + instance.handleUserMedia(null, stream) + ); + }) + .catch(e => { + DashWebCam.mountedInstances.forEach(instance => + instance.handleUserMedia(e) + ); + }); + }; + + if ("mediaDevices" in navigator) { + sourceSelected(props.audioConstraints, props.videoConstraints); + } else { + const optionalSource = (id: any) => ({ optional: [{ sourceId: id }] }); + + const constraintToSourceId = (constraint: any) => { + const { deviceId } = constraint; + + if (typeof deviceId === "string") { + return deviceId; + } + + if (Array.isArray(deviceId) && deviceId.length > 0) { + return deviceId[0]; + } + + if (typeof deviceId === "object" && deviceId.ideal) { + return deviceId.ideal; + } + + return null; + }; + + // @ts-ignore: deprecated api + MediaStreamTrack.getSources(sources => { + let audioSource = null; + let videoSource = null; + + sources.forEach((source: { kind: string; id: any; }) => { + if (source.kind === "audio") { + audioSource = source.id; + } else if (source.kind === "video") { + videoSource = source.id; + } + }); + + const audioSourceId = constraintToSourceId(props.audioConstraints); + if (audioSourceId) { + audioSource = audioSourceId; + } + + const videoSourceId = constraintToSourceId(props.videoConstraints); + if (videoSourceId) { + videoSource = videoSourceId; + } + + sourceSelected( + optionalSource(audioSource), + optionalSource(videoSource) + ); + }); + } + + DashWebCam.userMediaRequested = true; + } + + handleUserMedia(err: string | null, stream?: MediaStream) { + const { props } = this; + + if (err || !stream) { + this.setState({ hasUserMedia: false }); + // action(() => this.hasUserMedia = false); + props.onUserMediaError(err!); + + return; + } + + this.stream = stream; + + console.log("Stream done: ", stream); + + try { + if (this.video) { + this.video.srcObject = stream; + console.log("Source object: ", stream); + + } + this.setState({ hasUserMedia: true }); + // action(() => this.hasUserMedia = true); + + } catch (error) { + this.setState({ + hasUserMedia: true, + src: window.URL.createObjectURL(stream) + }); + console.log("State src set: ", this.state.src); + + // action(() => this.hasUserMedia = true); + // action(() => this.src = window.URL.createObjectURL(stream)); + } + + props.onUserMedia(); + } + + _ignore = 0; + onPreWheel = (e: React.WheelEvent) => { + this._ignore = e.timeStamp; + } + onPrePointer = (e: React.PointerEvent) => { + this._ignore = e.timeStamp; + } + onPostPointer = (e: React.PointerEvent) => { + if (this._ignore !== e.timeStamp) { + e.stopPropagation(); + } + } + onPostWheel = (e: React.WheelEvent) => { + if (this._ignore !== e.timeStamp) { + e.stopPropagation(); + } + } + + + + + public static LayoutString(fieldKey: string) { return FieldView.LayoutString(DashWebCam, fieldKey); } + + render() { + const { state, props } = this; + + + + const { + audio, + onUserMedia, + onUserMediaError, + screenshotFormat, + screenshotQuality, + minScreenshotWidth, + minScreenshotHeight, + audioConstraints, + videoConstraints, + imageSmoothing, + fieldKey, + fitToBox, + ContainingCollectionView, + Document, + DataDoc, + onClick, + isSelected, + select, + renderDepth, + addDocument, + addDocTab, + pinToPres, + removeDocument, + moveDocument, + ScreenToLocalTransform, + active, + whenActiveChanged, + focus, + PanelWidth, + PanelHeight, + setVideoBox, + ContentScaling, + ChromeHeight, + jitterRotation, + backgroundColor, + bringToFront, + zoomToScale, + getScale, + animateBetweenIcon, + layoutKey, + ...rest + } = props; + + console.log("Source produced: ", state.src); + + + + let content = + <div className="webcam-cont" style={{ width: "100%", height: "100%" }} onWheel={this.onPostWheel} onPointerDown={this.onPostPointer} onPointerMove={this.onPostPointer} onPointerUp={this.onPostPointer}> + <video + autoPlay + src={state.src} + muted={!audio} + playsInline + ref={ref => { + console.log("Source produced: ", state.src); + console.log("HasUser Media produced: ", state.hasUserMedia); + + this.video = ref; + }} + {...rest} + /> + </div>; + + + let frozen = !this.props.isSelected() || DocumentDecorations.Instance.Interacting; + let classname = "webBox-cont" + (this.props.isSelected() && !InkingControl.Instance.selectedTool && !DocumentDecorations.Instance.Interacting ? "-interactive" : ""); + + return ( + <> + <div className={classname} > + {content} + </div> + {!frozen ? (null) : <div className="webBox-overlay" onWheel={this.onPreWheel} onPointerDown={this.onPrePointer} onPointerMove={this.onPrePointer} onPointerUp={this.onPrePointer} />} + </>); + + } +}
\ No newline at end of file diff --git a/src/client/views/webcam/DashWebRTC.scss b/src/client/views/webcam/DashWebRTC.scss new file mode 100644 index 000000000..ddf4777a8 --- /dev/null +++ b/src/client/views/webcam/DashWebRTC.scss @@ -0,0 +1,23 @@ +.webcam-cont { + button { + margin: 10px; + position: relative; + top: 20%; + left: -60%; + } + + #localVideo { + margin: 10px; + position: relative; + width: 300px; + max-height: 300px; + } + + #remoteVideo { + margin: 10px; + position: relative; + width: 300px; + max-height: 300px; + + } +}
\ No newline at end of file diff --git a/src/client/views/webcam/DashWebRTC.ts b/src/client/views/webcam/DashWebRTC.ts new file mode 100644 index 000000000..801fd782d --- /dev/null +++ b/src/client/views/webcam/DashWebRTC.ts @@ -0,0 +1,321 @@ +import { DocServer } from '../../DocServer'; + + + +export namespace DashWebRTC { + + + let isChannelReady = false; + let isInitiator = false; + let isStarted = false; + let localStream: MediaStream | undefined; + let pc: any; + let remoteStream: MediaStream | undefined; + let turnReady; + let localVideo: HTMLVideoElement; + let remoteVideo: HTMLVideoElement; + + + let pcConfig = { + 'iceServers': [{ + 'urls': 'stun:stun.l.google.com:19302' + }] + }; + + // Set up audio and video regardless of what devices are present. + let sdpConstraints = { + offerToReceiveAudio: true, + offerToReceiveVideo: true + }; + + export function init() { + let room = 'test'; + + if (room !== '') { + DocServer._socket.emit('create or join', room); + console.log('Attempted to create or join room', room); + + } + + DocServer._socket.on('created', function (room: string) { + console.log('Created room ' + room); + isInitiator = true; + }); + + DocServer._socket.on('full', function (room: string) { + console.log('Room ' + room + ' is full'); + }); + + DocServer._socket.on('join', function (room: string) { + console.log('Another peer made a request to join room ' + room); + console.log('This peer is the initiator of room ' + room + '!'); + isChannelReady = true; + }); + + + DocServer._socket.on('joined', function (room: string) { + console.log('joined: ' + room); + isChannelReady = true; + }); + + + DocServer._socket.on('log', function (array: any) { + console.log.apply(console, array); + }); + + // This client receives a message + DocServer._socket.on('message', function (message: any) { + console.log('Client received message:', message); + if (message === 'got user media') { + maybeStart(); + } else if (message.type === 'offer') { + if (!isInitiator && !isStarted) { + maybeStart(); + } + pc.setRemoteDescription(new RTCSessionDescription(message)); + doAnswer(); + } else if (message.type === 'answer' && isStarted) { + pc.setRemoteDescription(new RTCSessionDescription(message)); + } else if (message.type === 'candidate' && isStarted) { + var candidate = new RTCIceCandidate({ + sdpMLineIndex: message.label, + candidate: message.candidate + }); + pc.addIceCandidate(candidate); + } else if (message === 'bye' && isStarted) { + handleRemoteHangup(); + } + }); + + navigator.mediaDevices.getUserMedia({ + audio: false, + video: true + }) + .then(gotStream) + .catch(function (e) { + alert('getUserMedia() error: ' + e.name); + }); + + //Trying this one out!!! + console.log('Getting user media with constraints', constraints); + + if (location.hostname !== 'localhost') { + requestTurn( + 'https://computeengineondemand.appspot.com/turn?username=41784574&key=4080218913' + ); + } + + + } + + + + + //let socket = io.connect(); + + + + + function sendMessage(message: any) { + console.log('Client sending message: ', message); + DocServer._socket.emit('message', message); + } + + + + + + export function setVideoObjects(localVideo: HTMLVideoElement, remoteVideo: HTMLVideoElement) { + localVideo = localVideo; + remoteVideo = remoteVideo; + } + + + + + function gotStream(stream: any) { + console.log('Adding local stream.'); + localStream = stream; + localVideo.srcObject = stream; + sendMessage('got user media'); + if (isInitiator) { + maybeStart(); + } + } + + let constraints = { + video: true, + audio: true + }; + + + + + + function maybeStart() { + console.log('>>>>>>> maybeStart() ', isStarted, localStream, isChannelReady); + if (!isStarted && typeof localStream !== 'undefined' && isChannelReady) { + console.log('>>>>>> creating peer connection'); + createPeerConnection(); + pc.addStream(localStream); + isStarted = true; + console.log('isInitiator', isInitiator); + if (isInitiator) { + doCall(); + } + } + } + + + //this will need to be changed to our version of hangUp + window.onbeforeunload = function () { + sendMessage('bye'); + }; + + function createPeerConnection() { + try { + pc = new RTCPeerConnection(undefined); + pc.onicecandidate = handleIceCandidate; + pc.onaddstream = handleRemoteStreamAdded; + pc.onremovestream = handleRemoteStreamRemoved; + console.log('Created RTCPeerConnnection'); + } catch (e) { + console.log('Failed to create PeerConnection, exception: ' + e.message); + alert('Cannot create RTCPeerConnection object.'); + return; + } + } + + function handleIceCandidate(event: any) { + console.log('icecandidate event: ', event); + if (event.candidate) { + sendMessage({ + type: 'candidate', + label: event.candidate.sdpMLineIndex, + id: event.candidate.sdpMid, + candidate: event.candidate.candidate + }); + } else { + console.log('End of candidates.'); + } + } + + function handleCreateOfferError(event: any) { + console.log('createOffer() error: ', event); + } + + function doCall() { + console.log('Sending offer to peer'); + pc.createOffer(setLocalAndSendMessage, handleCreateOfferError); + } + + function doAnswer() { + console.log('Sending answer to peer.'); + pc.createAnswer().then( + setLocalAndSendMessage, + onCreateSessionDescriptionError + ); + } + + function setLocalAndSendMessage(sessionDescription: any) { + pc.setLocalDescription(sessionDescription); + console.log('setLocalAndSendMessage sending message', sessionDescription); + sendMessage(sessionDescription); + } + + function onCreateSessionDescriptionError(error: any) { + console.log('Failed to create session description: ' + error.toString()); + } + + + function requestTurn(turnURL: any) { + var turnExists = false; + for (var i in pcConfig.iceServers) { + if (pcConfig.iceServers[i].urls.substr(0, 5) === 'turn:') { + turnExists = true; + turnReady = true; + break; + } + } + if (!turnExists) { + console.log('Getting TURN server from ', turnURL); + // No TURN server. Get one from computeengineondemand.appspot.com: + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function () { + if (xhr.readyState === 4 && xhr.status === 200) { + var turnServer = JSON.parse(xhr.responseText); + console.log('Got TURN server: ', turnServer); + pcConfig.iceServers.push({ + 'urls': 'turn:' + turnServer.username + '@' + turnServer.turn, + //'credential': turnServer.password + }); + turnReady = true; + } + }; + xhr.open('GET', turnURL, true); + xhr.send(); + } + } + + function handleRemoteStreamAdded(event: MediaStreamEvent) { + console.log('Remote stream added.'); + remoteStream = event.stream!; + remoteVideo.srcObject = remoteStream; + } + + function handleRemoteStreamRemoved(event: MediaStreamEvent) { + console.log('Remote stream removed. Event: ', event); + } + + function hangup() { + console.log('Hanging up.'); + stop(); + sendMessage('bye'); + } + + function handleRemoteHangup() { + console.log('Session terminated.'); + stop(); + isInitiator = false; + } + + function stop() { + isStarted = false; + pc.close(); + pc = null; + } + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +}
\ No newline at end of file diff --git a/src/client/views/webcam/DashWebRTCVideo.tsx b/src/client/views/webcam/DashWebRTCVideo.tsx new file mode 100644 index 000000000..62efd9730 --- /dev/null +++ b/src/client/views/webcam/DashWebRTCVideo.tsx @@ -0,0 +1,341 @@ +import { observer } from "mobx-react"; +import React = require("react"); +import { CollectionFreeFormDocumentViewProps } from "../nodes/CollectionFreeFormDocumentView"; +import { FieldViewProps, FieldView } from "../nodes/FieldView"; +import { observable } from "mobx"; +import { DocumentDecorations } from "../DocumentDecorations"; +import { InkingControl } from "../InkingControl"; +import "../../views/nodes/WebBox.scss"; +import "./DashWebRTC.scss"; +import adapter from 'webrtc-adapter'; +import { DashWebRTC } from "./DashWebRTC"; + + + + +const mediaStreamConstraints = { + video: true, +}; + +const offerOptions = { + offerToReceiveVideo: 1, +}; + + +@observer +export class DashWebRTCVideo extends React.Component<CollectionFreeFormDocumentViewProps & FieldViewProps> { + + @observable private localVideoEl: HTMLVideoElement | undefined; + @observable private peerVideoEl: HTMLVideoElement | undefined; + @observable private localStream: MediaStream | undefined; + @observable private startTime: any = null; + @observable private remoteStream: MediaStream | undefined; + @observable private localPeerConnection: any; + @observable private remotePeerConnection: any; + private callButton: HTMLButtonElement | undefined; + private startButton: HTMLButtonElement | undefined; + private hangupButton: HTMLButtonElement | undefined; + + componentDidMount() { + DashWebRTC.setVideoObjects(this.localVideoEl!, this.peerVideoEl!); + DashWebRTC.init(); + } + + + // componentDidMount() { + // this.callButton!.disabled = true; + // this.hangupButton!.disabled = true; + // // navigator.mediaDevices.getUserMedia(mediaStreamConstraints).then(this.gotLocalMediaStream).catch(this.handleLocalMediaStreamError); + // this.localVideoEl!.addEventListener('loadedmetadata', this.logVideoLoaded); + // this.peerVideoEl!.addEventListener('loadedmetadata', this.logVideoLoaded); + // this.peerVideoEl!.addEventListener('onresize', this.logResizedVideo); + // } + + + // gotLocalMediaStream = (mediaStream: MediaStream) => { + // this.localStream = mediaStream; + // if (this.localVideoEl) { + // this.localVideoEl.srcObject = mediaStream; + // } + // this.trace('Received local stream.'); + // this.callButton!.disabled = false; + + // } + + // gotRemoteMediaStream = (event: MediaStreamEvent) => { + // let mediaStream = event.stream; + // this.peerVideoEl!.srcObject = mediaStream; + // this.remoteStream = mediaStream!; + + // } + + // handleLocalMediaStreamError = (error: string) => { + // //console.log("navigator.getUserMedia error: ", error); + // this.trace(`navigator.getUserMedia error: ${error.toString()}.`); + + // } + + // logVideoLoaded = (event: any) => { + // let video = event.target; + // this.trace(`${video.id} videoWidth: ${video.videoWidth}px, ` + + // `videoHeight: ${video.videoHeight}px.`); + // } + + // logResizedVideo = (event: any) => { + // this.logVideoLoaded(event); + + // if (this.startTime) { + // let elapsedTime = window.performance.now() - this.startTime; + // this.startTime = null; + // this.trace(`Setup time: ${elapsedTime.toFixed(3)}ms.`); + // } + + // } + + // handleConnection = (event: any) => { + // let peerConnection = event.target; + // let iceCandidate = event.candidate; + + // if (iceCandidate) { + // let newIceCandidate: RTCIceCandidate = new RTCIceCandidate(iceCandidate); + // let otherPeer: any = this.getOtherPeer(peerConnection); + + // otherPeer.addIceCandidate(newIceCandidate).then(() => { + // this.handleConnectionSuccess(peerConnection); + // }).catch((error: any) => { + // this.handleConnectionFailure(peerConnection, error); + // }); + + // this.trace(`${this.getPeerName(peerConnection)} ICE candidate:\n` + + // `${event.candidate.candidate}.`); + + // } + // } + + // // Logs that the connection succeeded. + // handleConnectionSuccess = (peerConnection: any) => { + // this.trace(`${this.getPeerName(peerConnection)} addIceCandidate success.`); + // } + + // handleConnectionFailure = (peerConnection: any, error: any) => { + // this.trace(`${this.getPeerName(peerConnection)} failed to add ICE Candidate:\n` + + // `${error.toString()}.`); + // } + + // // Logs changes to the connection state. + // handleConnectionChange = (event: any) => { + // let peerConnection = event.target; + // console.log('ICE state change event: ', event); + // this.trace(`${this.getPeerName(peerConnection)} ICE state: ` + + // `${peerConnection.iceConnectionState}.`); + // } + + // // Logs error when setting session description fails. + // setSessionDescriptionError = (error: any) => { + // this.trace(`Failed to create session description: ${error.toString()}.`); + // } + + // // Logs success when setting session description. + // setDescriptionSuccess = (peerConnection: any, functionName: any) => { + // let peerName = this.getPeerName(peerConnection); + // this.trace(`${peerName} ${functionName} complete.`); + // } + + + // // Logs success when localDescription is set. + // setLocalDescriptionSuccess = (peerConnection: any) => { + // this.setDescriptionSuccess(peerConnection, 'setLocalDescription'); + // } + + // // Logs success when remoteDescription is set. + // setRemoteDescriptionSuccess = (peerConnection: any) => { + // this.setDescriptionSuccess(peerConnection, 'setRemoteDescription'); + // } + + // createdOffer = (description: any) => { + // this.trace(`Offer from localPeerConnection:\n${description.sdp}`); + // this.trace('localPeerConnection setLocalDescription start.'); + + // this.localPeerConnection.setLocalDescription(description).then(() => { + // this.setLocalDescriptionSuccess(this.localPeerConnection); + // }).catch(this.setSessionDescriptionError); + + + // this.trace('remotePeerConnection setRemoteDescription start.'); + // this.remotePeerConnection.setRemoteDescription(description) + // .then(() => { + // this.setRemoteDescriptionSuccess(this.remotePeerConnection); + // }).catch(this.setSessionDescriptionError); + + // this.trace('remotePeerConnection createAnswer start.'); + // this.remotePeerConnection.createAnswer() + // .then(this.createdAnswer) + // .catch(this.setSessionDescriptionError); + + // } + + // createdAnswer = (description: any) => { + // this.trace(`Answer from remotePeerConnection:\n${description.sdp}.`); + + // this.trace('remotePeerConnection setLocalDescription start.'); + // this.remotePeerConnection.setLocalDescription(description) + // .then(() => { + // this.setLocalDescriptionSuccess(this.remotePeerConnection); + // }).catch(this.setSessionDescriptionError); + + // this.trace('localPeerConnection setRemoteDescription start.'); + // this.localPeerConnection.setRemoteDescription(description) + // .then(() => { + // this.setRemoteDescriptionSuccess(this.localPeerConnection); + // }).catch(this.setSessionDescriptionError); + // } + + + // startAction = () => { + // this.startButton!.disabled = true; + // navigator.mediaDevices.getUserMedia(mediaStreamConstraints) + // .then(this.gotLocalMediaStream).catch(this.handleLocalMediaStreamError); + // this.trace('Requesting local stream.'); + // } + + + // // Handles call button action: creates peer connection. + // callAction = () => { + // this.callButton!.disabled = true; + // this.hangupButton!.disabled = false; + + // this.trace('Starting call.'); + // this.startTime = window.performance.now(); + + // // Get local media stream tracks. + // const videoTracks = this.localStream!.getVideoTracks(); + // const audioTracks = this.localStream!.getAudioTracks(); + // if (videoTracks.length > 0) { + // this.trace(`Using video device: ${videoTracks[0].label}.`); + // } + // if (audioTracks.length > 0) { + // this.trace(`Using audio device: ${audioTracks[0].label}.`); + // } + + // let servers: RTCConfiguration | undefined = undefined; // Allows for RTC server configuration. + + // // Create peer connections and add behavior. + // this.localPeerConnection = new RTCPeerConnection(servers); + // this.trace('Created local peer connection object localPeerConnection.'); + + // this.localPeerConnection.addEventListener('icecandidate', this.handleConnection); + // this.localPeerConnection.addEventListener( + // 'iceconnectionstatechange', this.handleConnectionChange); + + // this.remotePeerConnection = new RTCPeerConnection(servers); + // this.trace('Created remote peer connection object remotePeerConnection.'); + + // this.remotePeerConnection.addEventListener('icecandidate', this.handleConnection); + // this.remotePeerConnection.addEventListener( + // 'iceconnectionstatechange', this.handleConnectionChange); + // this.remotePeerConnection.addEventListener('addstream', this.gotRemoteMediaStream); + + // // Add local stream to connection and create offer to connect. + // this.localPeerConnection.addStream(this.localStream); + // this.trace('Added local stream to localPeerConnection.'); + + // this.trace('localPeerConnection createOffer start.'); + // this.localPeerConnection.createOffer(offerOptions) + // .then(this.createdOffer).catch(this.setSessionDescriptionError); + // } + + + // // Handles hangup action: ends up call, closes connections and resets peers. + // hangupAction = () => { + // this.localPeerConnection.close(); + // this.remotePeerConnection.close(); + // this.localPeerConnection = null; + // this.remotePeerConnection = null; + // this.hangupButton!.disabled = true; + // this.callButton!.disabled = false; + // this.trace('Ending call.'); + // } + + // // Gets the "other" peer connection. + // getOtherPeer = (peerConnection: any) => { + // return (peerConnection === this.localPeerConnection) ? + // this.remotePeerConnection : this.localPeerConnection; + // } + + // // Gets the name of a certain peer connection. + // getPeerName = (peerConnection: any) => { + // return (peerConnection === this.localPeerConnection) ? + // 'localPeerConnection' : 'remotePeerConnection'; + // } + + // // Logs an action (text) and the time when it happened on the console. + // trace = (text: string) => { + // text = text.trim(); + // const now = (window.performance.now() / 1000).toFixed(3); + + // console.log(now, text); + // } + + + + + + + + + + + + + + + + public static LayoutString(fieldKey: string) { return FieldView.LayoutString(DashWebRTCVideo, fieldKey); } + + + + _ignore = 0; + onPreWheel = (e: React.WheelEvent) => { + this._ignore = e.timeStamp; + } + onPrePointer = (e: React.PointerEvent) => { + this._ignore = e.timeStamp; + } + onPostPointer = (e: React.PointerEvent) => { + if (this._ignore !== e.timeStamp) { + e.stopPropagation(); + } + } + onPostWheel = (e: React.WheelEvent) => { + if (this._ignore !== e.timeStamp) { + e.stopPropagation(); + } + } + + + + render() { + let content = + <div className="webcam-cont" style={{ width: "100%", height: "100%" }} onWheel={this.onPostWheel} onPointerDown={this.onPostPointer} onPointerMove={this.onPostPointer} onPointerUp={this.onPostPointer}> + <video id="localVideo" autoPlay playsInline ref={(e) => this.localVideoEl = e!}></video> + <video id="remoteVideo" autoPlay playsInline ref={(e) => this.peerVideoEl = e!}></video> + {/* <button id="startButton" ref={(e) => this.startButton = e!} onClick={this.startAction}>Start</button> + <button id="callButton" ref={(e) => this.callButton = e!} onClick={this.callAction}>Call</button> + <button id="hangupButton" ref={(e) => this.hangupButton = e!} onClick={this.hangupAction}>Hang Up</button> */} + </div>; + + let frozen = !this.props.isSelected() || DocumentDecorations.Instance.Interacting; + let classname = "webBox-cont" + (this.props.isSelected() && !InkingControl.Instance.selectedTool && !DocumentDecorations.Instance.Interacting ? "-interactive" : ""); + + + return ( + <> + <div className={classname} > + {content} + </div> + {!frozen ? (null) : <div className="webBox-overlay" onWheel={this.onPreWheel} onPointerDown={this.onPrePointer} onPointerMove={this.onPrePointer} onPointerUp={this.onPrePointer} />} + </>); + } + + +}
\ No newline at end of file diff --git a/src/new_fields/URLField.ts b/src/new_fields/URLField.ts index 35ef6dd02..eabd04e0f 100644 --- a/src/new_fields/URLField.ts +++ b/src/new_fields/URLField.ts @@ -46,3 +46,5 @@ export const nullAudio = "https://actions.google.com/sounds/v1/alarms/beep_short @scriptingGlobal @Deserializable("pdf") export class PdfField extends URLField { } @scriptingGlobal @Deserializable("web") export class WebField extends URLField { } @scriptingGlobal @Deserializable("youtube") export class YoutubeField extends URLField { } +@scriptingGlobal @Deserializable("webcam") export class WebCamField extends URLField { } + diff --git a/src/server/Message.ts b/src/server/Message.ts index aaee143e8..60ffa215f 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -42,6 +42,11 @@ export interface Diff extends Reference { readonly diff: any; } +export interface RoomMessage { + readonly message: string; + readonly room: string; +} + export namespace MessageStore { export const Foo = new Message<string>("Foo"); export const Bar = new Message<string>("Bar"); @@ -58,4 +63,8 @@ export namespace MessageStore { export const YoutubeApiQuery = new Message<YoutubeQueryInput>("Youtube Api Query"); export const DeleteField = new Message<string>("Delete field"); export const DeleteFields = new Message<string[]>("Delete fields"); + export const NotifyRoommates = new Message<RoomMessage>("Notify Roommates"); + export const HangUpCall = new Message<string>("bye"); + + } diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 56ea5bfe1..55390989b 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -52,6 +52,7 @@ export class CurrentUserUtils { { title: "todo item", icon: "check", ignoreClick: true, drag: 'getCopy(this.dragFactory, true)', dragFactory: notes[notes.length - 1] }, { title: "web page", icon: "globe-asia", ignoreClick: true, drag: 'Docs.Create.WebDocument("https://en.wikipedia.org/wiki/Hedgehog", { width: 300, height: 300, title: "New Webpage" })' }, { title: "cat image", icon: "cat", ignoreClick: true, drag: 'Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg", { width: 200, title: "an image of a cat" })' }, + { title: "webcam", icon: "video", ignoreClick: true, drag: 'Docs.Create.WebCamDocument("", { width: 400, height: 400, title: "a test cam" })' }, { title: "record", icon: "microphone", ignoreClick: true, drag: `Docs.Create.AudioDocument("${nullAudio}", { width: 200, title: "ready to record audio" })` }, { title: "clickable button", icon: "bolt", ignoreClick: true, drag: 'Docs.Create.ButtonDocument({ width: 150, height: 50, title: "Button" })' }, { title: "presentation", icon: "tv", ignoreClick: true, drag: 'Doc.UserDoc().curPresentation = Docs.Create.PresDocument(new List<Doc>(), { width: 200, height: 500, title: "a presentation trail" })' }, diff --git a/src/server/index.ts b/src/server/index.ts index 1595781dc..e33fd4b71 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -24,7 +24,7 @@ import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLo import { DashUserModel } from './authentication/models/user_model'; import { Client } from './Client'; import { Database } from './database'; -import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message"; +import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput, RoomMessage } from "./Message"; import { RouteStore } from './RouteStore'; import v4 = require('uuid/v4'); const app = express(); @@ -914,6 +914,13 @@ server.on("connection", function (socket: Socket) { Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); + Utils.AddServerHandler(socket, MessageStore.NotifyRoommates, message => HandleRoommateNotification(socket, message)); + Utils.AddServerHandler(socket, MessageStore.HangUpCall, message => HandleHangUp(socket, message)); + Utils.AddRoomHandler(socket, "create or join", HandleCreateOrJoin); + + + + }); async function deleteFields() { @@ -975,6 +982,47 @@ function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any } } +function HandleRoommateNotification(socket: Socket, message: RoomMessage) { + //socket.broadcast.emit('message', message); + server.sockets.in(message.room).emit('message', message.message); + +} + +function HandleCreateOrJoin(socket: io.Socket, room: string) { + console.log("Received request to create or join room " + room); + + + let clientsInRoom = server.sockets.adapter.rooms[room]; + let numClients = clientsInRoom ? Object.keys(clientsInRoom.sockets).length : 0; + console.log('Room ' + room + ' now has ' + numClients + ' client(s)'); + + + if (numClients === 0) { + socket.join(room); + console.log('Client ID ' + socket.id + ' created room ' + room); + socket.emit('created', room, socket.id); + + } else if (numClients === 1) { + console.log('Client ID ' + socket.id + ' joined room ' + room); + server.sockets.in(room).emit('join', room); + socket.join(room); + socket.emit('joined', room, socket.id); + server.sockets.in(room).emit('ready'); + + } else { + socket.emit('full', room); + } + + + + + +} + +function HandleHangUp(socket: io.Socket, message: string) { + console.log("Receive bye from someone"); +} + const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); const EndpointHandlerMap = new Map<GoogleApiServerUtils.Action, GoogleApiServerUtils.ApiRouter>([ diff --git a/src/typings/index.d.ts b/src/typings/index.d.ts index 36d828fdb..26fc5a0b8 100644 --- a/src/typings/index.d.ts +++ b/src/typings/index.d.ts @@ -2,6 +2,9 @@ declare module 'googlephotos'; +declare module 'webrtc-adapter'; + + declare module '@react-pdf/renderer' { import * as React from 'react'; |