diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Utils.ts | 8 | ||||
| -rw-r--r-- | src/client/DocServer.ts | 2 | ||||
| -rw-r--r-- | src/client/documents/DocumentTypes.ts | 2 | ||||
| -rw-r--r-- | src/client/documents/Documents.ts | 9 | ||||
| -rw-r--r-- | src/client/northstar/utils/Utils.ts | 6 | ||||
| -rw-r--r-- | src/client/views/DocumentDecorations.tsx | 4 | ||||
| -rw-r--r-- | src/client/views/MainView.tsx | 1 | ||||
| -rw-r--r-- | src/client/views/nodes/DocumentContentsView.tsx | 4 | ||||
| -rw-r--r-- | src/client/views/webcam/DashWebCam.tsx | 396 | ||||
| -rw-r--r-- | src/client/views/webcam/DashWebRTC.scss | 23 | ||||
| -rw-r--r-- | src/client/views/webcam/DashWebRTC.ts | 305 | ||||
| -rw-r--r-- | src/client/views/webcam/DashWebRTCVideo.tsx | 385 | ||||
| -rw-r--r-- | src/new_fields/URLField.ts | 2 | ||||
| -rw-r--r-- | src/server/Message.ts | 9 | ||||
| -rw-r--r-- | src/server/authentication/models/current_user_utils.ts | 1 | ||||
| -rw-r--r-- | src/server/index.ts | 46 | ||||
| -rw-r--r-- | src/typings/index.d.ts | 3 |
17 files changed, 1203 insertions, 3 deletions
diff --git a/src/Utils.ts b/src/Utils.ts index 7bf05a6fc..8ab6207a9 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -1,6 +1,6 @@ import v4 = require('uuid/v4'); import v5 = require("uuid/v5"); -import { Socket } from 'socket.io'; +import { Socket, Room } from 'socket.io'; import { Message } from './server/Message'; export namespace Utils { @@ -294,6 +294,12 @@ export namespace Utils { handler([arg, loggingCallback('S sending', fn, message.Name)]); }); } + export type RoomHandler = (socket: Socket, room: string) => any; + export type UsedSockets = Socket | SocketIOClient.Socket; + export type RoomMessage = "create or join" | "created" | "joined"; + export function AddRoomHandler(socket: Socket, message: RoomMessage, handler: RoomHandler) { + socket.on(message, room => handler(socket, room)); + } } export function OmitKeys(obj: any, keys: string[], addKeyFunc?: (dup: any) => void): { omit: any, extract: any } { diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index d793b56af..7fbf30af8 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -21,7 +21,7 @@ import { Id, HandleUpdate } from '../new_fields/FieldSymbols'; */ export namespace DocServer { let _cache: { [id: string]: RefField | Promise<Opt<RefField>> } = {}; - let _socket: SocketIOClient.Socket; + export let _socket: SocketIOClient.Socket; // this client's distinct GUID created at initialization let GUID: string; // indicates whether or not a document is currently being udpated, and, if so, its id diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index 8f96b2fa6..46f27f958 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -17,6 +17,8 @@ export enum DocumentType { TEMPLATE = "template", EXTENSION = "extension", YOUTUBE = "youtube", + WEBCAM = "webcam", + DRAGBOX = "dragbox", FONTICON = "fonticonbox", PRES = "presentation", LINKFOLLOW = "linkfollow", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index c3f03aede..aaf3d3d11 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -44,7 +44,9 @@ import { ComputedField, ScriptField } from "../../new_fields/ScriptField"; import { ProxyField } from "../../new_fields/Proxy"; import { DocumentType } from "./DocumentTypes"; import { LinkFollowBox } from "../views/linking/LinkFollowBox"; +import { DashWebCam } from "../views/webcam/DashWebCam"; import { PresElementBox } from "../views/presentationview/PresElementBox"; +import { DashWebRTCVideo } from "../views/webcam/DashWebRTCVideo"; import { QueryBox } from "../views/nodes/QueryBox"; import { ColorBox } from "../views/nodes/ColorBox"; import { DocuLinkBox } from "../views/nodes/DocuLinkBox"; @@ -224,6 +226,9 @@ export namespace Docs { [DocumentType.LINKFOLLOW, { layout: { view: LinkFollowBox, dataField: data } }], + [DocumentType.WEBCAM, { + layout: { view: DashWebRTCVideo, dataField: data } + }], [DocumentType.PRESELEMENT, { layout: { view: PresElementBox, dataField: data } }], @@ -415,6 +420,10 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.YOUTUBE), new YoutubeField(new URL(url)), options); } + export function WebCamDocument(url: string, options: DocumentOptions = {}) { + return InstanceFromProto(Prototypes.get(DocumentType.WEBCAM), "", options); + } + export function AudioDocument(url: string, options: DocumentOptions = {}) { return InstanceFromProto(Prototypes.get(DocumentType.AUDIO), new AudioField(new URL(url)), options); } diff --git a/src/client/northstar/utils/Utils.ts b/src/client/northstar/utils/Utils.ts index d071dec62..09f83ce80 100644 --- a/src/client/northstar/utils/Utils.ts +++ b/src/client/northstar/utils/Utils.ts @@ -4,6 +4,12 @@ import { IBaseFilterProvider } from '../core/filter/IBaseFilterProvider'; import { AggregateFunction } from '../model/idea/idea'; export class Utils { + static Emit() { + throw new Error("Method not implemented."); + } + static EmitCallback() { + throw new Error("Method not implemented."); + } public static EqualityHelper(a: Object, b: Object): boolean { if (a === b) return true; diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 799b3695c..bacbcaa97 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -43,6 +43,8 @@ library.add(faCloudUploadAlt); library.add(faSyncAlt); library.add(faShare); +export type CloseCall = (toBeDeleted: DocumentView[]) => void; + @observer export class DocumentDecorations extends React.Component<{}, { value: string }> { static Instance: DocumentDecorations; @@ -71,6 +73,8 @@ export class DocumentDecorations extends React.Component<{}, { value: string }> @observable public pullIcon: IconProp = "arrow-alt-circle-down"; @observable public pullColor: string = "white"; @observable public openHover = false; + @observable private addedCloseCalls: CloseCall[] = []; + constructor(props: Readonly<{}>) { super(props); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 3648ddccf..b2d88a8c8 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -133,6 +133,7 @@ export class MainView extends React.Component { library.add(faArrowUp); library.add(faCloudUploadAlt); library.add(faBolt); + library.add(faVideo); library.add(faChevronRight); library.add(faEllipsisV); library.add(faMusic); diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index 66886165e..aa553afb7 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -33,6 +33,8 @@ import { VideoBox } from "./VideoBox"; import { WebBox } from "./WebBox"; import { InkingStroke } from "../InkingStroke"; import React = require("react"); +import { DashWebRTCVideo } from "../webcam/DashWebRTCVideo"; + import { TraceMobx } from "../../../new_fields/util"; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? @@ -103,7 +105,7 @@ export class DocumentContentsView extends React.Component<DocumentViewProps & { FormattedTextBox, ImageBox, IconBox, DirectoryImportBox, FontIconBox: FontIconBox, ButtonBox, FieldView, CollectionFreeFormView, CollectionDockingView, CollectionSchemaView, CollectionView, WebBox, KeyValueBox, PDFBox, VideoBox, AudioBox, HistogramBox, PresBox, YoutubeBox, LinkFollowBox, PresElementBox, QueryBox, - ColorBox, DocuLinkBox, InkingStroke, DocumentBox + ColorBox, DashWebRTCVideo, DocuLinkBox, InkingStroke, DocumentBox }} bindings={this.CreateBindings()} jsx={this.layout} diff --git a/src/client/views/webcam/DashWebCam.tsx b/src/client/views/webcam/DashWebCam.tsx new file mode 100644 index 000000000..a9669750f --- /dev/null +++ b/src/client/views/webcam/DashWebCam.tsx @@ -0,0 +1,396 @@ +import React = require("react"); +import { observer } from "mobx-react"; +import { FieldViewProps, FieldView } from "../nodes/FieldView"; +import { observable, action } from "mobx"; +import { DocumentDecorations } from "../DocumentDecorations"; +import { InkingControl } from "../InkingControl"; +import { CollectionFreeFormDocumentViewProps } from "../nodes/CollectionFreeFormDocumentView"; +import "../../views/nodes/WebBox.scss"; + + +//https://github.com/mozmorris/react-webcam is the source code used for the bigger part of this implementation. It's only modified to fit our current system. + + +function hasGetUserMedia() { + return !!( + (navigator.mediaDevices && navigator.mediaDevices.getUserMedia)); +} + +interface WebcamProps { + audio: boolean; + audioConstraints?: MediaStreamConstraints["audio"]; + imageSmoothing: boolean; + minScreenshotHeight?: number; + minScreenshotWidth?: number; + onUserMedia: () => void; + onUserMediaError: (error: string) => void; + screenshotFormat: "image/webp" | "image/png" | "image/jpeg"; + screenshotQuality: number; + videoConstraints?: MediaStreamConstraints["video"]; +} + +interface WebcamState { + hasUserMedia: boolean; + src?: string; +} + +@observer +export class DashWebCam extends React.Component<CollectionFreeFormDocumentViewProps & FieldViewProps & WebcamProps & React.HTMLAttributes<HTMLVideoElement> & { + layoutKey: string, +}, WebcamState> { + static defaultProps = { + audio: true, + imageSmoothing: true, + onUserMedia: () => { }, + onUserMediaError: () => { }, + screenshotFormat: "image/webp", + screenshotQuality: 0.92 + }; + + private static mountedInstances: DashWebCam[] = []; + private static userMediaRequested = false; + private canvas: HTMLCanvasElement | undefined; + private ctx: CanvasRenderingContext2D | null = null; + private stream: MediaStream | undefined; + private video: HTMLVideoElement | null | undefined; + + // @observable private hasUserMedia: boolean | undefined; + // @observable private src: string | undefined; + + constructor(props: any) { + super(props); + this.state = { + hasUserMedia: false + }; + } + + componentDidMount() { + if (!hasGetUserMedia()) return; + + const { state } = this; + + DashWebCam.mountedInstances.push(this); + + if (!state.hasUserMedia && !DashWebCam.userMediaRequested) { + this.requestUserMedia(); + } + } + + componentDidUpdate(nextProps: WebcamProps) { + const { props } = this; + if ( + JSON.stringify(nextProps.audioConstraints) !== + JSON.stringify(props.audioConstraints) || + JSON.stringify(nextProps.videoConstraints) !== + JSON.stringify(props.videoConstraints) + ) { + this.requestUserMedia(); + } + } + + componentWillUnmount() { + const { state } = this; + const index = DashWebCam.mountedInstances.indexOf(this); + DashWebCam.mountedInstances.splice(index, 1); + + DashWebCam.userMediaRequested = false; + if (DashWebCam.mountedInstances.length === 0 && state.hasUserMedia) { + if (this.stream!.getVideoTracks && this.stream!.getAudioTracks) { + this.stream!.getVideoTracks().map(track => track.stop()); + this.stream!.getAudioTracks().map(track => track.stop()); + } else { + ((this.stream as unknown) as MediaStreamTrack).stop(); + } + + if (state.src) { + window.URL.revokeObjectURL(state.src); + } + } + } + + //These are for screenshot if wanted. + + // getScreenshot() { + // const { state, props } = this; + + // if (!state.hasUserMedia) return null; + + // const canvas = this.getCanvas(); + // return ( + // canvas && + // canvas.toDataURL(props.screenshotFormat, props.screenshotQuality) + // ); + // } + + // getCanvas() { + // const { state, props } = this; + + // if (!this.video) { + // return null; + // } + + // if (!state.hasUserMedia || !this.video.videoHeight) return null; + + // if (!this.ctx) { + // const canvas = document.createElement("canvas"); + // const aspectRatio = this.video.videoWidth / this.video.videoHeight; + + // let canvasWidth = props.minScreenshotWidth || this.video.clientWidth; + // let canvasHeight = canvasWidth / aspectRatio; + + // if ( + // props.minScreenshotHeight && + // canvasHeight < props.minScreenshotHeight + // ) { + // canvasHeight = props.minScreenshotHeight; + // canvasWidth = canvasHeight * aspectRatio; + // } + + // canvas.width = canvasWidth; + // canvas.height = canvasHeight; + + // this.canvas = canvas; + // this.ctx = canvas.getContext("2d"); + // } + + // const { ctx, canvas } = this; + + // if (ctx) { + // ctx.imageSmoothingEnabled = props.imageSmoothing; + // ctx.drawImage(this.video, 0, 0, canvas!.width, canvas!.height); + // } + + // return canvas; + // } + + requestUserMedia() { + const { props } = this; + + navigator.getUserMedia = + navigator.mediaDevices.getUserMedia; + + const sourceSelected = (audioConstraints: any, videoConstraints: any) => { + const constraints: MediaStreamConstraints = { + video: typeof videoConstraints !== "undefined" ? videoConstraints : true + }; + + if (props.audio) { + constraints.audio = + typeof audioConstraints !== "undefined" ? audioConstraints : true; + } + + navigator.mediaDevices + .getUserMedia(constraints) + .then(stream => { + DashWebCam.mountedInstances.forEach(instance => + instance.handleUserMedia(null, stream) + ); + }) + .catch(e => { + DashWebCam.mountedInstances.forEach(instance => + instance.handleUserMedia(e) + ); + }); + }; + + if ("mediaDevices" in navigator) { + sourceSelected(props.audioConstraints, props.videoConstraints); + } else { + const optionalSource = (id: any) => ({ optional: [{ sourceId: id }] }); + + const constraintToSourceId = (constraint: any) => { + const { deviceId } = constraint; + + if (typeof deviceId === "string") { + return deviceId; + } + + if (Array.isArray(deviceId) && deviceId.length > 0) { + return deviceId[0]; + } + + if (typeof deviceId === "object" && deviceId.ideal) { + return deviceId.ideal; + } + + return null; + }; + + // @ts-ignore: deprecated api + MediaStreamTrack.getSources(sources => { + let audioSource = null; + let videoSource = null; + + sources.forEach((source: { kind: string; id: any; }) => { + if (source.kind === "audio") { + audioSource = source.id; + } else if (source.kind === "video") { + videoSource = source.id; + } + }); + + const audioSourceId = constraintToSourceId(props.audioConstraints); + if (audioSourceId) { + audioSource = audioSourceId; + } + + const videoSourceId = constraintToSourceId(props.videoConstraints); + if (videoSourceId) { + videoSource = videoSourceId; + } + + sourceSelected( + optionalSource(audioSource), + optionalSource(videoSource) + ); + }); + } + + DashWebCam.userMediaRequested = true; + } + + handleUserMedia(err: string | null, stream?: MediaStream) { + const { props } = this; + + if (err || !stream) { + this.setState({ hasUserMedia: false }); + // action(() => this.hasUserMedia = false); + props.onUserMediaError(err!); + + return; + } + + this.stream = stream; + + console.log("Stream done: ", stream); + + try { + if (this.video) { + this.video.srcObject = stream; + console.log("Source object: ", stream); + + } + this.setState({ hasUserMedia: true }); + // action(() => this.hasUserMedia = true); + + } catch (error) { + this.setState({ + hasUserMedia: true, + src: window.URL.createObjectURL(stream) + }); + console.log("State src set: ", this.state.src); + + // action(() => this.hasUserMedia = true); + // action(() => this.src = window.URL.createObjectURL(stream)); + } + + props.onUserMedia(); + } + + _ignore = 0; + onPreWheel = (e: React.WheelEvent) => { + this._ignore = e.timeStamp; + } + onPrePointer = (e: React.PointerEvent) => { + this._ignore = e.timeStamp; + } + onPostPointer = (e: React.PointerEvent) => { + if (this._ignore !== e.timeStamp) { + e.stopPropagation(); + } + } + onPostWheel = (e: React.WheelEvent) => { + if (this._ignore !== e.timeStamp) { + e.stopPropagation(); + } + } + + + + + public static LayoutString(fieldKey: string) { return FieldView.LayoutString(DashWebCam, fieldKey); } + + render() { + const { state, props } = this; + + + + const { + audio, + onUserMedia, + onUserMediaError, + screenshotFormat, + screenshotQuality, + minScreenshotWidth, + minScreenshotHeight, + audioConstraints, + videoConstraints, + imageSmoothing, + fieldKey, + fitToBox, + ContainingCollectionView, + Document, + DataDoc, + onClick, + isSelected, + select, + renderDepth, + addDocument, + addDocTab, + pinToPres, + removeDocument, + moveDocument, + ScreenToLocalTransform, + active, + whenActiveChanged, + focus, + PanelWidth, + PanelHeight, + setVideoBox, + ContentScaling, + ChromeHeight, + jitterRotation, + backgroundColor, + bringToFront, + zoomToScale, + getScale, + animateBetweenIcon, + layoutKey, + ...rest + } = props; + + console.log("Source produced: ", state.src); + + + + let content = + <div className="webcam-cont" style={{ width: "100%", height: "100%" }} onWheel={this.onPostWheel} onPointerDown={this.onPostPointer} onPointerMove={this.onPostPointer} onPointerUp={this.onPostPointer}> + <video + autoPlay + src={state.src} + muted={!audio} + playsInline + ref={ref => { + console.log("Source produced: ", state.src); + console.log("HasUser Media produced: ", state.hasUserMedia); + + this.video = ref; + }} + {...rest} + /> + </div>; + + + let frozen = !this.props.isSelected() || DocumentDecorations.Instance.Interacting; + let classname = "webBox-cont" + (this.props.isSelected() && !InkingControl.Instance.selectedTool && !DocumentDecorations.Instance.Interacting ? "-interactive" : ""); + + return ( + <> + <div className={classname} > + {content} + </div> + {!frozen ? (null) : <div className="webBox-overlay" onWheel={this.onPreWheel} onPointerDown={this.onPrePointer} onPointerMove={this.onPrePointer} onPointerUp={this.onPrePointer} />} + </>); + + } +}
\ No newline at end of file diff --git a/src/client/views/webcam/DashWebRTC.scss b/src/client/views/webcam/DashWebRTC.scss new file mode 100644 index 000000000..ddf4777a8 --- /dev/null +++ b/src/client/views/webcam/DashWebRTC.scss @@ -0,0 +1,23 @@ +.webcam-cont { + button { + margin: 10px; + position: relative; + top: 20%; + left: -60%; + } + + #localVideo { + margin: 10px; + position: relative; + width: 300px; + max-height: 300px; + } + + #remoteVideo { + margin: 10px; + position: relative; + width: 300px; + max-height: 300px; + + } +}
\ No newline at end of file diff --git a/src/client/views/webcam/DashWebRTC.ts b/src/client/views/webcam/DashWebRTC.ts new file mode 100644 index 000000000..6c7322a3d --- /dev/null +++ b/src/client/views/webcam/DashWebRTC.ts @@ -0,0 +1,305 @@ +import { DocServer } from '../../DocServer'; +import { Utils } from '../../../Utils'; +import { MessageStore } from '../../../server/Message'; + + + +/** + * This namespace will have the code required to have functionality code for the usage of webRTC. + */ +export namespace DashWebRTC { + + + let isChannelReady = false; + let isInitiator = false; + let isStarted = false; + let localStream: MediaStream | undefined; + let pc: any; + let remoteStream: MediaStream | undefined; + let turnReady; + let localVideo: HTMLVideoElement; + let remoteVideo: HTMLVideoElement; + let curRoom: string = ""; + + + let pcConfig = { + 'iceServers': [{ + 'urls': 'stun:stun.l.google.com:19302' + }] + }; + + // Set up audio and video regardless of what devices are present. + let sdpConstraints = { + offerToReceiveAudio: true, + offerToReceiveVideo: true + }; + + export function init(room: string) { + + curRoom = room; + + if (room !== '') { + DocServer._socket.emit('create or join', room); + console.log('Attempted to create or join room', room); + + } + + DocServer._socket.on('created', function (room: string) { + console.log('Created room ' + room); + isInitiator = true; + }); + + DocServer._socket.on('full', function (room: string) { + console.log('Room ' + room + ' is full'); + }); + + DocServer._socket.on('join', function (room: string) { + console.log('Another peer made a request to join room ' + room); + console.log('This peer is the initiator of room ' + room + '!'); + isChannelReady = true; + }); + + + DocServer._socket.on('joined', function (room: string) { + console.log('joined: ' + room); + isChannelReady = true; + }); + + + DocServer._socket.on('log', function (array: any) { + console.log.apply(console, array); + }); + + // This client receives a message + DocServer._socket.on('message', function (message: any) { + console.log('Client received message:', message); + if (message.message === 'got user media') { + maybeStart(); + } else if (message.message.type === 'offer') { + if (!isInitiator && !isStarted) { + maybeStart(); + } + pc.setRemoteDescription(new RTCSessionDescription(message.message)); + doAnswer(); + } else if (message.message.type === 'answer' && isStarted) { + pc.setRemoteDescription(new RTCSessionDescription(message.message)); + } else if (message.message.type === 'candidate' && isStarted) { + let candidate = new RTCIceCandidate({ + sdpMLineIndex: message.message.label, + candidate: message.message.candidate + }); + pc.addIceCandidate(candidate); + } else if (message === 'bye' && isStarted) { + handleRemoteHangup(); + } + }); + + navigator.mediaDevices.getUserMedia({ + audio: false, + video: true + }) + .then(gotStream) + .catch(function (e) { + alert('getUserMedia() error: ' + e.name); + }); + + //Trying this one out!!! + console.log('Getting user media with constraints', constraints); + + if (location.hostname !== 'localhost') { + requestTurn( + 'https://computeengineondemand.appspot.com/turn?username=41784574&key=4080218913' + ); + } + + + } + + + function sendMessage(message: any) { + console.log('Client sending message: ', message); + Utils.Emit(DocServer._socket, MessageStore.NotifyRoommates, { message: message, room: curRoom }); + //DocServer._socket.emit('message', message); + } + + + + + + export function setVideoObjects(localVideo: HTMLVideoElement, remoteVideo: HTMLVideoElement) { + localVideo = localVideo; + remoteVideo = remoteVideo; + } + + export function setLocalVideoObject(localVideoRef: HTMLVideoElement) { + localVideo = localVideoRef; + } + + export function setRemoteVideoObject(remoteVideoRef: HTMLVideoElement) { + remoteVideo = remoteVideoRef; + } + + + + + function gotStream(stream: any) { + console.log('Adding local stream.'); + localStream = stream; + localVideo.srcObject = stream; + sendMessage('got user media'); + if (isInitiator) { + maybeStart(); + } + } + + let constraints = { + video: true, + audio: true + }; + + + + + + function maybeStart() { + console.log('>>>>>>> maybeStart() ', isStarted, localStream, isChannelReady); + if (!isStarted && typeof localStream !== 'undefined' && isChannelReady) { + console.log('>>>>>> creating peer connection'); + createPeerConnection(); + pc.addStream(localStream); + isStarted = true; + console.log('isInitiator', isInitiator); + if (isInitiator) { + doCall(); + } + } + } + + + //this will need to be changed to our version of hangUp + window.onbeforeunload = function () { + sendMessage('bye'); + }; + + function createPeerConnection() { + try { + pc = new RTCPeerConnection(undefined); + pc.onicecandidate = handleIceCandidate; + pc.onaddstream = handleRemoteStreamAdded; + pc.onremovestream = handleRemoteStreamRemoved; + console.log('Created RTCPeerConnnection'); + } catch (e) { + console.log('Failed to create PeerConnection, exception: ' + e.message); + alert('Cannot create RTCPeerConnection object.'); + return; + } + } + + function handleIceCandidate(event: any) { + console.log('icecandidate event: ', event); + if (event.candidate) { + sendMessage({ + type: 'candidate', + label: event.candidate.sdpMLineIndex, + id: event.candidate.sdpMid, + candidate: event.candidate.candidate + }); + } else { + console.log('End of candidates.'); + } + } + + function handleCreateOfferError(event: any) { + console.log('createOffer() error: ', event); + } + + function doCall() { + console.log('Sending offer to peer'); + pc.createOffer(setLocalAndSendMessage, handleCreateOfferError); + } + + function doAnswer() { + console.log('Sending answer to peer.'); + pc.createAnswer().then( + setLocalAndSendMessage, + onCreateSessionDescriptionError + ); + } + + function setLocalAndSendMessage(sessionDescription: any) { + pc.setLocalDescription(sessionDescription); + console.log('setLocalAndSendMessage sending message', sessionDescription); + sendMessage(sessionDescription); + } + + function onCreateSessionDescriptionError(error: any) { + console.log('Failed to create session description: ' + error.toString()); + } + + + function requestTurn(turnURL: any) { + var turnExists = false; + for (var i in pcConfig.iceServers) { + if (pcConfig.iceServers[i].urls.substr(0, 5) === 'turn:') { + turnExists = true; + turnReady = true; + break; + } + } + if (!turnExists) { + console.log('Getting TURN server from ', turnURL); + // No TURN server. Get one from computeengineondemand.appspot.com: + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function () { + if (xhr.readyState === 4 && xhr.status === 200) { + var turnServer = JSON.parse(xhr.responseText); + console.log('Got TURN server: ', turnServer); + pcConfig.iceServers.push({ + 'urls': 'turn:' + turnServer.username + '@' + turnServer.turn, + //'credential': turnServer.password + }); + turnReady = true; + } + }; + xhr.open('GET', turnURL, true); + xhr.send(); + } + } + + function handleRemoteStreamAdded(event: MediaStreamEvent) { + console.log('Remote stream added.'); + remoteStream = event.stream!; + remoteVideo.srcObject = remoteStream; + } + + function handleRemoteStreamRemoved(event: MediaStreamEvent) { + console.log('Remote stream removed. Event: ', event); + } + + export function hangup() { + console.log('Hanging up.'); + if (pc) { + stop(); + sendMessage('bye'); + } + + if (localStream) { + localStream.getTracks().forEach(track => track.stop()); + } + + } + + function handleRemoteHangup() { + console.log('Session terminated.'); + stop(); + isInitiator = false; + } + + function stop() { + isStarted = false; + pc.close(); + pc = null; + } + + +}
\ No newline at end of file diff --git a/src/client/views/webcam/DashWebRTCVideo.tsx b/src/client/views/webcam/DashWebRTCVideo.tsx new file mode 100644 index 000000000..a9c67f026 --- /dev/null +++ b/src/client/views/webcam/DashWebRTCVideo.tsx @@ -0,0 +1,385 @@ +import { observer } from "mobx-react"; +import React = require("react"); +import { CollectionFreeFormDocumentViewProps } from "../nodes/CollectionFreeFormDocumentView"; +import { FieldViewProps, FieldView } from "../nodes/FieldView"; +import { observable } from "mobx"; +import { DocumentDecorations, CloseCall } from "../DocumentDecorations"; +import { InkingControl } from "../InkingControl"; +import "../../views/nodes/WebBox.scss"; +import "./DashWebRTC.scss"; +import adapter from 'webrtc-adapter'; +import { DashWebRTC } from "./DashWebRTC"; +import { DocServer } from "../../DocServer"; +import { DocumentView } from "../nodes/DocumentView"; +import { Utils } from "../../../Utils"; +import { MessageStore } from "../../../server/Message"; + + + + +const mediaStreamConstraints = { + video: true, +}; + +const offerOptions = { + offerToReceiveVideo: 1, +}; + + +/** + * This models the component that will be rendered, that can be used as a doc that will reflect the video cams. + */ +@observer +export class DashWebRTCVideo extends React.Component<CollectionFreeFormDocumentViewProps & FieldViewProps> { + + @observable private localVideoEl: HTMLVideoElement | undefined; + @observable private peerVideoEl: HTMLVideoElement | undefined; + @observable private localStream: MediaStream | undefined; + @observable private startTime: any = null; + @observable private remoteStream: MediaStream | undefined; + @observable private localPeerConnection: any; + @observable private remotePeerConnection: any; + private callButton: HTMLButtonElement | undefined; + private startButton: HTMLButtonElement | undefined; + private hangupButton: HTMLButtonElement | undefined; + private roomText: HTMLInputElement | undefined; + private roomOfCam: string = ""; + + componentDidMount() { + DocumentDecorations.Instance.addCloseCall(this.closeConnection); + } + + closeConnection: CloseCall = () => { + //Utils.Emit(DocServer._socket, MessageStore.NotifyRoommates, { message: 'bye', room: this.roomOfCam }); + DashWebRTC.hangup(); + } + + componentWillUnmount() { + // DocumentDecorations.Instance.removeCloseCall(this.closeConnection); + } + + + // componentDidMount() { + // // DashWebRTC.setVideoObjects(this.localVideoEl!, this.peerVideoEl!); + // //DashWebRTC.init(); + // } + + + // componentDidMount() { + // this.callButton!.disabled = true; + // this.hangupButton!.disabled = true; + // // navigator.mediaDevices.getUserMedia(mediaStreamConstraints).then(this.gotLocalMediaStream).catch(this.handleLocalMediaStreamError); + // this.localVideoEl!.addEventListener('loadedmetadata', this.logVideoLoaded); + // this.peerVideoEl!.addEventListener('loadedmetadata', this.logVideoLoaded); + // this.peerVideoEl!.addEventListener('onresize', this.logResizedVideo); + // } + + + // gotLocalMediaStream = (mediaStream: MediaStream) => { + // this.localStream = mediaStream; + // if (this.localVideoEl) { + // this.localVideoEl.srcObject = mediaStream; + // } + // this.trace('Received local stream.'); + // this.callButton!.disabled = false; + + // } + + // gotRemoteMediaStream = (event: MediaStreamEvent) => { + // let mediaStream = event.stream; + // this.peerVideoEl!.srcObject = mediaStream; + // this.remoteStream = mediaStream!; + + // } + + // handleLocalMediaStreamError = (error: string) => { + // //console.log("navigator.getUserMedia error: ", error); + // this.trace(`navigator.getUserMedia error: ${error.toString()}.`); + + // } + + // logVideoLoaded = (event: any) => { + // let video = event.target; + // this.trace(`${video.id} videoWidth: ${video.videoWidth}px, ` + + // `videoHeight: ${video.videoHeight}px.`); + // } + + // logResizedVideo = (event: any) => { + // this.logVideoLoaded(event); + + // if (this.startTime) { + // let elapsedTime = window.performance.now() - this.startTime; + // this.startTime = null; + // this.trace(`Setup time: ${elapsedTime.toFixed(3)}ms.`); + // } + + // } + + // handleConnection = (event: any) => { + // let peerConnection = event.target; + // let iceCandidate = event.candidate; + + // if (iceCandidate) { + // let newIceCandidate: RTCIceCandidate = new RTCIceCandidate(iceCandidate); + // let otherPeer: any = this.getOtherPeer(peerConnection); + + // otherPeer.addIceCandidate(newIceCandidate).then(() => { + // this.handleConnectionSuccess(peerConnection); + // }).catch((error: any) => { + // this.handleConnectionFailure(peerConnection, error); + // }); + + // this.trace(`${this.getPeerName(peerConnection)} ICE candidate:\n` + + // `${event.candidate.candidate}.`); + + // } + // } + + // // Logs that the connection succeeded. + // handleConnectionSuccess = (peerConnection: any) => { + // this.trace(`${this.getPeerName(peerConnection)} addIceCandidate success.`); + // } + + // handleConnectionFailure = (peerConnection: any, error: any) => { + // this.trace(`${this.getPeerName(peerConnection)} failed to add ICE Candidate:\n` + + // `${error.toString()}.`); + // } + + // // Logs changes to the connection state. + // handleConnectionChange = (event: any) => { + // let peerConnection = event.target; + // console.log('ICE state change event: ', event); + // this.trace(`${this.getPeerName(peerConnection)} ICE state: ` + + // `${peerConnection.iceConnectionState}.`); + // } + + // // Logs error when setting session description fails. + // setSessionDescriptionError = (error: any) => { + // this.trace(`Failed to create session description: ${error.toString()}.`); + // } + + // // Logs success when setting session description. + // setDescriptionSuccess = (peerConnection: any, functionName: any) => { + // let peerName = this.getPeerName(peerConnection); + // this.trace(`${peerName} ${functionName} complete.`); + // } + + + // // Logs success when localDescription is set. + // setLocalDescriptionSuccess = (peerConnection: any) => { + // this.setDescriptionSuccess(peerConnection, 'setLocalDescription'); + // } + + // // Logs success when remoteDescription is set. + // setRemoteDescriptionSuccess = (peerConnection: any) => { + // this.setDescriptionSuccess(peerConnection, 'setRemoteDescription'); + // } + + // createdOffer = (description: any) => { + // this.trace(`Offer from localPeerConnection:\n${description.sdp}`); + // this.trace('localPeerConnection setLocalDescription start.'); + + // this.localPeerConnection.setLocalDescription(description).then(() => { + // this.setLocalDescriptionSuccess(this.localPeerConnection); + // }).catch(this.setSessionDescriptionError); + + + // this.trace('remotePeerConnection setRemoteDescription start.'); + // this.remotePeerConnection.setRemoteDescription(description) + // .then(() => { + // this.setRemoteDescriptionSuccess(this.remotePeerConnection); + // }).catch(this.setSessionDescriptionError); + + // this.trace('remotePeerConnection createAnswer start.'); + // this.remotePeerConnection.createAnswer() + // .then(this.createdAnswer) + // .catch(this.setSessionDescriptionError); + + // } + + // createdAnswer = (description: any) => { + // this.trace(`Answer from remotePeerConnection:\n${description.sdp}.`); + + // this.trace('remotePeerConnection setLocalDescription start.'); + // this.remotePeerConnection.setLocalDescription(description) + // .then(() => { + // this.setLocalDescriptionSuccess(this.remotePeerConnection); + // }).catch(this.setSessionDescriptionError); + + // this.trace('localPeerConnection setRemoteDescription start.'); + // this.localPeerConnection.setRemoteDescription(description) + // .then(() => { + // this.setRemoteDescriptionSuccess(this.localPeerConnection); + // }).catch(this.setSessionDescriptionError); + // } + + + // startAction = () => { + // this.startButton!.disabled = true; + // navigator.mediaDevices.getUserMedia(mediaStreamConstraints) + // .then(this.gotLocalMediaStream).catch(this.handleLocalMediaStreamError); + // this.trace('Requesting local stream.'); + // } + + + // // Handles call button action: creates peer connection. + // callAction = () => { + // this.callButton!.disabled = true; + // this.hangupButton!.disabled = false; + + // this.trace('Starting call.'); + // this.startTime = window.performance.now(); + + // // Get local media stream tracks. + // const videoTracks = this.localStream!.getVideoTracks(); + // const audioTracks = this.localStream!.getAudioTracks(); + // if (videoTracks.length > 0) { + // this.trace(`Using video device: ${videoTracks[0].label}.`); + // } + // if (audioTracks.length > 0) { + // this.trace(`Using audio device: ${audioTracks[0].label}.`); + // } + + // let servers: RTCConfiguration | undefined = undefined; // Allows for RTC server configuration. + + // // Create peer connections and add behavior. + // this.localPeerConnection = new RTCPeerConnection(servers); + // this.trace('Created local peer connection object localPeerConnection.'); + + // this.localPeerConnection.addEventListener('icecandidate', this.handleConnection); + // this.localPeerConnection.addEventListener( + // 'iceconnectionstatechange', this.handleConnectionChange); + + // this.remotePeerConnection = new RTCPeerConnection(servers); + // this.trace('Created remote peer connection object remotePeerConnection.'); + + // this.remotePeerConnection.addEventListener('icecandidate', this.handleConnection); + // this.remotePeerConnection.addEventListener( + // 'iceconnectionstatechange', this.handleConnectionChange); + // this.remotePeerConnection.addEventListener('addstream', this.gotRemoteMediaStream); + + // // Add local stream to connection and create offer to connect. + // this.localPeerConnection.addStream(this.localStream); + // this.trace('Added local stream to localPeerConnection.'); + + // this.trace('localPeerConnection createOffer start.'); + // this.localPeerConnection.createOffer(offerOptions) + // .then(this.createdOffer).catch(this.setSessionDescriptionError); + // } + + + // // Handles hangup action: ends up call, closes connections and resets peers. + // hangupAction = () => { + // this.localPeerConnection.close(); + // this.remotePeerConnection.close(); + // this.localPeerConnection = null; + // this.remotePeerConnection = null; + // this.hangupButton!.disabled = true; + // this.callButton!.disabled = false; + // this.trace('Ending call.'); + // } + + // // Gets the "other" peer connection. + // getOtherPeer = (peerConnection: any) => { + // return (peerConnection === this.localPeerConnection) ? + // this.remotePeerConnection : this.localPeerConnection; + // } + + // // Gets the name of a certain peer connection. + // getPeerName = (peerConnection: any) => { + // return (peerConnection === this.localPeerConnection) ? + // 'localPeerConnection' : 'remotePeerConnection'; + // } + + // // Logs an action (text) and the time when it happened on the console. + // trace = (text: string) => { + // text = text.trim(); + // const now = (window.performance.now() / 1000).toFixed(3); + + // console.log(now, text); + // } + + + /** + * Function that submits the title entered by user on enter press. + */ + onEnterKeyDown = (e: React.KeyboardEvent) => { + if (e.keyCode === 13) { + let submittedTitle = this.roomText!.value; + this.roomText!.value = ""; + this.roomText!.blur(); + this.roomOfCam = submittedTitle; + DashWebRTC.init(submittedTitle); + } + } + + + + + + + + + + + + + + + + public static LayoutString(fieldKey: string) { return FieldView.LayoutString(DashWebRTCVideo, fieldKey); } + + + + _ignore = 0; + onPreWheel = (e: React.WheelEvent) => { + this._ignore = e.timeStamp; + } + onPrePointer = (e: React.PointerEvent) => { + this._ignore = e.timeStamp; + } + onPostPointer = (e: React.PointerEvent) => { + if (this._ignore !== e.timeStamp) { + e.stopPropagation(); + } + } + onPostWheel = (e: React.WheelEvent) => { + if (this._ignore !== e.timeStamp) { + e.stopPropagation(); + } + } + + + + render() { + let content = + <div className="webcam-cont" style={{ width: "100%", height: "100%" }} onWheel={this.onPostWheel} onPointerDown={this.onPostPointer} onPointerMove={this.onPostPointer} onPointerUp={this.onPostPointer}> + <input type="text" placeholder="Enter room name" ref={(e) => this.roomText = e!} onKeyDown={this.onEnterKeyDown} /> + <video id="localVideo" autoPlay playsInline ref={(e) => { + this.localVideoEl = e!; + DashWebRTC.setLocalVideoObject(e!); + }}></video> + <video id="remoteVideo" autoPlay playsInline ref={(e) => { + this.peerVideoEl = e!; + DashWebRTC.setRemoteVideoObject(e!); + }}></video> + {/* <button id="startButton" ref={(e) => this.startButton = e!} onClick={this.startAction}>Start</button> + <button id="callButton" ref={(e) => this.callButton = e!} onClick={this.callAction}>Call</button> + <button id="hangupButton" ref={(e) => this.hangupButton = e!} onClick={this.hangupAction}>Hang Up</button> */} + </div>; + + let frozen = !this.props.isSelected() || DocumentDecorations.Instance.Interacting; + let classname = "webBox-cont" + (this.props.isSelected() && !InkingControl.Instance.selectedTool && !DocumentDecorations.Instance.Interacting ? "-interactive" : ""); + + + return ( + <> + <div className={classname} > + {content} + </div> + {!frozen ? (null) : <div className="webBox-overlay" onWheel={this.onPreWheel} onPointerDown={this.onPrePointer} onPointerMove={this.onPrePointer} onPointerUp={this.onPrePointer} />} + </>); + } + + +}
\ No newline at end of file diff --git a/src/new_fields/URLField.ts b/src/new_fields/URLField.ts index 35ef6dd02..eabd04e0f 100644 --- a/src/new_fields/URLField.ts +++ b/src/new_fields/URLField.ts @@ -46,3 +46,5 @@ export const nullAudio = "https://actions.google.com/sounds/v1/alarms/beep_short @scriptingGlobal @Deserializable("pdf") export class PdfField extends URLField { } @scriptingGlobal @Deserializable("web") export class WebField extends URLField { } @scriptingGlobal @Deserializable("youtube") export class YoutubeField extends URLField { } +@scriptingGlobal @Deserializable("webcam") export class WebCamField extends URLField { } + diff --git a/src/server/Message.ts b/src/server/Message.ts index 621abfd1e..6ce5cd96a 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -42,6 +42,11 @@ export interface Diff extends Reference { readonly diff: any; } +export interface RoomMessage { + readonly message: string; + readonly room: string; +} + export namespace MessageStore { export const Foo = new Message<string>("Foo"); export const Bar = new Message<string>("Bar"); @@ -59,4 +64,8 @@ export namespace MessageStore { export const YoutubeApiQuery = new Message<YoutubeQueryInput>("Youtube Api Query"); export const DeleteField = new Message<string>("Delete field"); export const DeleteFields = new Message<string[]>("Delete fields"); + export const NotifyRoommates = new Message<RoomMessage>("message"); + export const HangUpCall = new Message<string>("bye"); + + } diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 36d4cd2f2..cc24cce71 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -51,6 +51,7 @@ export class CurrentUserUtils { { title: "todo item", icon: "check", ignoreClick: true, drag: 'getCopy(this.dragFactory, true)', dragFactory: notes[notes.length - 1] }, { title: "web page", icon: "globe-asia", ignoreClick: true, drag: 'Docs.Create.WebDocument("https://en.wikipedia.org/wiki/Hedgehog", { width: 300, height: 300, title: "New Webpage" })' }, { title: "cat image", icon: "cat", ignoreClick: true, drag: 'Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg", { width: 200, title: "an image of a cat" })' }, + { title: "webcam", icon: "video", ignoreClick: true, drag: 'Docs.Create.WebCamDocument("", { width: 400, height: 400, title: "a test cam" })' }, { title: "record", icon: "microphone", ignoreClick: true, drag: `Docs.Create.AudioDocument("${nullAudio}", { width: 200, title: "ready to record audio" })` }, { title: "clickable button", icon: "bolt", ignoreClick: true, drag: 'Docs.Create.ButtonDocument({ width: 150, height: 50, title: "Button" })' }, { title: "presentation", icon: "tv", ignoreClick: true, drag: 'Doc.UserDoc().curPresentation = Docs.Create.PresDocument(new List<Doc>(), { width: 200, height: 500, title: "a presentation trail" })' }, diff --git a/src/server/index.ts b/src/server/index.ts index 313a2f0e2..2b18fc578 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -86,12 +86,58 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: secureHandler: ({ res }) => res.redirect("/home") }); + // Utils.AddServerHandler(socket, MessageStore.NotifyRoommates, message => HandleRoommateNotification(socket, message)); + // // Utils.AddServerHandler(socket, MessageStore.HangUpCall, message => HandleHangUp(socket, message)); + // // Utils.AddRoomHandler(socket, "create or join", HandleCreateOrJoin); + + // function HandleRoommateNotification(socket: Socket, message: RoomMessage) { + // //socket.broadcast.emit('message', message); + // console.log("The room that sent this: ", message.room, " and message is : ", message.message); + // server.sockets.in(message.room).emit('message', message); + + // } + + // function HandleCreateOrJoin(socket: io.Socket, room: string) { + // console.log("Received request to create or join room " + room); + + + // let clientsInRoom = server.sockets.adapter.rooms[room]; + // let numClients = clientsInRoom ? Object.keys(clientsInRoom.sockets).length : 0; + // console.log('Room ' + room + ' now has ' + numClients + ' client(s)'); + + + // if (numClients === 0) { + // socket.join(room); + // console.log('Client ID ' + socket.id + ' created room ' + room); + // socket.emit('created', room, socket.id); + + // } else if (numClients === 1) { + // console.log('Client ID ' + socket.id + ' joined room ' + room); + // server.sockets.in(room).emit('join', room); + // socket.join(room); + // socket.emit('joined', room, socket.id); + // server.sockets.in(room).emit('ready'); + + // } else { + // socket.emit('full', room); + // } + + + + + + // } + + // function HandleHangUp(socket: io.Socket, message: string) { + // console.log("Receive bye from someone"); + // } addSupervisedRoute({ method: Method.GET, subscription: "/serverHeartbeat", secureHandler: ({ res }) => res.send(true) }); + const serve: PublicHandler = ({ req, res }) => { const detector = new mobileDetect(req.headers['user-agent'] || ""); const filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; diff --git a/src/typings/index.d.ts b/src/typings/index.d.ts index 281bb3217..850c533fc 100644 --- a/src/typings/index.d.ts +++ b/src/typings/index.d.ts @@ -4,6 +4,9 @@ declare module 'googlephotos'; declare module 'react-image-lightbox-with-rotate'; declare module 'cors'; +declare module 'webrtc-adapter'; + + declare module '@react-pdf/renderer' { import * as React from 'react'; |