_Bastler/partey_workadventure
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update screen sharing feature

Browse Source
This commit is contained in:
Gregoire Parant 2020-06-11 23:18:06 +02:00 committed by David Négrier
parent 0bbed7717a
commit a4f42111d7
4 changed files with 171 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
back/src/Controller/IoSocketController.ts
View File

@ -28,6 +28,7 @@ enum SockerIoEvent {
USER_MOVED = "user-moved", // From server to client USER_MOVED = "user-moved", // From server to client
USER_LEFT = "user-left", // From server to client USER_LEFT = "user-left", // From server to client
WEBRTC_SIGNAL = "webrtc-signal", WEBRTC_SIGNAL = "webrtc-signal",
WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal",
WEBRTC_START = "webrtc-start", WEBRTC_START = "webrtc-start",
WEBRTC_DISCONNECT = "webrtc-disconect", WEBRTC_DISCONNECT = "webrtc-disconect",
MESSAGE_ERROR = "message-error", MESSAGE_ERROR = "message-error",
@ -226,18 +227,11 @@ export class IoSocketController {
}); });
socket.on(SockerIoEvent.WEBRTC_SIGNAL, (data: unknown) => { socket.on(SockerIoEvent.WEBRTC_SIGNAL, (data: unknown) => {
if (!isWebRtcSignalMessageInterface(data)) { this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SIGNAL);
socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); });
console.warn('Invalid WEBRTC_SIGNAL message received: ', data);
return; socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => {
} this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL);
//send only at user
const client = this.sockets.get(data.receiverId);
if (client === undefined) {
console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition.");
return;
}
return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data);
}); });
socket.on(SockerIoEvent.DISCONNECT, () => { socket.on(SockerIoEvent.DISCONNECT, () => {
@ -284,6 +278,21 @@ export class IoSocketController {
}); });
} }
emit(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){
if (!isWebRtcSignalMessageInterface(data)) {
socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'});
console.warn('Invalid WEBRTC_SIGNAL message received: ', data);
return;
}
//send only at user
const client = this.sockets.get(data.receiverId);
if (client === undefined) {
console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition.");
return;
}
return client.emit(event, data);
}
searchClientByIdOrFail(userId: string): ExSocketInterface { searchClientByIdOrFail(userId: string): ExSocketInterface {
const client: ExSocketInterface|undefined = this.sockets.get(userId); const client: ExSocketInterface|undefined = this.sockets.get(userId);
if (client === undefined) { if (client === undefined) {

28
front/src/Connection.ts
View File

@ -9,9 +9,9 @@ import {PlayerAnimationNames} from "./Phaser/Player/Animation";
import {UserSimplePeerInterface} from "./WebRtc/SimplePeer"; import {UserSimplePeerInterface} from "./WebRtc/SimplePeer";
import {SignalData} from "simple-peer"; import {SignalData} from "simple-peer";
enum EventMessage{ enum EventMessage{
WEBRTC_SIGNAL = "webrtc-signal", WEBRTC_SIGNAL = "webrtc-signal",
WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal",
WEBRTC_START = "webrtc-start", WEBRTC_START = "webrtc-start",
JOIN_ROOM = "join-room", // bi-directional JOIN_ROOM = "join-room", // bi-directional
USER_POSITION = "user-position", // bi-directional USER_POSITION = "user-position", // bi-directional
@ -197,6 +197,15 @@ export class Connection implements Connection {
}); });
} }
sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId? : string|null, receiverId? : string) {
return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, {
userId: userId ? userId : this.userId,
receiverId: receiverId ? receiverId : this.userId,
roomId: roomId,
signal: signal
});
}
public receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) { public receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) {
this.socket.on(EventMessage.WEBRTC_START, callback); this.socket.on(EventMessage.WEBRTC_START, callback);
} }
@ -205,6 +214,23 @@ export class Connection implements Connection {
return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback);
} }
receiveWebrtcScreenSharingSignal(callback: Function) {
return this.getSocket().on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback);
}
private errorMessage(): void {
this.getSocket().on(EventMessage.MESSAGE_ERROR, (message: string) => {
console.error(EventMessage.MESSAGE_ERROR, message);
})
}
private disconnectServer(): void {
this.getSocket().on(EventMessage.CONNECT_ERROR, () => {
this.GameManager.switchToDisconnectedScene();
});
}
public onServerDisconnected(callback: (reason: string) => void): void { public onServerDisconnected(callback: (reason: string) => void): void {
this.socket.on('disconnect', (reason: string) => { this.socket.on('disconnect', (reason: string) => {
if (reason === 'io client disconnect') { if (reason === 'io client disconnect') {

14
front/src/WebRtc/MediaManager.ts
View File

@ -287,14 +287,13 @@ export class MediaManager {
* *
* @param userId * @param userId
*/ */
addScreenSharingActiveVideo(userId : string, userName: string = ""){ addScreenSharingActiveVideo(userId : string){
userId = `screen-sharing-${userId}`;
this.webrtcInAudio.play(); this.webrtcInAudio.play();
// FIXME: switch to DisplayManager! // FIXME: switch to DisplayManager!
let elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing"); let elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing");
userName = userName.toUpperCase();
let color = this.getColorByString(userName);
elementRemoteVideo.insertAdjacentHTML('beforeend', ` elementRemoteVideo.insertAdjacentHTML('beforeend', `
<div id="div-${userId}" class="screen-sharing-video-container" style="border-color: ${color};"> <div id="div-${userId}" class="screen-sharing-video-container">
<video id="${userId}" autoplay></video> <video id="${userId}" autoplay></video>
</div> </div>
`); `);
@ -302,6 +301,7 @@ export class MediaManager {
if(!activeHTMLVideoElement){ if(!activeHTMLVideoElement){
return; return;
} }
console.log(userId, (activeHTMLVideoElement as HTMLVideoElement));
this.remoteVideo.set(userId, (activeHTMLVideoElement as HTMLVideoElement)); this.remoteVideo.set(userId, (activeHTMLVideoElement as HTMLVideoElement));
} }
@ -372,6 +372,9 @@ export class MediaManager {
} }
remoteVideo.srcObject = stream; remoteVideo.srcObject = stream;
} }
addStreamRemoteScreenSharing(userId : string, stream : MediaStream){
this.addStreamRemoteVideo(`screen-sharing-${userId}`, stream);
}
/** /**
* *
@ -381,6 +384,9 @@ export class MediaManager {
layoutManager.remove(userId); layoutManager.remove(userId);
this.remoteVideo.delete(userId); this.remoteVideo.delete(userId);
} }
removeActiveScreenSharingVideo(userId : string) {
this.removeActiveVideo(`screen-sharing-${userId}`)
}
isConnecting(userId : string): void { isConnecting(userId : string): void {
const connectingSpinnerDiv = this.getSpinner(userId); const connectingSpinnerDiv = this.getSpinner(userId);

140
front/src/WebRtc/SimplePeer.ts
View File

@ -28,6 +28,7 @@ export class SimplePeer {
private WebRtcRoomId: string; private WebRtcRoomId: string;
private Users: Array<UserSimplePeerInterface> = new Array<UserSimplePeerInterface>(); private Users: Array<UserSimplePeerInterface> = new Array<UserSimplePeerInterface>();
private PeerScreenSharingConnectionArray: Map<string, SimplePeerNamespace.Instance> = new Map<string, SimplePeerNamespace.Instance>();
private PeerConnectionArray: Map<string, SimplePeerNamespace.Instance> = new Map<string, SimplePeerNamespace.Instance>(); private PeerConnectionArray: Map<string, SimplePeerNamespace.Instance> = new Map<string, SimplePeerNamespace.Instance>();
private readonly updateLocalStreamCallback: (media: MediaStream) => void; private readonly updateLocalStreamCallback: (media: MediaStream) => void;
private readonly updateScreenSharingCallback: (media: MediaStream) => void; private readonly updateScreenSharingCallback: (media: MediaStream) => void;
@ -62,6 +63,11 @@ export class SimplePeer {
this.receiveWebrtcSignal(message); this.receiveWebrtcSignal(message);
}); });
//receive signal by gemer
this.Connection.receiveWebrtcScreenSharingSignal((message: any) => {
this.receiveWebrtcScreenSharingSignal(message);
});
mediaManager.activeVisio(); mediaManager.activeVisio();
mediaManager.getCamera().then(() => { mediaManager.getCamera().then(() => {
@ -108,7 +114,8 @@ export class SimplePeer {
/** /**
* create peer connection to bind users * create peer connection to bind users
*/ */
private createPeerConnection(user : UserSimplePeerInterface) : SimplePeerNamespace.Instance | null{ private createPeerConnection(user : UserSimplePeerInterface, screenSharing: boolean = false) : SimplePeerNamespace.Instance | null{
console.log("createPeerConnection => screenSharing", screenSharing)
if(this.PeerConnectionArray.has(user.userId)) { if(this.PeerConnectionArray.has(user.userId)) {
return null; return null;
} }
@ -121,12 +128,11 @@ export class SimplePeer {
} }
} }
let screenSharing : boolean = name !== undefined && name.indexOf("screenSharing") > -1;
mediaManager.removeActiveVideo(user.userId); mediaManager.removeActiveVideo(user.userId);
if(!screenSharing) { if(screenSharing) {
mediaManager.addActiveVideo(user.userId, name); mediaManager.addScreenSharingActiveVideo(user.userId);
}else{ }else{
mediaManager.addScreenSharingActiveVideo(user.userId, name); mediaManager.addActiveVideo(user.userId, name);
} }
const peer : SimplePeerNamespace.Instance = new Peer({ const peer : SimplePeerNamespace.Instance = new Peer({
@ -145,10 +151,19 @@ export class SimplePeer {
] ]
}, },
}); });
this.PeerConnectionArray.set(user.userId, peer); if(screenSharing){
this.PeerScreenSharingConnectionArray.set(user.userId, peer);
}else {
this.PeerConnectionArray.set(user.userId, peer);
}
//start listen signal for the peer connection //start listen signal for the peer connection
peer.on('signal', (data: unknown) => { peer.on('signal', (data: unknown) => {
console.log("screenSharing", screenSharing);
if(screenSharing){
this.sendWebrtcScreenSharingSignal(data, user.userId);
return;
}
this.sendWebrtcSignal(data, user.userId); this.sendWebrtcSignal(data, user.userId);
}); });
@ -160,6 +175,9 @@ export class SimplePeer {
});*/ });*/
peer.on('close', () => { peer.on('close', () => {
if(screenSharing){
this.closeScreenSharingConnection(user.userId);
}
this.closeConnection(user.userId); this.closeConnection(user.userId);
}); });
@ -190,7 +208,11 @@ export class SimplePeer {
} }
}); });
this.addMedia(user.userId); if(screenSharing){
this.addMediaScreenSharing(user.userId);
}else {
this.addMedia(user.userId);
}
for (const peerConnectionListener of this.peerConnectionListeners) { for (const peerConnectionListener of this.peerConnectionListeners) {
peerConnectionListener.onConnect(user); peerConnectionListener.onConnect(user);
@ -225,6 +247,30 @@ export class SimplePeer {
} }
} }
/**
* This is triggered twice. Once by the server, and once by a remote client disconnecting
*
* @param userId
*/
private closeScreenSharingConnection(userId : string) {
try {
mediaManager.removeActiveScreenSharingVideo(userId);
let peer = this.PeerScreenSharingConnectionArray.get(userId);
if (peer === undefined) {
console.warn("Tried to close connection for user "+userId+" but could not find user")
return;
}
// FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray"
// I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel.
//console.log('Closing connection with '+userId);
peer.destroy();
this.PeerScreenSharingConnectionArray.delete(userId)
//console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size);
} catch (err) {
console.error("closeConnection", err)
}
}
public closeAllConnections() { public closeAllConnections() {
for (const userId of this.PeerConnectionArray.keys()) { for (const userId of this.PeerConnectionArray.keys()) {
this.closeConnection(userId); this.closeConnection(userId);
@ -244,6 +290,7 @@ export class SimplePeer {
* @param data * @param data
*/ */
private sendWebrtcSignal(data: unknown, userId : string) { private sendWebrtcSignal(data: unknown, userId : string) {
console.log("sendWebrtcSignal", data);
try { try {
this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId); this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId);
}catch (e) { }catch (e) {
@ -251,6 +298,20 @@ export class SimplePeer {
} }
} }
/**
*
* @param userId
* @param data
*/
private sendWebrtcScreenSharingSignal(data: any, userId : string) {
console.log("sendWebrtcScreenSharingSignal", data);
try {
this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, null, userId);
}catch (e) {
console.error(`sendWebrtcSignal => ${userId}`, e);
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
private receiveWebrtcSignal(data: WebRtcSignalMessageInterface) { private receiveWebrtcSignal(data: WebRtcSignalMessageInterface) {
try { try {
@ -269,6 +330,24 @@ export class SimplePeer {
} }
} }
private receiveWebrtcScreenSharingSignal(data: any) {
console.log("receiveWebrtcScreenSharingSignal", data);
try {
//if offer type, create peer connection
if(data.signal.type === "offer"){
this.createPeerConnection(data, true);
}
let peer = this.PeerConnectionArray.get(data.userId);
if (peer !== undefined) {
peer.signal(data.signal);
} else {
console.error('Could not find peer whose ID is "'+data.userId+'" in PeerConnectionArray');
}
} catch (e) {
console.error(`receiveWebrtcSignal => ${data.userId}`, e);
}
}
/** /**
* *
* @param userId * @param userId
@ -293,18 +372,8 @@ export class SimplePeer {
if (!PeerConnection) { if (!PeerConnection) {
throw new Error('While adding media, cannot find user with ID ' + userId); throw new Error('While adding media, cannot find user with ID ' + userId);
} }
if(userId.indexOf("screenSharing") > -1 && mediaManager.localScreenCapture){
for (const track of mediaManager.localScreenCapture.getTracks()) {
PeerConnection.addTrack(track, mediaManager.localScreenCapture);
}
return;
}
let localStream: MediaStream | null = mediaManager.localStream; let localStream: MediaStream | null = mediaManager.localStream;
let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; PeerConnection.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia)));
PeerConnection.write(new Buffer(JSON.stringify(Object.assign(this.MediaManager.constraintsMedia, {screen: localScreenCapture !== null}))));
if(!localStream){ if(!localStream){
return; return;
@ -319,6 +388,21 @@ export class SimplePeer {
} }
} }
private addMediaScreenSharing (userId : any = null) {
let PeerConnection = this.PeerScreenSharingConnectionArray.get(userId);
if (!PeerConnection) {
throw new Error('While adding media, cannot find user with ID ' + userId);
}
let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture;
if(!localScreenCapture){
return;
}
/*for (const track of localScreenCapture.getTracks()) {
PeerConnection.addTrack(track, localScreenCapture);
}*/
return;
}
updatedLocalStream(){ updatedLocalStream(){
this.Users.forEach((user: UserSimplePeerInterface) => { this.Users.forEach((user: UserSimplePeerInterface) => {
this.addMedia(user.userId); this.addMedia(user.userId);
@ -326,29 +410,31 @@ export class SimplePeer {
} }
updatedScreenSharing() { updatedScreenSharing() {
if (this.MediaManager.localScreenCapture) { if (mediaManager.localScreenCapture) {
if(!this.Connection.userId){
return;
}
let screenSharingUser: UserSimplePeerInterface = { let screenSharingUser: UserSimplePeerInterface = {
userId: `screenSharing-${this.Connection.userId}`, userId: this.Connection.userId,
name: 'screenSharing',
initiator: true initiator: true
}; };
let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser); let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true);
if (!PeerConnectionScreenSharing) { if (!PeerConnectionScreenSharing) {
return; return;
} }
try { try {
for (const track of this.MediaManager.localScreenCapture.getTracks()) { for (const track of mediaManager.localScreenCapture.getTracks()) {
PeerConnectionScreenSharing.addTrack(track, this.MediaManager.localScreenCapture); PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture);
} }
}catch (e) { }catch (e) {
console.error("updatedScreenSharing => ", e); console.error("updatedScreenSharing => ", e);
} }
this.MediaManager.addStreamRemoteVideo(screenSharingUser.userId, this.MediaManager.localScreenCapture); mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture);
} else { } else {
if (!this.PeerConnectionArray.has(`screenSharing-${this.Connection.userId}`)) { if (!this.Connection.userId || !this.PeerScreenSharingConnectionArray.has(this.Connection.userId)) {
return; return;
} }
let PeerConnectionScreenSharing = this.PeerConnectionArray.get(`screenSharing-${this.Connection.userId}`); let PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(this.Connection.userId);
if (!PeerConnectionScreenSharing) { if (!PeerConnectionScreenSharing) {
return; return;
} }