_Bastler/partey_workadventure
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update screen sharing feature

Browse Source
This commit is contained in:
Gregoire Parant 2020-06-11 23:18:06 +02:00 committed by David Négrier
parent 0bbed7717a
commit a4f42111d7
4 changed files with 171 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
back/src/Controller/IoSocketController.ts
View File

@ -28,6 +28,7 @@ enum SockerIoEvent {
USER_MOVED = "user-moved", // From server to client
USER_LEFT = "user-left", // From server to client
WEBRTC_SIGNAL = "webrtc-signal",
WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal",
WEBRTC_START = "webrtc-start",
WEBRTC_DISCONNECT = "webrtc-disconect",
MESSAGE_ERROR = "message-error",
@ -226,18 +227,11 @@ export class IoSocketController {
});
socket.on(SockerIoEvent.WEBRTC_SIGNAL, (data: unknown) => {
if (!isWebRtcSignalMessageInterface(data)) {
socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'});
console.warn('Invalid WEBRTC_SIGNAL message received: ', data);
return;
}
//send only at user
const client = this.sockets.get(data.receiverId);
if (client === undefined) {
console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition.");
return;
}
return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data);
this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SIGNAL);
});
socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => {
this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL);
});
socket.on(SockerIoEvent.DISCONNECT, () => {
@ -284,6 +278,21 @@ export class IoSocketController {
});
}
emit(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){
if (!isWebRtcSignalMessageInterface(data)) {
socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'});
console.warn('Invalid WEBRTC_SIGNAL message received: ', data);
return;
}
//send only at user
const client = this.sockets.get(data.receiverId);
if (client === undefined) {
console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition.");
return;
}
return client.emit(event, data);
}
searchClientByIdOrFail(userId: string): ExSocketInterface {
const client: ExSocketInterface|undefined = this.sockets.get(userId);
if (client === undefined) {

28
front/src/Connection.ts
View File

@ -9,9 +9,9 @@ import {PlayerAnimationNames} from "./Phaser/Player/Animation";
import {UserSimplePeerInterface} from "./WebRtc/SimplePeer";
import {SignalData} from "simple-peer";
enum EventMessage{
WEBRTC_SIGNAL = "webrtc-signal",
WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal",
WEBRTC_START = "webrtc-start",
JOIN_ROOM = "join-room", // bi-directional
USER_POSITION = "user-position", // bi-directional
@ -197,6 +197,15 @@ export class Connection implements Connection {
});
}
sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId? : string|null, receiverId? : string) {
return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, {
userId: userId ? userId : this.userId,
receiverId: receiverId ? receiverId : this.userId,
roomId: roomId,
signal: signal
});
}
public receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) {
this.socket.on(EventMessage.WEBRTC_START, callback);
}
@ -205,6 +214,23 @@ export class Connection implements Connection {
return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback);
}
receiveWebrtcScreenSharingSignal(callback: Function) {
return this.getSocket().on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback);
}
private errorMessage(): void {
this.getSocket().on(EventMessage.MESSAGE_ERROR, (message: string) => {
console.error(EventMessage.MESSAGE_ERROR, message);
})
}
private disconnectServer(): void {
this.getSocket().on(EventMessage.CONNECT_ERROR, () => {
this.GameManager.switchToDisconnectedScene();
});
}
public onServerDisconnected(callback: (reason: string) => void): void {
this.socket.on('disconnect', (reason: string) => {
if (reason === 'io client disconnect') {

14
front/src/WebRtc/MediaManager.ts
View File

@ -287,14 +287,13 @@ export class MediaManager {
*
* @param userId
*/
addScreenSharingActiveVideo(userId : string, userName: string = ""){
addScreenSharingActiveVideo(userId : string){
userId = `screen-sharing-${userId}`;
this.webrtcInAudio.play();
// FIXME: switch to DisplayManager!
let elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing");
userName = userName.toUpperCase();
let color = this.getColorByString(userName);
elementRemoteVideo.insertAdjacentHTML('beforeend', `
<div id="div-${userId}" class="screen-sharing-video-container" style="border-color: ${color};">
<div id="div-${userId}" class="screen-sharing-video-container">
<video id="${userId}" autoplay></video>
</div>
`);
@ -302,6 +301,7 @@ export class MediaManager {
if(!activeHTMLVideoElement){
return;
}
console.log(userId, (activeHTMLVideoElement as HTMLVideoElement));
this.remoteVideo.set(userId, (activeHTMLVideoElement as HTMLVideoElement));
}
@ -372,6 +372,9 @@ export class MediaManager {
}
remoteVideo.srcObject = stream;
}
addStreamRemoteScreenSharing(userId : string, stream : MediaStream){
this.addStreamRemoteVideo(`screen-sharing-${userId}`, stream);
}
/**
*
@ -381,6 +384,9 @@ export class MediaManager {
layoutManager.remove(userId);
this.remoteVideo.delete(userId);
}
removeActiveScreenSharingVideo(userId : string) {
this.removeActiveVideo(`screen-sharing-${userId}`)
}
isConnecting(userId : string): void {
const connectingSpinnerDiv = this.getSpinner(userId);

140
front/src/WebRtc/SimplePeer.ts
View File

@ -28,6 +28,7 @@ export class SimplePeer {
private WebRtcRoomId: string;
private Users: Array<UserSimplePeerInterface> = new Array<UserSimplePeerInterface>();
private PeerScreenSharingConnectionArray: Map<string, SimplePeerNamespace.Instance> = new Map<string, SimplePeerNamespace.Instance>();
private PeerConnectionArray: Map<string, SimplePeerNamespace.Instance> = new Map<string, SimplePeerNamespace.Instance>();
private readonly updateLocalStreamCallback: (media: MediaStream) => void;
private readonly updateScreenSharingCallback: (media: MediaStream) => void;
@ -62,6 +63,11 @@ export class SimplePeer {
this.receiveWebrtcSignal(message);
});
//receive signal by gemer
this.Connection.receiveWebrtcScreenSharingSignal((message: any) => {
this.receiveWebrtcScreenSharingSignal(message);
});
mediaManager.activeVisio();
mediaManager.getCamera().then(() => {
@ -108,7 +114,8 @@ export class SimplePeer {
/**
* create peer connection to bind users
*/
private createPeerConnection(user : UserSimplePeerInterface) : SimplePeerNamespace.Instance | null{
private createPeerConnection(user : UserSimplePeerInterface, screenSharing: boolean = false) : SimplePeerNamespace.Instance | null{
console.log("createPeerConnection => screenSharing", screenSharing)
if(this.PeerConnectionArray.has(user.userId)) {
return null;
}
@ -121,12 +128,11 @@ export class SimplePeer {
}
}
let screenSharing : boolean = name !== undefined && name.indexOf("screenSharing") > -1;
mediaManager.removeActiveVideo(user.userId);
if(!screenSharing) {
mediaManager.addActiveVideo(user.userId, name);
if(screenSharing) {
mediaManager.addScreenSharingActiveVideo(user.userId);
}else{
mediaManager.addScreenSharingActiveVideo(user.userId, name);
mediaManager.addActiveVideo(user.userId, name);
}
const peer : SimplePeerNamespace.Instance = new Peer({
@ -145,10 +151,19 @@ export class SimplePeer {
]
},
});
this.PeerConnectionArray.set(user.userId, peer);
if(screenSharing){
this.PeerScreenSharingConnectionArray.set(user.userId, peer);
}else {
this.PeerConnectionArray.set(user.userId, peer);
}
//start listen signal for the peer connection
peer.on('signal', (data: unknown) => {
console.log("screenSharing", screenSharing);
if(screenSharing){
this.sendWebrtcScreenSharingSignal(data, user.userId);
return;
}
this.sendWebrtcSignal(data, user.userId);
});
@ -160,6 +175,9 @@ export class SimplePeer {
});*/
peer.on('close', () => {
if(screenSharing){
this.closeScreenSharingConnection(user.userId);
}
this.closeConnection(user.userId);
});
@ -190,7 +208,11 @@ export class SimplePeer {
}
});
this.addMedia(user.userId);
if(screenSharing){
this.addMediaScreenSharing(user.userId);
}else {
this.addMedia(user.userId);
}
for (const peerConnectionListener of this.peerConnectionListeners) {
peerConnectionListener.onConnect(user);
@ -225,6 +247,30 @@ export class SimplePeer {
}
}
/**
* This is triggered twice. Once by the server, and once by a remote client disconnecting
*
* @param userId
*/
private closeScreenSharingConnection(userId : string) {
try {
mediaManager.removeActiveScreenSharingVideo(userId);
let peer = this.PeerScreenSharingConnectionArray.get(userId);
if (peer === undefined) {
console.warn("Tried to close connection for user "+userId+" but could not find user")
return;
}
// FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray"
// I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel.
//console.log('Closing connection with '+userId);
peer.destroy();
this.PeerScreenSharingConnectionArray.delete(userId)
//console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size);
} catch (err) {
console.error("closeConnection", err)
}
}
public closeAllConnections() {
for (const userId of this.PeerConnectionArray.keys()) {
this.closeConnection(userId);
@ -244,6 +290,7 @@ export class SimplePeer {
* @param data
*/
private sendWebrtcSignal(data: unknown, userId : string) {
console.log("sendWebrtcSignal", data);
try {
this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId);
}catch (e) {
@ -251,6 +298,20 @@ export class SimplePeer {
}
}
/**
*
* @param userId
* @param data
*/
private sendWebrtcScreenSharingSignal(data: any, userId : string) {
console.log("sendWebrtcScreenSharingSignal", data);
try {
this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, null, userId);
}catch (e) {
console.error(`sendWebrtcSignal => ${userId}`, e);
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
private receiveWebrtcSignal(data: WebRtcSignalMessageInterface) {
try {
@ -269,6 +330,24 @@ export class SimplePeer {
}
}
private receiveWebrtcScreenSharingSignal(data: any) {
console.log("receiveWebrtcScreenSharingSignal", data);
try {
//if offer type, create peer connection
if(data.signal.type === "offer"){
this.createPeerConnection(data, true);
}
let peer = this.PeerConnectionArray.get(data.userId);
if (peer !== undefined) {
peer.signal(data.signal);
} else {
console.error('Could not find peer whose ID is "'+data.userId+'" in PeerConnectionArray');
}
} catch (e) {
console.error(`receiveWebrtcSignal => ${data.userId}`, e);
}
}
/**
*
* @param userId
@ -293,18 +372,8 @@ export class SimplePeer {
if (!PeerConnection) {
throw new Error('While adding media, cannot find user with ID ' + userId);
}
if(userId.indexOf("screenSharing") > -1 && mediaManager.localScreenCapture){
for (const track of mediaManager.localScreenCapture.getTracks()) {
PeerConnection.addTrack(track, mediaManager.localScreenCapture);
}
return;
}
let localStream: MediaStream | null = mediaManager.localStream;
let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture;
PeerConnection.write(new Buffer(JSON.stringify(Object.assign(this.MediaManager.constraintsMedia, {screen: localScreenCapture !== null}))));
PeerConnection.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia)));
if(!localStream){
return;
@ -319,6 +388,21 @@ export class SimplePeer {
}
}
private addMediaScreenSharing (userId : any = null) {
let PeerConnection = this.PeerScreenSharingConnectionArray.get(userId);
if (!PeerConnection) {
throw new Error('While adding media, cannot find user with ID ' + userId);
}
let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture;
if(!localScreenCapture){
return;
}
/*for (const track of localScreenCapture.getTracks()) {
PeerConnection.addTrack(track, localScreenCapture);
}*/
return;
}
updatedLocalStream(){
this.Users.forEach((user: UserSimplePeerInterface) => {
this.addMedia(user.userId);
@ -326,29 +410,31 @@ export class SimplePeer {
}
updatedScreenSharing() {
if (this.MediaManager.localScreenCapture) {
if (mediaManager.localScreenCapture) {
if(!this.Connection.userId){
return;
}
let screenSharingUser: UserSimplePeerInterface = {
userId: `screenSharing-${this.Connection.userId}`,
name: 'screenSharing',
userId: this.Connection.userId,
initiator: true
};
let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser);
let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true);
if (!PeerConnectionScreenSharing) {
return;
}
try {
for (const track of this.MediaManager.localScreenCapture.getTracks()) {
PeerConnectionScreenSharing.addTrack(track, this.MediaManager.localScreenCapture);
for (const track of mediaManager.localScreenCapture.getTracks()) {
PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture);
}
}catch (e) {
console.error("updatedScreenSharing => ", e);
}
this.MediaManager.addStreamRemoteVideo(screenSharingUser.userId, this.MediaManager.localScreenCapture);
mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture);
} else {
if (!this.PeerConnectionArray.has(`screenSharing-${this.Connection.userId}`)) {
if (!this.Connection.userId || !this.PeerScreenSharingConnectionArray.has(this.Connection.userId)) {
return;
}
let PeerConnectionScreenSharing = this.PeerConnectionArray.get(`screenSharing-${this.Connection.userId}`);
let PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(this.Connection.userId);
if (!PeerConnectionScreenSharing) {
return;
}