_Bastler/partey_workadventure
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

First version of screen-sharing that works when a user is joining a group after screen sharing begun.

Browse Source
This commit is contained in:
David Négrier 2020-08-20 00:05:00 +02:00
parent 6c5772e849
commit 0119534283
6 changed files with 130 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
back/src/Controller/IoSocketController.ts
View File

@ -44,8 +44,6 @@ export class IoSocketController {
private nbClientsGauge: Gauge<string>;
private nbClientsPerRoomGauge: Gauge<string>;
private offerScreenSharingByClient: Map<string, Map<string, unknown>> = new Map<string, Map<string, unknown>>();
constructor(server: http.Server) {
this.Io = socketIO(server);
this.nbClientsGauge = new Gauge({
@ -229,11 +227,11 @@ export class IoSocketController {
});
socket.on(SockerIoEvent.WEBRTC_SIGNAL, (data: unknown) => {
this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SIGNAL);
this.emitVideo((socket as ExSocketInterface), data);
});
socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => {
this.emitScreenSharing((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL);
this.emitScreenSharing((socket as ExSocketInterface), data);
});
socket.on(SockerIoEvent.DISCONNECT, () => {
@ -280,7 +278,7 @@ export class IoSocketController {
});
}
emit(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){
emitVideo(socket: ExSocketInterface, data: unknown){
if (!isWebRtcSignalMessageInterface(data)) {
socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'});
console.warn('Invalid WEBRTC_SIGNAL message received: ', data);
@ -292,25 +290,22 @@ export class IoSocketController {
console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition.");
return;
}
return client.emit(event, data);
return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data);
}
emitScreenSharing(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){
if (!isWebRtcScreenSharingSignalMessageInterface(data)) {
socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'});
console.warn('Invalid WEBRTC_SIGNAL message received: ', data);
emitScreenSharing(socket: ExSocketInterface, data: unknown){
if (!isWebRtcSignalMessageInterface(data)) {
socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SCREEN_SHARING message.'});
console.warn('Invalid WEBRTC_SCREEN_SHARING message received: ', data);
return;
}
if(data.signal.type === "offer"){
let roomOffer = this.offerScreenSharingByClient.get(data.roomId);
if(!roomOffer){
roomOffer = new Map<string, unknown>();
}
roomOffer.set(data.userId, data.signal);
this.offerScreenSharingByClient.set(data.roomId, roomOffer);
//send only at user
const client = this.sockets.get(data.receiverId);
if (client === undefined) {
console.warn("While exchanging a WEBRTC_SCREEN_SHARING signal: client with id ", data.receiverId, " does not exist. This might be a race condition.");
return;
}
//share at all others clients send only at user
return socket.in(data.roomId).emit(event, data);
return client.emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, data);
}
searchClientByIdOrFail(userId: string): ExSocketInterface {
@ -393,13 +388,15 @@ export class IoSocketController {
if (this.Io.sockets.adapter.rooms[roomId].length < 2 /*|| this.Io.sockets.adapter.rooms[roomId].length >= 4*/) {
return;
}
// TODO: scanning all sockets is maybe not the most efficient
const clients: Array<ExSocketInterface> = (Object.values(this.Io.sockets.sockets) as Array<ExSocketInterface>)
.filter((client: ExSocketInterface) => client.webRtcRoomId && client.webRtcRoomId === roomId);
//send start at one client to initialise offer webrtc
//send all users in room to create PeerConnection in front
clients.forEach((client: ExSocketInterface, index: number) => {
const clientsId = clients.reduce((tabs: Array<UserInGroupInterface>, clientId: ExSocketInterface, indexClientId: number) => {
const peerClients = clients.reduce((tabs: Array<UserInGroupInterface>, clientId: ExSocketInterface, indexClientId: number) => {
if (!clientId.userId || clientId.userId === client.userId) {
return tabs;
}
@ -411,7 +408,7 @@ export class IoSocketController {
return tabs;
}, []);
client.emit(SockerIoEvent.WEBRTC_START, {clients: clientsId, roomId: roomId});
client.emit(SockerIoEvent.WEBRTC_START, {clients: peerClients, roomId: roomId});
});
}

7
back/src/Model/Websocket/WebRtcSignalMessage.ts
View File

@ -12,16 +12,9 @@ export const isWebRtcSignalMessageInterface =
roomId: tg.isString,
signal: isSignalData
}).get();
export const isWebRtcScreenSharingSignalMessageInterface =
new tg.IsInterface().withProperties({
userId: tg.isString,
roomId: tg.isString,
signal: isSignalData
}).get();
export const isWebRtcScreenSharingStartMessageInterface =
new tg.IsInterface().withProperties({
userId: tg.isString,
roomId: tg.isString
}).get();
export type WebRtcSignalMessageInterface = tg.GuardedType<typeof isWebRtcSignalMessageInterface>;
export type WebRtcScreenSharingMessageInterface = tg.GuardedType<typeof isWebRtcScreenSharingSignalMessageInterface>;

36
front/dist/resources/style/style.css vendored
View File

@ -365,39 +365,3 @@ body {
.chat-mode > div:last-child {
flex-grow: 5;
}
/*SCREEN SHARING*/
.active-screen-sharing video{
transform: scaleX(1);
}
.screen-sharing-video-container {
width: 25%;
position: absolute;
}
.active-screen-sharing .screen-sharing-video-container video:hover{
width: 200%;
z-index: 11;
}
.active-screen-sharing .screen-sharing-video-container video{
position: absolute;
width: 100%;
height: auto;
left: 0;
top: 0;
transition: all 0.2s ease;
z-index: 1;
}
.active-screen-sharing .screen-sharing-video-container:nth-child(1){
/*this is for camera of user*/
top: 0%;
}
.active-screen-sharing .screen-sharing-video-container:nth-child(2){
top: 25%;
}
.active-screen-sharing .screen-sharing-video-container:nth-child(3){
top: 50%;
}
.active-screen-sharing .screen-sharing-video-container:nth-child(4) {
top: 75%;
}

17
front/src/Connection.ts
View File

@ -80,14 +80,8 @@ export interface WebRtcDisconnectMessageInterface {
}
export interface WebRtcSignalMessageInterface {
userId: string,
receiverId: string, // TODO: is this needed? (can we merge this with WebRtcScreenSharingMessageInterface?)
roomId: string,
signal: SignalData
}
export interface WebRtcScreenSharingMessageInterface {
userId: string,
userId: string, // TODO: is this needed?
receiverId: string,
roomId: string,
signal: SignalData
}
@ -203,9 +197,10 @@ export class Connection implements Connection {
});
}
public sendWebrtcScreenSharingSignal(signal: unknown, roomId: string, userId? : string|null) {
public sendWebrtcScreenSharingSignal(signal: unknown, roomId: string, userId? : string|null, receiverId? : string) {
return this.socket.emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, {
userId: userId,
userId: userId ? userId : this.userId,
receiverId: receiverId ? receiverId : this.userId,
roomId: roomId,
signal: signal
});
@ -219,7 +214,7 @@ export class Connection implements Connection {
return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback);
}
public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcScreenSharingMessageInterface) => void) {
public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcSignalMessageInterface) => void) {
return this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback);
}

51
front/src/WebRtc/MediaManager.ts
View File

@ -42,13 +42,13 @@ export class MediaManager {
this.microphoneClose.style.display = "none";
this.microphoneClose.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.enabledMicrophone();
this.enableMicrophone();
//update tracking
});
this.microphone = this.getElementByIdOrFail<HTMLImageElement>('microphone');
this.microphone.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.disabledMicrophone();
this.disableMicrophone();
//update tracking
});
@ -56,13 +56,13 @@ export class MediaManager {
this.cinemaClose.style.display = "none";
this.cinemaClose.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.enabledCamera();
this.enableCamera();
//update tracking
});
this.cinema = this.getElementByIdOrFail<HTMLImageElement>('cinema');
this.cinema.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.disabledCamera();
this.disableCamera();
//update tracking
});
@ -70,24 +70,24 @@ export class MediaManager {
this.monitorClose.style.display = "block";
this.monitorClose.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.enabledMonitor();
this.enableScreenSharing();
//update tracking
});
this.monitor = this.getElementByIdOrFail<HTMLImageElement>('monitor');
this.monitor.style.display = "none";
this.monitor.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.disabledMonitor();
this.disableScreenSharing();
//update tracking
});
}
onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void {
public onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void {
this.updatedLocalStreamCallBacks.add(callback);
}
onUpdateScreenSharing(callback: UpdatedScreenSharingCallback): void {
public onUpdateScreenSharing(callback: UpdatedScreenSharingCallback): void {
this.updatedScreenSharingCallBacks.add(callback);
}
@ -108,12 +108,12 @@ export class MediaManager {
}
}
activeVisio(){
showGameOverlay(){
const gameOverlay = this.getElementByIdOrFail('game-overlay');
gameOverlay.classList.add('active');
}
enabledCamera() {
private enableCamera() {
this.cinemaClose.style.display = "none";
this.cinema.style.display = "block";
this.constraintsMedia.video = videoConstraint;
@ -122,7 +122,7 @@ export class MediaManager {
});
}
disabledCamera() {
private disableCamera() {
this.cinemaClose.style.display = "block";
this.cinema.style.display = "none";
this.constraintsMedia.video = false;
@ -137,7 +137,7 @@ export class MediaManager {
});
}
enabledMicrophone() {
private enableMicrophone() {
this.microphoneClose.style.display = "none";
this.microphone.style.display = "block";
this.constraintsMedia.audio = true;
@ -146,7 +146,7 @@ export class MediaManager {
});
}
disabledMicrophone() {
private disableMicrophone() {
this.microphoneClose.style.display = "block";
this.microphone.style.display = "none";
this.constraintsMedia.audio = false;
@ -160,7 +160,7 @@ export class MediaManager {
});
}
enabledMonitor() {
private enableScreenSharing() {
this.monitorClose.style.display = "none";
this.monitor.style.display = "block";
this.getScreenMedia().then((stream) => {
@ -168,7 +168,7 @@ export class MediaManager {
});
}
disabledMonitor() {
private disableScreenSharing() {
this.monitorClose.style.display = "block";
this.monitor.style.display = "none";
this.localScreenCapture?.getTracks().forEach((track: MediaStreamTrack) => {
@ -299,21 +299,18 @@ export class MediaManager {
* @param userId
*/
addScreenSharingActiveVideo(userId : string){
userId = `screen-sharing-${userId}`;
this.webrtcInAudio.play();
// FIXME: switch to DisplayManager!
const elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing");
elementRemoteVideo.insertAdjacentHTML('beforeend', `
<div id="div-${userId}" class="screen-sharing-video-container">
userId = `screen-sharing-${userId}`;
const html = `
<div id="div-${userId}" class="video-container">
<video id="${userId}" autoplay></video>
</div>
`);
const activeHTMLVideoElement : HTMLElement|null = document.getElementById(userId);
if(!activeHTMLVideoElement){
return;
}
console.log(userId, (activeHTMLVideoElement as HTMLVideoElement));
this.remoteVideo.set(userId, (activeHTMLVideoElement as HTMLVideoElement));
`;
layoutManager.add(DivImportance.Important, userId, html);
this.remoteVideo.set(userId, this.getElementByIdOrFail<HTMLVideoElement>(userId));
}
/**

148
front/src/WebRtc/SimplePeer.ts
View File

@ -1,6 +1,6 @@
import {
Connection,
WebRtcDisconnectMessageInterface, WebRtcScreenSharingMessageInterface,
WebRtcDisconnectMessageInterface,
WebRtcSignalMessageInterface,
WebRtcStartMessageInterface
} from "../Connection";
@ -30,18 +30,18 @@ export class SimplePeer {
private PeerScreenSharingConnectionArray: Map<string, SimplePeerNamespace.Instance> = new Map<string, SimplePeerNamespace.Instance>();
private PeerConnectionArray: Map<string, SimplePeerNamespace.Instance> = new Map<string, SimplePeerNamespace.Instance>();
private readonly updateLocalStreamCallback: (media: MediaStream) => void;
private readonly updateScreenSharingCallback: (media: MediaStream) => void;
private readonly sendLocalVideoStreamCallback: (media: MediaStream) => void;
private readonly sendLocalScreenSharingStreamCallback: (media: MediaStream) => void;
private readonly peerConnectionListeners: Array<PeerConnectionListener> = new Array<PeerConnectionListener>();
constructor(Connection: Connection, WebRtcRoomId: string = "test-webrtc") {
this.Connection = Connection;
this.WebRtcRoomId = WebRtcRoomId;
// We need to go through this weird bound function pointer in order to be able to "free" this reference later.
this.updateLocalStreamCallback = this.updatedLocalStream.bind(this);
this.updateScreenSharingCallback = this.updatedScreenSharing.bind(this);
mediaManager.onUpdateLocalStream(this.updateLocalStreamCallback);
mediaManager.onUpdateScreenSharing(this.updateScreenSharingCallback);
this.sendLocalVideoStreamCallback = this.sendLocalVideoStream.bind(this);
this.sendLocalScreenSharingStreamCallback = this.sendLocalScreenSharingStream.bind(this);
mediaManager.onUpdateLocalStream(this.sendLocalVideoStreamCallback);
mediaManager.onUpdateScreenSharing(this.sendLocalScreenSharingStreamCallback);
this.initialise();
}
@ -64,11 +64,11 @@ export class SimplePeer {
});
//receive signal by gemer
this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcScreenSharingMessageInterface) => {
this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcSignalMessageInterface) => {
this.receiveWebrtcScreenSharingSignal(message);
});
mediaManager.activeVisio();
mediaManager.showGameOverlay();
mediaManager.getCamera().then(() => {
//receive message start
@ -88,7 +88,7 @@ export class SimplePeer {
private receiveWebrtcStart(data: WebRtcStartMessageInterface) {
this.WebRtcRoomId = data.roomId;
this.Users = data.clients;
// Note: the clients array contain the list of all clients (event the ones we are already connected to in case a user joints a group)
// Note: the clients array contain the list of all clients (even the ones we are already connected to in case a user joints a group)
// So we can receive a request we already had before. (which will abort at the first line of createPeerConnection)
// TODO: refactor this to only send a message to connect to one user (rather than several users).
// This would be symmetrical to the way we handle disconnection.
@ -102,6 +102,7 @@ export class SimplePeer {
* server has two people connected, start the meet
*/
private startWebRtc() {
console.warn('startWebRtc startWebRtc');
this.Users.forEach((user: UserSimplePeerInterface) => {
//if it's not an initiator, peer connection will be created when gamer will receive offer signal
if(!user.initiator){
@ -131,8 +132,11 @@ export class SimplePeer {
}
if(screenSharing) {
mediaManager.removeActiveScreenSharingVideo(user.userId);
mediaManager.addScreenSharingActiveVideo(user.userId);
// We should display the screen sharing ONLY if we are not initiator
if (!user.initiator) {
mediaManager.removeActiveScreenSharingVideo(user.userId);
mediaManager.addScreenSharingActiveVideo(user.userId);
}
}else{
mediaManager.removeActiveVideo(user.userId);
mediaManager.addActiveVideo(user.userId, name);
@ -156,17 +160,18 @@ export class SimplePeer {
});
if(screenSharing){
this.PeerScreenSharingConnectionArray.set(user.userId, peer);
}else {
} else {
this.PeerConnectionArray.set(user.userId, peer);
}
//start listen signal for the peer connection
peer.on('signal', (data: unknown) => {
if(screenSharing){
//console.log('Sending WebRTC offer for screen sharing ', data, ' to ', user.userId);
this.sendWebrtcScreenSharingSignal(data, user.userId);
return;
} else {
this.sendWebrtcSignal(data, user.userId);
}
this.sendWebrtcSignal(data, user.userId);
});
peer.on('stream', (stream: MediaStream) => {
@ -197,6 +202,12 @@ export class SimplePeer {
peer.on('connect', () => {
mediaManager.isConnected(user.userId);
console.info(`connect => ${user.userId}`);
// When a connection is established to a video stream, and if a screen sharing is taking place,
// the user sharing screen should also initiate a connection to the remote user!
if (screenSharing === false && mediaManager.localScreenCapture) {
this.sendLocalScreenSharingStreamToUser(user.userId);
}
});
peer.on('data', (chunk: Buffer) => {
@ -217,9 +228,9 @@ export class SimplePeer {
});
if(screenSharing){
this.addMediaScreenSharing(user.userId);
this.pushScreenSharingToRemoteUser(user.userId);
}else {
this.addMedia(user.userId);
this.pushVideoToRemoteUser(user.userId);
}
for (const peerConnectionListener of this.peerConnectionListeners) {
@ -290,7 +301,7 @@ export class SimplePeer {
* Unregisters any held event handler.
*/
public unregister() {
mediaManager.removeUpdateLocalStreamEventListener(this.updateLocalStreamCallback);
mediaManager.removeUpdateLocalStreamEventListener(this.sendLocalVideoStreamCallback);
}
/**
@ -299,7 +310,6 @@ export class SimplePeer {
* @param data
*/
private sendWebrtcSignal(data: unknown, userId : string) {
console.log("sendWebrtcSignal", data);
try {
this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId);
}catch (e) {
@ -315,7 +325,7 @@ export class SimplePeer {
private sendWebrtcScreenSharingSignal(data: unknown, userId : string) {
console.log("sendWebrtcScreenSharingSignal", data);
try {
this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, userId);
this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, null, userId);
}catch (e) {
console.error(`sendWebrtcScreenSharingSignal => ${userId}`, e);
}
@ -339,7 +349,7 @@ export class SimplePeer {
}
}
private receiveWebrtcScreenSharingSignal(data: WebRtcScreenSharingMessageInterface) {
private receiveWebrtcScreenSharingSignal(data: WebRtcSignalMessageInterface) {
console.log("receiveWebrtcScreenSharingSignal", data);
try {
//if offer type, create peer connection
@ -384,7 +394,7 @@ export class SimplePeer {
*
* @param userId
*/
private addMedia (userId : string) {
private pushVideoToRemoteUser(userId : string) {
try {
const PeerConnection = this.PeerConnectionArray.get(userId);
if (!PeerConnection) {
@ -396,74 +406,80 @@ export class SimplePeer {
if(!localStream){
return;
}
if (localStream) {
for (const track of localStream.getTracks()) {
PeerConnection.addTrack(track, localStream);
}
for (const track of localStream.getTracks()) {
PeerConnection.addTrack(track, localStream);
}
}catch (e) {
console.error(`addMedia => addMedia => ${userId}`, e);
console.error(`pushVideoToRemoteUser => ${userId}`, e);
}
}
private addMediaScreenSharing(userId : string) {
private pushScreenSharingToRemoteUser(userId : string) {
const PeerConnection = this.PeerScreenSharingConnectionArray.get(userId);
if (!PeerConnection) {
throw new Error('While adding media, cannot find user with ID ' + userId);
throw new Error('While pushing screen sharing, cannot find user with ID ' + userId);
}
const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture;
if(!localScreenCapture){
return;
}
/*for (const track of localScreenCapture.getTracks()) {
for (const track of localScreenCapture.getTracks()) {
PeerConnection.addTrack(track, localScreenCapture);
}*/
}
return;
}
updatedLocalStream(){
public sendLocalVideoStream(){
this.Users.forEach((user: UserSimplePeerInterface) => {
this.addMedia(user.userId);
this.pushVideoToRemoteUser(user.userId);
})
}
updatedScreenSharing() {
/**
* Triggered locally when clicking on the screen sharing button
*/
public sendLocalScreenSharingStream() {
if (mediaManager.localScreenCapture) {
//this.Connection.sendWebrtcScreenSharingStart(this.WebRtcRoomId);
const userId = this.Connection.getUserId();
if(!userId){
return;
for (const user of this.Users) {
this.sendLocalScreenSharingStreamToUser(user.userId);
}
const screenSharingUser: UserSimplePeerInterface = {
userId,
initiator: true
};
const PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true);
if (!PeerConnectionScreenSharing) {
return;
}
try {
for (const track of mediaManager.localScreenCapture.getTracks()) {
PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture);
}
}catch (e) {
console.error("updatedScreenSharing => ", e);
}
mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture);
} else {
const userId = this.Connection.getUserId();
if (!userId || !this.PeerScreenSharingConnectionArray.has(userId)) {
return;
for (const user of this.Users) {
this.stopLocalScreenSharingStreamToUser(user.userId);
}
const PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId);
console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing);
if (!PeerConnectionScreenSharing) {
return;
}
PeerConnectionScreenSharing.destroy();
this.PeerScreenSharingConnectionArray.delete(userId);
}
}
private sendLocalScreenSharingStreamToUser(userId: string): void {
// If a connection already exists with user (because it is already sharing a screen with us... let's use this connection)
if (this.PeerScreenSharingConnectionArray.has(userId)) {
this.pushScreenSharingToRemoteUser(userId);
return;
}
const screenSharingUser: UserSimplePeerInterface = {
userId,
initiator: true
};
const PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true);
if (!PeerConnectionScreenSharing) {
return;
}
}
private stopLocalScreenSharingStreamToUser(userId: string): void {
const PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId);
if (!PeerConnectionScreenSharing) {
throw new Error('Weird, screen sharing connection to user ' + userId + 'not found')
}
console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing);
// FIXME: maybe we don't want to destroy the connexion if it is used in the other way around!
// FIXME: maybe we don't want to destroy the connexion if it is used in the other way around!
// FIXME: maybe we don't want to destroy the connexion if it is used in the other way around!
PeerConnectionScreenSharing.destroy();
this.PeerScreenSharingConnectionArray.delete(userId);
}
}