From 199ed1266a1ce419d398e8c527acb5b65a4483fa Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Sat, 24 Oct 2020 14:13:23 +0200 Subject: [PATCH 1/6] Switch off camera when user is not focused on WorkAdventure windows Feature to switch off camera when user is not focused on WorkAdventure windows after 10 seconds --- front/src/WebRtc/MediaManager.ts | 84 ++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 15 deletions(-) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 557f12fb..070f27f9 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -38,6 +38,9 @@ export class MediaManager { private cinemaBtn: HTMLDivElement; private monitorBtn: HTMLDivElement; + private previousConstraint : MediaStreamConstraints; + private timeoutBlurWindows?: NodeJS.Timeout; + constructor() { this.myCamVideo = this.getElementByIdOrFail('myCamVideo'); @@ -89,6 +92,23 @@ export class MediaManager { this.disableScreenSharing(); //update tracking }); + + this.previousConstraint = JSON.parse(JSON.stringify(this.constraintsMedia)); + window.addEventListener('blur', () => { + if(this.timeoutBlurWindows){ + clearTimeout(this.timeoutBlurWindows); + } + this.timeoutBlurWindows = setTimeout(() => { + this.previousConstraint = JSON.parse(JSON.stringify(this.constraintsMedia)); + this.disableCamera(); + }, 10000); + }); + window.addEventListener('focus', () => { + if(this.timeoutBlurWindows){ + clearTimeout(this.timeoutBlurWindows); + } + this.applyPreviousConfig(); + }); } public onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { @@ -136,9 +156,7 @@ export class MediaManager { } private enableCamera() { - this.cinemaClose.style.display = "none"; - this.cinemaBtn.classList.remove("disabled"); - this.cinema.style.display = "block"; + this.enableCameraStyle(); this.constraintsMedia.video = videoConstraint; this.getCamera().then((stream: MediaStream) => { this.triggerUpdatedLocalStreamCallbacks(stream); @@ -146,11 +164,7 @@ export class MediaManager { } private async disableCamera() { - this.cinemaClose.style.display = "block"; - this.cinema.style.display = "none"; - this.cinemaBtn.classList.add("disabled"); - this.constraintsMedia.video = false; - this.myCamVideo.srcObject = null; + this.disableCameraStyle(); this.stopCamera(); if (this.constraintsMedia.audio !== false) { @@ -162,9 +176,7 @@ export class MediaManager { } private enableMicrophone() { - this.microphoneClose.style.display = "none"; - this.microphone.style.display = "block"; - this.microphoneBtn.classList.remove("disabled"); + this.enableMicrophoneStyle(); this.constraintsMedia.audio = true; this.getCamera().then((stream) => { @@ -173,10 +185,7 @@ export class MediaManager { } private async disableMicrophone() { - this.microphoneClose.style.display = "block"; - this.microphone.style.display = "none"; - this.microphoneBtn.classList.add("disabled"); - this.constraintsMedia.audio = false; + this.disableMicrophoneStyle(); this.stopMicrophone(); if (this.constraintsMedia.video !== false) { @@ -187,6 +196,51 @@ export class MediaManager { } } + private applyPreviousConfig() { + this.constraintsMedia = this.previousConstraint; + if(!this.constraintsMedia.video){ + this.disableCameraStyle(); + }else{ + this.enableCameraStyle(); + } + if(!this.constraintsMedia.audio){ + this.disableMicrophoneStyle() + }else{ + this.enableMicrophoneStyle() + } + + this.getCamera().then((stream: MediaStream) => { + this.triggerUpdatedLocalStreamCallbacks(stream); + }); + } + + private enableCameraStyle(){ + this.cinemaClose.style.display = "none"; + this.cinemaBtn.classList.remove("disabled"); + this.cinema.style.display = "block"; + } + + private disableCameraStyle(){ + this.cinemaClose.style.display = "block"; + this.cinema.style.display = "none"; + this.cinemaBtn.classList.add("disabled"); + this.constraintsMedia.video = false; + this.myCamVideo.srcObject = null; + } + + private enableMicrophoneStyle(){ + this.microphoneClose.style.display = "none"; + this.microphone.style.display = "block"; + this.microphoneBtn.classList.remove("disabled"); + } + + private disableMicrophoneStyle(){ + this.microphoneClose.style.display = "block"; + this.microphone.style.display = "none"; + this.microphoneBtn.classList.add("disabled"); + this.constraintsMedia.audio = false; + } + private enableScreenSharing() { this.monitorClose.style.display = "none"; this.monitor.style.display = "block"; From 69f3e511aba0a6c3d89c71ef93f20193bde9dc4c Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Sat, 24 Oct 2020 14:40:51 +0200 Subject: [PATCH 2/6] Ping status camera and microphone --- front/src/WebRtc/MediaManager.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 070f27f9..28771f63 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -109,6 +109,7 @@ export class MediaManager { } this.applyPreviousConfig(); }); + this.pingCameraStatus(); } public onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { @@ -619,6 +620,17 @@ export class MediaManager { mainContainer.appendChild(divReport); } + //ping always 30 seconds data on stream + private pingCameraStatus(){ + setTimeout(() => { + console.log('ping camera status'); + this.getCamera().then((stream: MediaStream) => { + this.triggerUpdatedLocalStreamCallbacks(stream); + this.pingCameraStatus(); + }); + }, 30000); + } + } From 997acd17ad8216ba2d4e0f641fa36823a2fdb0dc Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Mon, 26 Oct 2020 22:39:52 +0100 Subject: [PATCH 3/6] Update to use update function scene --- front/src/Phaser/Game/GameScene.ts | 7 ++-- front/src/WebRtc/MediaManager.ts | 60 +++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/front/src/Phaser/Game/GameScene.ts b/front/src/Phaser/Game/GameScene.ts index bfc6402e..46efc5e5 100644 --- a/front/src/Phaser/Game/GameScene.ts +++ b/front/src/Phaser/Game/GameScene.ts @@ -324,7 +324,7 @@ export class GameScene extends ResizableScene implements CenterListener { // Let's alter browser history let path = this.room.id; if (this.room.hash) { - path += '#'+this.room.hash; + path += '#' + this.room.hash; } window.history.pushState({}, 'WorkAdventure', path); @@ -486,7 +486,7 @@ export class GameScene extends ResizableScene implements CenterListener { this.stopJitsi(); } else { if (JITSI_PRIVATE_MODE) { - const adminTag = allProps.get("jitsiRoomAdminTag") as string|undefined; + const adminTag = allProps.get("jitsiRoomAdminTag") as string | undefined; this.connection.emitQueryJitsiJwtMessage(this.instance.replace('/', '-') + "-" + newValue, adminTag); } else { @@ -634,7 +634,6 @@ export class GameScene extends ResizableScene implements CenterListener { this.gameMap.setPosition(event.x, event.y); }) - this.scene.wake(); this.scene.sleep(ReconnectingSceneName); @@ -973,6 +972,8 @@ export class GameScene extends ResizableScene implements CenterListener { this.scene.remove(this.scene.key); this.scene.start(nextSceneKey.key); } + + mediaManager.setLastUpdateScene(); } private checkToExit(): {key: string, hash: string} | null { diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 28771f63..1cb1bdbb 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -39,7 +39,10 @@ export class MediaManager { private monitorBtn: HTMLDivElement; private previousConstraint : MediaStreamConstraints; - private timeoutBlurWindows?: NodeJS.Timeout; + private focused : boolean = true; + + private lastUpdateScene : Date = new Date(); + private setTimeOutlastUpdateScene? : NodeJS.Timeout; constructor() { @@ -94,22 +97,30 @@ export class MediaManager { }); this.previousConstraint = JSON.parse(JSON.stringify(this.constraintsMedia)); - window.addEventListener('blur', () => { - if(this.timeoutBlurWindows){ - clearTimeout(this.timeoutBlurWindows); - } - this.timeoutBlurWindows = setTimeout(() => { - this.previousConstraint = JSON.parse(JSON.stringify(this.constraintsMedia)); - this.disableCamera(); - }, 10000); - }); - window.addEventListener('focus', () => { - if(this.timeoutBlurWindows){ - clearTimeout(this.timeoutBlurWindows); - } - this.applyPreviousConfig(); - }); this.pingCameraStatus(); + + this.checkActiveUser(); + } + + public setLastUpdateScene(){ + this.lastUpdateScene = new Date(); + } + + public blurCamera() { + if(!this.focused){ + return; + } + this.focused = false; + this.previousConstraint = JSON.parse(JSON.stringify(this.constraintsMedia)); + this.disableCamera(); + } + + public focusCamera() { + if(this.focused){ + return; + } + this.focused = true; + this.applyPreviousConfig(); } public onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { @@ -631,7 +642,22 @@ export class MediaManager { }, 30000); } - + //check if user is active + private checkActiveUser(){ + if(this.setTimeOutlastUpdateScene){ + clearTimeout(this.setTimeOutlastUpdateScene); + } + this.setTimeOutlastUpdateScene = setTimeout(() => { + const now = new Date(); + //if last update is more of 10 sec + if( (now.getTime() - this.lastUpdateScene.getTime()) > 10000) { + this.blurCamera(); + }else{ + this.focusCamera(); + } + this.checkActiveUser(); + }, this.focused ? 10000 : 1000); + } } export const mediaManager = new MediaManager(); From df49ea856a9d446c9f87742443eb763bac2e7045 Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Tue, 27 Oct 2020 20:51:11 +0100 Subject: [PATCH 4/6] Fix conflict error --- front/src/WebRtc/MediaManager.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 1cb1bdbb..672b09ae 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -167,7 +167,7 @@ export class MediaManager { gameOverlay.classList.remove('active'); } - private enableCamera() { + public enableCamera() { this.enableCameraStyle(); this.constraintsMedia.video = videoConstraint; this.getCamera().then((stream: MediaStream) => { @@ -175,7 +175,7 @@ export class MediaManager { }); } - private async disableCamera() { + public async disableCamera() { this.disableCameraStyle(); this.stopCamera(); @@ -187,7 +187,7 @@ export class MediaManager { } } - private enableMicrophone() { + public enableMicrophone() { this.enableMicrophoneStyle(); this.constraintsMedia.audio = true; @@ -196,7 +196,7 @@ export class MediaManager { }); } - private async disableMicrophone() { + public async disableMicrophone() { this.disableMicrophoneStyle(); this.stopMicrophone(); @@ -574,7 +574,7 @@ export class MediaManager { return elem as T; } - private showReportModal(userId: string, userName: string, reportCallBack: ReportCallback){ + public showReportModal(userId: string, userName: string, reportCallBack: ReportCallback){ //create report text area const mainContainer = this.getElementByIdOrFail('main-container'); From 47a049ecafd689ece1cb2e7fc1b4edc2a1ec6414 Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Wed, 4 Nov 2020 12:42:33 +0100 Subject: [PATCH 5/6] Feedback comment @moufmouf --- front/src/WebRtc/MediaManager.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 672b09ae..37a23578 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -631,7 +631,10 @@ export class MediaManager { mainContainer.appendChild(divReport); } - //ping always 30 seconds data on stream + /** + * For some reasons, the microphone muted icon or the stream is not always up to date. + * Here, every 30 seconds, we are "reseting" the streams and sending again the constraints to the other peers via the data channel again (see SimplePeer::pushVideoToRemoteUser) + **/ private pingCameraStatus(){ setTimeout(() => { console.log('ping camera status'); From 39cdc3fbd90e190e95f7ede431110e696d00305d Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Sat, 21 Nov 2020 18:47:38 +0100 Subject: [PATCH 6/6] Fix trgger stream and update scene record --- front/src/Phaser/Game/GameScene.ts | 3 +-- front/src/WebRtc/MediaManager.ts | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/front/src/Phaser/Game/GameScene.ts b/front/src/Phaser/Game/GameScene.ts index 9553e584..6a0b86d9 100644 --- a/front/src/Phaser/Game/GameScene.ts +++ b/front/src/Phaser/Game/GameScene.ts @@ -906,6 +906,7 @@ export class GameScene extends ResizableScene implements CenterListener { * @param delta The delta time in ms since the last frame. This is a smoothed and capped value based on the FPS rate. */ update(time: number, delta: number) : void { + mediaManager.setLastUpdateScene(); this.currentTick = time; this.CurrentPlayer.moveUser(delta); @@ -959,8 +960,6 @@ export class GameScene extends ResizableScene implements CenterListener { this.CurrentPlayer.x = this.startX; this.CurrentPlayer.y = this.startY; } - - mediaManager.setLastUpdateScene(); } private checkToExit(): {key: string, hash: string} | null { diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 044a14ee..9dfbf7c4 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -661,10 +661,8 @@ export class MediaManager { private pingCameraStatus(){ setTimeout(() => { console.log('ping camera status'); - this.getCamera().then((stream: MediaStream) => { - this.triggerUpdatedLocalStreamCallbacks(stream); - this.pingCameraStatus(); - }); + this.triggerUpdatedLocalStreamCallbacks(this.localStream); + this.pingCameraStatus(); }, 30000); }