From 80a698d89127b7ddf44a5117d11e9ccb495375eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 3 Jun 2021 14:53:58 +0200 Subject: [PATCH 1/3] Preventing crash on iOS On iOS, audio elements must be triggered by a user gesture. We are catching exceptions thrown in order not to crash. In the future, we need to find another way to play sounds. --- front/src/WebRtc/MediaManager.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 7b527962..7cdf1de0 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -129,7 +129,11 @@ export class MediaManager { } addActiveVideo(user: UserSimplePeerInterface, userName: string = ""){ - this.webrtcInAudio.play(); + try { + this.webrtcInAudio.play(); + } catch(e) { + console.error(e); + } const userId = ''+user.userId userName = userName.toUpperCase(); @@ -278,7 +282,11 @@ export class MediaManager { } playWebrtcOutSound(): void { - this.webrtcOutAudio.play(); + try { + this.webrtcOutAudio.play(); + } catch(e) { + console.error(e); + } } isConnecting(userId: string): void { From 50e994c674213e1f0cdc9956cc8c725fd0f504e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 3 Jun 2021 15:40:44 +0200 Subject: [PATCH 2/3] Attempt to switch bubble sound playing into Phaser In iOS, we cannot trigger a playing sound if it does not start from a user gesture. This is a huge bummer for a notification sound! This is an attempt to switch sound playing to Phaser, which is using under the hood the WebAudio API. This might solve the issue. --- front/dist/index.tmpl.html | 6 ------ front/src/Phaser/Game/GameScene.ts | 22 ++++++++++++++++++++++ front/src/WebRtc/MediaManager.ts | 20 -------------------- front/src/WebRtc/SimplePeer.ts | 1 - 4 files changed, 22 insertions(+), 27 deletions(-) diff --git a/front/dist/index.tmpl.html b/front/dist/index.tmpl.html index 7f541145..aa63229f 100644 --- a/front/dist/index.tmpl.html +++ b/front/dist/index.tmpl.html @@ -91,12 +91,6 @@
- - diff --git a/front/src/Phaser/Game/GameScene.ts b/front/src/Phaser/Game/GameScene.ts index 2de7747f..b4f79401 100644 --- a/front/src/Phaser/Game/GameScene.ts +++ b/front/src/Phaser/Game/GameScene.ts @@ -160,6 +160,7 @@ export class GameScene extends DirtyScene implements CenterListener { private createPromise: Promise; private createPromiseResolve!: (value?: void | PromiseLike) => void; private iframeSubscriptionList! : Array; + private peerStoreUnsubscribe!: () => void; MapUrlFile: string; RoomId: string; instance: string; @@ -228,6 +229,11 @@ export class GameScene extends DirtyScene implements CenterListener { this.load.image(joystickBaseKey, joystickBaseImg); this.load.image(joystickThumbKey, joystickThumbImg); } + this.load.audio('audio-webrtc-in', '/resources/objects/webrtc-in.mp3'); + this.load.audio('audio-webrtc-out', '/resources/objects/webrtc-out.mp3'); + //this.load.audio('audio-report-message', '/resources/objects/report-message.mp3'); + this.sound.pauseOnBlur = false; + this.load.on(FILE_LOAD_ERROR, (file: {src: string}) => { // If we happen to be in HTTP and we are trying to load a URL in HTTPS only... (this happens only in dev environments) if (window.location.protocol === 'http:' && file.src === this.MapUrlFile && file.src.startsWith('http:') && this.originalMapUrl === undefined) { @@ -519,6 +525,21 @@ export class GameScene extends DirtyScene implements CenterListener { } this.emoteManager = new EmoteManager(this); + + let oldPeerNumber = 0; + this.peerStoreUnsubscribe = peerStore.subscribe((peers) => { + const newPeerNumber = peers.size; + if (newPeerNumber > oldPeerNumber) { + this.sound.play('audio-webrtc-in', { + volume: 0.2 + }); + } else if (newPeerNumber < oldPeerNumber) { + this.sound.play('audio-webrtc-out', { + volume: 0.2 + }); + } + oldPeerNumber = newPeerNumber; + }); } /** @@ -960,6 +981,7 @@ ${escapedMessage} this.userInputManager.destroy(); this.pinchManager?.destroy(); this.emoteManager.destroy(); + this.peerStoreUnsubscribe(); mediaManager.hideGameOverlay(); diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 7cdf1de0..7b537920 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -23,10 +23,8 @@ export type HelpCameraSettingsCallBack = () => void; export class MediaManager { private remoteVideo: Map = new Map(); - webrtcInAudio: HTMLAudioElement; //FIX ME SOUNDMETER: check stalability of sound meter calculation //mySoundMeterElement: HTMLDivElement; - private webrtcOutAudio: HTMLAudioElement; startScreenSharingCallBacks : Set = new Set(); stopScreenSharingCallBacks : Set = new Set(); showReportModalCallBacks : Set = new Set(); @@ -44,11 +42,6 @@ export class MediaManager { constructor() { - this.webrtcInAudio = HtmlUtils.getElementByIdOrFail('audio-webrtc-in'); - this.webrtcOutAudio = HtmlUtils.getElementByIdOrFail('audio-webrtc-out'); - this.webrtcInAudio.volume = 0.2; - this.webrtcOutAudio.volume = 0.2; - this.pingCameraStatus(); //FIX ME SOUNDMETER: check stability of sound meter calculation @@ -129,11 +122,6 @@ export class MediaManager { } addActiveVideo(user: UserSimplePeerInterface, userName: string = ""){ - try { - this.webrtcInAudio.play(); - } catch(e) { - console.error(e); - } const userId = ''+user.userId userName = userName.toUpperCase(); @@ -281,14 +269,6 @@ export class MediaManager { this.removeActiveVideo(this.getScreenSharingId(userId)) } - playWebrtcOutSound(): void { - try { - this.webrtcOutAudio.play(); - } catch(e) { - console.error(e); - } - } - isConnecting(userId: string): void { const connectingSpinnerDiv = this.getSpinner(userId); if (connectingSpinnerDiv === null) { diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 9193f18b..2a502bab 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -246,7 +246,6 @@ export class SimplePeer { * This is triggered twice. Once by the server, and once by a remote client disconnecting */ private closeConnection(userId : number) { - mediaManager.playWebrtcOutSound(); try { const peer = this.PeerConnectionArray.get(userId); if (peer === undefined) { From 46666d17dcbe8ef51e8541fd102eeba7074c607c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 3 Jun 2021 15:56:06 +0200 Subject: [PATCH 3/3] Making sure the report audio sound does not break the application on iOS --- front/src/Administration/TypeMessage.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/front/src/Administration/TypeMessage.ts b/front/src/Administration/TypeMessage.ts index 07f330fd..d735ba58 100644 --- a/front/src/Administration/TypeMessage.ts +++ b/front/src/Administration/TypeMessage.ts @@ -44,7 +44,13 @@ export class TypeMessageExt implements TypeMessageInterface{ mainSectionDiv.appendChild(div); const reportMessageAudio = HtmlUtils.getElementByIdOrFail('report-message'); - reportMessageAudio.play(); + // FIXME: this will fail on iOS + // We should move the sound playing into the GameScene and listen to the event of a report using a store + try { + reportMessageAudio.play(); + } catch (e) { + console.error(e); + } this.nbSecond = this.maxNbSecond; setTimeout((c) => {