Merge pull request #1820 from thecodingmachine/feature-voice-chat-indicator

Feature voice chat indicator

front/dist/.gitignore

@@ -4,3 +4,4 @@ index.html
 style.*.css
 !env-config.template.js
 *.png
+!/resources/**

front/src/Components/EnableCamera/EnableCameraScene.svelte

@@ -5,6 +5,7 @@
         audioConstraintStore,
         cameraListStore,
         localStreamStore,
+        localVolumeStore,
         microphoneListStore,
         videoConstraintStore,
     } from "../../Stores/MediaStore";
@@ -38,7 +39,7 @@
 
     let stream: MediaStream | null;
 
-    const unsubscribe = localStreamStore.subscribe((value) => {
+    const unsubscribeLocalStreamStore = localStreamStore.subscribe((value) => {
         if (value.type === "success") {
             stream = value.stream;
 
@@ -59,7 +60,9 @@
         }
     });
 
-    onDestroy(unsubscribe);
+    onDestroy(() => {
+        unsubscribeLocalStreamStore();
+    });
 
     function normalizeDeviceName(label: string): string {
         // remove IDs (that can appear in Chrome, like: "HD Pro Webcam (4df7:4eda)"
@@ -86,7 +89,7 @@
             <img class="background-img" src={cinemaCloseImg} alt="" />
         </div>
     {/if}
-    <HorizontalSoundMeterWidget {stream} />
+    <HorizontalSoundMeterWidget volume={$localVolumeStore} />
 
     <section class="selectWebcamForm">
         {#if $cameraListStore.length > 1}

front/src/Components/EnableCamera/HorizontalSoundMeterWidget.svelte

@@ -1,50 +1,8 @@
 <script lang="typescript">
-    import { AudioContext } from "standardized-audio-context";
+    export let volume = 0;
-    import { SoundMeter } from "../../Phaser/Components/SoundMeter";
-    import { onDestroy } from "svelte";
-
-    export let stream: MediaStream | null;
-    let volume = 0;
 
     const NB_BARS = 20;
 
-    let timeout: ReturnType<typeof setTimeout>;
-    const soundMeter = new SoundMeter();
-    let display = false;
-    let error = false;
-
-    $: {
-        if (stream && stream.getAudioTracks().length > 0) {
-            display = true;
-            soundMeter.connectToSource(stream, new AudioContext());
-
-            if (timeout) {
-                clearInterval(timeout);
-                error = false;
-            }
-
-            timeout = setInterval(() => {
-                try {
-                    volume = parseInt(((soundMeter.getVolume() / 100) * NB_BARS).toFixed(0));
-                } catch (err) {
-                    if (!error) {
-                        console.error(err);
-                        error = true;
-                    }
-                }
-            }, 100);
-        } else {
-            display = false;
-        }
-    }
-
-    onDestroy(() => {
-        soundMeter.stop();
-        if (timeout) {
-            clearInterval(timeout);
-        }
-    });
-
     function color(i: number, volume: number) {
         const red = (255 * i) / NB_BARS;
         const green = 255 * (1 - i / NB_BARS);
@@ -58,7 +16,7 @@
     }
 </script>
 
-<div class="horizontal-sound-meter" class:active={display}>
+<div class="horizontal-sound-meter" class:active={volume !== undefined}>
     {#each [...Array(NB_BARS).keys()] as i (i)}
         <div style={color(i, volume)} />
     {/each}

front/src/Components/MyCamera.svelte

@@ -1,5 +1,5 @@
 <script lang="typescript">
-    import { obtainedMediaConstraintStore } from "../Stores/MediaStore";
+    import { localVolumeStore, obtainedMediaConstraintStore } from "../Stores/MediaStore";
     import { localStreamStore, isSilentStore } from "../Stores/MediaStore";
     import SoundMeterWidget from "./SoundMeterWidget.svelte";
     import { onDestroy, onMount } from "svelte";
@@ -8,7 +8,7 @@
 
     let stream: MediaStream | null;
 
-    const unsubscribe = localStreamStore.subscribe((value) => {
+    const unsubscribeLocalStreamStore = localStreamStore.subscribe((value) => {
         if (value.type === "success") {
             stream = value.stream;
         } else {
@@ -16,7 +16,9 @@
         }
     });
 
-    onDestroy(unsubscribe);
+    onDestroy(() => {
+        unsubscribeLocalStreamStore();
+    });
 
     let isSilent: boolean;
     const unsubscribeIsSilent = isSilentStore.subscribe((value) => {
@@ -51,7 +53,7 @@
         <div class="is-silent">{$LL.camera.my.silentZone()}</div>
     {:else if $localStreamStore.type === "success" && $localStreamStore.stream}
         <video class="my-cam-video" use:srcObject={stream} autoplay muted playsinline />
-        <SoundMeterWidget {stream} />
+        <SoundMeterWidget volume={$localVolumeStore} />
     {/if}
 </div>
 

front/src/Components/SoundMeterWidget.svelte

@@ -1,47 +1,6 @@
 <script lang="typescript">
-    import { AudioContext } from "standardized-audio-context";
+    export let volume = 0;
-    import { SoundMeter } from "../Phaser/Components/SoundMeter";
+    let display = true;
-    import { onDestroy } from "svelte";
-
-    export let stream: MediaStream | null;
-    let volume = 0;
-
-    let timeout: ReturnType<typeof setTimeout>;
-    const soundMeter = new SoundMeter();
-    let display = false;
-    let error = false;
-
-    $: {
-        if (stream && stream.getAudioTracks().length > 0) {
-            display = true;
-            soundMeter.connectToSource(stream, new AudioContext());
-
-            if (timeout) {
-                clearInterval(timeout);
-                error = false;
-            }
-
-            timeout = setInterval(() => {
-                try {
-                    volume = soundMeter.getVolume();
-                } catch (err) {
-                    if (!error) {
-                        console.error(err);
-                        error = true;
-                    }
-                }
-            }, 100);
-        } else {
-            display = false;
-        }
-    }
-
-    onDestroy(() => {
-        soundMeter.stop();
-        if (timeout) {
-            clearInterval(timeout);
-        }
-    });
 </script>
 
 <div class="sound-progress" class:active={display}>

front/src/Components/Video/VideoMediaBox.svelte

@@ -18,6 +18,7 @@
 
     export let peer: VideoPeer;
     let streamStore = peer.streamStore;
+    let volumeStore = peer.volumeStore;
     let name = peer.userName;
     let statusStore = peer.statusStore;
     let constraintStore = peer.constraintsStore;
@@ -93,7 +94,7 @@
     />
     <img src={blockSignImg} draggable="false" on:dragstart|preventDefault={noDrag} class="block-logo" alt="Block" />
     {#if $constraintStore && $constraintStore.audio !== false}
-        <SoundMeterWidget stream={$streamStore} />
+        <SoundMeterWidget volume={$volumeStore} />
     {/if}
 </div>
 

front/src/Phaser/Components/SoundMeter.ts

@@ -1,4 +1,4 @@
-import type { IAnalyserNode, IAudioContext, IMediaStreamAudioSourceNode } from "standardized-audio-context";
+import { AudioContext, IAnalyserNode, IAudioContext, IMediaStreamAudioSourceNode } from "standardized-audio-context";
 
 /**
  * Class to measure the sound volume of a media stream
@@ -6,41 +6,15 @@ import type { IAnalyserNode, IAudioContext, IMediaStreamAudioSourceNode } from "
 export class SoundMeter {
     private instant: number;
     private clip: number;
-    //private script: ScriptProcessorNode;
     private analyser: IAnalyserNode<IAudioContext> | undefined;
     private dataArray: Uint8Array | undefined;
     private context: IAudioContext | undefined;
     private source: IMediaStreamAudioSourceNode<IAudioContext> | undefined;
 
-    constructor() {
+    constructor(mediaStream: MediaStream) {
         this.instant = 0.0;
         this.clip = 0.0;
-        //this.script = context.createScriptProcessor(2048, 1, 1);
+        this.connectToSource(mediaStream, new AudioContext());
-    }
-
-    private init(context: IAudioContext) {
-        this.context = context;
-        this.analyser = this.context.createAnalyser();
-
-        this.analyser.fftSize = 2048;
-        const bufferLength = this.analyser.fftSize;
-        this.dataArray = new Uint8Array(bufferLength);
-    }
-
-    public connectToSource(stream: MediaStream, context: IAudioContext): void {
-        if (this.source !== undefined) {
-            this.stop();
-        }
-
-        this.init(context);
-
-        this.source = this.context?.createMediaStreamSource(stream);
-        if (this.analyser !== undefined) {
-            this.source?.connect(this.analyser);
-        }
-        //analyser.connect(distortion);
-        //distortion.connect(this.context.destination);
-        //this.analyser.connect(this.context.destination);
     }
 
     public getVolume(): number {
@@ -78,4 +52,29 @@ export class SoundMeter {
         this.dataArray = undefined;
         this.source = undefined;
     }
+
+    private init(context: IAudioContext) {
+        this.context = context;
+        this.analyser = this.context.createAnalyser();
+
+        this.analyser.fftSize = 2048;
+        const bufferLength = this.analyser.fftSize;
+        this.dataArray = new Uint8Array(bufferLength);
+    }
+
+    private connectToSource(stream: MediaStream, context: IAudioContext): void {
+        if (this.source !== undefined) {
+            this.stop();
+        }
+
+        this.init(context);
+
+        this.source = this.context?.createMediaStreamSource(stream);
+        if (this.analyser !== undefined) {
+            this.source?.connect(this.analyser);
+        }
+        //analyser.connect(distortion);
+        //distortion.connect(this.context.destination);
+        //this.analyser.connect(this.context.destination);
+    }
 }

front/src/Phaser/Components/TalkIcon.ts

@@ -0,0 +1,58 @@
+import { Easing } from "../../types";
+
+export class TalkIcon extends Phaser.GameObjects.Image {
+    private shown: boolean;
+    private showAnimationTween?: Phaser.Tweens.Tween;
+
+    private defaultPosition: { x: number; y: number };
+    private defaultScale: number;
+
+    constructor(scene: Phaser.Scene, x: number, y: number) {
+        super(scene, x, y, "iconTalk");
+
+        this.defaultPosition = { x, y };
+        this.defaultScale = 0.3;
+
+        this.shown = false;
+        this.setAlpha(0);
+        this.setScale(this.defaultScale);
+
+        this.scene.add.existing(this);
+    }
+
+    public show(show: boolean = true, forceClose: boolean = false): void {
+        if (this.shown === show && !forceClose) {
+            return;
+        }
+        this.showAnimation(show, forceClose);
+    }
+
+    private showAnimation(show: boolean = true, forceClose: boolean = false) {
+        if (forceClose && !show) {
+            this.showAnimationTween?.stop();
+        } else if (this.showAnimationTween?.isPlaying()) {
+            return;
+        }
+        this.shown = show;
+        if (show) {
+            this.y += 50;
+            this.scale = 0.05;
+            this.alpha = 0;
+        }
+        this.showAnimationTween = this.scene.tweens.add({
+            targets: [this],
+            duration: 350,
+            alpha: show ? 1 : 0,
+            y: this.defaultPosition.y,
+            scale: this.defaultScale,
+            ease: Easing.BackEaseOut,
+            onComplete: () => {
+                this.showAnimationTween = undefined;
+            },
+        });
+    }
+
+    public isShown(): boolean {
+        return this.shown;
+    }
+}

front/src/Phaser/Entity/Character.ts

@@ -17,6 +17,7 @@ import { Unsubscriber, Writable, writable } from "svelte/store";
 import { createColorStore } from "../../Stores/OutlineColorStore";
 import type { OutlineableInterface } from "../Game/OutlineableInterface";
 import type CancelablePromise from "cancelable-promise";
+import { TalkIcon } from "../Components/TalkIcon";
 
 const playerNameY = -25;
 
@@ -33,6 +34,7 @@ const interactiveRadius = 35;
 export abstract class Character extends Container implements OutlineableInterface {
     private bubble: SpeechBubble | null = null;
     private readonly playerNameText: Text;
+    private readonly talkIcon: TalkIcon;
     public playerName: string;
     public sprites: Map<string, Sprite>;
     protected lastDirection: PlayerAnimationDirections = PlayerAnimationDirections.Down;
@@ -102,6 +104,17 @@ export abstract class Character extends Container implements OutlineableInterfac
                 fontSize: 35,
             },
         });
+
+        this.talkIcon = new TalkIcon(scene, 0, -45);
+        this.add(this.talkIcon);
+
+        if (isClickable) {
+            this.setInteractive({
+                hitArea: new Phaser.Geom.Circle(0, 0, interactiveRadius),
+                hitAreaCallback: Phaser.Geom.Circle.Contains, //eslint-disable-line @typescript-eslint/unbound-method
+                useHandCursor: true,
+            });
+        }
         this.playerNameText.setOrigin(0.5).setDepth(DEPTH_INGAME_TEXT_INDEX);
         this.add(this.playerNameText);
 
@@ -200,6 +213,10 @@ export abstract class Character extends Container implements OutlineableInterfac
         });
     }
 
+    public showTalkIcon(show: boolean = true, forceClose: boolean = false): void {
+        this.talkIcon.show(show, forceClose);
+    }
+
     public addCompanion(name: string, texturePromise?: CancelablePromise<string>): void {
         if (typeof texturePromise !== "undefined") {
             this.companion = new Companion(this.scene, this.x, this.y, name, texturePromise);

front/src/Phaser/Game/GameScene.ts

@@ -91,6 +91,7 @@ import { MapStore } from "../../Stores/Utils/MapStore";
 import { followUsersColorStore } from "../../Stores/FollowStore";
 import { GameSceneUserInputHandler } from "../UserInput/GameSceneUserInputHandler";
 import { locale } from "../../i18n/i18n-svelte";
+import { localVolumeStore } from "../../Stores/MediaStore";
 import { StringUtils } from "../../Utils/StringUtils";
 import { startLayerNamesStore } from "../../Stores/StartLayerNamesStore";
 import { JitsiCoWebsite } from "../../WebRtc/CoWebsite/JitsiCoWebsite";
@@ -172,6 +173,9 @@ export class GameScene extends DirtyScene {
     private peerStoreUnsubscribe!: Unsubscriber;
     private emoteUnsubscribe!: Unsubscriber;
     private emoteMenuUnsubscribe!: Unsubscriber;
+
+    private volumeStoreUnsubscribers: Map<number, Unsubscriber> = new Map<number, Unsubscriber>();
+    private localVolumeStoreUnsubscriber: Unsubscriber | undefined;
     private followUsersColorStoreUnsubscribe!: Unsubscriber;
 
     private biggestAvailableAreaStoreUnsubscribe!: () => void;
@@ -247,6 +251,7 @@ export class GameScene extends DirtyScene {
                 loadCustomTexture(this.load, texture).catch((e) => console.error(e));
             }
         }
+        this.load.image("iconTalk", "/resources/icons/icon_talking.png");
 
         if (touchScreenManager.supportTouchScreen) {
             this.load.image(joystickBaseKey, joystickBaseImg);
@@ -636,14 +641,45 @@ export class GameScene extends DirtyScene {
             this.connect();
         }
 
+        const talkIconVolumeTreshold = 10;
         let oldPeerNumber = 0;
         this.peerStoreUnsubscribe = peerStore.subscribe((peers) => {
+            this.volumeStoreUnsubscribers.forEach((unsubscribe) => unsubscribe());
+            this.volumeStoreUnsubscribers.clear();
+
+            for (const [key, videoStream] of peers) {
+                this.volumeStoreUnsubscribers.set(
+                    key,
+                    videoStream.volumeStore.subscribe((volume) => {
+                        if (volume) {
+                            this.MapPlayersByKey.get(key)?.showTalkIcon(volume > talkIconVolumeTreshold);
+                        }
+                    })
+                );
+            }
+
             const newPeerNumber = peers.size;
             if (newPeerNumber > oldPeerNumber) {
                 this.playSound("audio-webrtc-in");
             } else if (newPeerNumber < oldPeerNumber) {
                 this.playSound("audio-webrtc-out");
             }
+            if (newPeerNumber > 0) {
+                if (!this.localVolumeStoreUnsubscriber) {
+                    this.localVolumeStoreUnsubscriber = localVolumeStore.subscribe((volume) => {
+                        if (volume) {
+                            this.CurrentPlayer.showTalkIcon(volume > talkIconVolumeTreshold);
+                        }
+                    });
+                }
+            } else {
+                this.CurrentPlayer.showTalkIcon(false, true);
+                this.MapPlayersByKey.forEach((remotePlayer) => remotePlayer.showTalkIcon(false, true));
+                if (this.localVolumeStoreUnsubscriber) {
+                    this.localVolumeStoreUnsubscriber();
+                    this.localVolumeStoreUnsubscriber = undefined;
+                }
+            }
             oldPeerNumber = newPeerNumber;
         });
 

front/src/Stores/MediaStore.ts

@@ -10,6 +10,8 @@ import { myCameraVisibilityStore } from "./MyCameraStoreVisibility";
 import { peerStore } from "./PeerStore";
 import { privacyShutdownStore } from "./PrivacyShutdownStore";
 import { MediaStreamConstraintsError } from "./Errors/MediaStreamConstraintsError";
+import { SoundMeter } from "../Phaser/Components/SoundMeter";
+import { AudioContext } from "standardized-audio-context";
 
 /**
  * A store that contains the camera state requested by the user (on or off).
@@ -541,6 +543,41 @@ export const obtainedMediaConstraintStore = derived<Readable<MediaStreamConstrai
     }
 );
 
+export const localVolumeStore = readable<number | undefined>(undefined, (set) => {
+    let timeout: ReturnType<typeof setTimeout>;
+    const unsubscribe = localStreamStore.subscribe((localStreamStoreValue) => {
+        clearInterval(timeout);
+        if (localStreamStoreValue.type === "error") {
+            set(undefined);
+            return;
+        }
+        const mediaStream = localStreamStoreValue.stream;
+
+        if (mediaStream === null || mediaStream.getAudioTracks().length <= 0) {
+            set(undefined);
+            return;
+        }
+        const soundMeter = new SoundMeter(mediaStream);
+        let error = false;
+
+        timeout = setInterval(() => {
+            try {
+                set(soundMeter.getVolume());
+            } catch (err) {
+                if (!error) {
+                    console.error(err);
+                    error = true;
+                }
+            }
+        }, 100);
+    });
+
+    return () => {
+        unsubscribe();
+        clearInterval(timeout);
+    };
+});
+
 /**
  * Device list
  */

front/src/WebRtc/VideoPeer.ts

@@ -10,6 +10,9 @@ import { playersStore } from "../Stores/PlayersStore";
 import { chatMessagesStore, newChatMessageSubject } from "../Stores/ChatStore";
 import { getIceServersConfig } from "../Components/Video/utils";
 import { isMediaBreakpointUp } from "../Utils/BreakpointsUtils";
+import { SoundMeter } from "../Phaser/Components/SoundMeter";
+import { AudioContext } from "standardized-audio-context";
+import { Console } from "console";
 
 const Peer: SimplePeerNamespace.SimplePeer = require("simple-peer");
 
@@ -33,6 +36,7 @@ export class VideoPeer extends Peer {
     private onBlockSubscribe: Subscription;
     private onUnBlockSubscribe: Subscription;
     public readonly streamStore: Readable<MediaStream | null>;
+    public readonly volumeStore: Readable<number | undefined>;
     public readonly statusStore: Readable<PeerStatus>;
     public readonly constraintsStore: Readable<ObtainedMediaStreamConstraints | null>;
     private newMessageSubscribtion: Subscription | undefined;
@@ -69,6 +73,34 @@ export class VideoPeer extends Peer {
             };
         });
 
+        this.volumeStore = readable<number | undefined>(undefined, (set) => {
+            let timeout: ReturnType<typeof setTimeout>;
+            const unsubscribe = this.streamStore.subscribe((mediaStream) => {
+                if (mediaStream === null || mediaStream.getAudioTracks().length <= 0) {
+                    set(undefined);
+                    return;
+                }
+                const soundMeter = new SoundMeter(mediaStream);
+                let error = false;
+
+                timeout = setInterval(() => {
+                    try {
+                        set(soundMeter.getVolume());
+                    } catch (err) {
+                        if (!error) {
+                            console.error(err);
+                            error = true;
+                        }
+                    }
+                }, 100);
+            });
+
+            return () => {
+                unsubscribe();
+                clearInterval(timeout);
+            };
+        });
+
         this.constraintsStore = readable<ObtainedMediaStreamConstraints | null>(null, (set) => {
             const onData = (chunk: Buffer) => {
                 const message = JSON.parse(chunk.toString("utf8"));