FEATURE: improved the mediaStore code to disable tracks instead of deleting them

front/src/Components/MyCamera.svelte

@@ -1,27 +1,11 @@
 <script lang="typescript">
+    import {obtainedMediaConstraintStore} from "../Stores/MediaStore";
     import {localStreamStore} from "../Stores/MediaStore";
     import SoundMeterWidget from "./SoundMeterWidget.svelte";
     import {onDestroy} from "svelte";
-
+    import {srcObject} from "./Video/utils";
-    function srcObject(node: HTMLVideoElement, stream: MediaStream) {
-        node.srcObject = stream;
-        return {
-            update(newStream: MediaStream) {
-                if (node.srcObject != newStream) {
-                    node.srcObject = newStream
-                }
-            }
-        }
-    }
 
     let stream : MediaStream|null;
-    /*$: {
-        if ($localStreamStore.type === 'success') {
-            stream = $localStreamStore.stream;
-        } else {
-            stream = null;
-        }
-    }*/
 
     const unsubscribe = localStreamStore.subscribe(value => {
         if (value.type === 'success') {
@@ -37,9 +21,9 @@
 
 
 <div>
-    <div class="video-container div-myCamVideo" class:hide={!$localStreamStore.constraints.video}>
+    <div class="video-container div-myCamVideo" class:hide={!$obtainedMediaConstraintStore.video}>
-        {#if $localStreamStore.type === "success" && $localStreamStore.stream }
+        {#if $localStreamStore.type === "success" && $localStreamStore.stream}
-        <video class="myCamVideo" use:srcObject={$localStreamStore.stream} autoplay muted playsinline></video>
+        <video class="myCamVideo" use:srcObject={stream} autoplay muted playsinline></video>
         <SoundMeterWidget stream={stream}></SoundMeterWidget>
         {/if}
     </div>

front/src/Phaser/Game/GameScene.ts

@@ -712,7 +712,7 @@ export class GameScene extends DirtyScene {
                 });
 
                 // When connection is performed, let's connect SimplePeer
-                this.simplePeer = new SimplePeer(this.connection, !this.room.isPublic, this.playerName);
+                this.simplePeer = new SimplePeer(this.connection);
                 peerStore.connectToSimplePeer(this.simplePeer);
                 screenSharingPeerStore.connectToSimplePeer(this.simplePeer);
                 videoFocusStore.connectToSimplePeer(this.simplePeer);

front/src/Stores/MediaStore.ts

@@ -324,14 +324,11 @@ export type LocalStreamStoreValue = StreamSuccessValue | StreamErrorValue;
 interface StreamSuccessValue {
     type: "success";
     stream: MediaStream | null;
-    // The constraints that we got (and not the one that have been requested)
-    constraints: MediaStreamConstraints;
 }
 
 interface StreamErrorValue {
     type: "error";
     error: Error;
-    constraints: MediaStreamConstraints;
 }
 
 let currentStream: MediaStream | null = null;
@@ -339,10 +336,13 @@ let currentStream: MediaStream | null = null;
 /**
  * Stops the camera from filming
  */
-function stopCamera(): void {
+function applyCameraConstraints(currentStream: MediaStream | null, constraints: MediaTrackConstraints | boolean): void {
     if (currentStream) {
         for (const track of currentStream.getVideoTracks()) {
-            track.stop();
+            track.enabled = constraints !== false;
+            if (constraints && constraints !== true) {
+                track.applyConstraints(constraints);
+            }
         }
     }
 }
@@ -350,10 +350,16 @@ function stopCamera(): void {
 /**
  * Stops the microphone from listening
  */
-function stopMicrophone(): void {
+function applyMicrophoneConstraints(
+    currentStream: MediaStream | null,
+    constraints: MediaTrackConstraints | boolean
+): void {
     if (currentStream) {
         for (const track of currentStream.getAudioTracks()) {
-            track.stop();
+            track.enabled = constraints !== false;
+            if (constraints && constraints !== true) {
+                track.applyConstraints(constraints);
+            }
         }
     }
 }
@@ -372,122 +378,96 @@ export const localStreamStore = derived<Readable<MediaStreamConstraints>, LocalS
                 set({
                     type: "error",
                     error: new Error("Unable to access your camera or microphone. You need to use a HTTPS connection."),
-                    constraints,
                 });
                 return;
             } else if (isIOS()) {
                 set({
                     type: "error",
                     error: new WebviewOnOldIOS(),
-                    constraints,
                 });
                 return;
             } else {
                 set({
                     type: "error",
                     error: new BrowserTooOldError(),
-                    constraints,
                 });
                 return;
             }
         }
 
-        if (constraints.audio === false) {
+        applyMicrophoneConstraints(currentStream, constraints.audio || false);
-            stopMicrophone();
+        applyCameraConstraints(currentStream, constraints.video || false);
-        }
-        if (constraints.video === false) {
-            stopCamera();
-        }
 
-        if (constraints.audio === false && constraints.video === false) {
+        if (currentStream === null) {
-            currentStream = null;
+            // we need to assign a first value to the stream because getUserMedia is async
             set({
                 type: "success",
                 stream: null,
-                constraints,
             });
-            return;
+            (async () => {
-        }
-
-        (async () => {
-            try {
-                stopMicrophone();
-                stopCamera();
-                currentStream = await navigator.mediaDevices.getUserMedia(constraints);
-                set({
-                    type: "success",
-                    stream: currentStream,
-                    constraints,
-                });
-                return;
-            } catch (e) {
-                if (constraints.video !== false) {
-                    console.info(
-                        "Error. Unable to get microphone and/or camera access. Trying audio only.",
-                        $mediaStreamConstraintsStore,
-                        e
-                    );
-                    // TODO: does it make sense to pop this error when retrying?
-                    set({
-                        type: "error",
-                        error: e,
-                        constraints,
-                    });
-                    // Let's try without video constraints
-                    requestedCameraState.disableWebcam();
-                } else {
-                    console.info(
-                        "Error. Unable to get microphone and/or camera access.",
-                        $mediaStreamConstraintsStore,
-                        e
-                    );
-                    set({
-                        type: "error",
-                        error: e,
-                        constraints,
-                    });
-                }
-
-                /*constraints.video = false;
-            if (constraints.audio === false) {
-                console.info("Error. Unable to get microphone and/or camera access.", $mediaStreamConstraintsStore, e);
-                set({
-                    type: 'error',
-                    error: e,
-                    constraints
-                });
-                // Let's make as if the user did not ask.
-                requestedCameraState.disableWebcam();
-            } else {
-                console.info("Error. Unable to get microphone and/or camera access. Trying audio only.", $mediaStreamConstraintsStore, e);
                 try {
                     currentStream = await navigator.mediaDevices.getUserMedia(constraints);
                     set({
-                        type: 'success',
+                        type: "success",
                         stream: currentStream,
-                        constraints
                     });
                     return;
-                } catch (e2) {
+                } catch (e) {
-                    console.info("Error. Unable to get microphone fallback access.", $mediaStreamConstraintsStore, e2);
+                    if (constraints.video !== false) {
-                    set({
+                        console.info(
-                        type: 'error',
+                            "Error. Unable to get microphone and/or camera access. Trying audio only.",
-                        error: e,
+                            $mediaStreamConstraintsStore,
-                        constraints
+                            e
-                    });
+                        );
+                        // TODO: does it make sense to pop this error when retrying?
+                        set({
+                            type: "error",
+                            error: e,
+                        });
+                        // Let's try without video constraints
+                        requestedCameraState.disableWebcam();
+                    } else {
+                        console.info(
+                            "Error. Unable to get microphone and/or camera access.",
+                            $mediaStreamConstraintsStore,
+                            e
+                        );
+                        set({
+                            type: "error",
+                            error: e,
+                        });
+                    }
                 }
-            }*/
+            })();
-            }
+        }
-        })();
     }
 );
 
+export interface ObtainedMediaStreamConstraints {
+    video: boolean;
+    audio: boolean;
+}
+
+let obtainedMediaConstraint: ObtainedMediaStreamConstraints = {
+    audio: false,
+    video: false,
+};
 /**
- * A store containing the real active media constrained (not the one requested by the user, but the one we got from the system)
+ * A store containing the actual states of audio and video (activated or deactivated)
  */
-export const obtainedMediaConstraintStore = derived(localStreamStore, ($localStreamStore) => {
+export const obtainedMediaConstraintStore = derived<Readable<MediaStreamConstraints>, ObtainedMediaStreamConstraints>(
-    return $localStreamStore.constraints;
+    mediaStreamConstraintsStore,
-});
+    ($mediaStreamConstraintsStore, set) => {
+        const newObtainedMediaConstraint = {
+            video: !!$mediaStreamConstraintsStore.video,
+            audio: !!$mediaStreamConstraintsStore.audio,
+        };
+        if (newObtainedMediaConstraint !== obtainedMediaConstraint) {
+            obtainedMediaConstraint = newObtainedMediaConstraint;
+            set(obtainedMediaConstraint);
+        }
+    }
+);
 
 /**
  * Device list

front/src/Stores/ScreenSharingStore.ts

@@ -1,7 +1,6 @@
-import { derived, get, Readable, readable, writable, Writable } from "svelte/store";
+import { derived, Readable, readable, writable } from "svelte/store";
 import { peerStore } from "./PeerStore";
 import type { LocalStreamStoreValue } from "./MediaStore";
-import { DivImportance } from "../WebRtc/LayoutManager";
 import { gameOverlayVisibilityStore } from "./GameOverlayStoreVisibility";
 
 declare const navigator: any; // eslint-disable-line @typescript-eslint/no-explicit-any
@@ -106,7 +105,6 @@ export const screenSharingLocalStreamStore = derived<Readable<MediaStreamConstra
             set({
                 type: "success",
                 stream: null,
-                constraints,
             });
             return;
         }
@@ -121,7 +119,6 @@ export const screenSharingLocalStreamStore = derived<Readable<MediaStreamConstra
             set({
                 type: "error",
                 error: new Error("Your browser does not support sharing screen"),
-                constraints,
             });
             return;
         }
@@ -141,10 +138,6 @@ export const screenSharingLocalStreamStore = derived<Readable<MediaStreamConstra
                         set({
                             type: "success",
                             stream: null,
-                            constraints: {
-                                video: false,
-                                audio: false,
-                            },
                         });
                     };
                 }
@@ -152,7 +145,6 @@ export const screenSharingLocalStreamStore = derived<Readable<MediaStreamConstra
                 set({
                     type: "success",
                     stream: currentStream,
-                    constraints,
                 });
                 return;
             } catch (e) {
@@ -162,7 +154,6 @@ export const screenSharingLocalStreamStore = derived<Readable<MediaStreamConstra
                 set({
                     type: "error",
                     error: e,
-                    constraints,
                 });
             }
         })();
@@ -184,7 +175,6 @@ export const screenSharingAvailableStore = derived(peerStore, ($peerStore, set)
 export interface ScreenSharingLocalMedia {
     uniqueId: string;
     stream: MediaStream | null;
-    //subscribe(this: void, run: Subscriber<ScreenSharingLocalMedia>, invalidate?: (value?: ScreenSharingLocalMedia) => void): Unsubscriber;
 }
 
 /**

front/src/WebRtc/SimplePeer.ts

@@ -4,16 +4,12 @@ import type {
 } from "../Connexion/ConnexionModels";
 import { mediaManager, StartScreenSharingCallback, StopScreenSharingCallback } from "./MediaManager";
 import { ScreenSharingPeer } from "./ScreenSharingPeer";
-import { MESSAGE_TYPE_BLOCKED, MESSAGE_TYPE_CONSTRAINT, MESSAGE_TYPE_MESSAGE, VideoPeer } from "./VideoPeer";
+import { VideoPeer } from "./VideoPeer";
 import type { RoomConnection } from "../Connexion/RoomConnection";
 import { blackListManager } from "./BlackListManager";
 import { get } from "svelte/store";
-import { localStreamStore, LocalStreamStoreValue, obtainedMediaConstraintStore } from "../Stores/MediaStore";
 import { screenSharingLocalStreamStore } from "../Stores/ScreenSharingStore";
-import { discussionManager } from "./DiscussionManager";
 import { playersStore } from "../Stores/PlayersStore";
-import { newChatMessageStore } from "../Stores/ChatStore";
-import { isMobile } from "../Enum/EnvironmentVariable";
 
 export interface UserSimplePeerInterface {
     userId: number;
@@ -46,19 +42,14 @@ export class SimplePeer {
     private lastWebrtcUserName: string | undefined;
     private lastWebrtcPassword: string | undefined;
 
-    constructor(private Connection: RoomConnection, private enableReporting: boolean, private myName: string) {
+    constructor(private Connection: RoomConnection) {
         // We need to go through this weird bound function pointer in order to be able to "free" this reference later.
         this.sendLocalScreenSharingStreamCallback = this.sendLocalScreenSharingStream.bind(this);
         this.stopLocalScreenSharingStreamCallback = this.stopLocalScreenSharingStream.bind(this);
 
-        this.unsubscribers.push(
-            localStreamStore.subscribe((streamResult) => {
-                this.sendLocalVideoStream(streamResult);
-            })
-        );
-
         let localScreenCapture: MediaStream | null = null;
 
+        //todo
         this.unsubscribers.push(
             screenSharingLocalStreamStore.subscribe((streamResult) => {
                 if (streamResult.type === "error") {
@@ -126,19 +117,14 @@ export class SimplePeer {
         if (!user.initiator) {
             return;
         }
-        const streamResult = get(localStreamStore);
-        let stream: MediaStream | null = null;
-        if (streamResult.type === "success" && streamResult.stream) {
-            stream = streamResult.stream;
-        }
 
-        this.createPeerConnection(user, stream);
+        this.createPeerConnection(user);
     }
 
     /**
      * create peer connection to bind users
      */
-    private createPeerConnection(user: UserSimplePeerInterface, localStream: MediaStream | null): VideoPeer | null {
+    private createPeerConnection(user: UserSimplePeerInterface): VideoPeer | null {
         const peerConnection = this.PeerConnectionArray.get(user.userId);
         if (peerConnection) {
             if (peerConnection.destroyed) {
@@ -160,7 +146,7 @@ export class SimplePeer {
         this.lastWebrtcUserName = user.webRtcUser;
         this.lastWebrtcPassword = user.webRtcPassword;
 
-        const peer = new VideoPeer(user, user.initiator ? user.initiator : false, name, this.Connection, localStream);
+        const peer = new VideoPeer(user, user.initiator ? user.initiator : false, name, this.Connection);
 
         peer.toClose = false;
         // When a connection is established to a video stream, and if a screen sharing is taking place,
@@ -204,7 +190,7 @@ export class SimplePeer {
                 if (!peerConnexionDeleted) {
                     throw "Error to delete peer connection";
                 }
-                this.createPeerConnection(user, stream);
+                this.createPeerConnection(user);
             } else {
                 peerConnection.toClose = false;
             }
@@ -282,7 +268,6 @@ export class SimplePeer {
      */
     private closeScreenSharingConnection(userId: number) {
         try {
-            //mediaManager.removeActiveScreenSharingVideo("" + userId);
             const peer = this.PeerScreenSharingConnectionArray.get(userId);
             if (peer === undefined) {
                 console.warn(
@@ -295,12 +280,6 @@ export class SimplePeer {
             // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray"
             // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel.
             peer.destroy();
-
-            //Comment this peer connection because if we delete and try to reshare screen, the RTCPeerConnection send renegotiate event. This array will be remove when user left circle discussion
-            /*if(!this.PeerScreenSharingConnectionArray.delete(userId)){
-                throw 'Couln\'t delete peer screen sharing connexion';
-            }*/
-            //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size);
         } catch (err) {
             console.error("closeConnection", err);
         }
@@ -330,13 +309,7 @@ export class SimplePeer {
         try {
             //if offer type, create peer connection
             if (data.signal.type === "offer") {
-                const streamResult = get(localStreamStore);
+                this.createPeerConnection(data);
-                let stream: MediaStream | null = null;
-                if (streamResult.type === "success" && streamResult.stream) {
-                    stream = streamResult.stream;
-                }
-
-                this.createPeerConnection(data, stream);
             }
             const peer = this.PeerConnectionArray.get(data.userId);
             if (peer !== undefined) {
@@ -379,42 +352,10 @@ export class SimplePeer {
         } catch (e) {
             console.error(`receiveWebrtcSignal => ${data.userId}`, e);
             //Comment this peer connection because if we delete and try to reshare screen, the RTCPeerConnection send renegotiate event. This array will be remove when user left circle discussion
-            //this.PeerScreenSharingConnectionArray.delete(data.userId);
             this.receiveWebrtcScreenSharingSignal(data);
         }
     }
 
-    private pushVideoToRemoteUser(userId: number, streamResult: LocalStreamStoreValue) {
-        try {
-            const PeerConnection = this.PeerConnectionArray.get(userId);
-            if (!PeerConnection) {
-                throw new Error("While adding media, cannot find user with ID " + userId);
-            }
-
-            PeerConnection.write(
-                new Buffer(
-                    JSON.stringify({
-                        type: MESSAGE_TYPE_CONSTRAINT,
-                        ...streamResult.constraints,
-                        isMobile: isMobile(),
-                    })
-                )
-            );
-
-            if (streamResult.type === "error") {
-                return;
-            }
-            const localStream: MediaStream | null = streamResult.stream;
-
-            if (!localStream) {
-                return;
-            }
-            PeerConnection.addStream(localStream);
-        } catch (e) {
-            console.error(`pushVideoToRemoteUser => ${userId}`, e);
-        }
-    }
-
     private pushScreenSharingToRemoteUser(userId: number, localScreenCapture: MediaStream) {
         const PeerConnection = this.PeerScreenSharingConnectionArray.get(userId);
         if (!PeerConnection) {
@@ -427,12 +368,6 @@ export class SimplePeer {
         return;
     }
 
-    public sendLocalVideoStream(streamResult: LocalStreamStoreValue) {
-        for (const user of this.Users) {
-            this.pushVideoToRemoteUser(user.userId, streamResult);
-        }
-    }
-
     /**
      * Triggered locally when clicking on the screen sharing button
      */
@@ -486,8 +421,6 @@ export class SimplePeer {
 
         if (!PeerConnectionScreenSharing.isReceivingScreenSharingStream()) {
             PeerConnectionScreenSharing.destroy();
-            //Comment this peer connection because if we delete and try to reshare screen, the RTCPeerConnection send renegotiate event. This array will be remove when user left circle discussion
-            //this.PeerScreenSharingConnectionArray.delete(userId);
         }
     }
 }

front/src/WebRtc/VideoPeer.ts

@@ -5,7 +5,12 @@ import { blackListManager } from "./BlackListManager";
 import type { Subscription } from "rxjs";
 import type { UserSimplePeerInterface } from "./SimplePeer";
 import { get, readable, Readable, Unsubscriber } from "svelte/store";
-import { obtainedMediaConstraintIsMobileStore, obtainedMediaConstraintStore } from "../Stores/MediaStore";
+import {
+    localStreamStore,
+    obtainedMediaConstraintIsMobileStore,
+    obtainedMediaConstraintStore,
+    ObtainedMediaStreamConstraints,
+} from "../Stores/MediaStore";
 import { playersStore } from "../Stores/PlayersStore";
 import { chatMessagesStore, chatVisibilityStore, newChatMessageStore } from "../Stores/ChatStore";
 import { getIceServersConfig } from "../Components/Video/utils";
@@ -34,16 +39,17 @@ export class VideoPeer extends Peer {
     private onUnBlockSubscribe: Subscription;
     public readonly streamStore: Readable<MediaStream | null>;
     public readonly statusStore: Readable<PeerStatus>;
-    public readonly constraintsStore: Readable<MediaStreamConstraints | null>;
+    public readonly constraintsStore: Readable<ObtainedMediaStreamConstraints | null>;
     private newMessageunsubscriber: Unsubscriber | null = null;
     private closing: Boolean = false; //this is used to prevent destroy() from being called twice
+    private localStreamStoreSubscribe: Unsubscriber;
+    private obtainedMediaConstraintStoreSubscribe: Unsubscriber;
 
     constructor(
         public user: UserSimplePeerInterface,
         initiator: boolean,
         public readonly userName: string,
-        private connection: RoomConnection,
+        private connection: RoomConnection
-        localStream: MediaStream | null
     ) {
         super({
             initiator,
@@ -60,27 +66,15 @@ export class VideoPeer extends Peer {
             const onStream = (stream: MediaStream | null) => {
                 set(stream);
             };
-            const onData = (chunk: Buffer) => {
-                this.on("data", (chunk: Buffer) => {
-                    const message = JSON.parse(chunk.toString("utf8"));
-                    if (message.type === MESSAGE_TYPE_CONSTRAINT) {
-                        if (!message.video) {
-                            set(null);
-                        }
-                    }
-                });
-            };
 
             this.on("stream", onStream);
-            this.on("data", onData);
 
             return () => {
                 this.off("stream", onStream);
-                this.off("data", onData);
             };
         });
 
-        this.constraintsStore = readable<MediaStreamConstraints | null>(null, (set) => {
+        this.constraintsStore = readable<ObtainedMediaStreamConstraints | null>(null, (set) => {
             const onData = (chunk: Buffer) => {
                 const message = JSON.parse(chunk.toString("utf8"));
                 if (message.type === MESSAGE_TYPE_CONSTRAINT) {
@@ -191,7 +185,6 @@ export class VideoPeer extends Peer {
             this._onFinish();
         });
 
-        this.pushVideoToRemoteUser(localStream);
         this.onBlockSubscribe = blackListManager.onBlockStream.subscribe((userUuid) => {
             if (userUuid === this.userUuid) {
                 this.toggleRemoteStream(false);
@@ -208,6 +201,21 @@ export class VideoPeer extends Peer {
         if (blackListManager.isBlackListed(this.userUuid)) {
             this.sendBlockMessage(true);
         }
+
+        this.localStreamStoreSubscribe = localStreamStore.subscribe((streamValue) => {
+            if (streamValue.type === "success" && streamValue.stream) this.addStream(streamValue.stream);
+        });
+        this.obtainedMediaConstraintStoreSubscribe = obtainedMediaConstraintStore.subscribe((constraints) => {
+            this.write(
+                new Buffer(
+                    JSON.stringify({
+                        type: MESSAGE_TYPE_CONSTRAINT,
+                        ...constraints,
+                        isMobile: isMobile(),
+                    })
+                )
+            );
+        });
     }
 
     private sendBlockMessage(blocking: boolean) {
@@ -264,6 +272,8 @@ export class VideoPeer extends Peer {
             this.onUnBlockSubscribe.unsubscribe();
             if (this.newMessageunsubscriber) this.newMessageunsubscriber();
             chatMessagesStore.addOutcomingUser(this.userId);
+            if (this.localStreamStoreSubscribe) this.localStreamStoreSubscribe();
+            if (this.obtainedMediaConstraintStoreSubscribe) this.obtainedMediaConstraintStoreSubscribe();
             super.destroy();
         } catch (err) {
             console.error("VideoPeer::destroy", err);
@@ -281,28 +291,4 @@ export class VideoPeer extends Peer {
             this.once("connect", destroySoon);
         }
     }
-
-    private pushVideoToRemoteUser(localStream: MediaStream | null) {
-        try {
-            this.write(
-                new Buffer(
-                    JSON.stringify({
-                        type: MESSAGE_TYPE_CONSTRAINT,
-                        ...get(obtainedMediaConstraintStore),
-                        isMobile: isMobile(),
-                    })
-                )
-            );
-
-            if (!localStream) {
-                return;
-            }
-
-            for (const track of localStream.getTracks()) {
-                this.addTrack(track, localStream);
-            }
-        } catch (e) {
-            console.error(`pushVideoToRemoteUser => ${this.userId}`, e);
-        }
-    }
 }