From e7b0f859a567f21dffe9e88a831206b399dbeb31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Fri, 11 Jun 2021 11:29:36 +0200 Subject: [PATCH 01/21] Migrating the video overlay in Svelte (WIP) --- front/src/Components/App.svelte | 6 +- .../HorizontalSoundMeterWidget.svelte | 2 +- front/src/Components/SoundMeterWidget.svelte | 10 +- .../Components/Video/LocalStreamMedia.svelte | 20 ++++ front/src/Components/Video/Peer.svelte | 20 ++++ .../Video/ScreenSharingMedia.svelte | 57 +++++++++ front/src/Components/Video/VideoMedia.svelte | 75 ++++++++++++ .../src/Components/Video/VideoOverlay.svelte | 32 +++++ .../src/Components/Video/images/blockSign.svg | 22 ++++ front/src/Components/Video/images/report.svg | 1 + front/src/Phaser/Game/GameScene.ts | 7 +- .../src/Stores/GameOverlayStoreVisibility.ts | 17 +++ front/src/Stores/LayoutStore.ts | 58 +++++++++ front/src/Stores/MediaStore.ts | 17 +-- front/src/Stores/PeerStore.ts | 111 ++++++++++++++++-- front/src/Stores/ScreenSharingStore.ts | 46 ++++++-- front/src/WebRtc/MediaManager.ts | 37 ++---- front/src/WebRtc/ScreenSharingPeer.ts | 73 ++++++++++-- front/src/WebRtc/SimplePeer.ts | 36 ++++-- front/src/WebRtc/VideoPeer.ts | 80 ++++++++++++- 20 files changed, 630 insertions(+), 97 deletions(-) create mode 100644 front/src/Components/Video/LocalStreamMedia.svelte create mode 100644 front/src/Components/Video/Peer.svelte create mode 100644 front/src/Components/Video/ScreenSharingMedia.svelte create mode 100644 front/src/Components/Video/VideoMedia.svelte create mode 100644 front/src/Components/Video/VideoOverlay.svelte create mode 100644 front/src/Components/Video/images/blockSign.svg create mode 100644 front/src/Components/Video/images/report.svg create mode 100644 front/src/Stores/GameOverlayStoreVisibility.ts create mode 100644 front/src/Stores/LayoutStore.ts diff --git a/front/src/Components/App.svelte b/front/src/Components/App.svelte index c973a6c2..1d492ab7 100644 --- a/front/src/Components/App.svelte +++ b/front/src/Components/App.svelte @@ -1,5 +1,5 @@
@@ -66,6 +69,7 @@ --> {#if $gameOverlayVisibilityStore}
+
diff --git a/front/src/Components/EnableCamera/HorizontalSoundMeterWidget.svelte b/front/src/Components/EnableCamera/HorizontalSoundMeterWidget.svelte index a22da2fa..79ad1810 100644 --- a/front/src/Components/EnableCamera/HorizontalSoundMeterWidget.svelte +++ b/front/src/Components/EnableCamera/HorizontalSoundMeterWidget.svelte @@ -58,7 +58,7 @@
- {#each [...Array(NB_BARS).keys()] as i} + {#each [...Array(NB_BARS).keys()] as i (i)}
{/each}
diff --git a/front/src/Components/SoundMeterWidget.svelte b/front/src/Components/SoundMeterWidget.svelte index 30650e3f..94e0cdd2 100644 --- a/front/src/Components/SoundMeterWidget.svelte +++ b/front/src/Components/SoundMeterWidget.svelte @@ -23,7 +23,7 @@ timeout = setInterval(() => { try{ - volume = parseInt((soundMeter.getVolume() / 100 * NB_BARS).toFixed(0)); + volume = soundMeter.getVolume(); //console.log(volume); }catch(err){ @@ -45,9 +45,9 @@
- 1}> - 2}> - 3}> - 4}> 5}> + 10}> + 15}> + 40}> + 70}>
diff --git a/front/src/Components/Video/LocalStreamMedia.svelte b/front/src/Components/Video/LocalStreamMedia.svelte new file mode 100644 index 00000000..43b1d117 --- /dev/null +++ b/front/src/Components/Video/LocalStreamMedia.svelte @@ -0,0 +1,20 @@ + + + +
+ +
diff --git a/front/src/Components/Video/Peer.svelte b/front/src/Components/Video/Peer.svelte new file mode 100644 index 00000000..c73d620e --- /dev/null +++ b/front/src/Components/Video/Peer.svelte @@ -0,0 +1,20 @@ + + +
+ {#if peer instanceof VideoPeer} + + {:else if peer instanceof ScreenSharingPeer} + + {:else} + + {/if} +
diff --git a/front/src/Components/Video/ScreenSharingMedia.svelte b/front/src/Components/Video/ScreenSharingMedia.svelte new file mode 100644 index 00000000..e16fac58 --- /dev/null +++ b/front/src/Components/Video/ScreenSharingMedia.svelte @@ -0,0 +1,57 @@ + + +
+ {#if $statusStore === 'connecting'} + + {/if} + {#if $statusStore === 'error'} +
+ {/if} + {#if $streamStore === null} + {name} + {/if} + +
+ + diff --git a/front/src/Components/Video/VideoMedia.svelte b/front/src/Components/Video/VideoMedia.svelte new file mode 100644 index 00000000..fbc5c6f7 --- /dev/null +++ b/front/src/Components/Video/VideoMedia.svelte @@ -0,0 +1,75 @@ + + +
+ {#if $statusStore === 'connecting'} + + {/if} + {#if $statusStore === 'error'} +
+ {/if} + {#if !$constraintStore || $constraintStore.video === false} + {name} + {/if} + {#if $constraintStore && $constraintStore.audio === false} + Muted + {/if} + + + + {#if $constraintStore && $constraintStore.audio !== false} + + {/if} +
+ + diff --git a/front/src/Components/Video/VideoOverlay.svelte b/front/src/Components/Video/VideoOverlay.svelte new file mode 100644 index 00000000..5ba7fbc7 --- /dev/null +++ b/front/src/Components/Video/VideoOverlay.svelte @@ -0,0 +1,32 @@ + + +
+
+ {#each [...$layoutStore.get(DivImportance.Important).values()] as peer (peer.uniqueId)} + + {/each} +
+ + + + +
+ + diff --git a/front/src/Components/Video/images/blockSign.svg b/front/src/Components/Video/images/blockSign.svg new file mode 100644 index 00000000..c64ba294 --- /dev/null +++ b/front/src/Components/Video/images/blockSign.svg @@ -0,0 +1,22 @@ + + + + + + + image/svg+xml + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/front/src/Components/Video/images/report.svg b/front/src/Components/Video/images/report.svg new file mode 100644 index 00000000..14753256 --- /dev/null +++ b/front/src/Components/Video/images/report.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/front/src/Phaser/Game/GameScene.ts b/front/src/Phaser/Game/GameScene.ts index b5876d5a..1bec39fc 100644 --- a/front/src/Phaser/Game/GameScene.ts +++ b/front/src/Phaser/Game/GameScene.ts @@ -30,7 +30,7 @@ import {PlayerMovement} from "./PlayerMovement"; import {PlayersPositionInterpolator} from "./PlayersPositionInterpolator"; import {RemotePlayer} from "../Entity/RemotePlayer"; import {Queue} from 'queue-typescript'; -import {SimplePeer, UserSimplePeerInterface} from "../../WebRtc/SimplePeer"; +import {SimplePeer} from "../../WebRtc/SimplePeer"; import {ReconnectingSceneName} from "../Reconnecting/ReconnectingScene"; import {lazyLoadPlayerCharacterTextures, loadCustomTexture} from "../Entity/PlayerTexturesLoadingManager"; import { @@ -93,7 +93,7 @@ import {PinchManager} from "../UserInput/PinchManager"; import {joystickBaseImg, joystickBaseKey, joystickThumbImg, joystickThumbKey} from "../Components/MobileJoystick"; import {DEPTH_OVERLAY_INDEX} from "./DepthIndexes"; import {waScaleManager} from "../Services/WaScaleManager"; -import {peerStore} from "../../Stores/PeerStore"; +import {peerStore, screenSharingPeerStore} from "../../Stores/PeerStore"; import {EmoteManager} from "./EmoteManager"; export interface GameSceneInitInterface { @@ -646,12 +646,13 @@ export class GameScene extends DirtyScene implements CenterListener { // When connection is performed, let's connect SimplePeer this.simplePeer = new SimplePeer(this.connection, !this.room.isPublic, this.playerName); peerStore.connectToSimplePeer(this.simplePeer); + screenSharingPeerStore.connectToSimplePeer(this.simplePeer); this.GlobalMessageManager = new GlobalMessageManager(this.connection); userMessageManager.setReceiveBanListener(this.bannedUser.bind(this)); const self = this; this.simplePeer.registerPeerConnectionListener({ - onConnect(user: UserSimplePeerInterface) { + onConnect(peer) { self.presentationModeSprite.setVisible(true); self.chatModeSprite.setVisible(true); self.openChatIcon.setVisible(true); diff --git a/front/src/Stores/GameOverlayStoreVisibility.ts b/front/src/Stores/GameOverlayStoreVisibility.ts new file mode 100644 index 00000000..c58c929d --- /dev/null +++ b/front/src/Stores/GameOverlayStoreVisibility.ts @@ -0,0 +1,17 @@ +import {writable} from "svelte/store"; + +/** + * A store that contains whether the game overlay is shown or not. + * Typically, the overlay is hidden when entering Jitsi meet. + */ +function createGameOverlayVisibilityStore() { + const { subscribe, set, update } = writable(false); + + return { + subscribe, + showGameOverlay: () => set(true), + hideGameOverlay: () => set(false), + }; +} + +export const gameOverlayVisibilityStore = createGameOverlayVisibilityStore(); diff --git a/front/src/Stores/LayoutStore.ts b/front/src/Stores/LayoutStore.ts new file mode 100644 index 00000000..45e6f604 --- /dev/null +++ b/front/src/Stores/LayoutStore.ts @@ -0,0 +1,58 @@ +import {derived, get} from "svelte/store"; +import {ScreenSharingLocalMedia, screenSharingLocalMedia} from "./ScreenSharingStore"; +import {DivImportance} from "../WebRtc/LayoutManager"; +import { peerStore, screenSharingStreamStore} from "./PeerStore"; +import type {RemotePeer} from "../WebRtc/SimplePeer"; + +export type DisplayableMedia = RemotePeer | ScreenSharingLocalMedia; + +/** + * A store that contains the layout of the streams + */ +function createLayoutStore() { + + let unsubscribes: (()=>void)[] = []; + + return derived([ + screenSharingStreamStore, + peerStore, + screenSharingLocalMedia, + ], ([ + $screenSharingStreamStore, + $peerStore, + $screenSharingLocalMedia, + ], set) => { + for (const unsubscribe of unsubscribes) { + unsubscribe(); + } + unsubscribes = []; + + const peers = new Map>(); + peers.set(DivImportance.Normal, new Map()); + peers.set(DivImportance.Important, new Map()); + + const addPeer = (peer: DisplayableMedia) => { + const importance = get(peer.importanceStore); + + peers.get(importance)?.set(peer.uniqueId, peer); + + unsubscribes.push(peer.importanceStore.subscribe((importance) => { + peers.forEach((category) => { + category.delete(peer.uniqueId); + }); + peers.get(importance)?.set(peer.uniqueId, peer); + set(peers); + })); + }; + + $screenSharingStreamStore.forEach(addPeer); + $peerStore.forEach(addPeer); + if ($screenSharingLocalMedia?.stream) { + addPeer($screenSharingLocalMedia); + } + + set(peers); + }); +} + +export const layoutStore = createLayoutStore(); diff --git a/front/src/Stores/MediaStore.ts b/front/src/Stores/MediaStore.ts index d622511e..b2cd9f42 100644 --- a/front/src/Stores/MediaStore.ts +++ b/front/src/Stores/MediaStore.ts @@ -1,13 +1,13 @@ import {derived, get, Readable, readable, writable, Writable} from "svelte/store"; import {peerStore} from "./PeerStore"; import {localUserStore} from "../Connexion/LocalUserStore"; -import {ITiledMapGroupLayer, ITiledMapObjectLayer, ITiledMapTileLayer} from "../Phaser/Map/ITiledMap"; import {userMovingStore} from "./GameStore"; import {HtmlUtils} from "../WebRtc/HtmlUtils"; import {BrowserTooOldError} from "./Errors/BrowserTooOldError"; import {errorStore} from "./ErrorStore"; import {isIOS} from "../WebRtc/DeviceUtils"; import {WebviewOnOldIOS} from "./Errors/WebviewOnOldIOS"; +import {gameOverlayVisibilityStore} from "./GameOverlayStoreVisibility"; /** * A store that contains the camera state requested by the user (on or off). @@ -50,20 +50,6 @@ export const visibilityStore = readable(document.visibilityState === 'visible', }; }); -/** - * A store that contains whether the game overlay is shown or not. - * Typically, the overlay is hidden when entering Jitsi meet. - */ -function createGameOverlayVisibilityStore() { - const { subscribe, set, update } = writable(false); - - return { - subscribe, - showGameOverlay: () => set(true), - hideGameOverlay: () => set(false), - }; -} - /** * A store that contains whether the EnableCameraScene is shown or not. */ @@ -79,7 +65,6 @@ function createEnableCameraSceneVisibilityStore() { export const requestedCameraState = createRequestedCameraState(); export const requestedMicrophoneState = createRequestedMicrophoneState(); -export const gameOverlayVisibilityStore = createGameOverlayVisibilityStore(); export const enableCameraSceneVisibilityStore = createEnableCameraSceneVisibilityStore(); /** diff --git a/front/src/Stores/PeerStore.ts b/front/src/Stores/PeerStore.ts index a582e692..df8b17ce 100644 --- a/front/src/Stores/PeerStore.ts +++ b/front/src/Stores/PeerStore.ts @@ -1,26 +1,64 @@ -import { derived, writable, Writable } from "svelte/store"; -import type {UserSimplePeerInterface} from "../WebRtc/SimplePeer"; -import type {SimplePeer} from "../WebRtc/SimplePeer"; +import {derived, get, readable, writable} from "svelte/store"; +import type {RemotePeer, SimplePeer} from "../WebRtc/SimplePeer"; +import {VideoPeer} from "../WebRtc/VideoPeer"; +import {ScreenSharingPeer} from "../WebRtc/ScreenSharingPeer"; /** - * A store that contains the camera state requested by the user (on or off). + * A store that contains the list of (video) peers we are connected to. */ function createPeerStore() { - let users = new Map(); + let peers = new Map(); - const { subscribe, set, update } = writable(users); + const { subscribe, set, update } = writable(peers); return { subscribe, connectToSimplePeer: (simplePeer: SimplePeer) => { - users = new Map(); - set(users); + peers = new Map(); + set(peers); simplePeer.registerPeerConnectionListener({ - onConnect(user: UserSimplePeerInterface) { + onConnect(peer: RemotePeer) { + if (peer instanceof VideoPeer) { + update(users => { + users.set(peer.userId, peer); + return users; + }); + } + console.log('CONNECT VIDEO', peers); + }, + onDisconnect(userId: number) { update(users => { - users.set(user.userId, user); + users.delete(userId); return users; }); + console.log('DISCONNECT VIDEO', peers); + } + }) + } + }; +} + +/** + * A store that contains the list of screen sharing peers we are connected to. + */ +function createScreenSharingPeerStore() { + let peers = new Map(); + + const { subscribe, set, update } = writable(peers); + + return { + subscribe, + connectToSimplePeer: (simplePeer: SimplePeer) => { + peers = new Map(); + set(peers); + simplePeer.registerPeerConnectionListener({ + onConnect(peer: RemotePeer) { + if (peer instanceof ScreenSharingPeer) { + update(users => { + users.set(peer.userId, peer); + return users; + }); + } }, onDisconnect(userId: number) { update(users => { @@ -34,3 +72,56 @@ function createPeerStore() { } export const peerStore = createPeerStore(); +export const screenSharingPeerStore = createScreenSharingPeerStore(); + +/** + * A store that contains ScreenSharingPeer, ONLY if those ScreenSharingPeer are emitting a stream towards us! + */ +function createScreenSharingStreamStore() { + let peers = new Map(); + + return readable>(peers, function start(set) { + + let unsubscribes: (()=>void)[] = []; + + const unsubscribe = screenSharingPeerStore.subscribe((screenSharingPeers) => { + for (const unsubscribe of unsubscribes) { + unsubscribe(); + } + unsubscribes = []; + + peers = new Map(); + + screenSharingPeers.forEach((screenSharingPeer: ScreenSharingPeer, key: number) => { + + if (screenSharingPeer.isReceivingScreenSharingStream()) { + peers.set(key, screenSharingPeer); + } + + unsubscribes.push(screenSharingPeer.streamStore.subscribe((stream) => { + if (stream) { + peers.set(key, screenSharingPeer); + } else { + peers.delete(key); + } + set(peers); + })); + + }); + + set(peers); + + }); + + return function stop() { + unsubscribe(); + for (const unsubscribe of unsubscribes) { + unsubscribe(); + } + }; + }) +} + +export const screenSharingStreamStore = createScreenSharingStreamStore(); + + diff --git a/front/src/Stores/ScreenSharingStore.ts b/front/src/Stores/ScreenSharingStore.ts index ec5aa46f..41d450c2 100644 --- a/front/src/Stores/ScreenSharingStore.ts +++ b/front/src/Stores/ScreenSharingStore.ts @@ -1,16 +1,10 @@ import {derived, get, Readable, readable, writable, Writable} from "svelte/store"; import {peerStore} from "./PeerStore"; -import {localUserStore} from "../Connexion/LocalUserStore"; -import {ITiledMapGroupLayer, ITiledMapObjectLayer, ITiledMapTileLayer} from "../Phaser/Map/ITiledMap"; -import {userMovingStore} from "./GameStore"; -import {HtmlUtils} from "../WebRtc/HtmlUtils"; -import { - audioConstraintStore, cameraEnergySavingStore, - enableCameraSceneVisibilityStore, - gameOverlayVisibilityStore, LocalStreamStoreValue, privacyShutdownStore, - requestedCameraState, - requestedMicrophoneState, videoConstraintStore +import type { + LocalStreamStoreValue, } from "./MediaStore"; +import {DivImportance} from "../WebRtc/LayoutManager"; +import {gameOverlayVisibilityStore} from "./GameOverlayStoreVisibility"; declare const navigator:any; // eslint-disable-line @typescript-eslint/no-explicit-any @@ -191,3 +185,35 @@ export const screenSharingAvailableStore = derived(peerStore, ($peerStore, set) set($peerStore.size !== 0); }); + +export interface ScreenSharingLocalMedia { + uniqueId: string; + importanceStore: Writable; + stream: MediaStream|null; + //subscribe(this: void, run: Subscriber, invalidate?: (value?: ScreenSharingLocalMedia) => void): Unsubscriber; +} + +/** + * The representation of the screen sharing stream. + */ +export const screenSharingLocalMedia = readable(null, function start(set) { + + const localMedia: ScreenSharingLocalMedia = { + uniqueId: "localScreenSharingStream", + importanceStore: writable(DivImportance.Normal), + stream: null + } + + const unsubscribe = screenSharingLocalStreamStore.subscribe((screenSharingLocalStream) => { + if (screenSharingLocalStream.type === "success") { + localMedia.stream = screenSharingLocalStream.stream; + } else { + localMedia.stream = null; + } + set(localMedia); + }); + + return function stop() { + unsubscribe(); + }; +}) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index efc9660a..a71618e0 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -7,7 +7,7 @@ import type {UserSimplePeerInterface} from "./SimplePeer"; import {SoundMeter} from "../Phaser/Components/SoundMeter"; import {DISABLE_NOTIFICATIONS} from "../Enum/EnvironmentVariable"; import { - gameOverlayVisibilityStore, localStreamStore, + localStreamStore, } from "../Stores/MediaStore"; import { screenSharingLocalStreamStore @@ -22,6 +22,7 @@ export type ShowReportCallBack = (userId: string, userName: string|undefined) => export type HelpCameraSettingsCallBack = () => void; import {cowebsiteCloseButtonId} from "./CoWebsiteManager"; +import {gameOverlayVisibilityStore} from "../Stores/GameOverlayStoreVisibility"; export class MediaManager { private remoteVideo: Map = new Map(); @@ -65,7 +66,7 @@ export class MediaManager { } }); - let isScreenSharing = false; + //let isScreenSharing = false; screenSharingLocalStreamStore.subscribe((result) => { if (result.type === 'error') { console.error(result.error); @@ -75,7 +76,7 @@ export class MediaManager { return; } - if (result.stream !== null) { + /*if (result.stream !== null) { isScreenSharing = true; this.addScreenSharingActiveVideo('me', DivImportance.Normal); HtmlUtils.getElementByIdOrFail('screen-sharing-me').srcObject = result.stream; @@ -84,7 +85,7 @@ export class MediaManager { isScreenSharing = false; this.removeActiveScreenSharingVideo('me'); } - } + }*/ }); @@ -134,7 +135,7 @@ export class MediaManager { gameOverlayVisibilityStore.hideGameOverlay(); } - addActiveVideo(user: UserSimplePeerInterface, userName: string = ""){ + /*addActiveVideo(user: UserSimplePeerInterface, userName: string = ""){ const userId = ''+user.userId userName = userName.toUpperCase(); @@ -194,7 +195,7 @@ export class MediaManager { layoutManager.add(divImportance, userId, html); this.remoteVideo.set(userId, HtmlUtils.getElementByIdOrFail(userId)); - } + }*/ private getScreenSharingId(userId: string): string { return `screen-sharing-${userId}`; @@ -242,19 +243,12 @@ export class MediaManager { const blockLogoElement = HtmlUtils.getElementByIdOrFail('blocking-'+userId); show ? blockLogoElement.classList.add('active') : blockLogoElement.classList.remove('active'); } - addStreamRemoteVideo(userId: string, stream : MediaStream): void { + /*addStreamRemoteVideo(userId: string, stream : MediaStream): void { const remoteVideo = this.remoteVideo.get(userId); if (remoteVideo === undefined) { throw `Unable to find video for ${userId}`; } remoteVideo.srcObject = stream; - - //FIX ME SOUNDMETER: check stalability of sound meter calculation - //sound metter - /*const soundMeter = new SoundMeter(); - soundMeter.connectToSource(stream, new AudioContext()); - this.soundMeters.set(userId, soundMeter); - this.soundMeterElements.set(userId, HtmlUtils.getElementByIdOrFail('soundMeter-'+userId));*/ } addStreamRemoteScreenSharing(userId: string, stream : MediaStream){ // In the case of screen sharing (going both ways), we may need to create the HTML element if it does not exist yet @@ -264,23 +258,18 @@ export class MediaManager { } this.addStreamRemoteVideo(this.getScreenSharingId(userId), stream); - } + }*/ removeActiveVideo(userId: string){ - layoutManager.remove(userId); - this.remoteVideo.delete(userId); - - //FIX ME SOUNDMETER: check stalability of sound meter calculation - /*this.soundMeters.get(userId)?.stop(); - this.soundMeters.delete(userId); - this.soundMeterElements.delete(userId);*/ + //layoutManager.remove(userId); + //this.remoteVideo.delete(userId); //permit to remove user in discussion part this.removeParticipant(userId); } - removeActiveScreenSharingVideo(userId: string) { + /*removeActiveScreenSharingVideo(userId: string) { this.removeActiveVideo(this.getScreenSharingId(userId)) - } + }*/ isConnecting(userId: string): void { const connectingSpinnerDiv = this.getSpinner(userId); diff --git a/front/src/WebRtc/ScreenSharingPeer.ts b/front/src/WebRtc/ScreenSharingPeer.ts index d797f59b..49026971 100644 --- a/front/src/WebRtc/ScreenSharingPeer.ts +++ b/front/src/WebRtc/ScreenSharingPeer.ts @@ -1,9 +1,11 @@ import type * as SimplePeerNamespace from "simple-peer"; import {mediaManager} from "./MediaManager"; -import {STUN_SERVER, TURN_SERVER, TURN_USER, TURN_PASSWORD} from "../Enum/EnvironmentVariable"; +import {STUN_SERVER, TURN_PASSWORD, TURN_SERVER, TURN_USER} from "../Enum/EnvironmentVariable"; import type {RoomConnection} from "../Connexion/RoomConnection"; import {MESSAGE_TYPE_CONSTRAINT} from "./VideoPeer"; import type {UserSimplePeerInterface} from "./SimplePeer"; +import {Readable, readable, writable, Writable} from "svelte/store"; +import {DivImportance} from "./LayoutManager"; const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); @@ -17,9 +19,13 @@ export class ScreenSharingPeer extends Peer { private isReceivingStream:boolean = false; public toClose: boolean = false; public _connected: boolean = false; - private userId: number; + public readonly userId: number; + public readonly uniqueId: string; + public readonly streamStore: Readable; + public readonly importanceStore: Writable; + public readonly statusStore: Readable<"connecting" | "connected" | "error" | "closed">; - constructor(user: UserSimplePeerInterface, initiator: boolean, private connection: RoomConnection, stream: MediaStream | null) { + constructor(user: UserSimplePeerInterface, initiator: boolean, public readonly userName: string, private connection: RoomConnection, stream: MediaStream | null) { super({ initiator: initiator ? initiator : false, //reconnectTimer: 10000, @@ -38,6 +44,56 @@ export class ScreenSharingPeer extends Peer { }); this.userId = user.userId; + this.uniqueId = 'screensharing_'+this.userId; + + this.streamStore = readable(null, (set) => { + const onStream = (stream: MediaStream|null) => { + set(stream); + }; + const onData = (chunk: Buffer) => { + // We unfortunately need to rely on an event to let the other party know a stream has stopped. + // It seems there is no native way to detect that. + // TODO: we might rely on the "ended" event: https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/ended_event + const message = JSON.parse(chunk.toString('utf8')); + if (message.streamEnded !== true) { + console.error('Unexpected message on screen sharing peer connection'); + return; + } + set(null); + } + + this.on('stream', onStream); + this.on('data', onData); + + return () => { + this.off('stream', onStream); + this.off('data', onData); + }; + }); + + this.importanceStore = writable(DivImportance.Important); + + this.statusStore = readable<"connecting" | "connected" | "error" | "closed">("connecting", (set) => { + const onConnect = () => { + set('connected'); + }; + const onError = () => { + set('error'); + }; + const onClose = () => { + set('closed'); + }; + + this.on('connect', onConnect); + this.on('error', onError); + this.on('close', onClose); + + return () => { + this.off('connect', onConnect); + this.off('error', onError); + this.off('close', onClose); + }; + }); //start listen signal for the peer connection this.on('signal', (data: unknown) => { @@ -54,16 +110,17 @@ export class ScreenSharingPeer extends Peer { this.destroy(); }); - this.on('data', (chunk: Buffer) => { + /*this.on('data', (chunk: Buffer) => { // We unfortunately need to rely on an event to let the other party know a stream has stopped. // It seems there is no native way to detect that. + // TODO: we might rely on the "ended" event: https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/ended_event const message = JSON.parse(chunk.toString('utf8')); if (message.streamEnded !== true) { console.error('Unexpected message on screen sharing peer connection'); return; } mediaManager.removeActiveScreenSharingVideo("" + this.userId); - }); + });*/ // eslint-disable-next-line @typescript-eslint/no-explicit-any this.on('error', (err: any) => { @@ -103,10 +160,10 @@ export class ScreenSharingPeer extends Peer { //console.log(`ScreenSharingPeer::stream => ${this.userId}`, stream); //console.log(`stream => ${this.userId} => `, stream); if(!stream){ - mediaManager.removeActiveScreenSharingVideo("" + this.userId); + //mediaManager.removeActiveScreenSharingVideo("" + this.userId); this.isReceivingStream = false; } else { - mediaManager.addStreamRemoteScreenSharing("" + this.userId, stream); + //mediaManager.addStreamRemoteScreenSharing("" + this.userId, stream); this.isReceivingStream = true; } } @@ -121,7 +178,7 @@ export class ScreenSharingPeer extends Peer { if(!this.toClose){ return; } - mediaManager.removeActiveScreenSharingVideo("" + this.userId); + //mediaManager.removeActiveScreenSharingVideo("" + this.userId); // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 2a502bab..7c6264a9 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -28,8 +28,10 @@ export interface UserSimplePeerInterface{ webRtcPassword?: string|undefined; } +export type RemotePeer = VideoPeer | ScreenSharingPeer; + export interface PeerConnectionListener { - onConnect(user: UserSimplePeerInterface): void; + onConnect(user: RemotePeer): void; onDisconnect(userId: number): void; } @@ -159,20 +161,17 @@ export class SimplePeer { let name = user.name; if (!name) { - const userSearch = this.Users.find((userSearch: UserSimplePeerInterface) => userSearch.userId === user.userId); - if (userSearch) { - name = userSearch.name; - } + name = this.getName(user.userId); } mediaManager.removeActiveVideo("" + user.userId); - mediaManager.addActiveVideo(user, name); + //mediaManager.addActiveVideo(user, name); this.lastWebrtcUserName = user.webRtcUser; this.lastWebrtcPassword = user.webRtcPassword; - const peer = new VideoPeer(user, user.initiator ? user.initiator : false, this.Connection, localStream); + const peer = new VideoPeer(user, user.initiator ? user.initiator : false, name, this.Connection, localStream); //permit to send message mediaManager.addSendMessageCallback(user.userId,(message: string) => { @@ -196,11 +195,20 @@ export class SimplePeer { this.PeerConnectionArray.set(user.userId, peer); for (const peerConnectionListener of this.peerConnectionListeners) { - peerConnectionListener.onConnect(user); + peerConnectionListener.onConnect(peer); } return peer; } + private getName(userId: number): string { + const userSearch = this.Users.find((userSearch: UserSimplePeerInterface) => userSearch.userId === userId); + if (userSearch) { + return userSearch.name || ''; + } else { + return ''; + } + } + /** * create peer connection to bind users */ @@ -222,10 +230,10 @@ export class SimplePeer { } // We should display the screen sharing ONLY if we are not initiator - if (!user.initiator) { +/* if (!user.initiator) { mediaManager.removeActiveScreenSharingVideo("" + user.userId); mediaManager.addScreenSharingActiveVideo("" + user.userId); - } + }*/ // Enrich the user with last known credentials (if they are not set in the user object, which happens when a user triggers the screen sharing) if (user.webRtcUser === undefined) { @@ -233,11 +241,13 @@ export class SimplePeer { user.webRtcPassword = this.lastWebrtcPassword; } - const peer = new ScreenSharingPeer(user, user.initiator ? user.initiator : false, this.Connection, stream); + const name = this.getName(user.userId); + + const peer = new ScreenSharingPeer(user, user.initiator ? user.initiator : false, name, this.Connection, stream); this.PeerScreenSharingConnectionArray.set(user.userId, peer); for (const peerConnectionListener of this.peerConnectionListeners) { - peerConnectionListener.onConnect(user); + peerConnectionListener.onConnect(peer); } return peer; } @@ -288,7 +298,7 @@ export class SimplePeer { */ private closeScreenSharingConnection(userId : number) { try { - mediaManager.removeActiveScreenSharingVideo("" + userId); + //mediaManager.removeActiveScreenSharingVideo("" + userId); const peer = this.PeerScreenSharingConnectionArray.get(userId); if (peer === undefined) { console.warn("closeScreenSharingConnection => Tried to close connection for user "+userId+" but could not find user") diff --git a/front/src/WebRtc/VideoPeer.ts b/front/src/WebRtc/VideoPeer.ts index 5ca8952c..c69bc4cd 100644 --- a/front/src/WebRtc/VideoPeer.ts +++ b/front/src/WebRtc/VideoPeer.ts @@ -5,8 +5,9 @@ import type {RoomConnection} from "../Connexion/RoomConnection"; import {blackListManager} from "./BlackListManager"; import type {Subscription} from "rxjs"; import type {UserSimplePeerInterface} from "./SimplePeer"; -import {get} from "svelte/store"; +import {get, readable, Readable, writable, Writable} from "svelte/store"; import {obtainedMediaConstraintStore} from "../Stores/MediaStore"; +import {DivImportance} from "./LayoutManager"; const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); @@ -22,12 +23,16 @@ export class VideoPeer extends Peer { public _connected: boolean = false; private remoteStream!: MediaStream; private blocked: boolean = false; - private userId: number; - private userName: string; + public readonly userId: number; + public readonly uniqueId: string; private onBlockSubscribe: Subscription; private onUnBlockSubscribe: Subscription; + public readonly streamStore: Readable; + public readonly importanceStore: Writable; + public readonly statusStore: Readable<"connecting" | "connected" | "error" | "closed">; + public readonly constraintsStore: Readable; - constructor(public user: UserSimplePeerInterface, initiator: boolean, private connection: RoomConnection, localStream: MediaStream | null) { + constructor(public user: UserSimplePeerInterface, initiator: boolean, public readonly userName: string, private connection: RoomConnection, localStream: MediaStream | null) { super({ initiator: initiator ? initiator : false, //reconnectTimer: 10000, @@ -46,7 +51,70 @@ export class VideoPeer extends Peer { }); this.userId = user.userId; - this.userName = user.name || ''; + this.uniqueId = 'video_'+this.userId; + + this.streamStore = readable(null, (set) => { + const onStream = (stream: MediaStream|null) => { + set(stream); + }; + const onData = (chunk: Buffer) => { + this.on('data', (chunk: Buffer) => { + const message = JSON.parse(chunk.toString('utf8')); + if (message.type === MESSAGE_TYPE_CONSTRAINT) { + if (!message.video) { + set(null); + } + } + }); + } + + this.on('stream', onStream); + this.on('data', onData); + + return () => { + this.off('stream', onStream); + this.off('data', onData); + }; + }); + + this.constraintsStore = readable(null, (set) => { + const onData = (chunk: Buffer) => { + const message = JSON.parse(chunk.toString('utf8')); + if(message.type === MESSAGE_TYPE_CONSTRAINT) { + set(message); + } + } + + this.on('data', onData); + + return () => { + this.off('data', onData); + }; + }); + + this.importanceStore = writable(DivImportance.Normal); + + this.statusStore = readable<"connecting" | "connected" | "error" | "closed">("connecting", (set) => { + const onConnect = () => { + set('connected'); + }; + const onError = () => { + set('error'); + }; + const onClose = () => { + set('closed'); + }; + + this.on('connect', onConnect); + this.on('error', onError); + this.on('close', onClose); + + return () => { + this.off('connect', onConnect); + this.off('error', onError); + this.off('close', onClose); + }; + }); //start listen signal for the peer connection this.on('signal', (data: unknown) => { @@ -152,7 +220,7 @@ export class VideoPeer extends Peer { if (blackListManager.isBlackListed(this.userId) || this.blocked) { this.toggleRemoteStream(false); } - mediaManager.addStreamRemoteVideo("" + this.userId, stream); + //mediaManager.addStreamRemoteVideo("" + this.userId, stream); }catch (err){ console.error(err); } From ac7fa164b6e39052bd8354e8c917418949abbf94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Mon, 14 Jun 2021 17:59:50 +0200 Subject: [PATCH 02/21] Adding importance handling --- .../Components/Video/LocalStreamMedia.svelte | 7 +++-- front/src/Components/Video/Peer.svelte | 2 +- .../Video/ScreenSharingMedia.svelte | 2 +- front/src/Components/Video/VideoMedia.svelte | 5 +++- front/src/Stores/ImportanceStore.ts | 26 +++++++++++++++++++ front/src/Stores/ScreenSharingStore.ts | 5 ++-- front/src/WebRtc/ScreenSharingPeer.ts | 6 +++-- front/src/WebRtc/VideoPeer.ts | 6 +++-- 8 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 front/src/Stores/ImportanceStore.ts diff --git a/front/src/Components/Video/LocalStreamMedia.svelte b/front/src/Components/Video/LocalStreamMedia.svelte index 43b1d117..375612a6 100644 --- a/front/src/Components/Video/LocalStreamMedia.svelte +++ b/front/src/Components/Video/LocalStreamMedia.svelte @@ -1,4 +1,6 @@
- +
diff --git a/front/src/Components/Video/Peer.svelte b/front/src/Components/Video/Peer.svelte index c73d620e..8d5637f6 100644 --- a/front/src/Components/Video/Peer.svelte +++ b/front/src/Components/Video/Peer.svelte @@ -15,6 +15,6 @@ {:else if peer instanceof ScreenSharingPeer} {:else} - + {/if}
diff --git a/front/src/Components/Video/ScreenSharingMedia.svelte b/front/src/Components/Video/ScreenSharingMedia.svelte index e16fac58..0dbc3c16 100644 --- a/front/src/Components/Video/ScreenSharingMedia.svelte +++ b/front/src/Components/Video/ScreenSharingMedia.svelte @@ -45,7 +45,7 @@ {#if $streamStore === null} {name} {/if} - + diff --git a/front/src/Components/Video/VideoOverlay.svelte b/front/src/Components/Video/VideoOverlay.svelte index 5ba7fbc7..88c1cd42 100644 --- a/front/src/Components/Video/VideoOverlay.svelte +++ b/front/src/Components/Video/VideoOverlay.svelte @@ -3,18 +3,23 @@ import {DivImportance} from "../../WebRtc/LayoutManager"; import Peer from "./Peer.svelte"; import {layoutStore} from "../../Stores/LayoutStore"; + import {videoFocusStore} from "../../Stores/VideoFocusStore";
- {#each [...$layoutStore.get(DivImportance.Important).values()] as peer (peer.uniqueId)} - + {#each [...$layoutStore.values()] as peer (peer.uniqueId)} + {#if $videoFocusStore && peer === $videoFocusStore } + + {/if} {/each}