merged latest dev

This commit is contained in:
_Bastler
2021-05-30 15:29:28 +02:00
18 changed files with 359 additions and 311 deletions
+2 -200
View File
@@ -8,33 +8,11 @@ import {SoundMeter} from "../Phaser/Components/SoundMeter";
import {DISABLE_NOTIFICATIONS} from "../Enum/EnvironmentVariable";
import {
gameOverlayVisibilityStore, localStreamStore,
mediaStreamConstraintsStore,
requestedCameraState,
requestedMicrophoneState
} from "../Stores/MediaStore";
import {
requestedScreenSharingState,
screenSharingAvailableStore,
screenSharingLocalStreamStore
} from "../Stores/ScreenSharingStore";
declare const navigator:any; // eslint-disable-line @typescript-eslint/no-explicit-any
const videoConstraint: boolean|MediaTrackConstraints = {
width: { min: 640, ideal: 1280, max: 1920 },
height: { min: 400, ideal: 720 },
frameRate: { ideal: localUserStore.getVideoQualityValue() },
facingMode: "user",
resizeMode: 'crop-and-scale',
aspectRatio: 1.777777778
};
const audioConstraint: boolean|MediaTrackConstraints = {
//TODO: make these values configurable in the game settings menu and store them in localstorage
autoGainControl: false,
echoCancellation: true,
noiseSuppression: false
};
export type UpdatedLocalStreamCallback = (media: MediaStream|null) => void;
export type StartScreenSharingCallback = (media: MediaStream) => void;
export type StopScreenSharingCallback = (media: MediaStream) => void;
@@ -43,30 +21,16 @@ export type ShowReportCallBack = (userId: string, userName: string|undefined) =>
export type HelpCameraSettingsCallBack = () => void;
export class MediaManager {
localStream: MediaStream|null = null;
localScreenCapture: MediaStream|null = null;
private remoteVideo: Map<string, HTMLVideoElement> = new Map<string, HTMLVideoElement>();
myCamVideo: HTMLVideoElement;
cinemaClose: HTMLImageElement;
cinema: HTMLImageElement;
monitorClose: HTMLImageElement;
monitor: HTMLImageElement;
microphoneClose: HTMLImageElement;
microphone: HTMLImageElement;
webrtcInAudio: HTMLAudioElement;
//FIX ME SOUNDMETER: check stalability of sound meter calculation
//mySoundMeterElement: HTMLDivElement;
private webrtcOutAudio: HTMLAudioElement;
updatedLocalStreamCallBacks : Set<UpdatedLocalStreamCallback> = new Set<UpdatedLocalStreamCallback>();
startScreenSharingCallBacks : Set<StartScreenSharingCallback> = new Set<StartScreenSharingCallback>();
stopScreenSharingCallBacks : Set<StopScreenSharingCallback> = new Set<StopScreenSharingCallback>();
showReportModalCallBacks : Set<ShowReportCallBack> = new Set<ShowReportCallBack>();
helpCameraSettingsCallBacks : Set<HelpCameraSettingsCallBack> = new Set<HelpCameraSettingsCallBack>();
private microphoneBtn: HTMLDivElement;
private cinemaBtn: HTMLDivElement;
private monitorBtn: HTMLDivElement;
private focused : boolean = true;
private triggerCloseJistiFrame : Map<String, Function> = new Map<String, Function>();
@@ -80,54 +44,11 @@ export class MediaManager {
constructor() {
this.myCamVideo = HtmlUtils.getElementByIdOrFail<HTMLVideoElement>('myCamVideo');
this.webrtcInAudio = HtmlUtils.getElementByIdOrFail<HTMLAudioElement>('audio-webrtc-in');
this.webrtcOutAudio = HtmlUtils.getElementByIdOrFail<HTMLAudioElement>('audio-webrtc-out');
this.webrtcInAudio.volume = 0.2;
this.webrtcOutAudio.volume = 0.2;
this.microphoneBtn = HtmlUtils.getElementByIdOrFail<HTMLDivElement>('btn-micro');
this.microphoneClose = HtmlUtils.getElementByIdOrFail<HTMLImageElement>('microphone-close');
this.microphoneClose.style.display = "none";
this.microphoneClose.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
requestedMicrophoneState.enableMicrophone();
});
this.microphone = HtmlUtils.getElementByIdOrFail<HTMLImageElement>('microphone');
this.microphone.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
requestedMicrophoneState.disableMicrophone();
});
this.cinemaBtn = HtmlUtils.getElementByIdOrFail<HTMLDivElement>('btn-video');
this.cinemaClose = HtmlUtils.getElementByIdOrFail<HTMLImageElement>('cinema-close');
this.cinemaClose.style.display = "none";
this.cinemaClose.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
requestedCameraState.enableWebcam();
});
this.cinema = HtmlUtils.getElementByIdOrFail<HTMLImageElement>('cinema');
this.cinema.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
requestedCameraState.disableWebcam();
});
this.monitorBtn = HtmlUtils.getElementByIdOrFail<HTMLDivElement>('btn-monitor');
this.monitorClose = HtmlUtils.getElementByIdOrFail<HTMLImageElement>('monitor-close');
this.monitorClose.style.display = "block";
this.monitorClose.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
//this.enableScreenSharing();
requestedScreenSharingState.enableScreenSharing();
});
this.monitor = HtmlUtils.getElementByIdOrFail<HTMLImageElement>('monitor');
this.monitor.style.display = "none";
this.monitor.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
//this.disableScreenSharing();
requestedScreenSharingState.disableScreenSharing();
});
this.pingCameraStatus();
//FIX ME SOUNDMETER: check stability of sound meter calculation
@@ -147,40 +68,8 @@ export class MediaManager {
}, this.userInputManager);
return;
}
if (result.constraints.video !== false) {
HtmlUtils.getElementByIdOrFail('div-myCamVideo').classList.remove('hide');
} else {
HtmlUtils.getElementByIdOrFail('div-myCamVideo').classList.add('hide');
}/*
if (result.constraints.audio !== false) {
this.enableMicrophoneStyle();
} else {
this.disableMicrophoneStyle();
}*/
this.localStream = result.stream;
this.myCamVideo.srcObject = this.localStream;
// TODO: migrate all listeners to the store directly.
this.triggerUpdatedLocalStreamCallbacks(result.stream);
});
requestedCameraState.subscribe((enabled) => {
if (enabled) {
this.enableCameraStyle();
} else {
this.disableCameraStyle();
}
});
requestedMicrophoneState.subscribe((enabled) => {
if (enabled) {
this.enableMicrophoneStyle();
} else {
this.disableMicrophoneStyle();
}
});
//let screenSharingStream : MediaStream|null;
screenSharingLocalStreamStore.subscribe((result) => {
if (result.type === 'error') {
console.error(result.error);
@@ -191,38 +80,21 @@ export class MediaManager {
}
if (result.stream !== null) {
this.enableScreenSharingStyle();
mediaManager.localScreenCapture = result.stream;
// TODO: migrate this out of MediaManager
this.triggerStartedScreenSharingCallbacks(result.stream);
//screenSharingStream = result.stream;
this.addScreenSharingActiveVideo('me', DivImportance.Normal);
HtmlUtils.getElementByIdOrFail<HTMLVideoElement>('screen-sharing-me').srcObject = result.stream;
} else {
this.disableScreenSharingStyle();
this.removeActiveScreenSharingVideo('me');
// FIXME: we need the old stream that is being stopped!
if (this.localScreenCapture) {
this.triggerStoppedScreenSharingCallbacks(this.localScreenCapture);
this.localScreenCapture = null;
}
//screenSharingStream = null;
}
});
screenSharingAvailableStore.subscribe((available) => {
/*screenSharingAvailableStore.subscribe((available) => {
if (available) {
document.querySelector('.btn-monitor')?.classList.remove('hide');
} else {
document.querySelector('.btn-monitor')?.classList.add('hide');
}
});
});*/
}
public updateScene(){
@@ -230,40 +102,6 @@ export class MediaManager {
//this.updateSoudMeter();
}
public onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void {
this.updatedLocalStreamCallBacks.add(callback);
}
public onStartScreenSharing(callback: StartScreenSharingCallback): void {
this.startScreenSharingCallBacks.add(callback);
}
public onStopScreenSharing(callback: StopScreenSharingCallback): void {
this.stopScreenSharingCallBacks.add(callback);
}
removeUpdateLocalStreamEventListener(callback: UpdatedLocalStreamCallback): void {
this.updatedLocalStreamCallBacks.delete(callback);
}
private triggerUpdatedLocalStreamCallbacks(stream: MediaStream|null): void {
for (const callback of this.updatedLocalStreamCallBacks) {
callback(stream);
}
}
private triggerStartedScreenSharingCallbacks(stream: MediaStream): void {
for (const callback of this.startScreenSharingCallBacks) {
callback(stream);
}
}
private triggerStoppedScreenSharingCallbacks(stream: MediaStream): void {
for (const callback of this.stopScreenSharingCallBacks) {
callback(stream);
}
}
public showGameOverlay(): void {
const gameOverlay = HtmlUtils.getElementByIdOrFail('game-overlay');
gameOverlay.classList.add('active');
@@ -290,42 +128,6 @@ export class MediaManager {
gameOverlayVisibilityStore.hideGameOverlay();
}
private enableCameraStyle(){
this.cinemaClose.style.display = "none";
this.cinemaBtn.classList.remove("disabled");
this.cinema.style.display = "block";
}
private disableCameraStyle(){
this.cinemaClose.style.display = "block";
this.cinema.style.display = "none";
this.cinemaBtn.classList.add("disabled");
}
private enableMicrophoneStyle(){
this.microphoneClose.style.display = "none";
this.microphone.style.display = "block";
this.microphoneBtn.classList.remove("disabled");
}
private disableMicrophoneStyle(){
this.microphoneClose.style.display = "block";
this.microphone.style.display = "none";
this.microphoneBtn.classList.add("disabled");
}
private enableScreenSharingStyle(){
this.monitorClose.style.display = "none";
this.monitor.style.display = "block";
this.monitorBtn.classList.add("enabled");
}
private disableScreenSharingStyle(){
this.monitorClose.style.display = "block";
this.monitor.style.display = "none";
this.monitorBtn.classList.remove("enabled");
}
addActiveVideo(user: UserSimplePeerInterface, userName: string = ""){
this.webrtcInAudio.play();
const userId = ''+user.userId
+4 -12
View File
@@ -19,7 +19,7 @@ export class ScreenSharingPeer extends Peer {
public _connected: boolean = false;
private userId: number;
constructor(user: UserSimplePeerInterface, initiator: boolean, private connection: RoomConnection) {
constructor(user: UserSimplePeerInterface, initiator: boolean, private connection: RoomConnection, stream: MediaStream | null) {
super({
initiator: initiator ? initiator : false,
//reconnectTimer: 10000,
@@ -81,7 +81,9 @@ export class ScreenSharingPeer extends Peer {
this._onFinish();
});
this.pushScreenSharingToRemoteUser();
if (stream) {
this.addStream(stream);
}
}
private sendWebrtcScreenSharingSignal(data: unknown) {
@@ -141,16 +143,6 @@ export class ScreenSharingPeer extends Peer {
}
}
private pushScreenSharingToRemoteUser() {
const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture;
if(!localScreenCapture){
return;
}
this.addStream(localScreenCapture);
return;
}
public stopPushingScreenSharingToRemoteUser(stream: MediaStream) {
this.removeStream(stream);
this.write(new Buffer(JSON.stringify({type: MESSAGE_TYPE_CONSTRAINT, streamEnded: true})));
+78 -44
View File
@@ -15,7 +15,10 @@ import {connectionManager} from "../Connexion/ConnectionManager";
import {GameConnexionTypes} from "../Url/UrlManager";
import {blackListManager} from "./BlackListManager";
import {get} from "svelte/store";
import {localStreamStore, obtainedMediaConstraintStore} from "../Stores/MediaStore";
import {localStreamStore, LocalStreamStoreValue, obtainedMediaConstraintStore} from "../Stores/MediaStore";
import {screenSharingLocalStreamStore} from "../Stores/ScreenSharingStore";
import {DivImportance, layoutManager} from "./LayoutManager";
import {HtmlUtils} from "./HtmlUtils";
export interface UserSimplePeerInterface{
userId: number;
@@ -39,9 +42,9 @@ export class SimplePeer {
private PeerScreenSharingConnectionArray: Map<number, ScreenSharingPeer> = new Map<number, ScreenSharingPeer>();
private PeerConnectionArray: Map<number, VideoPeer> = new Map<number, VideoPeer>();
private readonly sendLocalVideoStreamCallback: UpdatedLocalStreamCallback;
private readonly sendLocalScreenSharingStreamCallback: StartScreenSharingCallback;
private readonly stopLocalScreenSharingStreamCallback: StopScreenSharingCallback;
private readonly unsubscribers: (() => void)[] = [];
private readonly peerConnectionListeners: Array<PeerConnectionListener> = new Array<PeerConnectionListener>();
private readonly userId: number;
private lastWebrtcUserName: string|undefined;
@@ -49,13 +52,32 @@ export class SimplePeer {
constructor(private Connection: RoomConnection, private enableReporting: boolean, private myName: string) {
// We need to go through this weird bound function pointer in order to be able to "free" this reference later.
this.sendLocalVideoStreamCallback = this.sendLocalVideoStream.bind(this);
this.sendLocalScreenSharingStreamCallback = this.sendLocalScreenSharingStream.bind(this);
this.stopLocalScreenSharingStreamCallback = this.stopLocalScreenSharingStream.bind(this);
mediaManager.onUpdateLocalStream(this.sendLocalVideoStreamCallback);
mediaManager.onStartScreenSharing(this.sendLocalScreenSharingStreamCallback);
mediaManager.onStopScreenSharing(this.stopLocalScreenSharingStreamCallback);
this.unsubscribers.push(localStreamStore.subscribe((streamResult) => {
this.sendLocalVideoStream(streamResult);
}));
let localScreenCapture: MediaStream|null = null;
this.unsubscribers.push(screenSharingLocalStreamStore.subscribe((streamResult) => {
if (streamResult.type === 'error') {
// Let's ignore screen sharing errors, we will deal with those in a different way.
return;
}
if (streamResult.stream !== null) {
localScreenCapture = streamResult.stream;
this.sendLocalScreenSharingStream(localScreenCapture);
} else {
if (localScreenCapture) {
this.stopLocalScreenSharingStream(localScreenCapture);
localScreenCapture = null;
}
}
}));
this.userId = Connection.getUserId();
this.initialise();
}
@@ -106,13 +128,19 @@ export class SimplePeer {
if(!user.initiator){
return;
}
this.createPeerConnection(user);
const streamResult = get(localStreamStore);
let stream : MediaStream | null = null;
if (streamResult.type === 'success' && streamResult.stream) {
stream = streamResult.stream;
}
this.createPeerConnection(user, stream);
}
/**
* create peer connection to bind users
*/
private createPeerConnection(user : UserSimplePeerInterface) : VideoPeer | null {
private createPeerConnection(user : UserSimplePeerInterface, localStream: MediaStream | null) : VideoPeer | null {
const peerConnection = this.PeerConnectionArray.get(user.userId)
if (peerConnection) {
if (peerConnection.destroyed) {
@@ -122,11 +150,11 @@ export class SimplePeer {
if (!peerConnexionDeleted) {
throw 'Error to delete peer connection';
}
this.createPeerConnection(user);
//return this.createPeerConnection(user, localStream);
} else {
peerConnection.toClose = false;
return null;
}
return null;
}
let name = user.name;
@@ -144,7 +172,7 @@ export class SimplePeer {
this.lastWebrtcUserName = user.webRtcUser;
this.lastWebrtcPassword = user.webRtcPassword;
const peer = new VideoPeer(user, user.initiator ? user.initiator : false, this.Connection);
const peer = new VideoPeer(user, user.initiator ? user.initiator : false, this.Connection, localStream);
//permit to send message
mediaManager.addSendMessageCallback(user.userId,(message: string) => {
@@ -155,8 +183,9 @@ export class SimplePeer {
// When a connection is established to a video stream, and if a screen sharing is taking place,
// the user sharing screen should also initiate a connection to the remote user!
peer.on('connect', () => {
if (mediaManager.localScreenCapture) {
this.sendLocalScreenSharingStreamToUser(user.userId);
const streamResult = get(screenSharingLocalStreamStore);
if (streamResult.type === 'success' && streamResult.stream !== null) {
this.sendLocalScreenSharingStreamToUser(user.userId, streamResult.stream);
}
});
@@ -175,7 +204,7 @@ export class SimplePeer {
/**
* create peer connection to bind users
*/
private createPeerScreenSharingConnection(user : UserSimplePeerInterface) : ScreenSharingPeer | null{
private createPeerScreenSharingConnection(user : UserSimplePeerInterface, stream: MediaStream | null) : ScreenSharingPeer | null{
const peerConnection = this.PeerScreenSharingConnectionArray.get(user.userId);
if(peerConnection){
if(peerConnection.destroyed){
@@ -185,7 +214,7 @@ export class SimplePeer {
if(!peerConnexionDeleted){
throw 'Error to delete peer connection';
}
this.createPeerConnection(user);
this.createPeerConnection(user, stream);
}else {
peerConnection.toClose = false;
}
@@ -204,7 +233,7 @@ export class SimplePeer {
user.webRtcPassword = this.lastWebrtcPassword;
}
const peer = new ScreenSharingPeer(user, user.initiator ? user.initiator : false, this.Connection);
const peer = new ScreenSharingPeer(user, user.initiator ? user.initiator : false, this.Connection, stream);
this.PeerScreenSharingConnectionArray.set(user.userId, peer);
for (const peerConnectionListener of this.peerConnectionListeners) {
@@ -294,7 +323,9 @@ export class SimplePeer {
* Unregisters any held event handler.
*/
public unregister() {
mediaManager.removeUpdateLocalStreamEventListener(this.sendLocalVideoStreamCallback);
for (const unsubscriber of this.unsubscribers) {
unsubscriber();
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -302,7 +333,13 @@ export class SimplePeer {
try {
//if offer type, create peer connection
if(data.signal.type === "offer"){
this.createPeerConnection(data);
const streamResult = get(localStreamStore);
let stream : MediaStream | null = null;
if (streamResult.type === 'success' && streamResult.stream) {
stream = streamResult.stream;
}
this.createPeerConnection(data, stream);
}
const peer = this.PeerConnectionArray.get(data.userId);
if (peer !== undefined) {
@@ -318,18 +355,26 @@ export class SimplePeer {
private receiveWebrtcScreenSharingSignal(data: WebRtcSignalReceivedMessageInterface) {
if (blackListManager.isBlackListed(data.userId)) return;
console.log("receiveWebrtcScreenSharingSignal", data);
const streamResult = get(screenSharingLocalStreamStore);
let stream : MediaStream | null = null;
if (streamResult.type === 'success' && streamResult.stream !== null) {
stream = streamResult.stream;
}
try {
//if offer type, create peer connection
if(data.signal.type === "offer"){
this.createPeerScreenSharingConnection(data);
this.createPeerScreenSharingConnection(data, stream);
}
const peer = this.PeerScreenSharingConnectionArray.get(data.userId);
if (peer !== undefined) {
peer.signal(data.signal);
} else {
console.error('Could not find peer whose ID is "'+data.userId+'" in receiveWebrtcScreenSharingSignal');
console.info('tentative to create new peer connexion');
this.sendLocalScreenSharingStreamToUser(data.userId);
console.info('Attempt to create new peer connexion');
if (stream) {
this.sendLocalScreenSharingStreamToUser(data.userId, stream);
}
}
} catch (e) {
console.error(`receiveWebrtcSignal => ${data.userId}`, e);
@@ -339,21 +384,19 @@ export class SimplePeer {
}
}
private pushVideoToRemoteUser(userId : number) {
private pushVideoToRemoteUser(userId: number, streamResult: LocalStreamStoreValue) {
try {
const PeerConnection = this.PeerConnectionArray.get(userId);
if (!PeerConnection) {
throw new Error('While adding media, cannot find user with ID ' + userId);
}
const result = get(localStreamStore);
PeerConnection.write(new Buffer(JSON.stringify({type: MESSAGE_TYPE_CONSTRAINT, ...streamResult.constraints})));
PeerConnection.write(new Buffer(JSON.stringify({type: MESSAGE_TYPE_CONSTRAINT, ...result.constraints})));
if (result.type === 'error') {
if (streamResult.type === 'error') {
return;
}
const localStream: MediaStream | null = result.stream;
const localStream: MediaStream | null = streamResult.stream;
if(!localStream){
return;
@@ -370,15 +413,11 @@ export class SimplePeer {
}
}
private pushScreenSharingToRemoteUser(userId : number) {
private pushScreenSharingToRemoteUser(userId: number, localScreenCapture: MediaStream) {
const PeerConnection = this.PeerScreenSharingConnectionArray.get(userId);
if (!PeerConnection) {
throw new Error('While pushing screen sharing, cannot find user with ID ' + userId);
}
const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture;
if(!localScreenCapture){
return;
}
for (const track of localScreenCapture.getTracks()) {
PeerConnection.addTrack(track, localScreenCapture);
@@ -386,23 +425,18 @@ export class SimplePeer {
return;
}
public sendLocalVideoStream(){
public sendLocalVideoStream(streamResult: LocalStreamStoreValue){
for (const user of this.Users) {
this.pushVideoToRemoteUser(user.userId);
this.pushVideoToRemoteUser(user.userId, streamResult);
}
}
/**
* Triggered locally when clicking on the screen sharing button
*/
public sendLocalScreenSharingStream() {
if (!mediaManager.localScreenCapture) {
console.error('Could not find localScreenCapture to share')
return;
}
public sendLocalScreenSharingStream(localScreenCapture: MediaStream) {
for (const user of this.Users) {
this.sendLocalScreenSharingStreamToUser(user.userId);
this.sendLocalScreenSharingStreamToUser(user.userId, localScreenCapture);
}
}
@@ -415,11 +449,11 @@ export class SimplePeer {
}
}
private sendLocalScreenSharingStreamToUser(userId: number): void {
private sendLocalScreenSharingStreamToUser(userId: number, localScreenCapture: MediaStream): void {
if (blackListManager.isBlackListed(userId)) return;
// If a connection already exists with user (because it is already sharing a screen with us... let's use this connection)
if (this.PeerScreenSharingConnectionArray.has(userId)) {
this.pushScreenSharingToRemoteUser(userId);
this.pushScreenSharingToRemoteUser(userId, localScreenCapture);
return;
}
@@ -427,7 +461,7 @@ export class SimplePeer {
userId,
initiator: true
};
const PeerConnectionScreenSharing = this.createPeerScreenSharingConnection(screenSharingUser);
const PeerConnectionScreenSharing = this.createPeerScreenSharingConnection(screenSharingUser, localScreenCapture);
if (!PeerConnectionScreenSharing) {
return;
}
+3 -4
View File
@@ -27,7 +27,7 @@ export class VideoPeer extends Peer {
private onBlockSubscribe: Subscription;
private onUnBlockSubscribe: Subscription;
constructor(public user: UserSimplePeerInterface, initiator: boolean, private connection: RoomConnection) {
constructor(public user: UserSimplePeerInterface, initiator: boolean, private connection: RoomConnection, localStream: MediaStream | null) {
super({
initiator: initiator ? initiator : false,
//reconnectTimer: 10000,
@@ -107,7 +107,7 @@ export class VideoPeer extends Peer {
this._onFinish();
});
this.pushVideoToRemoteUser();
this.pushVideoToRemoteUser(localStream);
this.onBlockSubscribe = blackListManager.onBlockStream.subscribe((userId) => {
if (userId === this.userId) {
this.toggleRemoteStream(false);
@@ -190,9 +190,8 @@ export class VideoPeer extends Peer {
}
}
private pushVideoToRemoteUser() {
private pushVideoToRemoteUser(localStream: MediaStream | null) {
try {
const localStream: MediaStream | null = mediaManager.localStream;
this.write(new Buffer(JSON.stringify({type: MESSAGE_TYPE_CONSTRAINT, ...get(obtainedMediaConstraintStore)})));
if(!localStream){