From d78006e10611c69fe0e790c77e06240bb25c8a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Tue, 23 Jun 2020 14:56:57 +0200 Subject: [PATCH 01/17] Fixing memory leak with listeners The listeners from MediaManager and SimplePeer were never removed, leading to a huge amount of listeners all over the applications when switching regularly of scene. --- front/src/Phaser/Game/GameScene.ts | 2 ++ front/src/WebRtc/MediaManager.ts | 36 ++++++++++++++++------ front/src/WebRtc/SimplePeer.ts | 48 +++++++++++++++++------------- 3 files changed, 56 insertions(+), 30 deletions(-) diff --git a/front/src/Phaser/Game/GameScene.ts b/front/src/Phaser/Game/GameScene.ts index 8763f913..44c4feb6 100644 --- a/front/src/Phaser/Game/GameScene.ts +++ b/front/src/Phaser/Game/GameScene.ts @@ -190,6 +190,7 @@ export class GameScene extends Phaser.Scene { console.log('Player disconnected from server. Reloading scene.'); this.simplePeer.closeAllConnections(); + this.simplePeer.unregister(); const key = 'somekey'+Math.round(Math.random()*10000); const game : Phaser.Scene = GameScene.createFromUrl(this.MapUrlFile, this.instance, key); @@ -610,6 +611,7 @@ export class GameScene extends Phaser.Scene { if(nextSceneKey){ // We are completely destroying the current scene to avoid using a half-backed instance when coming back to the same map. this.connection.closeConnection(); + this.simplePeer.unregister(); this.scene.stop(); this.scene.remove(this.scene.key); this.scene.start(nextSceneKey.key, { diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 03736e6e..8be141ec 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -3,7 +3,10 @@ const videoConstraint: boolean|MediaTrackConstraints = { height: { ideal: 720 }, facingMode: "user" }; -export class MediaManager { + +type UpdatedLocalStreamCallback = (media: MediaStream) => void; + +class MediaManager { localStream: MediaStream|null = null; private remoteVideo: Map = new Map(); myCamVideo: HTMLVideoElement; @@ -16,11 +19,9 @@ export class MediaManager { audio: true, video: videoConstraint }; - updatedLocalStreamCallBack : (media: MediaStream) => void; - - constructor(updatedLocalStreamCallBack : (media: MediaStream) => void) { - this.updatedLocalStreamCallBack = updatedLocalStreamCallBack; + updatedLocalStreamCallBacks : Set = new Set(); + constructor() { this.myCamVideo = this.getElementByIdOrFail('myCamVideo'); this.webrtcInAudio = this.getElementByIdOrFail('audio-webrtc-in'); this.webrtcInAudio.volume = 0.2; @@ -54,6 +55,21 @@ export class MediaManager { }); } + onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { + + this.updatedLocalStreamCallBacks.add(callback); + } + + removeUpdateLocalStreamEventListener(callback: UpdatedLocalStreamCallback): void { + this.updatedLocalStreamCallBacks.delete(callback); + } + + private triggerUpdatedLocalStreamCallbacks(stream: MediaStream): void { + for (const callback of this.updatedLocalStreamCallBacks) { + callback(stream); + } + } + activeVisio(){ const webRtc = this.getElementByIdOrFail('webRtc'); webRtc.classList.add('active'); @@ -64,7 +80,7 @@ export class MediaManager { this.cinema.style.display = "block"; this.constraintsMedia.video = videoConstraint; this.getCamera().then((stream: MediaStream) => { - this.updatedLocalStreamCallBack(stream); + this.triggerUpdatedLocalStreamCallbacks(stream); }); } @@ -79,7 +95,7 @@ export class MediaManager { }); } this.getCamera().then((stream) => { - this.updatedLocalStreamCallBack(stream); + this.triggerUpdatedLocalStreamCallbacks(stream); }); } @@ -88,7 +104,7 @@ export class MediaManager { this.microphone.style.display = "block"; this.constraintsMedia.audio = true; this.getCamera().then((stream) => { - this.updatedLocalStreamCallBack(stream); + this.triggerUpdatedLocalStreamCallbacks(stream); }); } @@ -102,7 +118,7 @@ export class MediaManager { }); } this.getCamera().then((stream) => { - this.updatedLocalStreamCallBack(stream); + this.triggerUpdatedLocalStreamCallbacks(stream); }); } @@ -308,3 +324,5 @@ export class MediaManager { } } + +export const mediaManager = new MediaManager(); diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 381b3ac2..553c9307 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -4,7 +4,7 @@ import { WebRtcSignalMessageInterface, WebRtcStartMessageInterface } from "../Connection"; -import {MediaManager} from "./MediaManager"; +import { mediaManager } from "./MediaManager"; import * as SimplePeerNamespace from "simple-peer"; const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); @@ -22,16 +22,15 @@ export class SimplePeer { private WebRtcRoomId: string; private Users: Array = new Array(); - private MediaManager: MediaManager; - private PeerConnectionArray: Map = new Map(); + private readonly updateLocalStreamCallback: (media: MediaStream) => void; constructor(Connection: Connection, WebRtcRoomId: string = "test-webrtc") { this.Connection = Connection; this.WebRtcRoomId = WebRtcRoomId; - this.MediaManager = new MediaManager((stream : MediaStream) => { - this.updatedLocalStream(); - }); + // We need to go through this weird bound function pointer in order to be able to "free" this reference later. + this.updateLocalStreamCallback = this.updatedLocalStream.bind(this); + mediaManager.onUpdateLocalStream(this.updateLocalStreamCallback); this.initialise(); } @@ -45,8 +44,8 @@ export class SimplePeer { this.receiveWebrtcSignal(message); }); - this.MediaManager.activeVisio(); - this.MediaManager.getCamera().then(() => { + mediaManager.activeVisio(); + mediaManager.getCamera().then(() => { //receive message start this.Connection.receiveWebrtcStart((message: WebRtcStartMessageInterface) => { @@ -105,8 +104,8 @@ export class SimplePeer { name = userSearch.name; } } - this.MediaManager.removeActiveVideo(user.userId); - this.MediaManager.addActiveVideo(user.userId, name); + mediaManager.removeActiveVideo(user.userId); + mediaManager.addActiveVideo(user.userId, name); const peer : SimplePeerNamespace.Instance = new Peer({ initiator: user.initiator ? user.initiator : false, @@ -143,15 +142,15 @@ export class SimplePeer { } }); if(microphoneActive){ - this.MediaManager.enabledMicrophoneByUserId(user.userId); + mediaManager.enabledMicrophoneByUserId(user.userId); }else{ - this.MediaManager.disabledMicrophoneByUserId(user.userId); + mediaManager.disabledMicrophoneByUserId(user.userId); } if(videoActive){ - this.MediaManager.enabledVideoByUserId(user.userId); + mediaManager.enabledVideoByUserId(user.userId); }else{ - this.MediaManager.disabledVideoByUserId(user.userId); + mediaManager.disabledVideoByUserId(user.userId); } this.stream(user.userId, stream); }); @@ -167,11 +166,11 @@ export class SimplePeer { // eslint-disable-next-line @typescript-eslint/no-explicit-any peer.on('error', (err: any) => { console.error(`error => ${user.userId} => ${err.code}`, err); - this.MediaManager.isError(user.userId); + mediaManager.isError(user.userId); }); peer.on('connect', () => { - this.MediaManager.isConnected(user.userId); + mediaManager.isConnected(user.userId); console.info(`connect => ${user.userId}`); }); @@ -192,7 +191,7 @@ export class SimplePeer { */ private closeConnection(userId : string) { try { - this.MediaManager.removeActiveVideo(userId); + mediaManager.removeActiveVideo(userId); const peer = this.PeerConnectionArray.get(userId); if (peer === undefined) { console.warn("Tried to close connection for user "+userId+" but could not find user") @@ -215,6 +214,13 @@ export class SimplePeer { } } + /** + * Unregisters any held event handler. + */ + public unregister() { + mediaManager.removeUpdateLocalStreamEventListener(this.updateLocalStreamCallback); + } + /** * * @param userId @@ -253,11 +259,11 @@ export class SimplePeer { */ private stream(userId : string, stream: MediaStream) { if(!stream){ - this.MediaManager.disabledVideoByUserId(userId); - this.MediaManager.disabledMicrophoneByUserId(userId); + mediaManager.disabledVideoByUserId(userId); + mediaManager.disabledMicrophoneByUserId(userId); return; } - this.MediaManager.addStreamRemoteVideo(userId, stream); + mediaManager.addStreamRemoteVideo(userId, stream); } /** @@ -266,7 +272,7 @@ export class SimplePeer { */ private addMedia (userId : string) { try { - const localStream: MediaStream|null = this.MediaManager.localStream; + const localStream: MediaStream|null = mediaManager.localStream; const peer = this.PeerConnectionArray.get(userId); if(localStream === null) { //send fake signal From 3de37bafedc6a14a875d2b7f0283685b5414a047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Tue, 23 Jun 2020 12:24:36 +0200 Subject: [PATCH 02/17] Adding a scene to configure the webcam --- front/dist/index.html | 4 + front/dist/resources/objects/arrow_right.png | Bin 0 -> 224 bytes front/dist/resources/objects/arrow_up.png | Bin 0 -> 149 bytes front/dist/resources/style/style.css | 26 ++ front/src/Phaser/Components/SoundMeter.ts | 138 ++++++++ .../src/Phaser/Components/SoundMeterSprite.ts | 44 +++ front/src/Phaser/Login/EnableCameraScene.ts | 294 ++++++++++++++++++ .../src/Phaser/Login/SelectCharacterScene.ts | 9 +- front/src/WebRtc/MediaManager.ts | 24 +- front/src/index.ts | 3 +- 10 files changed, 537 insertions(+), 5 deletions(-) create mode 100644 front/dist/resources/objects/arrow_right.png create mode 100644 front/dist/resources/objects/arrow_up.png create mode 100644 front/src/Phaser/Components/SoundMeter.ts create mode 100644 front/src/Phaser/Components/SoundMeterSprite.ts create mode 100644 front/src/Phaser/Login/EnableCameraScene.ts diff --git a/front/dist/index.html b/front/dist/index.html index 6e9735f0..a680c59a 100644 --- a/front/dist/index.html +++ b/front/dist/index.html @@ -59,6 +59,10 @@ --> +
+ + +
diff --git a/front/dist/resources/objects/arrow_right.png b/front/dist/resources/objects/arrow_right.png new file mode 100644 index 0000000000000000000000000000000000000000..58df21bbe375fc91281e5052007e9df7eb37ddc0 GIT binary patch literal 224 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=fdz&H|6fVg?4jBOuH;Rhv&5C^*T} z#W5tJ_3m|Rz5@mv%mJ5w-?xn2#onyzo$7aD@~1|I=@#*Kj`==*$ycJ-6}x9uRPho9 z^(L<;4i`MHbObV~CfP)`M!Xd5_$rqtIBD`8#}n^9izwD!RpZc1Xqvzr(ZYH_;nvd| zqW^CoAqX8ESa$GRLAj~9osLbi!htoIW%nAa;7eI(&`(>vkzEl t1tff9W4GwEz33SrY3#YWZ}l5~$#eV`Y_Ij6eF7TE;OXk;vd$@?2>{{yFwg)1 literal 0 HcmV?d00001 diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index 458dde9c..eef6216f 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -207,3 +207,29 @@ video{ opacity: 0; } } + +.webrtcsetup{ + display: none; + position: absolute; + top: 230px; + left: 0; + right: 0; + margin-left: auto; + margin-right: auto; + height: 50%; + width: 50%; +} +.webrtcsetup .background-img { + position: relative; + display: block; + width: 40%; + height: 60%; + margin-left: auto; + margin-right: auto; +} +#myCamVideoSetup { + width: 100%; +} +.webrtcsetup.active{ + display: block; +} diff --git a/front/src/Phaser/Components/SoundMeter.ts b/front/src/Phaser/Components/SoundMeter.ts new file mode 100644 index 00000000..af75940e --- /dev/null +++ b/front/src/Phaser/Components/SoundMeter.ts @@ -0,0 +1,138 @@ +/** + * Class to measure the sound volume of a media stream + */ +export class SoundMeter { + private instant: number; + private clip: number; + //private script: ScriptProcessorNode; + private analyser: AnalyserNode|undefined; + private dataArray: Uint8Array|undefined; + private context: AudioContext|undefined; + private source: MediaStreamAudioSourceNode|undefined; + + constructor() { + this.instant = 0.0; + this.clip = 0.0; + //this.script = context.createScriptProcessor(2048, 1, 1); + } + + private init(context: AudioContext) { + if (this.context === undefined) { + this.context = context; + this.analyser = this.context.createAnalyser(); + + this.analyser.fftSize = 2048; + const bufferLength = this.analyser.fftSize; + this.dataArray = new Uint8Array(bufferLength); + } + } + + public connectToSource(stream: MediaStream, context: AudioContext): void + { + this.init(context); + + this.source = this.context?.createMediaStreamSource(stream); + if (this.analyser !== undefined) { + this.source?.connect(this.analyser); + } + //analyser.connect(distortion); + //distortion.connect(this.context.destination); + //this.analyser.connect(this.context.destination); + + + } + + public getVolume(): number { + if (this.context === undefined || this.dataArray === undefined || this.analyser === undefined) { + return 0; + } + this.analyser.getByteFrequencyData(this.dataArray); + + + const input = this.dataArray; + let i; + let sum = 0.0; + //let clipcount = 0; + for (i = 0; i < input.length; ++i) { + sum += input[i] * input[i]; + // if (Math.abs(input[i]) > 0.99) { + // clipcount += 1; + // } + } + this.instant = Math.sqrt(sum / input.length); + //this.slow = 0.95 * that.slow + 0.05 * that.instant; + //this.clip = clipcount / input.length; + + //console.log('instant', this.instant, 'clip', this.clip); + + return this.instant; + } + + public stop(): void { + if (this.context === undefined) { + return; + } + if (this.source !== undefined) { + this.source.disconnect(); + } + this.context = undefined; + this.analyser = undefined; + this.dataArray = undefined; + this.source = undefined; + } + +} + + +// Meter class that generates a number correlated to audio volume. +// The meter class itself displays nothing, but it makes the +// instantaneous and time-decaying volumes available for inspection. +// It also reports on the fraction of samples that were at or near +// the top of the measurement range. +/*function SoundMeter(context) { + this.context = context; + this.instant = 0.0; + this.slow = 0.0; + this.clip = 0.0; + this.script = context.createScriptProcessor(2048, 1, 1); + const that = this; + this.script.onaudioprocess = function(event) { + const input = event.inputBuffer.getChannelData(0); + let i; + let sum = 0.0; + let clipcount = 0; + for (i = 0; i < input.length; ++i) { + sum += input[i] * input[i]; + if (Math.abs(input[i]) > 0.99) { + clipcount += 1; + } + } + that.instant = Math.sqrt(sum / input.length); + that.slow = 0.95 * that.slow + 0.05 * that.instant; + that.clip = clipcount / input.length; + }; +} + +SoundMeter.prototype.connectToSource = function(stream, callback) { + console.log('SoundMeter connecting'); + try { + this.mic = this.context.createMediaStreamSource(stream); + this.mic.connect(this.script); + // necessary to make sample run, but should not be. + this.script.connect(this.context.destination); + if (typeof callback !== 'undefined') { + callback(null); + } + } catch (e) { + console.error(e); + if (typeof callback !== 'undefined') { + callback(e); + } + } +}; + +SoundMeter.prototype.stop = function() { + this.mic.disconnect(); + this.script.disconnect(); +}; +*/ diff --git a/front/src/Phaser/Components/SoundMeterSprite.ts b/front/src/Phaser/Components/SoundMeterSprite.ts new file mode 100644 index 00000000..2787059d --- /dev/null +++ b/front/src/Phaser/Components/SoundMeterSprite.ts @@ -0,0 +1,44 @@ +import Container = Phaser.GameObjects.Container; +import {Scene} from "phaser"; +import GameObject = Phaser.GameObjects.GameObject; +import Rectangle = Phaser.GameObjects.Rectangle; + + +export class SoundMeterSprite extends Container { + private rectangles: Rectangle[] = new Array(); + private static readonly NB_BARS = 20; + + constructor(scene: Scene, x?: number, y?: number, children?: GameObject[]) { + super(scene, x, y, children); + + for (let i = 0; i < SoundMeterSprite.NB_BARS; i++) { + const rectangle = new Rectangle(scene, i * 13, 0, 10, 20, (Math.round(255 - i * 255 / SoundMeterSprite.NB_BARS) << 8) + (Math.round(i * 255 / SoundMeterSprite.NB_BARS) << 16)); + this.add(rectangle); + this.rectangles.push(rectangle); + } + } + + /** + * A number between 0 and 100 + * + * @param volume + */ + public setVolume(volume: number): void { + + const normalizedVolume = volume / 100 * SoundMeterSprite.NB_BARS; + for (let i = 0; i < SoundMeterSprite.NB_BARS; i++) { + if (normalizedVolume < i) { + this.rectangles[i].alpha = 0.5; + } else { + this.rectangles[i].alpha = 1; + } + } + } + + public getWidth(): number { + return SoundMeterSprite.NB_BARS * 13; + } + + + +} diff --git a/front/src/Phaser/Login/EnableCameraScene.ts b/front/src/Phaser/Login/EnableCameraScene.ts new file mode 100644 index 00000000..64145804 --- /dev/null +++ b/front/src/Phaser/Login/EnableCameraScene.ts @@ -0,0 +1,294 @@ +import {gameManager} from "../Game/GameManager"; +import {TextField} from "../Components/TextField"; +import {ClickButton} from "../Components/ClickButton"; +import Image = Phaser.GameObjects.Image; +import Rectangle = Phaser.GameObjects.Rectangle; +import {PLAYER_RESOURCES, PlayerResourceDescriptionInterface} from "../Entity/Character"; +import {GameSceneInitInterface} from "../Game/GameScene"; +import {StartMapInterface} from "../../Connection"; +import {mediaManager, MediaManager} from "../../WebRtc/MediaManager"; +import {RESOLUTION} from "../../Enum/EnvironmentVariable"; +import {SoundMeter} from "../Components/SoundMeter"; +import {SoundMeterSprite} from "../Components/SoundMeterSprite"; + +export const EnableCameraSceneName = "EnableCameraScene"; +enum LoginTextures { + playButton = "play_button", + icon = "icon", + mainFont = "main_font", + arrowRight = "arrow_right", + arrowUp = "arrow_up" +} + +export class EnableCameraScene extends Phaser.Scene { + private textField: TextField; + private pressReturnField: TextField; + private cameraNameField: TextField; + private logo: Image; + private arrowLeft: Image; + private arrowRight: Image; + private arrowDown: Image; + private arrowUp: Image; + private microphonesList: InputDeviceInfo[] = new Array(); + private camerasList: InputDeviceInfo[] = new Array(); + private cameraSelected: number = 0; + private microphoneSelected: number = 0; + private soundMeter: SoundMeter; + private soundMeterSprite: SoundMeterSprite; + private microphoneNameField: TextField; + + constructor() { + super({ + key: EnableCameraSceneName + }); + this.soundMeter = new SoundMeter(); + } + + preload() { + this.load.image(LoginTextures.playButton, "resources/objects/play_button.png"); + this.load.image(LoginTextures.icon, "resources/logos/tcm_full.png"); + this.load.image(LoginTextures.arrowRight, "resources/objects/arrow_right.png"); + this.load.image(LoginTextures.arrowUp, "resources/objects/arrow_up.png"); + // Note: arcade.png from the Phaser 3 examples at: https://github.com/photonstorm/phaser3-examples/tree/master/public/assets/fonts/bitmap + this.load.bitmapFont(LoginTextures.mainFont, 'resources/fonts/arcade.png', 'resources/fonts/arcade.xml'); + } + + create() { + this.textField = new TextField(this, this.game.renderer.width / 2, 50, 'Turn on your camera and microphone'); + this.textField.setOrigin(0.5).setCenterAlign(); + + this.pressReturnField = new TextField(this, this.game.renderer.width / 2, this.game.renderer.height - 30, 'Press enter to start'); + this.pressReturnField.setOrigin(0.5).setCenterAlign(); + + this.cameraNameField = new TextField(this, this.game.renderer.width / 2, this.game.renderer.height - 60, ''); + this.cameraNameField.setOrigin(0.5).setCenterAlign(); + + this.microphoneNameField = new TextField(this, this.game.renderer.width / 2, this.game.renderer.height - 40, ''); + this.microphoneNameField.setOrigin(0.5).setCenterAlign(); + + this.arrowRight = new Image(this, 0, 0, LoginTextures.arrowRight); + this.arrowRight.setOrigin(0.5, 0.5); + this.arrowRight.setVisible(false); + this.add.existing(this.arrowRight); + + this.arrowLeft = new Image(this, 0, 0, LoginTextures.arrowRight); + this.arrowLeft.setOrigin(0.5, 0.5); + this.arrowLeft.setVisible(false); + this.arrowLeft.flipX = true; + this.add.existing(this.arrowLeft); + + this.arrowUp = new Image(this, 0, 0, LoginTextures.arrowUp); + this.arrowUp.setOrigin(0.5, 0.5); + this.arrowUp.setVisible(false); + this.add.existing(this.arrowUp); + + this.arrowDown = new Image(this, 0, 0, LoginTextures.arrowUp); + this.arrowDown.setOrigin(0.5, 0.5); + this.arrowDown.setVisible(false); + this.arrowDown.flipY = true; + this.add.existing(this.arrowDown); + + this.logo = new Image(this, this.game.renderer.width - 30, this.game.renderer.height - 20, LoginTextures.icon); + this.add.existing(this.logo); + + this.input.keyboard.on('keyup-ENTER', () => { + return this.login(); + }); + + this.getElementByIdOrFail('webRtcSetup').classList.add('active'); + + const mediaPromise = mediaManager.getCamera(); + mediaPromise.then(this.getDevices.bind(this)); + mediaPromise.then(this.setupStream.bind(this)); + + this.input.keyboard.on('keydown-RIGHT', () => { + if (this.cameraSelected === this.camerasList.length - 1) { + return; + } + this.cameraSelected++; + // TODO: the change of camera should be OBSERVED (reactive) + mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this)); + }); + this.input.keyboard.on('keydown-LEFT', () => { + if (this.cameraSelected === 0) { + return; + } + this.cameraSelected--; + mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this)); + }); + this.input.keyboard.on('keydown-DOWN', () => { + if (this.microphoneSelected === this.microphonesList.length - 1) { + return; + } + this.microphoneSelected++; + // TODO: the change of camera should be OBSERVED (reactive) + mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this)); + }); + this.input.keyboard.on('keydown-UP', () => { + if (this.microphoneSelected === 0) { + return; + } + this.microphoneSelected--; + mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this)); + }); + + this.soundMeterSprite = new SoundMeterSprite(this, 50, 50); + this.soundMeterSprite.setVisible(false); + this.add.existing(this.soundMeterSprite); + } + + /** + * Function called each time a camera is changed + */ + private setupStream(stream: MediaStream): void { + const img = this.getElementByIdOrFail('webRtcSetupNoVideo'); + img.style.display = 'none'; + + const div = this.getElementByIdOrFail('myCamVideoSetup'); + div.srcObject = stream; + + this.soundMeter.connectToSource(stream, new window.AudioContext()); + + const bounds = div.getBoundingClientRect(); + this.soundMeterSprite.x = this.game.renderer.width / 2 - this.soundMeterSprite.getWidth() / 2; + this.soundMeterSprite.y = bounds.bottom / RESOLUTION + 64; + this.soundMeterSprite.setVisible(true); + + this.updateWebCamName(); + } + + private updateWebCamName(): void { + if (this.camerasList.length > 1) { + const div = this.getElementByIdOrFail('myCamVideoSetup'); + const bounds = div.getBoundingClientRect(); + + let label = this.camerasList[this.cameraSelected].label; + // remove text in parenthesis + label = label.replace(/\([^()]*\)/g, '').trim(); + // remove accents + label = label.normalize("NFD").replace(/[\u0300-\u036f]/g, ""); + this.cameraNameField.text = label; + this.cameraNameField.y = bounds.bottom / RESOLUTION + 30; + + if (this.cameraSelected < this.camerasList.length - 1) { + this.arrowRight.x = bounds.right / RESOLUTION + 16; + this.arrowRight.y = (bounds.top + bounds.height / 2) / RESOLUTION; + this.arrowRight.setVisible(true); + } else { + this.arrowRight.setVisible(false); + } + + if (this.cameraSelected > 0) { + this.arrowLeft.x = bounds.left / RESOLUTION - 16; + this.arrowLeft.y = (bounds.top + bounds.height / 2) / RESOLUTION; + this.arrowLeft.setVisible(true); + } else { + this.arrowLeft.setVisible(false); + } + } + + if (this.microphonesList.length > 1) { + let label = this.microphonesList[this.microphoneSelected].label; + // remove text in parenthesis + label = label.replace(/\([^()]*\)/g, '').trim(); + // remove accents + label = label.normalize("NFD").replace(/[\u0300-\u036f]/g, ""); + + this.microphoneNameField.text = label; + this.microphoneNameField.y = this.soundMeterSprite.y + 22; + + if (this.microphoneSelected < this.microphonesList.length - 1) { + this.arrowDown.x = this.microphoneNameField.x + this.microphoneNameField.width / 2 + 16; + this.arrowDown.y = this.microphoneNameField.y; + this.arrowDown.setVisible(true); + } else { + this.arrowDown.setVisible(false); + } + + if (this.microphoneSelected > 0) { + this.arrowUp.x = this.microphoneNameField.x - this.microphoneNameField.width / 2 - 16; + this.arrowUp.y = this.microphoneNameField.y; + this.arrowUp.setVisible(true); + } else { + this.arrowUp.setVisible(false); + } + + } + } + + update(time: number, delta: number): void { + this.pressReturnField.setVisible(!!(Math.floor(time / 500) % 2)); + + console.log(this.soundMeter.getVolume()); + this.soundMeterSprite.setVolume(this.soundMeter.getVolume()); + } + + private async login(): Promise { + this.getElementByIdOrFail('webRtcSetup').style.display = 'none'; + this.soundMeter.stop(); + + // Do we have a start URL in the address bar? If so, let's redirect to this address + const instanceAndMapUrl = this.findMapUrl(); + if (instanceAndMapUrl !== null) { + const [mapUrl, instance] = instanceAndMapUrl; + const key = gameManager.loadMap(mapUrl, this.scene, instance); + this.scene.start(key, { + startLayerName: window.location.hash ? window.location.hash.substr(1) : undefined + } as GameSceneInitInterface); + return { + mapUrlStart: mapUrl, + startInstance: instance + }; + } else { + // If we do not have a map address in the URL, let's ask the server for a start map. + return gameManager.loadStartMap().then((startMap: StartMapInterface) => { + const key = gameManager.loadMap(window.location.protocol + "//" + startMap.mapUrlStart, this.scene, startMap.startInstance); + this.scene.start(key); + return startMap; + }).catch((err) => { + console.error(err); + throw err; + }); + } + } + + /** + * Returns the map URL and the instance from the current URL + */ + private findMapUrl(): [string, string]|null { + const path = window.location.pathname; + if (!path.startsWith('/_/')) { + return null; + } + const instanceAndMap = path.substr(3); + const firstSlash = instanceAndMap.indexOf('/'); + if (firstSlash === -1) { + return null; + } + const instance = instanceAndMap.substr(0, firstSlash); + return [window.location.protocol+'//'+instanceAndMap.substr(firstSlash+1), instance]; + } + + private async getDevices() { + const mediaDeviceInfos = await navigator.mediaDevices.enumerateDevices(); + for (const mediaDeviceInfo of mediaDeviceInfos) { + if (mediaDeviceInfo instanceof InputDeviceInfo) { + if (mediaDeviceInfo.kind === 'audioinput') { + this.microphonesList.push(mediaDeviceInfo); + } else if (mediaDeviceInfo.kind === 'videoinput') { + this.camerasList.push(mediaDeviceInfo); + } + } + } + this.updateWebCamName(); + } + + private getElementByIdOrFail(id: string): T { + const elem = document.getElementById(id); + if (elem === null) { + throw new Error("Cannot find HTML element with id '"+id+"'"); + } + // FIXME: does not check the type of the returned type + return elem as T; + } +} diff --git a/front/src/Phaser/Login/SelectCharacterScene.ts b/front/src/Phaser/Login/SelectCharacterScene.ts index 5175a7b8..ddfd5c3b 100644 --- a/front/src/Phaser/Login/SelectCharacterScene.ts +++ b/front/src/Phaser/Login/SelectCharacterScene.ts @@ -6,6 +6,7 @@ import Rectangle = Phaser.GameObjects.Rectangle; import {PLAYER_RESOURCES, PlayerResourceDescriptionInterface} from "../Entity/Character"; import {GameSceneInitInterface} from "../Game/GameScene"; import {StartMapInterface} from "../../Connection"; +import {EnableCameraSceneName} from "./EnableCameraScene"; //todo: put this constants in a dedicated file export const SelectCharacterSceneName = "SelectCharacterScene"; @@ -116,11 +117,13 @@ export class SelectCharacterScene extends Phaser.Scene { this.pressReturnField.setVisible(!!(Math.floor(time / 500) % 2)); } - private async login(name: string): Promise { + private login(name: string): void { gameManager.storePlayerDetails(name, this.selectedPlayer.texture.key); + this.scene.start(EnableCameraSceneName); + // Do we have a start URL in the address bar? If so, let's redirect to this address - const instanceAndMapUrl = this.findMapUrl(); + /*const instanceAndMapUrl = this.findMapUrl(); if (instanceAndMapUrl !== null) { const [mapUrl, instance] = instanceAndMapUrl; const key = gameManager.loadMap(mapUrl, this.scene, instance); @@ -141,7 +144,7 @@ export class SelectCharacterScene extends Phaser.Scene { console.error(err); throw err; }); - } + }*/ } /** diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 8be141ec..a56e0c3c 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -6,7 +6,9 @@ const videoConstraint: boolean|MediaTrackConstraints = { type UpdatedLocalStreamCallback = (media: MediaStream) => void; -class MediaManager { +// TODO: Split MediaManager in 2 classes: MediaManagerUI (in charge of HTML) and MediaManager (singleton in charge of the camera only) +// TODO: verify that microphone event listeners are not triggered plenty of time NOW (since MediaManager is created many times!!!!) +export class MediaManager { localStream: MediaStream|null = null; private remoteVideo: Map = new Map(); myCamVideo: HTMLVideoElement; @@ -154,6 +156,26 @@ class MediaManager { return promise; } + setCamera(id: string): Promise { + let video = this.constraintsMedia.video; + if (typeof(video) === 'boolean' || video === undefined) { + video = {} + } + video.deviceId = id; + + return this.getCamera(); + } + + setMicrophone(id: string): Promise { + let audio = this.constraintsMedia.audio; + if (typeof(audio) === 'boolean' || audio === undefined) { + audio = {} + } + audio.deviceId = id; + + return this.getCamera(); + } + /** * * @param userId diff --git a/front/src/index.ts b/front/src/index.ts index 6221d3ad..15c2e0e8 100644 --- a/front/src/index.ts +++ b/front/src/index.ts @@ -6,13 +6,14 @@ import {LoginScene} from "./Phaser/Login/LoginScene"; import {ReconnectingScene} from "./Phaser/Reconnecting/ReconnectingScene"; import {gameManager} from "./Phaser/Game/GameManager"; import {SelectCharacterScene} from "./Phaser/Login/SelectCharacterScene"; +import {EnableCameraScene} from "./Phaser/Login/EnableCameraScene"; const config: GameConfig = { title: "Office game", width: window.innerWidth / RESOLUTION, height: window.innerHeight / RESOLUTION, parent: "game", - scene: [LoginScene, SelectCharacterScene, ReconnectingScene], + scene: [LoginScene, SelectCharacterScene, EnableCameraScene, ReconnectingScene], zoom: RESOLUTION, physics: { default: "arcade", From 253108eba08031b793fcaa0959a92104eb2d48a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Wed, 24 Jun 2020 17:29:23 +0200 Subject: [PATCH 03/17] - Making the EnableCameraScene responsive - Enabling click on arrows --- front/dist/resources/style/style.css | 3 +- front/src/Phaser/Login/EnableCameraScene.ts | 126 ++++++++++++-------- 2 files changed, 79 insertions(+), 50 deletions(-) diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index eef6216f..5bf3559d 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -211,7 +211,7 @@ video{ .webrtcsetup{ display: none; position: absolute; - top: 230px; + top: 140px; left: 0; right: 0; margin-left: auto; @@ -229,6 +229,7 @@ video{ } #myCamVideoSetup { width: 100%; + height: 100%; } .webrtcsetup.active{ display: block; diff --git a/front/src/Phaser/Login/EnableCameraScene.ts b/front/src/Phaser/Login/EnableCameraScene.ts index 64145804..074582ea 100644 --- a/front/src/Phaser/Login/EnableCameraScene.ts +++ b/front/src/Phaser/Login/EnableCameraScene.ts @@ -36,6 +36,7 @@ export class EnableCameraScene extends Phaser.Scene { private soundMeter: SoundMeter; private soundMeterSprite: SoundMeterSprite; private microphoneNameField: TextField; + private repositionCallback: (this: Window, ev: UIEvent) => any; constructor() { super({ @@ -54,7 +55,7 @@ export class EnableCameraScene extends Phaser.Scene { } create() { - this.textField = new TextField(this, this.game.renderer.width / 2, 50, 'Turn on your camera and microphone'); + this.textField = new TextField(this, this.game.renderer.width / 2, 20, 'Turn on your camera and microphone'); this.textField.setOrigin(0.5).setCenterAlign(); this.pressReturnField = new TextField(this, this.game.renderer.width / 2, this.game.renderer.height - 30, 'Press enter to start'); @@ -69,23 +70,27 @@ export class EnableCameraScene extends Phaser.Scene { this.arrowRight = new Image(this, 0, 0, LoginTextures.arrowRight); this.arrowRight.setOrigin(0.5, 0.5); this.arrowRight.setVisible(false); + this.arrowRight.setInteractive().on('pointerdown', this.nextCam.bind(this)); this.add.existing(this.arrowRight); this.arrowLeft = new Image(this, 0, 0, LoginTextures.arrowRight); this.arrowLeft.setOrigin(0.5, 0.5); this.arrowLeft.setVisible(false); this.arrowLeft.flipX = true; + this.arrowLeft.setInteractive().on('pointerdown', this.previousCam.bind(this)); this.add.existing(this.arrowLeft); this.arrowUp = new Image(this, 0, 0, LoginTextures.arrowUp); this.arrowUp.setOrigin(0.5, 0.5); this.arrowUp.setVisible(false); + this.arrowUp.setInteractive().on('pointerdown', this.previousMic.bind(this)); this.add.existing(this.arrowUp); this.arrowDown = new Image(this, 0, 0, LoginTextures.arrowUp); this.arrowDown.setOrigin(0.5, 0.5); this.arrowDown.setVisible(false); this.arrowDown.flipY = true; + this.arrowDown.setInteractive().on('pointerdown', this.nextMic.bind(this)); this.add.existing(this.arrowDown); this.logo = new Image(this, this.game.renderer.width - 30, this.game.renderer.height - 20, LoginTextures.icon); @@ -101,40 +106,51 @@ export class EnableCameraScene extends Phaser.Scene { mediaPromise.then(this.getDevices.bind(this)); mediaPromise.then(this.setupStream.bind(this)); - this.input.keyboard.on('keydown-RIGHT', () => { - if (this.cameraSelected === this.camerasList.length - 1) { - return; - } - this.cameraSelected++; - // TODO: the change of camera should be OBSERVED (reactive) - mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this)); - }); - this.input.keyboard.on('keydown-LEFT', () => { - if (this.cameraSelected === 0) { - return; - } - this.cameraSelected--; - mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this)); - }); - this.input.keyboard.on('keydown-DOWN', () => { - if (this.microphoneSelected === this.microphonesList.length - 1) { - return; - } - this.microphoneSelected++; - // TODO: the change of camera should be OBSERVED (reactive) - mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this)); - }); - this.input.keyboard.on('keydown-UP', () => { - if (this.microphoneSelected === 0) { - return; - } - this.microphoneSelected--; - mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this)); - }); + this.input.keyboard.on('keydown-RIGHT', this.nextCam.bind(this)); + this.input.keyboard.on('keydown-LEFT', this.previousCam.bind(this)); + this.input.keyboard.on('keydown-DOWN', this.nextMic.bind(this)); + this.input.keyboard.on('keydown-UP', this.previousMic.bind(this)); this.soundMeterSprite = new SoundMeterSprite(this, 50, 50); this.soundMeterSprite.setVisible(false); this.add.existing(this.soundMeterSprite); + + this.repositionCallback = this.reposition.bind(this); + window.addEventListener('resize', this.repositionCallback); + } + + private previousCam(): void { + if (this.cameraSelected === 0) { + return; + } + this.cameraSelected--; + mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this)); + } + + private nextCam(): void { + if (this.cameraSelected === this.camerasList.length - 1) { + return; + } + this.cameraSelected++; + // TODO: the change of camera should be OBSERVED (reactive) + mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this)); + } + + private previousMic(): void { + if (this.microphoneSelected === 0) { + return; + } + this.microphoneSelected--; + mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this)); + } + + private nextMic(): void { + if (this.microphoneSelected === this.microphonesList.length - 1) { + return; + } + this.microphoneSelected++; + // TODO: the change of camera should be OBSERVED (reactive) + mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this)); } /** @@ -149,18 +165,12 @@ export class EnableCameraScene extends Phaser.Scene { this.soundMeter.connectToSource(stream, new window.AudioContext()); - const bounds = div.getBoundingClientRect(); - this.soundMeterSprite.x = this.game.renderer.width / 2 - this.soundMeterSprite.getWidth() / 2; - this.soundMeterSprite.y = bounds.bottom / RESOLUTION + 64; - this.soundMeterSprite.setVisible(true); - this.updateWebCamName(); } private updateWebCamName(): void { if (this.camerasList.length > 1) { const div = this.getElementByIdOrFail('myCamVideoSetup'); - const bounds = div.getBoundingClientRect(); let label = this.camerasList[this.cameraSelected].label; // remove text in parenthesis @@ -168,25 +178,19 @@ export class EnableCameraScene extends Phaser.Scene { // remove accents label = label.normalize("NFD").replace(/[\u0300-\u036f]/g, ""); this.cameraNameField.text = label; - this.cameraNameField.y = bounds.bottom / RESOLUTION + 30; if (this.cameraSelected < this.camerasList.length - 1) { - this.arrowRight.x = bounds.right / RESOLUTION + 16; - this.arrowRight.y = (bounds.top + bounds.height / 2) / RESOLUTION; this.arrowRight.setVisible(true); } else { this.arrowRight.setVisible(false); } if (this.cameraSelected > 0) { - this.arrowLeft.x = bounds.left / RESOLUTION - 16; - this.arrowLeft.y = (bounds.top + bounds.height / 2) / RESOLUTION; this.arrowLeft.setVisible(true); } else { this.arrowLeft.setVisible(false); } } - if (this.microphonesList.length > 1) { let label = this.microphonesList[this.microphoneSelected].label; // remove text in parenthesis @@ -195,37 +199,61 @@ export class EnableCameraScene extends Phaser.Scene { label = label.normalize("NFD").replace(/[\u0300-\u036f]/g, ""); this.microphoneNameField.text = label; - this.microphoneNameField.y = this.soundMeterSprite.y + 22; if (this.microphoneSelected < this.microphonesList.length - 1) { - this.arrowDown.x = this.microphoneNameField.x + this.microphoneNameField.width / 2 + 16; - this.arrowDown.y = this.microphoneNameField.y; this.arrowDown.setVisible(true); } else { this.arrowDown.setVisible(false); } if (this.microphoneSelected > 0) { - this.arrowUp.x = this.microphoneNameField.x - this.microphoneNameField.width / 2 - 16; - this.arrowUp.y = this.microphoneNameField.y; this.arrowUp.setVisible(true); } else { this.arrowUp.setVisible(false); } } + this.reposition(); + } + + private reposition(): void { + const div = this.getElementByIdOrFail('myCamVideoSetup'); + const bounds = div.getBoundingClientRect(); + + this.cameraNameField.y = bounds.top / RESOLUTION - 8; + + this.soundMeterSprite.x = this.game.renderer.width / 2 - this.soundMeterSprite.getWidth() / 2; + this.soundMeterSprite.y = bounds.bottom / RESOLUTION + 16; + this.soundMeterSprite.setVisible(true); + + this.microphoneNameField.y = this.soundMeterSprite.y + 22; + + this.arrowRight.x = bounds.right / RESOLUTION + 16; + this.arrowRight.y = (bounds.top + bounds.height / 2) / RESOLUTION; + + this.arrowLeft.x = bounds.left / RESOLUTION - 16; + this.arrowLeft.y = (bounds.top + bounds.height / 2) / RESOLUTION; + + this.arrowDown.x = this.microphoneNameField.x + this.microphoneNameField.width / 2 + 16; + this.arrowDown.y = this.microphoneNameField.y; + + this.arrowUp.x = this.microphoneNameField.x - this.microphoneNameField.width / 2 - 16; + this.arrowUp.y = this.microphoneNameField.y; + + this.pressReturnField.y = Math.max(this.game.renderer.height - 30, this.microphoneNameField.y + 20); + this.logo.y = Math.max(this.game.renderer.height - 20, this.microphoneNameField.y + 30); } update(time: number, delta: number): void { this.pressReturnField.setVisible(!!(Math.floor(time / 500) % 2)); - console.log(this.soundMeter.getVolume()); this.soundMeterSprite.setVolume(this.soundMeter.getVolume()); } private async login(): Promise { this.getElementByIdOrFail('webRtcSetup').style.display = 'none'; this.soundMeter.stop(); + window.removeEventListener('resize', this.repositionCallback); // Do we have a start URL in the address bar? If so, let's redirect to this address const instanceAndMapUrl = this.findMapUrl(); From b3c18702bb7eb1dd2eccf068ce1130d57f41bc12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Wed, 24 Jun 2020 17:46:41 +0200 Subject: [PATCH 04/17] Adding borders, centering camera, fixing small bug on resize when no camera is enabled --- front/dist/resources/style/style.css | 3 +++ front/src/Phaser/Login/EnableCameraScene.ts | 12 ++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index 5bf3559d..5f0e1cab 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -218,6 +218,7 @@ video{ margin-right: auto; height: 50%; width: 50%; + border: white 6px solid; } .webrtcsetup .background-img { position: relative; @@ -226,6 +227,8 @@ video{ height: 60%; margin-left: auto; margin-right: auto; + top: 50%; + transform: translateY(-50%); } #myCamVideoSetup { width: 100%; diff --git a/front/src/Phaser/Login/EnableCameraScene.ts b/front/src/Phaser/Login/EnableCameraScene.ts index 074582ea..9a5100af 100644 --- a/front/src/Phaser/Login/EnableCameraScene.ts +++ b/front/src/Phaser/Login/EnableCameraScene.ts @@ -36,7 +36,7 @@ export class EnableCameraScene extends Phaser.Scene { private soundMeter: SoundMeter; private soundMeterSprite: SoundMeterSprite; private microphoneNameField: TextField; - private repositionCallback: (this: Window, ev: UIEvent) => any; + private repositionCallback: (this: Window, ev: UIEvent) => void; constructor() { super({ @@ -164,6 +164,7 @@ export class EnableCameraScene extends Phaser.Scene { div.srcObject = stream; this.soundMeter.connectToSource(stream, new window.AudioContext()); + this.soundMeterSprite.setVisible(true); this.updateWebCamName(); } @@ -217,14 +218,17 @@ export class EnableCameraScene extends Phaser.Scene { } private reposition(): void { - const div = this.getElementByIdOrFail('myCamVideoSetup'); - const bounds = div.getBoundingClientRect(); + let div = this.getElementByIdOrFail('myCamVideoSetup'); + let bounds = div.getBoundingClientRect(); + if (!div.srcObject) { + div = this.getElementByIdOrFail('webRtcSetup'); + bounds = div.getBoundingClientRect(); + } this.cameraNameField.y = bounds.top / RESOLUTION - 8; this.soundMeterSprite.x = this.game.renderer.width / 2 - this.soundMeterSprite.getWidth() / 2; this.soundMeterSprite.y = bounds.bottom / RESOLUTION + 16; - this.soundMeterSprite.setVisible(true); this.microphoneNameField.y = this.soundMeterSprite.y + 22; From a52b1f612bdfacb1ae7b95b82c6a5aea43113360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Wed, 24 Jun 2020 18:09:59 +0200 Subject: [PATCH 05/17] Reorganizing on x axis too. --- front/src/Phaser/Login/EnableCameraScene.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/front/src/Phaser/Login/EnableCameraScene.ts b/front/src/Phaser/Login/EnableCameraScene.ts index 9a5100af..735bfb19 100644 --- a/front/src/Phaser/Login/EnableCameraScene.ts +++ b/front/src/Phaser/Login/EnableCameraScene.ts @@ -225,6 +225,12 @@ export class EnableCameraScene extends Phaser.Scene { bounds = div.getBoundingClientRect(); } + this.textField.x = this.game.renderer.width / 2; + this.cameraNameField.x = this.game.renderer.width / 2; + this.microphoneNameField.x = this.game.renderer.width / 2; + this.pressReturnField.x = this.game.renderer.width / 2; + this.pressReturnField.x = this.game.renderer.width / 2; + this.cameraNameField.y = bounds.top / RESOLUTION - 8; this.soundMeterSprite.x = this.game.renderer.width / 2 - this.soundMeterSprite.getWidth() / 2; @@ -245,6 +251,7 @@ export class EnableCameraScene extends Phaser.Scene { this.arrowUp.y = this.microphoneNameField.y; this.pressReturnField.y = Math.max(this.game.renderer.height - 30, this.microphoneNameField.y + 20); + this.logo.x = this.game.renderer.width - 30; this.logo.y = Math.max(this.game.renderer.height - 20, this.microphoneNameField.y + 30); } From 371b4f0063bbda340e287417f4745b2da0f2dcd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 09:28:00 +0200 Subject: [PATCH 06/17] Fixing Firefox compatibility by remove references to InputDeviceInfo --- front/src/Phaser/Login/EnableCameraScene.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/front/src/Phaser/Login/EnableCameraScene.ts b/front/src/Phaser/Login/EnableCameraScene.ts index 735bfb19..264173fd 100644 --- a/front/src/Phaser/Login/EnableCameraScene.ts +++ b/front/src/Phaser/Login/EnableCameraScene.ts @@ -29,8 +29,8 @@ export class EnableCameraScene extends Phaser.Scene { private arrowRight: Image; private arrowDown: Image; private arrowUp: Image; - private microphonesList: InputDeviceInfo[] = new Array(); - private camerasList: InputDeviceInfo[] = new Array(); + private microphonesList: MediaDeviceInfo[] = new Array(); + private camerasList: MediaDeviceInfo[] = new Array(); private cameraSelected: number = 0; private microphoneSelected: number = 0; private soundMeter: SoundMeter; @@ -311,12 +311,10 @@ export class EnableCameraScene extends Phaser.Scene { private async getDevices() { const mediaDeviceInfos = await navigator.mediaDevices.enumerateDevices(); for (const mediaDeviceInfo of mediaDeviceInfos) { - if (mediaDeviceInfo instanceof InputDeviceInfo) { - if (mediaDeviceInfo.kind === 'audioinput') { - this.microphonesList.push(mediaDeviceInfo); - } else if (mediaDeviceInfo.kind === 'videoinput') { - this.camerasList.push(mediaDeviceInfo); - } + if (mediaDeviceInfo.kind === 'audioinput') { + this.microphonesList.push(mediaDeviceInfo); + } else if (mediaDeviceInfo.kind === 'videoinput') { + this.camerasList.push(mediaDeviceInfo); } } this.updateWebCamName(); From d18abaf0678c7e84cd23d98ed63676f0cf219cc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 10:32:47 +0200 Subject: [PATCH 07/17] Enabling optional https in development --- .github/workflows/continuous_integration.yml | 2 +- deeployer.libsonnet | 2 +- docker-compose.yaml | 26 ++++++++++++++++++-- front/src/Enum/EnvironmentVariable.ts | 2 +- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index a326bb1b..7c74fb66 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -29,7 +29,7 @@ jobs: - name: "Build" run: yarn run build env: - API_URL: "http://localhost:8080" + API_URL: "localhost:8080" working-directory: "front" - name: "Lint" diff --git a/deeployer.libsonnet b/deeployer.libsonnet index 975686be..69bc8fcf 100644 --- a/deeployer.libsonnet +++ b/deeployer.libsonnet @@ -24,7 +24,7 @@ }, "ports": [80], "env": { - "API_URL": "https://api."+url + "API_URL": "api."+url } }, "website": { diff --git a/docker-compose.yaml b/docker-compose.yaml index b2093f0c..85027a3d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,9 +2,14 @@ version: "3" services: reverse-proxy: image: traefik:v2.0 - command: --api.insecure=true --providers.docker + command: + - --api.insecure=true + - --providers.docker + - --entryPoints.web.address=:80 + - --entryPoints.websecure.address=:443 ports: - "80:80" + - "443:443" # The Web UI (enabled by --api.insecure=true) - "8080:8080" depends_on: @@ -19,14 +24,20 @@ services: DEBUG_MODE: "$DEBUG_MODE" HOST: "0.0.0.0" NODE_ENV: development - API_URL: http://api.workadventure.localhost + API_URL: api.workadventure.localhost STARTUP_COMMAND_1: yarn install command: yarn run start volumes: - ./front:/usr/src/app labels: - "traefik.http.routers.front.rule=Host(`play.workadventure.localhost`)" + - "traefik.http.routers.front.entryPoints=web" - "traefik.http.services.front.loadbalancer.server.port=8080" + - "traefik.http.routers.front-ssl.rule=Host(`play.workadventure.localhost`)" + - "traefik.http.routers.front-ssl.entryPoints=websecure" + - "traefik.http.routers.front-ssl.tls=true" + - "traefik.http.routers.front-ssl.service=front" + back: image: thecodingmachine/nodejs:12 @@ -39,7 +50,13 @@ services: - ./back:/usr/src/app labels: - "traefik.http.routers.back.rule=Host(`api.workadventure.localhost`)" + - "traefik.http.routers.back.entryPoints=web" - "traefik.http.services.back.loadbalancer.server.port=8080" + - "traefik.http.routers.back-ssl.rule=Host(`api.workadventure.localhost`)" + - "traefik.http.routers.back-ssl.entryPoints=websecure" + - "traefik.http.routers.back-ssl.tls=true" + - "traefik.http.routers.back-ssl.service=back" + website: image: thecodingmachine/nodejs:12-apache @@ -51,4 +68,9 @@ services: - ./website:/var/www/html labels: - "traefik.http.routers.website.rule=Host(`workadventure.localhost`)" + - "traefik.http.routers.website.entryPoints=web" - "traefik.http.services.website.loadbalancer.server.port=80" + - "traefik.http.routers.website-ssl.rule=Host(`workadventure.localhost`)" + - "traefik.http.routers.website-ssl.entryPoints=websecure" + - "traefik.http.routers.website-ssl.tls=true" + - "traefik.http.routers.website-ssl.service=website" diff --git a/front/src/Enum/EnvironmentVariable.ts b/front/src/Enum/EnvironmentVariable.ts index 6e0edd8f..c1f9efe4 100644 --- a/front/src/Enum/EnvironmentVariable.ts +++ b/front/src/Enum/EnvironmentVariable.ts @@ -1,5 +1,5 @@ const DEBUG_MODE: boolean = process.env.DEBUG_MODE == "true"; -const API_URL = process.env.API_URL || "http://api.workadventure.localhost"; +const API_URL = window.location.protocol + '//' + process.env.API_URL || "http://api.workadventure.localhost"; const RESOLUTION = 3; const ZOOM_LEVEL = 1/*3/4*/; const POSITION_DELAY = 200; // Wait 200ms between sending position events From 8db615551a43d03b33b9dde00bc9e9eacec58d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 10:33:26 +0200 Subject: [PATCH 08/17] Fixing device switching in Firefox --- front/src/WebRtc/MediaManager.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index a56e0c3c..50f4bf15 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -161,7 +161,9 @@ export class MediaManager { if (typeof(video) === 'boolean' || video === undefined) { video = {} } - video.deviceId = id; + video.deviceId = { + exact: id + }; return this.getCamera(); } From 7e2bf38562ed6dfa7209a073a3ac398087aa65a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 10:43:10 +0200 Subject: [PATCH 09/17] Fixing hot reloading --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 85027a3d..74bbafbf 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -31,7 +31,7 @@ services: - ./front:/usr/src/app labels: - "traefik.http.routers.front.rule=Host(`play.workadventure.localhost`)" - - "traefik.http.routers.front.entryPoints=web" + - "traefik.http.routers.front.entryPoints=web,traefik" - "traefik.http.services.front.loadbalancer.server.port=8080" - "traefik.http.routers.front-ssl.rule=Host(`play.workadventure.localhost`)" - "traefik.http.routers.front-ssl.entryPoints=websecure" From c322de4412a3b6edb34cb0314e64537a4871fff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 10:43:27 +0200 Subject: [PATCH 10/17] Moving to async/await --- front/src/WebRtc/MediaManager.ts | 42 +++++++++++++++----------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 50f4bf15..cdee2ae8 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -125,35 +125,33 @@ export class MediaManager { } //get camera - getCamera(): Promise { - let promise = null; - + async getCamera(): Promise { if (navigator.mediaDevices === undefined) { - return Promise.reject(new Error('Unable to access your camera or microphone. Your browser is too old (or you are running a development version of WorkAdventure on Firefox)')); + if (window.location.protocol === 'http:') { + throw new Error('Unable to access your camera or microphone. You need to use a HTTPS connection.'); + } else { + throw new Error('Unable to access your camera or microphone. Your browser is too old.'); + } } try { - promise = navigator.mediaDevices.getUserMedia(this.constraintsMedia) - .then((stream: MediaStream) => { - this.localStream = stream; - this.myCamVideo.srcObject = this.localStream; + let stream = await navigator.mediaDevices.getUserMedia(this.constraintsMedia); - //TODO resize remote cam - /*console.log(this.localStream.getTracks()); - let videoMediaStreamTrack = this.localStream.getTracks().find((media : MediaStreamTrack) => media.kind === "video"); - let {width, height} = videoMediaStreamTrack.getSettings(); - console.info(`${width}x${height}`); // 6*/ + this.localStream = stream; + this.myCamVideo.srcObject = this.localStream; - return stream; - }).catch((err) => { - console.info("error get media ", this.constraintsMedia.video, this.constraintsMedia.audio, err); - this.localStream = null; - throw err; - }); - } catch (e) { - promise = Promise.reject(e); + return stream; + + //TODO resize remote cam + /*console.log(this.localStream.getTracks()); + let videoMediaStreamTrack = this.localStream.getTracks().find((media : MediaStreamTrack) => media.kind === "video"); + let {width, height} = videoMediaStreamTrack.getSettings(); + console.info(`${width}x${height}`); // 6*/ + } catch (err) { + console.info("error get media ", this.constraintsMedia.video, this.constraintsMedia.audio, err); + this.localStream = null; + throw err; } - return promise; } setCamera(id: string): Promise { From e3e7b92c6a721b6fa3adf0da1393cdf1d434fc99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 10:43:42 +0200 Subject: [PATCH 11/17] Fixing errors when arrows touched and no cam --- front/src/Phaser/Login/EnableCameraScene.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/front/src/Phaser/Login/EnableCameraScene.ts b/front/src/Phaser/Login/EnableCameraScene.ts index 264173fd..6d96459e 100644 --- a/front/src/Phaser/Login/EnableCameraScene.ts +++ b/front/src/Phaser/Login/EnableCameraScene.ts @@ -120,7 +120,7 @@ export class EnableCameraScene extends Phaser.Scene { } private previousCam(): void { - if (this.cameraSelected === 0) { + if (this.cameraSelected === 0 || this.camerasList.length === 0) { return; } this.cameraSelected--; @@ -128,7 +128,7 @@ export class EnableCameraScene extends Phaser.Scene { } private nextCam(): void { - if (this.cameraSelected === this.camerasList.length - 1) { + if (this.cameraSelected === this.camerasList.length - 1 || this.camerasList.length === 0) { return; } this.cameraSelected++; @@ -137,7 +137,7 @@ export class EnableCameraScene extends Phaser.Scene { } private previousMic(): void { - if (this.microphoneSelected === 0) { + if (this.microphoneSelected === 0 || this.microphonesList.length === 0) { return; } this.microphoneSelected--; @@ -145,7 +145,7 @@ export class EnableCameraScene extends Phaser.Scene { } private nextMic(): void { - if (this.microphoneSelected === this.microphonesList.length - 1) { + if (this.microphoneSelected === this.microphonesList.length - 1 || this.microphonesList.length === 0) { return; } this.microphoneSelected++; From 5f5cec93ea7e8dd70044f54a23d217ee4ba29f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 11:26:55 +0200 Subject: [PATCH 12/17] Audio device => exact mode --- front/src/WebRtc/MediaManager.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index cdee2ae8..debe7730 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -171,7 +171,9 @@ export class MediaManager { if (typeof(audio) === 'boolean' || audio === undefined) { audio = {} } - audio.deviceId = id; + audio.deviceId = { + exact: id + }; return this.getCamera(); } From 1978c1a324f2659f0c791e4d65b53a649da16b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 11:35:20 +0200 Subject: [PATCH 13/17] Lint --- front/src/WebRtc/MediaManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index debe7730..e69850a2 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -135,7 +135,7 @@ export class MediaManager { } try { - let stream = await navigator.mediaDevices.getUserMedia(this.constraintsMedia); + const stream = await navigator.mediaDevices.getUserMedia(this.constraintsMedia); this.localStream = stream; this.myCamVideo.srcObject = this.localStream; From bb6b07536e504ba1e303a242b27bd87b378e0475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 11:39:18 +0200 Subject: [PATCH 14/17] Fixing Jasmine tests --- front/src/Enum/EnvironmentVariable.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front/src/Enum/EnvironmentVariable.ts b/front/src/Enum/EnvironmentVariable.ts index c1f9efe4..6d84df3c 100644 --- a/front/src/Enum/EnvironmentVariable.ts +++ b/front/src/Enum/EnvironmentVariable.ts @@ -1,5 +1,5 @@ const DEBUG_MODE: boolean = process.env.DEBUG_MODE == "true"; -const API_URL = window.location.protocol + '//' + process.env.API_URL || "http://api.workadventure.localhost"; +const API_URL = (window !== undefined ? window.location.protocol : 'http://') + '//' + (process.env.API_URL || "http://api.workadventure.localhost"); const RESOLUTION = 3; const ZOOM_LEVEL = 1/*3/4*/; const POSITION_DELAY = 200; // Wait 200ms between sending position events From bfec2317c94a6b255218e6c03ee7fadf3a88b59e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 11:39:38 +0200 Subject: [PATCH 15/17] Fixing default value --- front/src/Enum/EnvironmentVariable.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front/src/Enum/EnvironmentVariable.ts b/front/src/Enum/EnvironmentVariable.ts index 6d84df3c..ae51921b 100644 --- a/front/src/Enum/EnvironmentVariable.ts +++ b/front/src/Enum/EnvironmentVariable.ts @@ -1,5 +1,5 @@ const DEBUG_MODE: boolean = process.env.DEBUG_MODE == "true"; -const API_URL = (window !== undefined ? window.location.protocol : 'http://') + '//' + (process.env.API_URL || "http://api.workadventure.localhost"); +const API_URL = (window !== undefined ? window.location.protocol : 'http://') + '//' + (process.env.API_URL || "api.workadventure.localhost"); const RESOLUTION = 3; const ZOOM_LEVEL = 1/*3/4*/; const POSITION_DELAY = 200; // Wait 200ms between sending position events From 63e2e176d309df14e6ad3b2904a5b2cd88ea4b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 11:56:11 +0200 Subject: [PATCH 16/17] Fixing tests --- front/src/Enum/EnvironmentVariable.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front/src/Enum/EnvironmentVariable.ts b/front/src/Enum/EnvironmentVariable.ts index ae51921b..e35818bc 100644 --- a/front/src/Enum/EnvironmentVariable.ts +++ b/front/src/Enum/EnvironmentVariable.ts @@ -1,5 +1,5 @@ const DEBUG_MODE: boolean = process.env.DEBUG_MODE == "true"; -const API_URL = (window !== undefined ? window.location.protocol : 'http://') + '//' + (process.env.API_URL || "api.workadventure.localhost"); +const API_URL = (typeof(window) !== 'undefined' ? window.location.protocol : 'http:') + '//' + (process.env.API_URL || "api.workadventure.localhost"); const RESOLUTION = 3; const ZOOM_LEVEL = 1/*3/4*/; const POSITION_DELAY = 200; // Wait 200ms between sending position events From 56302dafd6663e7031704584b7073f6628868be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 25 Jun 2020 12:07:43 +0200 Subject: [PATCH 17/17] Adding version number to Deeployer --- deeployer.libsonnet | 1 + 1 file changed, 1 insertion(+) diff --git a/deeployer.libsonnet b/deeployer.libsonnet index 69bc8fcf..09074148 100644 --- a/deeployer.libsonnet +++ b/deeployer.libsonnet @@ -4,6 +4,7 @@ local tag = namespace, local url = if namespace == "master" then "workadventu.re" else namespace+".workadventure.test.thecodingmachine.com", "$schema": "https://raw.githubusercontent.com/thecodingmachine/deeployer/master/deeployer.schema.json", + "version": "1.0", "containers": { "back": { "image": "thecodingmachine/workadventure-back:"+tag,