_Bastler/partey_workadventure
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1820 from thecodingmachine/feature-voice-chat-indicator

Browse Source 
Feature voice chat indicator
This commit is contained in:
David Négrier 2022-02-16 16:46:13 +01:00 committed by GitHub
commit a7264954ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 227 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
front/dist/.gitignore vendored
View File

@ -4,3 +4,4 @@ index.html
style.*.css
!env-config.template.js
*.png
!/resources/**

BIN
front/dist/resources/icons/icon_talking.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 11 KiB

9
front/src/Components/EnableCamera/EnableCameraScene.svelte
View File

@ -5,6 +5,7 @@
audioConstraintStore,
cameraListStore,
localStreamStore,
localVolumeStore,
microphoneListStore,
videoConstraintStore,
} from "../../Stores/MediaStore";
@ -38,7 +39,7 @@
let stream: MediaStream | null;
const unsubscribe = localStreamStore.subscribe((value) => {
const unsubscribeLocalStreamStore = localStreamStore.subscribe((value) => {
if (value.type === "success") {
stream = value.stream;
@ -59,7 +60,9 @@
}
});
onDestroy(unsubscribe);
onDestroy(() => {
unsubscribeLocalStreamStore();
});
function normalizeDeviceName(label: string): string {
// remove IDs (that can appear in Chrome, like: "HD Pro Webcam (4df7:4eda)"
@ -86,7 +89,7 @@
<img class="background-img" src={cinemaCloseImg} alt="" />
</div>
{/if}
<HorizontalSoundMeterWidget {stream} />
<HorizontalSoundMeterWidget volume={$localVolumeStore} />
<section class="selectWebcamForm">
{#if $cameraListStore.length > 1}

46
front/src/Components/EnableCamera/HorizontalSoundMeterWidget.svelte
View File

@ -1,50 +1,8 @@
<script lang="typescript">
import { AudioContext } from "standardized-audio-context";
import { SoundMeter } from "../../Phaser/Components/SoundMeter";
import { onDestroy } from "svelte";
export let stream: MediaStream | null;
let volume = 0;
export let volume = 0;
const NB_BARS = 20;
let timeout: ReturnType<typeof setTimeout>;
const soundMeter = new SoundMeter();
let display = false;
let error = false;
$: {
if (stream && stream.getAudioTracks().length > 0) {
display = true;
soundMeter.connectToSource(stream, new AudioContext());
if (timeout) {
clearInterval(timeout);
error = false;
}
timeout = setInterval(() => {
try {
volume = parseInt(((soundMeter.getVolume() / 100) * NB_BARS).toFixed(0));
} catch (err) {
if (!error) {
console.error(err);
error = true;
}
}
}, 100);
} else {
display = false;
}
}
onDestroy(() => {
soundMeter.stop();
if (timeout) {
clearInterval(timeout);
}
});
function color(i: number, volume: number) {
const red = (255 * i) / NB_BARS;
const green = 255 * (1 - i / NB_BARS);
@ -58,7 +16,7 @@
}
</script>
<div class="horizontal-sound-meter" class:active={display}>
<div class="horizontal-sound-meter" class:active={volume !== undefined}>
{#each [...Array(NB_BARS).keys()] as i (i)}
<div style={color(i, volume)} />
{/each}

10
front/src/Components/MyCamera.svelte
View File

@ -1,5 +1,5 @@
<script lang="typescript">
import { obtainedMediaConstraintStore } from "../Stores/MediaStore";
import { localVolumeStore, obtainedMediaConstraintStore } from "../Stores/MediaStore";
import { localStreamStore, isSilentStore } from "../Stores/MediaStore";
import SoundMeterWidget from "./SoundMeterWidget.svelte";
import { onDestroy, onMount } from "svelte";
@ -8,7 +8,7 @@
let stream: MediaStream | null;
const unsubscribe = localStreamStore.subscribe((value) => {
const unsubscribeLocalStreamStore = localStreamStore.subscribe((value) => {
if (value.type === "success") {
stream = value.stream;
} else {
@ -16,7 +16,9 @@
}
});
onDestroy(unsubscribe);
onDestroy(() => {
unsubscribeLocalStreamStore();
});
let isSilent: boolean;
const unsubscribeIsSilent = isSilentStore.subscribe((value) => {
@ -51,7 +53,7 @@
<div class="is-silent">{$LL.camera.my.silentZone()}</div>
{:else if $localStreamStore.type === "success" && $localStreamStore.stream}
<video class="my-cam-video" use:srcObject={stream} autoplay muted playsinline />
<SoundMeterWidget {stream} />
<SoundMeterWidget volume={$localVolumeStore} />
{/if}
</div>

45
front/src/Components/SoundMeterWidget.svelte
View File

@ -1,47 +1,6 @@
<script lang="typescript">
import { AudioContext } from "standardized-audio-context";
import { SoundMeter } from "../Phaser/Components/SoundMeter";
import { onDestroy } from "svelte";
export let stream: MediaStream | null;
let volume = 0;
let timeout: ReturnType<typeof setTimeout>;
const soundMeter = new SoundMeter();
let display = false;
let error = false;
$: {
if (stream && stream.getAudioTracks().length > 0) {
display = true;
soundMeter.connectToSource(stream, new AudioContext());
if (timeout) {
clearInterval(timeout);
error = false;
}
timeout = setInterval(() => {
try {
volume = soundMeter.getVolume();
} catch (err) {
if (!error) {
console.error(err);
error = true;
}
}
}, 100);
} else {
display = false;
}
}
onDestroy(() => {
soundMeter.stop();
if (timeout) {
clearInterval(timeout);
}
});
export let volume = 0;
let display = true;
</script>
<div class="sound-progress" class:active={display}>

3
front/src/Components/Video/VideoMediaBox.svelte
View File

@ -18,6 +18,7 @@
export let peer: VideoPeer;
let streamStore = peer.streamStore;
let volumeStore = peer.volumeStore;
let name = peer.userName;
let statusStore = peer.statusStore;
let constraintStore = peer.constraintsStore;
@ -93,7 +94,7 @@
/>
<img src={blockSignImg} draggable="false" on:dragstart|preventDefault={noDrag} class="block-logo" alt="Block" />
{#if $constraintStore && $constraintStore.audio !== false}
<SoundMeterWidget stream={$streamStore} />
<SoundMeterWidget volume={$volumeStore} />
{/if}
</div>

57
front/src/Phaser/Components/SoundMeter.ts
View File

@ -1,4 +1,4 @@
import type { IAnalyserNode, IAudioContext, IMediaStreamAudioSourceNode } from "standardized-audio-context";
import { AudioContext, IAnalyserNode, IAudioContext, IMediaStreamAudioSourceNode } from "standardized-audio-context";
/**
* Class to measure the sound volume of a media stream
@ -6,41 +6,15 @@ import type { IAnalyserNode, IAudioContext, IMediaStreamAudioSourceNode } from "
export class SoundMeter {
private instant: number;
private clip: number;
//private script: ScriptProcessorNode;
private analyser: IAnalyserNode<IAudioContext> | undefined;
private dataArray: Uint8Array | undefined;
private context: IAudioContext | undefined;
private source: IMediaStreamAudioSourceNode<IAudioContext> | undefined;
constructor() {
constructor(mediaStream: MediaStream) {
this.instant = 0.0;
this.clip = 0.0;
//this.script = context.createScriptProcessor(2048, 1, 1);
}
private init(context: IAudioContext) {
this.context = context;
this.analyser = this.context.createAnalyser();
this.analyser.fftSize = 2048;
const bufferLength = this.analyser.fftSize;
this.dataArray = new Uint8Array(bufferLength);
}
public connectToSource(stream: MediaStream, context: IAudioContext): void {
if (this.source !== undefined) {
this.stop();
}
this.init(context);
this.source = this.context?.createMediaStreamSource(stream);
if (this.analyser !== undefined) {
this.source?.connect(this.analyser);
}
//analyser.connect(distortion);
//distortion.connect(this.context.destination);
//this.analyser.connect(this.context.destination);
this.connectToSource(mediaStream, new AudioContext());
}
public getVolume(): number {
@ -78,4 +52,29 @@ export class SoundMeter {
this.dataArray = undefined;
this.source = undefined;
}
private init(context: IAudioContext) {
this.context = context;
this.analyser = this.context.createAnalyser();
this.analyser.fftSize = 2048;
const bufferLength = this.analyser.fftSize;
this.dataArray = new Uint8Array(bufferLength);
}
private connectToSource(stream: MediaStream, context: IAudioContext): void {
if (this.source !== undefined) {
this.stop();
}
this.init(context);
this.source = this.context?.createMediaStreamSource(stream);
if (this.analyser !== undefined) {
this.source?.connect(this.analyser);
}
//analyser.connect(distortion);
//distortion.connect(this.context.destination);
//this.analyser.connect(this.context.destination);
}
}

58
front/src/Phaser/Components/TalkIcon.ts Normal file
View File

@ -0,0 +1,58 @@
import { Easing } from "../../types";
export class TalkIcon extends Phaser.GameObjects.Image {
private shown: boolean;
private showAnimationTween?: Phaser.Tweens.Tween;
private defaultPosition: { x: number; y: number };
private defaultScale: number;
constructor(scene: Phaser.Scene, x: number, y: number) {
super(scene, x, y, "iconTalk");
this.defaultPosition = { x, y };
this.defaultScale = 0.3;
this.shown = false;
this.setAlpha(0);
this.setScale(this.defaultScale);
this.scene.add.existing(this);
}
public show(show: boolean = true, forceClose: boolean = false): void {
if (this.shown === show && !forceClose) {
return;
}
this.showAnimation(show, forceClose);
}
private showAnimation(show: boolean = true, forceClose: boolean = false) {
if (forceClose && !show) {
this.showAnimationTween?.stop();
} else if (this.showAnimationTween?.isPlaying()) {
return;
}
this.shown = show;
if (show) {
this.y += 50;
this.scale = 0.05;
this.alpha = 0;
}
this.showAnimationTween = this.scene.tweens.add({
targets: [this],
duration: 350,
alpha: show ? 1 : 0,
y: this.defaultPosition.y,
scale: this.defaultScale,
ease: Easing.BackEaseOut,
onComplete: () => {
this.showAnimationTween = undefined;
},
});
}
public isShown(): boolean {
return this.shown;
}
}

17
front/src/Phaser/Entity/Character.ts
View File

@ -17,6 +17,7 @@ import { Unsubscriber, Writable, writable } from "svelte/store";
import { createColorStore } from "../../Stores/OutlineColorStore";
import type { OutlineableInterface } from "../Game/OutlineableInterface";
import type CancelablePromise from "cancelable-promise";
import { TalkIcon } from "../Components/TalkIcon";
const playerNameY = -25;
@ -33,6 +34,7 @@ const interactiveRadius = 35;
export abstract class Character extends Container implements OutlineableInterface {
private bubble: SpeechBubble | null = null;
private readonly playerNameText: Text;
private readonly talkIcon: TalkIcon;
public playerName: string;
public sprites: Map<string, Sprite>;
protected lastDirection: PlayerAnimationDirections = PlayerAnimationDirections.Down;
@ -102,6 +104,17 @@ export abstract class Character extends Container implements OutlineableInterfac
fontSize: 35,
},
});
this.talkIcon = new TalkIcon(scene, 0, -45);
this.add(this.talkIcon);
if (isClickable) {
this.setInteractive({
hitArea: new Phaser.Geom.Circle(0, 0, interactiveRadius),
hitAreaCallback: Phaser.Geom.Circle.Contains, //eslint-disable-line @typescript-eslint/unbound-method
useHandCursor: true,
});
}
this.playerNameText.setOrigin(0.5).setDepth(DEPTH_INGAME_TEXT_INDEX);
this.add(this.playerNameText);
@ -200,6 +213,10 @@ export abstract class Character extends Container implements OutlineableInterfac
});
}
public showTalkIcon(show: boolean = true, forceClose: boolean = false): void {
this.talkIcon.show(show, forceClose);
}
public addCompanion(name: string, texturePromise?: CancelablePromise<string>): void {
if (typeof texturePromise !== "undefined") {
this.companion = new Companion(this.scene, this.x, this.y, name, texturePromise);

36
front/src/Phaser/Game/GameScene.ts
View File

@ -91,6 +91,7 @@ import { MapStore } from "../../Stores/Utils/MapStore";
import { followUsersColorStore } from "../../Stores/FollowStore";
import { GameSceneUserInputHandler } from "../UserInput/GameSceneUserInputHandler";
import { locale } from "../../i18n/i18n-svelte";
import { localVolumeStore } from "../../Stores/MediaStore";
import { StringUtils } from "../../Utils/StringUtils";
import { startLayerNamesStore } from "../../Stores/StartLayerNamesStore";
import { JitsiCoWebsite } from "../../WebRtc/CoWebsite/JitsiCoWebsite";
@ -172,6 +173,9 @@ export class GameScene extends DirtyScene {
private peerStoreUnsubscribe!: Unsubscriber;
private emoteUnsubscribe!: Unsubscriber;
private emoteMenuUnsubscribe!: Unsubscriber;
private volumeStoreUnsubscribers: Map<number, Unsubscriber> = new Map<number, Unsubscriber>();
private localVolumeStoreUnsubscriber: Unsubscriber | undefined;
private followUsersColorStoreUnsubscribe!: Unsubscriber;
private biggestAvailableAreaStoreUnsubscribe!: () => void;
@ -247,6 +251,7 @@ export class GameScene extends DirtyScene {
loadCustomTexture(this.load, texture).catch((e) => console.error(e));
}
}
this.load.image("iconTalk", "/resources/icons/icon_talking.png");
if (touchScreenManager.supportTouchScreen) {
this.load.image(joystickBaseKey, joystickBaseImg);
@ -636,14 +641,45 @@ export class GameScene extends DirtyScene {
this.connect();
}
const talkIconVolumeTreshold = 10;
let oldPeerNumber = 0;
this.peerStoreUnsubscribe = peerStore.subscribe((peers) => {
this.volumeStoreUnsubscribers.forEach((unsubscribe) => unsubscribe());
this.volumeStoreUnsubscribers.clear();
for (const [key, videoStream] of peers) {
this.volumeStoreUnsubscribers.set(
key,
videoStream.volumeStore.subscribe((volume) => {
if (volume) {
this.MapPlayersByKey.get(key)?.showTalkIcon(volume > talkIconVolumeTreshold);
}
})
);
}
const newPeerNumber = peers.size;
if (newPeerNumber > oldPeerNumber) {
this.playSound("audio-webrtc-in");
} else if (newPeerNumber < oldPeerNumber) {
this.playSound("audio-webrtc-out");
}
if (newPeerNumber > 0) {
if (!this.localVolumeStoreUnsubscriber) {
this.localVolumeStoreUnsubscriber = localVolumeStore.subscribe((volume) => {
if (volume) {
this.CurrentPlayer.showTalkIcon(volume > talkIconVolumeTreshold);
}
});
}
} else {
this.CurrentPlayer.showTalkIcon(false, true);
this.MapPlayersByKey.forEach((remotePlayer) => remotePlayer.showTalkIcon(false, true));
if (this.localVolumeStoreUnsubscriber) {
this.localVolumeStoreUnsubscriber();
this.localVolumeStoreUnsubscriber = undefined;
}
}
oldPeerNumber = newPeerNumber;
});

37
front/src/Stores/MediaStore.ts
View File

@ -10,6 +10,8 @@ import { myCameraVisibilityStore } from "./MyCameraStoreVisibility";
import { peerStore } from "./PeerStore";
import { privacyShutdownStore } from "./PrivacyShutdownStore";
import { MediaStreamConstraintsError } from "./Errors/MediaStreamConstraintsError";
import { SoundMeter } from "../Phaser/Components/SoundMeter";
import { AudioContext } from "standardized-audio-context";
/**
* A store that contains the camera state requested by the user (on or off).
@ -541,6 +543,41 @@ export const obtainedMediaConstraintStore = derived<Readable<MediaStreamConstrai
}
);
export const localVolumeStore = readable<number | undefined>(undefined, (set) => {
let timeout: ReturnType<typeof setTimeout>;
const unsubscribe = localStreamStore.subscribe((localStreamStoreValue) => {
clearInterval(timeout);
if (localStreamStoreValue.type === "error") {
set(undefined);
return;
}
const mediaStream = localStreamStoreValue.stream;
if (mediaStream === null || mediaStream.getAudioTracks().length <= 0) {
set(undefined);
return;
}
const soundMeter = new SoundMeter(mediaStream);
let error = false;
timeout = setInterval(() => {
try {
set(soundMeter.getVolume());
} catch (err) {
if (!error) {
console.error(err);
error = true;
}
}
}, 100);
});
return () => {
unsubscribe();
clearInterval(timeout);
};
});
/**
* Device list
*/

32
front/src/WebRtc/VideoPeer.ts
View File

@ -10,6 +10,9 @@ import { playersStore } from "../Stores/PlayersStore";
import { chatMessagesStore, newChatMessageSubject } from "../Stores/ChatStore";
import { getIceServersConfig } from "../Components/Video/utils";
import { isMediaBreakpointUp } from "../Utils/BreakpointsUtils";
import { SoundMeter } from "../Phaser/Components/SoundMeter";
import { AudioContext } from "standardized-audio-context";
import { Console } from "console";
const Peer: SimplePeerNamespace.SimplePeer = require("simple-peer");
@ -33,6 +36,7 @@ export class VideoPeer extends Peer {
private onBlockSubscribe: Subscription;
private onUnBlockSubscribe: Subscription;
public readonly streamStore: Readable<MediaStream | null>;
public readonly volumeStore: Readable<number | undefined>;
public readonly statusStore: Readable<PeerStatus>;
public readonly constraintsStore: Readable<ObtainedMediaStreamConstraints | null>;
private newMessageSubscribtion: Subscription | undefined;
@ -69,6 +73,34 @@ export class VideoPeer extends Peer {
};
});
this.volumeStore = readable<number | undefined>(undefined, (set) => {
let timeout: ReturnType<typeof setTimeout>;
const unsubscribe = this.streamStore.subscribe((mediaStream) => {
if (mediaStream === null || mediaStream.getAudioTracks().length <= 0) {
set(undefined);
return;
}
const soundMeter = new SoundMeter(mediaStream);
let error = false;
timeout = setInterval(() => {
try {
set(soundMeter.getVolume());
} catch (err) {
if (!error) {
console.error(err);
error = true;
}
}
}, 100);
});
return () => {
unsubscribe();
clearInterval(timeout);
};
});
this.constraintsStore = readable<ObtainedMediaStreamConstraints | null>(null, (set) => {
const onData = (chunk: Buffer) => {
const message = JSON.parse(chunk.toString("utf8"));