Moved benchmark to its own directory and added multicore testing + a README

This commit is contained in:
David Négrier 2020-09-11 09:56:05 +02:00
parent d4fe59d154
commit b37a8f63be
11 changed files with 160 additions and 1337 deletions

View File

@ -41,7 +41,6 @@
"@types/jsonwebtoken": "^8.3.8",
"@types/socket.io": "^2.1.4",
"@types/uuidv4": "^5.0.0",
"artillery": "^1.6.1",
"body-parser": "^1.19.0",
"express": "^4.17.1",
"generic-type-guard": "^3.2.0",

View File

@ -4,7 +4,7 @@ import * as http from "http";
import {MessageUserPosition, Point} from "../Model/Websocket/MessageUserPosition"; //TODO fix import by "_Model/.."
import {ExSocketInterface} from "../Model/Websocket/ExSocketInterface"; //TODO fix import by "_Model/.."
import Jwt, {JsonWebTokenError} from "jsonwebtoken";
import {SECRET_KEY, MINIMUM_DISTANCE, GROUP_RADIUS} from "../Enum/EnvironmentVariable"; //TODO fix import by "_Enum/..."
import {SECRET_KEY, MINIMUM_DISTANCE, GROUP_RADIUS, ALLOW_ARTILLERY} from "../Enum/EnvironmentVariable"; //TODO fix import by "_Enum/..."
import {World} from "../Model/World";
import {Group} from "_Model/Group";
import {UserInterface} from "_Model/UserInterface";
@ -67,12 +67,19 @@ export class IoSocketController {
return next(new Error('Authentication error'));
}
if(socket.handshake.query.token === 'test'){
(socket as ExSocketInterface).token = socket.handshake.query.token;
(socket as ExSocketInterface).userId = uuid();
console.log((socket as ExSocketInterface).userId);
next();
return;
if (ALLOW_ARTILLERY) {
(socket as ExSocketInterface).token = socket.handshake.query.token;
(socket as ExSocketInterface).userId = uuid();
(socket as ExSocketInterface).isArtillery = true;
console.log((socket as ExSocketInterface).userId);
next();
return;
} else {
console.warn("In order to perform a load-testing test on this environment, you must set the ALLOW_ARTILLERY environment variable to 'true'");
next();
}
}
(socket as ExSocketInterface).isArtillery = false;
if(this.searchClientByToken(socket.handshake.query.token)){
console.error('An authentication error happened, a user tried to connect while its token is already connected.');
return next(new Error('Authentication error'));
@ -201,7 +208,7 @@ export class IoSocketController {
}
return new MessageUserPosition(user.id, player.name, player.characterLayers, player.position);
}).filter((item: MessageUserPosition|null) => item !== null);
//answerFn(listOfUsers);
answerFn(listOfUsers);
} catch (e) {
console.error('An error occurred on "join_room" event');
console.error(e);
@ -285,7 +292,10 @@ export class IoSocketController {
const Client = (socket as ExSocketInterface);
Client.name = playerDetails.name;
Client.characterLayers = playerDetails.characterLayers;
//answerFn(Client.userId);
// Artillery fails when receiving an acknowledgement that is not a JSON object
if (!Client.isArtillery) {
answerFn(Client.userId);
}
});
socket.on(SockerIoEvent.SET_SILENT, (silent: unknown) => {

View File

@ -2,10 +2,12 @@ const SECRET_KEY = process.env.SECRET_KEY || "THECODINGMACHINE_SECRET_KEY";
const URL_ROOM_STARTED = "/Floor0/floor0.json";
const MINIMUM_DISTANCE = process.env.MINIMUM_DISTANCE ? Number(process.env.MINIMUM_DISTANCE) : 64;
const GROUP_RADIUS = process.env.GROUP_RADIUS ? Number(process.env.GROUP_RADIUS) : 48;
const ALLOW_ARTILLERY = process.env.ALLOW_ARTILLERY ? process.env.ALLOW_ARTILLERY == 'true' : false;
export {
SECRET_KEY,
URL_ROOM_STARTED,
MINIMUM_DISTANCE,
GROUP_RADIUS
GROUP_RADIUS,
ALLOW_ARTILLERY
}

View File

@ -11,4 +11,5 @@ export interface ExSocketInterface extends Socket, Identificable {
name: string;
characterLayers: string[];
position: PointInterface;
isArtillery: boolean; // Whether this socket is opened by Artillery for load testing (hack)
}

File diff suppressed because it is too large Load Diff

69
benchmark/README.md Normal file
View File

@ -0,0 +1,69 @@
# Load testing
Load testing is performed with Artillery.
Install:
```bash
cd benchmark
npm install
```
Running the tests (on one core):
```bash
cd benchmark
npm run start
```
You can adapt the `socketio-load-test.yaml` file to increase/decrease load.
Default settings are:
```yaml
phases:
- duration: 20
arrivalRate: 2
```
which means: during 20 seconds, 2 users will be added every second (peaking at 40 simultaneous users).
Important: don't go above 40 simultaneous users for Artillery, otherwise, it is Artillery that will fail to run the tests properly.
To know, simply run "top". The "node" process for Artillery should never reach 100%.
Reports are generated in `artillery_output.html`.
# Multicore tests
You will want to test with Artillery running on multiple cores.
You can use
```bash
./artillery_multi_core.sh
```
This will trigger 4 Artillery instances in parallel.
Beware, the report generated is generated for only one instance.
# How to test, what to track?
While testing, you can check:
- CPU load of WorkAdventure API node process (it should not reach 100%)
- Get metrics at the end of the run: `http://api.workadventure.localhost/metrics`
In particular, look for:
```
# HELP nodejs_eventloop_lag_max_seconds The maximum recorded event loop delay.
# TYPE nodejs_eventloop_lag_max_seconds gauge
nodejs_eventloop_lag_max_seconds 23.991418879
```
This is the maximum time it took Node to process an event (you need to restart node after each test to reset this counter)
- Generate a profiling using "node --prof" by switching the command in docker-compose.yaml:
```
#command: yarn dev
command: yarn run profile
```
Read https://nodejs.org/en/docs/guides/simple-profiling/ on how to generate a profile.

View File

@ -0,0 +1,17 @@
#!/bin/bash
npm run start &
pid1=$!
npm run start:nooutput &
pid2=$!
npm run start:nooutput &
pid3=$!
npm run start:nooutput &
pid4=$!
wait $pid1
wait $pid2
wait $pid3
wait $pid4

27
benchmark/package.json Normal file
View File

@ -0,0 +1,27 @@
{
"name": "workadventure-artillery",
"version": "1.0.0",
"description": "Load testing for WorkAdventure",
"scripts": {
"start": "artillery run socketio-load-test.yaml -o artillery_output.json && artillery report --output artillery_output.html artillery_output.json",
"start:nooutput": "artillery run socketio-load-test.yaml"
},
"contributors": [
{
"name": "Grégoire Parant",
"email": "g.parant@thecodingmachine.com"
},
{
"name": "David Négrier",
"email": "d.negrier@thecodingmachine.com"
},
{
"name": "Arthmaël Poly",
"email": "a.poly@thecodingmachine.com"
}
],
"license": "SEE LICENSE IN LICENSE.txt",
"dependencies": {
"artillery": "^1.6.1"
}
}

View File

@ -5,21 +5,14 @@ config:
query:
token: "test"
phases:
- duration: 10
arrivalRate: 10
- duration: 10
arrivalRate: 10
- duration: 20
arrivalRate: 2
processor: "./socketioLoadTest.js"
scenarios:
- name: "Connect and send a bunch of messages"
- name: "Connects and moves player for 20 seconds"
weight: 90
engine: "socketio"
flow:
#- loop:
#- emit:
# channel: "connection"
# data: "hello world!"
#- think: 5
- emit:
channel: "set-player-details"
data:
@ -45,6 +38,6 @@ scenarios:
y: "{{ y }}"
direction: 'down'
moving: false
- think: 1
count: 10
- think: 0.2
count: 100
- think: 10

View File

@ -1,7 +1,3 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
'use strict';
module.exports = {
@ -12,4 +8,4 @@ function setYRandom(context, events, done) {
context.vars.x = (883 + Math.round(Math.random() * 300));
context.vars.y = (270 + Math.round(Math.random() * 300));
return done();
}
}

View File

@ -45,11 +45,12 @@ services:
back:
image: thecodingmachine/nodejs:12
command: yarn dev
#command: yarn run profile
#command: yarn dev
command: yarn run profile
environment:
STARTUP_COMMAND_1: yarn install
SECRET_KEY: yourSecretKey
ALLOW_ARTILLERY: "true"
volumes:
- ./back:/usr/src/app
labels: