Moved benchmark to its own directory and added multicore testing + a README

back/package.json

@@ -41,7 +41,6 @@
     "@types/jsonwebtoken": "^8.3.8",
     "@types/socket.io": "^2.1.4",
     "@types/uuidv4": "^5.0.0",
-    "artillery": "^1.6.1",
     "body-parser": "^1.19.0",
     "express": "^4.17.1",
     "generic-type-guard": "^3.2.0",

back/src/Controller/IoSocketController.ts

@@ -4,7 +4,7 @@ import * as http from "http";
 import {MessageUserPosition, Point} from "../Model/Websocket/MessageUserPosition"; //TODO fix import by "_Model/.."
 import {ExSocketInterface} from "../Model/Websocket/ExSocketInterface"; //TODO fix import by "_Model/.."
 import Jwt, {JsonWebTokenError} from "jsonwebtoken";
-import {SECRET_KEY, MINIMUM_DISTANCE, GROUP_RADIUS} from "../Enum/EnvironmentVariable"; //TODO fix import by "_Enum/..."
+import {SECRET_KEY, MINIMUM_DISTANCE, GROUP_RADIUS, ALLOW_ARTILLERY} from "../Enum/EnvironmentVariable"; //TODO fix import by "_Enum/..."
 import {World} from "../Model/World";
 import {Group} from "_Model/Group";
 import {UserInterface} from "_Model/UserInterface";
@@ -67,12 +67,19 @@ export class IoSocketController {
                 return next(new Error('Authentication error'));
             }
             if(socket.handshake.query.token === 'test'){
+                if (ALLOW_ARTILLERY) {
                     (socket as ExSocketInterface).token = socket.handshake.query.token;
                     (socket as ExSocketInterface).userId = uuid();
+                    (socket as ExSocketInterface).isArtillery = true;
                     console.log((socket as ExSocketInterface).userId);
                     next();
                     return;
+                } else {
+                    console.warn("In order to perform a load-testing test on this environment, you must set the ALLOW_ARTILLERY environment variable to 'true'");
+                    next();
                 }
+            }
+            (socket as ExSocketInterface).isArtillery = false;
             if(this.searchClientByToken(socket.handshake.query.token)){
                 console.error('An authentication error happened, a user tried to connect while its token is already connected.');
                 return next(new Error('Authentication error'));
@@ -201,7 +208,7 @@ export class IoSocketController {
                         }
                         return new MessageUserPosition(user.id, player.name, player.characterLayers, player.position);
                     }).filter((item: MessageUserPosition|null) => item !== null);
-                    //answerFn(listOfUsers);
+                    answerFn(listOfUsers);
                 } catch (e) {
                     console.error('An error occurred on "join_room" event');
                     console.error(e);
@@ -285,7 +292,10 @@ export class IoSocketController {
                 const Client = (socket as ExSocketInterface);
                 Client.name = playerDetails.name;
                 Client.characterLayers = playerDetails.characterLayers;
-                //answerFn(Client.userId);
+                // Artillery fails when receiving an acknowledgement that is not a JSON object
+                if (!Client.isArtillery) {
+                    answerFn(Client.userId);
+                }
             });
 
             socket.on(SockerIoEvent.SET_SILENT, (silent: unknown) => {

back/src/Enum/EnvironmentVariable.ts

@@ -2,10 +2,12 @@ const SECRET_KEY = process.env.SECRET_KEY || "THECODINGMACHINE_SECRET_KEY";
 const URL_ROOM_STARTED = "/Floor0/floor0.json";
 const MINIMUM_DISTANCE = process.env.MINIMUM_DISTANCE ? Number(process.env.MINIMUM_DISTANCE) : 64;
 const GROUP_RADIUS = process.env.GROUP_RADIUS ? Number(process.env.GROUP_RADIUS) : 48;
+const ALLOW_ARTILLERY = process.env.ALLOW_ARTILLERY ? process.env.ALLOW_ARTILLERY == 'true' : false;
 
 export {
     SECRET_KEY,
     URL_ROOM_STARTED,
     MINIMUM_DISTANCE,
-    GROUP_RADIUS
+    GROUP_RADIUS,
+    ALLOW_ARTILLERY
 }

back/src/Model/Websocket/ExSocketInterface.ts

@@ -11,4 +11,5 @@ export interface ExSocketInterface extends Socket, Identificable {
     name: string;
     characterLayers: string[];
     position: PointInterface;
+    isArtillery: boolean; // Whether this socket is opened by Artillery for load testing (hack)
 }

benchmark/README.md

@@ -0,0 +1,69 @@
+# Load testing
+
+Load testing is performed with Artillery.
+
+Install:
+
+```bash
+cd benchmark
+npm install
+```
+
+Running the tests (on one core):
+
+```bash
+cd benchmark
+npm run start
+```
+
+You can adapt the `socketio-load-test.yaml` file to increase/decrease load.
+
+Default settings are:
+
+```yaml
+  phases:
+    - duration: 20
+      arrivalRate: 2
+```
+
+which means: during 20 seconds, 2 users will be added every second (peaking at 40 simultaneous users).
+
+Important: don't go above 40 simultaneous users for Artillery, otherwise, it is Artillery that will fail to run the tests properly.
+To know, simply run "top". The "node" process for Artillery should never reach 100%.
+
+Reports are generated in `artillery_output.html`.
+
+# Multicore tests
+
+You will want to test with Artillery running on multiple cores.
+
+You can use
+
+```bash
+./artillery_multi_core.sh
+```
+
+This will trigger 4 Artillery instances in parallel.
+
+Beware, the report generated is generated for only one instance.
+
+# How to test, what to track?
+
+While testing, you can check:
+
+- CPU load of WorkAdventure API node process (it should not reach 100%)
+- Get metrics at the end of the run: `http://api.workadventure.localhost/metrics`  
+  In particular, look for:
+  ```
+  # HELP nodejs_eventloop_lag_max_seconds The maximum recorded event loop delay.
+  # TYPE nodejs_eventloop_lag_max_seconds gauge
+  nodejs_eventloop_lag_max_seconds 23.991418879
+  ```
+  This is the maximum time it took Node to process an event (you need to restart node after each test to reset this counter)
+- Generate a profiling using "node --prof" by switching the command in docker-compose.yaml:
+  ```
+      #command: yarn dev
+      command: yarn run profile
+  ```
+  Read https://nodejs.org/en/docs/guides/simple-profiling/ on how to generate a profile.
+

benchmark/artillery_multi_core.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+npm run start &
+pid1=$!
+npm run start:nooutput &
+pid2=$!
+npm run start:nooutput &
+pid3=$!
+npm run start:nooutput &
+pid4=$!
+
+wait $pid1
+wait $pid2
+wait $pid3
+wait $pid4
+
+

benchmark/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "workadventure-artillery",
+  "version": "1.0.0",
+  "description": "Load testing for WorkAdventure",
+  "scripts": {
+    "start": "artillery run socketio-load-test.yaml -o artillery_output.json && artillery report --output artillery_output.html artillery_output.json",
+    "start:nooutput": "artillery run socketio-load-test.yaml"
+  },
+  "contributors": [
+    {
+      "name": "Grégoire Parant",
+      "email": "g.parant@thecodingmachine.com"
+    },
+    {
+      "name": "David Négrier",
+      "email": "d.negrier@thecodingmachine.com"
+    },
+    {
+      "name": "Arthmaël Poly",
+      "email": "a.poly@thecodingmachine.com"
+    }
+  ],
+  "license": "SEE LICENSE IN LICENSE.txt",
+  "dependencies": {
+    "artillery": "^1.6.1"
+  }
+}

benchmark/socketio-load-test.yaml

@@ -5,21 +5,14 @@ config:
     query:
       token: "test"
   phases:
-    - duration: 10
+    - duration: 20
-      arrivalRate: 10
+      arrivalRate: 2
-    - duration: 10
-      arrivalRate: 10
   processor: "./socketioLoadTest.js"
 scenarios:
-  - name: "Connect and send a bunch of messages"
+  - name: "Connects and moves player for 20 seconds"
     weight: 90
     engine: "socketio"
     flow:
-      #- loop:
-      #- emit:
-      #    channel: "connection"
-      #    data: "hello world!"
-      #- think: 5
       - emit:
           channel: "set-player-details"
           data:
@@ -45,6 +38,6 @@ scenarios:
               y: "{{ y }}"
               direction: 'down'
               moving: false
-        - think: 1
+        - think: 0.2
-        count: 10
+        count: 100
       - think: 10

benchmark/socketioLoadTest.js

@@ -1,7 +1,3 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
 'use strict';
 
 module.exports = {

docker-compose.yaml

@@ -45,11 +45,12 @@ services:
 
   back:
     image: thecodingmachine/nodejs:12
-    command: yarn dev
+    #command: yarn dev
-    #command: yarn run profile
+    command: yarn run profile
     environment:
       STARTUP_COMMAND_1: yarn install
       SECRET_KEY: yourSecretKey
+      ALLOW_ARTILLERY: "true"
     volumes:
       - ./back:/usr/src/app
     labels: