Merge pull request #488 from thecodingmachine/apiscaling

Add scaling capability to the API servers with this one weird trick

back/server.ts

@@ -3,11 +3,13 @@ import App from "./src/App";
 import grpc from "grpc";
 import {roomManager} from "./src/RoomManager";
 import {IRoomManagerServer, RoomManagerService} from "./src/Messages/generated/messages_grpc_pb";
+import {HTTP_PORT, GRPC_PORT} from "./src/Enum/EnvironmentVariable";
 
-App.listen(8080, () => console.log(`WorkAdventure starting on port 8080!`))
+App.listen(HTTP_PORT, () => console.log(`WorkAdventure HTTP API starting on port %d!`, HTTP_PORT))
 
 const server = new grpc.Server();
 server.addService<IRoomManagerServer>(RoomManagerService, roomManager);
 
-server.bind('0.0.0.0:50051', grpc.ServerCredentials.createInsecure());
+server.bind('0.0.0.0:'+GRPC_PORT, grpc.ServerCredentials.createInsecure());
 server.start();
+console.log('WorkAdventure HTTP/2 API starting on port %d!', GRPC_PORT);

back/src/Enum/EnvironmentVariable.ts

@@ -10,6 +10,8 @@ const CPU_OVERHEAT_THRESHOLD = Number(process.env.CPU_OVERHEAT_THRESHOLD) || 80;
 const JITSI_URL : string|undefined = (process.env.JITSI_URL === '') ? undefined : process.env.JITSI_URL;
 const JITSI_ISS = process.env.JITSI_ISS || '';
 const SECRET_JITSI_KEY = process.env.SECRET_JITSI_KEY || '';
+const HTTP_PORT = parseInt(process.env.HTTP_PORT || '8080') || 8080;
+const GRPC_PORT = parseInt(process.env.GRPC_PORT || '50051') || 50051;
 export const SOCKET_IDLE_TIMER = parseInt(process.env.SOCKET_IDLE_TIMER as string) || 30; // maximum time (in second) without activity before a socket is closed
 
 export {
@@ -18,6 +20,8 @@ export {
     MINIMUM_DISTANCE,
     ADMIN_API_URL,
     ADMIN_API_TOKEN,
+    HTTP_PORT,
+    GRPC_PORT,
     MAX_USERS_PER_ROOM,
     GROUP_RADIUS,
     ALLOW_ARTILLERY,

deeployer.libsonnet

@@ -7,10 +7,28 @@
   "$schema": "https://raw.githubusercontent.com/thecodingmachine/deeployer/master/deeployer.schema.json",
   "version": "1.0",
   "containers": {
-     "back": {
+     "back1": {
        "image": "thecodingmachine/workadventure-back:"+tag,
        "host": {
-         "url": "api."+url,
+         "url": "api1."+url,
+         "https": "enable",
+         "containerPort": 8080
+       },
+       "ports": [8080, 50051],
+       "env": {
+         "SECRET_KEY": "tempSecretKeyNeedsToChange",
+         "ADMIN_API_TOKEN": env.ADMIN_API_TOKEN,
+         "JITSI_ISS": env.JITSI_ISS,
+         "JITSI_URL": env.JITSI_URL,
+         "SECRET_JITSI_KEY": env.SECRET_JITSI_KEY,
+       } + if adminUrl != null then {
+         "ADMIN_API_URL": adminUrl,
+       } else {}
+     },
+     "back2": {
+            "image": "thecodingmachine/workadventure-back:"+tag,
+            "host": {
+              "url": "api2."+url,
               "https": "enable",
               "containerPort": 8080
             },
@@ -38,7 +56,7 @@
               "ADMIN_API_TOKEN": env.ADMIN_API_TOKEN,
               "JITSI_ISS": env.JITSI_ISS,
               "JITSI_URL": env.JITSI_URL,
-              "API_URL": "back:50051",
+              "API_URL": "back1:50051,back2:50051",
               "SECRET_JITSI_KEY": env.SECRET_JITSI_KEY,
             } + if adminUrl != null then {
               "ADMIN_API_URL": adminUrl,
@@ -100,7 +118,35 @@
     },
     k8sextension(k8sConf)::
         k8sConf + {
-          back+: {
+          back1+: {
+            deployment+: {
+              spec+: {
+                template+: {
+                  metadata+: {
+                    annotations+: {
+                      "prometheus.io/port": "8080",
+                      "prometheus.io/scrape": "true"
+                    }
+                  }
+                }
+              }
+            }
+          },
+          back2+: {
+            deployment+: {
+              spec+: {
+                template+: {
+                  metadata+: {
+                    annotations+: {
+                      "prometheus.io/port": "8080",
+                      "prometheus.io/scrape": "true"
+                    }
+                  }
+                }
+              }
+            }
+          },
+          pusher+: {
             deployment+: {
               spec+: {
                 template+: {

pusher/src/Services/ApiClientRepository.ts

@@ -3,17 +3,30 @@
  */
 import {RoomManagerClient} from "../Messages/generated/messages_grpc_pb";
 import grpc from 'grpc';
+import crypto from 'crypto';
 import {API_URL} from "../Enum/EnvironmentVariable";
 
+import Debug from "debug";
+
+const debug = Debug('apiClientRespository');
 
 class ApiClientRepository {
-    private roomManagerClient: RoomManagerClient|null = null;
+    private roomManagerClients: RoomManagerClient[] = [];
+
+    public constructor(private apiUrls: string[]) {
+    }
 
     public async getClient(roomId: string): Promise<RoomManagerClient> {
-        if (this.roomManagerClient === null) {
+        const array = new Uint32Array(crypto.createHash('md5').update(roomId).digest());
-            this.roomManagerClient = new RoomManagerClient(API_URL, grpc.credentials.createInsecure());
+        const index = array[0] % this.apiUrls.length;
+
+        let client = this.roomManagerClients[index];
+        if (client === undefined) {
+            this.roomManagerClients[index] = client = new RoomManagerClient(this.apiUrls[index], grpc.credentials.createInsecure());
+            debug('Mapping room %s to API server %s', roomId, this.apiUrls[index])
         }
-        return Promise.resolve(this.roomManagerClient);
+
+        return Promise.resolve(client);
     }
 
     public async getAllClients(): Promise<RoomManagerClient[]> {
@@ -21,6 +34,6 @@ class ApiClientRepository {
     }
 }
 
-const apiClientRepository = new ApiClientRepository();
+const apiClientRepository = new ApiClientRepository(API_URL.split(','));
 
 export { apiClientRepository };