misc

options.json

@@ -3,6 +3,7 @@
     "shardOptions": {
         "respawn": false
     },
+    "shardCount": 1,
     "serverOptions": {
         "serveFiles": "./files",
         "callbackURL": "/api/login",

src/controller/Controller.js

@@ -38,6 +38,8 @@ class Controller extends EventEmitter {
             this.logger.error(ex.stack);
         });
 
+        process.on('SIGINT', this.#shutdown.bind(this));
+
         this.on('built', () => {
             this.logger.info(`API Controller built`);
             this._built = true;
@@ -90,6 +92,16 @@ class Controller extends EventEmitter {
         shard.on('message', (msg) => this._handleMessage(shard, msg));
     }
 
+    async #shutdown () {
+        this.logger.info('Received SIGINT, shutting down');
+        const promises = this.shards.map(shard => shard.awaitShutdown());
+        await Promise.all(promises);
+        this.logger.info(`Shutdown completed, goodbye`);
+
+        // eslint-disable-next-line no-process-exit
+        process.exit();
+    }
+
 }
 
 module.exports = Controller;

src/controller/Shard.js

@@ -2,7 +2,8 @@ const ChildProcess = require('node:child_process');
 const { EventEmitter } = require('node:events');
 
 const { Util } = require('../util');
-
+// Give subprocess 90s to shut down before being forcibly killed
+const KillTO = 90 * 1000;
 class Shard extends EventEmitter {
 
     constructor (controller, id, options = {}) {
@@ -31,6 +32,8 @@ class Shard extends EventEmitter {
         // Set in the spawn method
         this.spawnedAt = null;
 
+        this._awaitingShutdown = null;
+
     }
 
     async spawn (waitForReady = false) {
@@ -85,7 +88,7 @@ class Shard extends EventEmitter {
                 const to = setTimeout(() => {
                     this.process.kill();
                     resolve();
-                }, 5000);
+                }, KillTO);
                 // Gracefully handle exit
                 this.process.once('exit', (code, signal) => {
                     clearTimeout(to);
@@ -118,6 +121,13 @@ class Shard extends EventEmitter {
         });
     }
 
+    awaitShutdown () {
+        this._respawn = false;
+        return new Promise((resolve) => {
+            this._awaitingShutdown = resolve;
+        });
+    }
+
     _handleMessage (message) {
         if (message) {
             if (message._ready) {
@@ -125,6 +135,10 @@ class Shard extends EventEmitter {
                 this.emit('ready');
                 return;
             } else if (message._shutdown) {
+                setTimeout(() => {
+                    if (this.process) 
+                        this.process.kill('SIGKILL');
+                }, KillTO);
                 this.ready = false;
                 this.emit('shutdown');
                 return;
@@ -150,6 +164,8 @@ class Shard extends EventEmitter {
             this.process.removeAllListeners();
         
         this.emit('death');
+        if (this._awaitingShutdown) 
+            this._awaitingShutdown();
 
         if (code !== 0) 
             this.crashes.push(Date.now() - this.spawnedAt);

src/server/Server.js

@@ -155,6 +155,7 @@ class Server extends EventEmitter {
         this.app.use(this.#ready.bind(this)); // denies requests before the server is ready
 
         process.on('message', this._handleMessage.bind(this));
+        process.on('SIGINT', this.shutdown.bind(this));
 
     }
 
@@ -209,17 +210,19 @@ class Server extends EventEmitter {
 
     async shutdown () {
         this.logger.info(`Received shutdown command, initiating graceful shutdown`);
+        // Tells the manager the shard is shutting down, will set a timeout to kill the shard if it hangs
         process.send({ _shutdown: true });
 
-        this._ready = false; // stops any new connections
+        this._ready = false;
+        this.server.close(async () => {
+            await this.mongodb.close();
+            await this.mariadb.close();
+            this.logger.status('DB shutdowns complete.');
 
-        await this.mongodb.close();
+            this.logger.status('Shutdown complete. Goodbye');
-        await this.mariadb.close();
+            // eslint-disable-next-line no-process-exit
-        this.logger.status('DB shutdowns complete.');
+            process.exit();
-
+        });
-        this.logger.status('Shutdown complete. Goodbye');
-        // eslint-disable-next-line no-process-exit
-        process.exit();
 
     }
 

src/server/database/MariaDB.js

@@ -3,13 +3,15 @@ const mysql = require('mysql');
 const { inspect } = require('node:util');
 const { Util } = require('../../util');
 
+const SAFE_TO_RETRY = [ 'ER_LOCK_DEADLOCK' ];
+
 class MariaDB {
 
     constructor (server, config) {
 
-        if (!server) 
+        if (!server)
             Util.fatal(new Error('Missing reference to server!'));
-        if (!config) 
+        if (!config)
             Util.fatal(new Error('No config options provided!'));
 
         this.config = config;
@@ -19,7 +21,7 @@ class MariaDB {
         for (const host of hosts) {
             this._credentials.push({ host, user, port, password, database });
         }
-        
+
         this.pool = null;
         this.ready = false;
         this.debug = true;
@@ -37,11 +39,10 @@ class MariaDB {
         this.logger.status(`Creating${this._cluster ? ' cluster' : ''} connection pool`);
         if (this._cluster) {
             this.pool = mysql.createPoolCluster(this.config.options.cluster);
-            for (const creds of this._credentials) 
+            for (const creds of this._credentials)
                 this.pool.add({ ...this.config.options.client, ...creds });
-            
         } else {
-            this.pool = mysql.createPool({ ...this.config.options.client, ...this._credentials[0] }); 
+            this.pool = mysql.createPool({ ...this.config.options.client, ...this._credentials[0] });
         }
 
         this.pool.on('connection', (connection) => {
@@ -59,17 +60,17 @@ class MariaDB {
 
         this.pool.on('release', (connection) => {
             this.activeQueries--;
-            
+
-            if (!this.ready && !this.activeQueries && this.afterLastQuery) 
+            if (!this.ready && !this.activeQueries && this.afterLastQuery)
                 this.afterLastQuery();
-            
+
             this.logger.debug(`Connection released: ${connection?.threadId || null}`);
         });
 
         this.logger.info(`Testing MariaDB connection`);
         await new Promise((resolve, reject) => {
             this.pool.getConnection((err, conn) => {
-                if (err) 
+                if (err)
                     return reject(err);
                 conn.release();
                 return resolve();
@@ -106,30 +107,56 @@ class MariaDB {
 
     }
 
-    query (query, values) {
+    getConnection () {
-
         return new Promise((resolve, reject) => {
-
+            this.pool.getConnection((err, conn) => {
-            if (!this.ready) 
-                return reject(new Error('MariaDB not ready'));
-
-            let batch = false;
-            if (values && typeof values.some === 'function') 
-                batch = values.some(val => val instanceof Array);
-            this.logger.debug(`Incoming query (batch: ${batch})\n${query}\n${inspect(values)}`);
-
-            // If connected to a cluster, pick one of the node connection pools
-            // By default the config is set to use round robin for picking a pool when connected to a cluster
-            const pool = this._cluster ? this.pool.of('*') : this.pool;
-
-            return pool.query(query, values, (err, results, fields) => {
                 if (err) 
                     return reject(err);
-                if (results) 
+                resolve(conn);
-                    return resolve(results);
-                resolve(fields);
             });
         });
+    }
+
+    /** 
+     * Retry certain queries that are safe to retry, e.g. deadlock
+     * 
+     * @private 
+     * */
+    async #_query (query, values, attempts = 0) {
+        const connection = await this.getConnection();
+        try {
+            const result = await new Promise((resolve, reject) => {
+                const q = connection.query(query, values, (err, results, fields) => {
+                    if (err) 
+                        reject(err);
+                    else if (results) 
+                        resolve(results);
+                    else 
+                        resolve(fields);
+                    connection.release();
+                });
+                this.logger.debug(`Constructed query: ${q.sql}`);
+            });
+            return Promise.resolve(result);
+        } catch (err) {
+            // Retry safe errors 
+            if (SAFE_TO_RETRY.includes(err.code) && attempts < 5) //  
+                return this.#_query(query, values, ++attempts);
+            return Promise.reject(err);
+        }
+    }
+
+    async query (query, values) {
+
+        if (!this.ready)
+            return Promise.reject(new Error('MariaDB not ready'));
+
+        let batch = false;
+        if (values && typeof values.some === 'function')
+            batch = values.some(val => val instanceof Array);
+        this.logger.debug(`Incoming query (batch: ${batch})\n${query}\n${inspect(values)}`);
+
+        return this.#_query(query, values);
 
     }
 

src/util/Util.js

@@ -11,9 +11,9 @@ class Util {
         return Math.floor(Date.now() / 1000);
     }
 
-    static wait (time) {
+    static wait (timeMs) {
         return new Promise((resolve) => {
-            setTimeout(resolve, time);
+            setTimeout(resolve, timeMs);
         });
     }
 

start.json

@@ -0,0 +1,14 @@
+{
+    "apps": [
+        {
+            "name": "framework",
+            "script": "index.js",
+            "interpreter_args": "",
+            "error_file": "/dev/null",
+            "out_file": "/dev/null",
+            "kill_timeout": 90000,
+            "max_restarts": 5,
+            "restart_delay": 5000
+        }
+    ]
+}