Navy.gif/wrappers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add more debug statements to broker

Browse Source 
Upgrade packages
This commit is contained in:
Erik 2024-10-09 00:03:42 +03:00
parent d781458cee
commit 0dac59b426
13 changed files with 1247 additions and 1149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintignore Normal file
View File

@ -0,0 +1 @@
tests

81
.eslintrc.json
View File

@ -12,11 +12,19 @@
"plugins": [ "@typescript-eslint" ],
"parserOptions": {
"ecmaVersion": 2022,
"sourceType": "module"
"sourceType": "module",
"project": "./tsconfig.json"
},
"rules": {
"@typescript-eslint/no-unused-vars": "off",
"@typescript-eslint/no-non-null-assertion":"off",
"@typescript-eslint/no-non-null-assertion": "off",
"@typescript-eslint/no-misused-promises": ["error", {
"checksVoidReturn": {
"arguments": false
}
}],
"@typescript-eslint/no-shadow": "error",
"@typescript-eslint/no-use-before-define": "error",
"accessor-pairs": "warn",
"array-callback-return": "warn",
"array-bracket-newline": [
@ -74,7 +82,7 @@
"warn",
"never"
],
"eqeqeq": "warn",
"eqeqeq": "error",
"func-call-spacing": "warn",
"func-name-matching": "warn",
"func-names": "warn",
@ -87,18 +95,38 @@
"id-blacklist": "warn",
"id-match": "warn",
"implicit-arrow-linebreak": "warn",
"indent": "warn",
"init-declarations": "warn",
"quotes": ["error" , "single"],
"jsx-quotes": [
"indent": [
"warn",
4,
{
"SwitchCase": 1,
"VariableDeclarator": "first",
"FunctionDeclaration": {
"parameters": "first"
},
"CallExpression": {
"arguments": "first"
},
"ArrayExpression": "first",
"ObjectExpression": "first",
"ImportDeclaration": "first"
}
],
"init-declarations": "warn",
"quotes": [
"error",
"single"
],
"jsx-quotes": [
"error",
"prefer-single"
],
"key-spacing": [
"warn",
{
"beforeColon": false,
"afterColon": true
"afterColon": true,
"align": "value"
}
],
"keyword-spacing": [
@ -128,7 +156,7 @@
],
"max-lines-per-function": [
"warn",
100
140
],
"max-depth": [
"warn",
@ -137,16 +165,16 @@
"new-parens": "warn",
"no-alert": "warn",
"no-array-constructor": "warn",
// "no-bitwise": "warn",
"no-buffer-constructor": "warn",
"no-caller": "warn",
"no-console": "warn",
"no-constant-binary-expression": "error",
"no-div-regex": "warn",
"no-dupe-else-if": "warn",
"no-duplicate-imports": "warn",
"no-else-return": "warn",
"no-empty-function": "warn",
"no-eq-null": "warn",
"no-eq-null": "off",
"no-eval": "warn",
"no-extend-native": "warn",
"no-extra-bind": "warn",
@ -164,7 +192,15 @@
"no-loop-func": "warn",
"no-mixed-requires": "warn",
"no-multi-assign": "warn",
"no-multi-spaces": "warn",
"no-multi-spaces": [
"warn",
{
"ignoreEOLComments": true,
"exceptions": {
"ImportDeclaration": true
}
}
],
"no-multi-str": "warn",
"no-multiple-empty-lines": "warn",
"no-native-reassign": "warn",
@ -185,23 +221,25 @@
"no-restricted-modules": "warn",
"no-restricted-properties": "warn",
"no-restricted-syntax": "warn",
"no-return-assign": "warn",
"no-return-assign": [
"warn",
"except-parens"
],
"no-return-await": "warn",
"no-script-url": "warn",
"no-self-compare": "warn",
"no-sequences": "warn",
"no-setter-return": "warn",
"no-spaced-func": "warn",
"@typescript-eslint/no-shadow": "error",
"no-tabs": "warn",
"no-tabs": "error",
"no-template-curly-in-string": "error",
"no-throw-literal": "warn",
"no-trailing-spaces": "warn",
"no-undef-init": "error",
"no-undefined": "error",
"no-unmodified-loop-condition": "warn",
"no-unneeded-ternary": "error",
"no-unused-expressions": "warn",
"@typescript-eslint/no-use-before-define": "error",
"no-useless-call": "warn",
"no-useless-computed-key": "warn",
"no-useless-concat": "warn",
@ -209,7 +247,6 @@
"no-useless-rename": "warn",
"no-useless-return": "warn",
"no-var": "warn",
// "no-void": "warn",
"no-whitespace-before-property": "error",
"nonblock-statement-body-position": [
"warn",
@ -293,6 +330,16 @@
"yoda": [
"warn",
"never"
],
"no-warning-comments": [
1,
{
"terms": [
"todo",
"fixme"
],
"location": "anywhere"
}
]
}
}

893
.yarn/releases/yarn-4.1.1.cjs vendored
View File

File diff suppressed because one or more lines are too long

925
.yarn/releases/yarn-4.5.0.cjs vendored Executable file
View File

File diff suppressed because one or more lines are too long

4
.yarnrc.yml
View File

@ -1,5 +1,5 @@
nodeLinker: node-modules
yarnPath: .yarn/releases/yarn-4.1.1.cjs
npmRegistryServer: "https://registry.corgi.wtf"
yarnPath: .yarn/releases/yarn-4.5.0.cjs

2
package.json
View File

@ -54,5 +54,5 @@
"optional": true
}
},
"packageManager": "yarn@4.1.1"
"packageManager": "yarn@4.5.0"
}

44
src/MariaDB.ts
View File

@ -6,7 +6,7 @@ import { ILogger, IServer } from './interfaces/index.js';
const SAFE_TO_RETRY = [ 'ER_LOCK_DEADLOCK', 'PROTOCOL_CONNECTION_LOST' ];
type Credentials = {
interface Credentials {
user: string,
password: string,
host: string,
@ -18,7 +18,7 @@ type ExtendedCredentials = {
node: string
} & Credentials
export type MariaOptions = {
export interface MariaOptions {
load?: boolean,
cluster?: PoolClusterConfig,
client?: PoolConfig,
@ -40,7 +40,7 @@ type MariaError = {
type Value = (string | number | null | boolean | object)
type Values = (Value | Value[] | Value[][])[];
type QueryOptions = {
interface QueryOptions {
node?: string,
timeout?: number,
errorIfNodeUnavailable?: boolean
@ -54,14 +54,14 @@ type StatusString =
| 'donor'
| 'joiner'
| 'joined'
type Node = {
interface Node {
name: string,
host: string,
uuid?: string,
status?: StatusString
}
type StatusUpdate = {
interface StatusUpdate {
status: StatusString,
uuid: string,
primary: string,
@ -145,7 +145,7 @@ class MariaDB
return this.#_activeQueries;
}
// eslint-disable-next-line max-lines-per-function
async init ()
{
if (!this.#load)
@ -197,13 +197,17 @@ class MariaDB
await new Promise<void>((resolve, reject) =>
{
if (!this.#pool)
return reject(new Error('Missing connection pool'));
{
reject(new Error('Missing connection pool')); return;
}
this.#pool.getConnection((err, conn) =>
{
if (err)
return reject(err);
{
reject(err); return;
}
conn.release();
return resolve();
resolve();
});
});
@ -246,9 +250,9 @@ class MariaDB
{
this.#logger?.info('Setting up metric recording');
MariaDB.#queryHistogram = new Prometheus.Histogram({
name: 'sql_queries',
help: 'Tracks query duration in seconds and frequency',
buckets: this.#config.metricsBuckets ?? Prometheus?.exponentialBuckets(0.005, 2, 10),
name: 'sql_queries',
help: 'Tracks query duration in seconds and frequency',
buckets: this.#config.metricsBuckets ?? Prometheus?.exponentialBuckets(0.005, 2, 10),
labelNames: [ 'type' ] as const
});
if (this.#server.registerMetric)
@ -274,7 +278,9 @@ class MariaDB
return new Promise<void>(resolve =>
{
if (!this.#pool)
return resolve();
{
resolve(); return;
}
this.#pool.end(() =>
{
this.#pool?.removeAllListeners();
@ -359,7 +365,9 @@ class MariaDB
return new Promise((resolve, reject) =>
{
if (!this.#pool)
return reject(new Error('Pool closed'));
{
reject(new Error('Pool closed')); return;
}
// Get node by name
const pool = this.#pool;
if (nodeName)
@ -386,11 +394,13 @@ class MariaDB
}
}
this.#logger?.debug(`Selected node ${nodeName} for query`);
return pool.of(nodeName ?? '*').getConnection((err, conn) =>
pool.of(nodeName ?? '*').getConnection((err, conn) =>
{
if (err)
return reject(err);
return resolve(conn);
{
reject(err); return;
}
resolve(conn);
});
});

60
src/MessageBroker.ts
View File

@ -2,15 +2,15 @@ import { ILogger, IServer } from './interfaces/index.js';
import amqp, { AmqpConnectionManager, ChannelWrapper } from 'amqp-connection-manager';
import { Channel, ConfirmChannel, ConsumeMessage, Options } from 'amqplib';
type ExchangeDef = {
interface ExchangeDef {
durable?: boolean,
internal?: boolean,
autoDelete?: boolean,
type?: 'direct' | 'topic' | 'headers' | 'fanout' | 'match' | string,
type?: 'direct' | 'topic' | 'headers' | 'fanout' | 'match',
arguments?: object
}
type QueueDef = {
interface QueueDef {
durable?: boolean,
messageTtl?: number,
exclusive?: boolean,
@ -22,22 +22,18 @@ type QueueDef = {
maxPriority?: number
}
export type BrokerOptions = {
export interface BrokerOptions {
load?: boolean,
host: string,
port: number
user: string,
pass: string,
vhost: string,
exchanges?: {
[key: string]: ExchangeDef
},
queues?: {
[key: string]: QueueDef
}
exchanges?: Record<string, ExchangeDef>,
queues?: Record<string, QueueDef>
}
type InternalMessage = {
interface InternalMessage {
properties: object,
content: object
}
@ -54,23 +50,21 @@ type InternalQueueMsg = {
type Consumer<T = unknown> = (content: T, msg: ConsumeMessage) => Promise<void> | void
type Subscriber<T = unknown> = (content: T, msg: ConsumeMessage) => Promise<void> | void
export type SubscriptionOptions = {
export interface SubscriptionOptions {
exchangeType?: 'direct' | 'topic' | 'headers' | 'fanout' | 'match',
routingKey?: string
}
class MessageBroker
{
// Broker definitions
#load: boolean;
#hosts: string[];
#port: number;
#username: string;
#password: string;
#vhost: string;
#exchanges: { [key: string]: ExchangeDef };
#queues: {[key: string]: QueueDef};
#exchanges: Record<string, ExchangeDef>;
#queues: Record<string, QueueDef>;
// Wrapper related
#connection: AmqpConnectionManager | null;
@ -88,7 +82,6 @@ class MessageBroker
constructor (server: IServer, options: BrokerOptions)
{
this.#load = options.load ?? true;
this.#hosts = options.host.split(',');
this.#username = options.user;
@ -119,7 +112,6 @@ class MessageBroker
this.#_pQueue = [];
this.#_qQueue = [];
this.#_qTO = null;
}
async init ()
@ -130,7 +122,7 @@ class MessageBroker
this.#logger?.info('Initialising message broker');
const credentials = this.#username ? `${this.#username}:${this.#password}@` : '';
const connectionStrings = this.#hosts.map(host => `amqp://${credentials}${host}:${this.#port}/${this.#vhost}`);
this.#connection = await amqp.connect(connectionStrings);
this.#connection = amqp.connect(connectionStrings);
this.#connection.on('disconnect', async ({ err }) =>
{
@ -163,24 +155,27 @@ class MessageBroker
await this.createChannel();
this.#connection.on('connect', this.createChannel.bind(this));
}
async close ()
{
this.#logger?.status('Closing broker');
if (this.#channel)
{
await this.#channel.close();
this.#logger?.info('Closing channel');
this.#channel.removeAllListeners();
await this.#channel.close();
}
if (this.#connection)
{
this.#logger?.info('Closing connection');
await this.#connection.close();
// No clue why the removeAllListeners function isn't exposed from the connection manager, but it exists since it's an EventEmitter
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
this.#connection.removeAllListeners();
}
this.#logger?.status('Broker closed');
}
async createChannel ()
@ -188,14 +183,16 @@ class MessageBroker
const exchanges = Object.entries(this.#exchanges);
const queues = Object.entries(this.#queues);
if (!this.#connection)
throw new Error('Broker not connected');
if (this.#channel)
{
this.#logger?.debug('Closing old channel');
this.#channel.removeAllListeners();
await this.#channel.close().catch(err => this.#logger?.error(err.stack));
}
if (!this.#connection)
throw new Error();
this.#logger?.debug('Creating channel');
this.#channel = this.#connection.createChannel({
setup: async (channel: Channel | ConfirmChannel) =>
@ -218,7 +215,9 @@ class MessageBroker
await new Promise<void>((resolve, reject) =>
{
if (!this.#channel)
return reject(new Error('Missing channel?'));
{
reject(new Error('Missing channel?')); return;
}
this.#channel.once('connect', () =>
{
this.#channel?.removeListener('error', reject);
@ -255,6 +254,7 @@ class MessageBroker
return Promise.reject(new Error('Channel doesn\'t exist'));
await this.#channel.consume(queue, async (msg: ConsumeMessage) =>
{
this.#logger?.debug(`Consumer for ${queue} (${consumer.name}) fired`);
if (msg.content)
await consumer(JSON.parse(msg.content.toString()), msg);
this.#channel?.ack(msg);
@ -276,7 +276,6 @@ class MessageBroker
const list = this.#subscribers.get(name) ?? [];
list.push({ subscriber, options });
this.#subscribers.set(name, list);
}
private async _subscribe<T> (name: string, listener: Subscriber<T>, options: SubscriptionOptions): Promise<void>
@ -291,6 +290,7 @@ class MessageBroker
await this.#channel.bindQueue(queue.queue, name, options.routingKey ?? '');
await this.#channel.consume(queue.queue, async (msg) =>
{
this.#logger?.debug(`Subscriber for ${name} (${listener.name}) fired`);
if (msg.content && msg.content.toString().startsWith('{'))
await listener(JSON.parse(msg.content.toString()), msg);
else
@ -304,15 +304,19 @@ class MessageBroker
// Add item to queue
async enqueue (queue: string, content: object, headers?: object): Promise<void | number>
{
this.#logger?.debug(`Adding to queue ${queue}`);
const properties = {
persistent: true,
persistent: true,
contentType: 'application/json',
headers
};
// The channel is null while the failover is occurring, so enqueue messages for publishing once the connection is restored
if (!this.#channel)
{
this.#logger?.debug('Channel is missing, adding to internal queue');
return this.#_qQueue.push({ queue, content, properties });
}
try
{
@ -329,6 +333,7 @@ class MessageBroker
// Publish to exchange
async publish (exchange: string, content: object, { headers, routingKey = '' }: {headers?: string, routingKey?: string} = {}): Promise<void|number>
{
this.#logger?.debug(`Publishing to ${exchange}`);
const properties = {
contentType: 'application/json',
headers
@ -336,7 +341,10 @@ class MessageBroker
// The channel is null while the failover is occurring, so enqueue messages for publishing once the connection is restored
if (!this.#channel)
{
this.#logger?.debug('Channel is missing, adding to internal queue');
return this.#_pQueue.push({ exchange, content, routingKey, properties });
}
try
{

16
src/MongoDB.ts
View File

@ -2,7 +2,7 @@ import { inspect } from 'node:util';
import { MongoClient, MongoClientOptions, Db, Document, WithId, ObjectId, Filter, IndexSpecification, CreateIndexesOptions, FindOptions, ModifyResult, DistinctOptions } from 'mongodb';
import { IServer, ILogger } from './interfaces/index.js';
type Credentials = {
interface Credentials {
URI?: string,
user?: string,
password?: string,
@ -12,12 +12,12 @@ type Credentials = {
authDb?: string
}
type MongoQuery = {
interface MongoQuery {
_id?: unknown,
[key: string]: unknown
}
export type MongoOptions = {
export interface MongoOptions {
credentials: Credentials,
loggerOptions?: object,
client?: MongoClientOptions,
@ -26,7 +26,7 @@ export type MongoOptions = {
metricsBuckets?: number[],
}
type StringIndexable = {[key: string]: boolean | string | number | Document | object}
type StringIndexable = Record<string, boolean | string | number | Document | object>;
const objIsSubset = (superObj: StringIndexable, subObj: StringIndexable): boolean =>
{
@ -67,7 +67,7 @@ class MongoDB
throw new Error('No config options provided!');
const { user, password, host, port, database, URI, authDb } = config.credentials;
if ((!host?.length || !port || !database?.length) && !URI)
if ((!host?.length || !port || !database.length) && !URI)
throw new Error('Must provide host, port, and database OR URI parameters!');
this.#config = config;
@ -149,9 +149,9 @@ class MongoDB
{
this.#logger?.info('Setting up metric recording');
MongoDB.#queryHistogram = new Prometheus.Histogram({
name: 'mongo_queries',
help: 'Tracks query duration in seconds and frequency',
buckets: this.#config.metricsBuckets ?? Prometheus?.exponentialBuckets(0.005, 2, 10),
name: 'mongo_queries',
help: 'Tracks query duration in seconds and frequency',
buckets: this.#config.metricsBuckets ?? Prometheus?.exponentialBuckets(0.005, 2, 10),
labelNames: [ 'type' ] as const
});
if (this.#server.registerMetric)

8
tests/testBroker.js
View File

@ -6,11 +6,11 @@ const broker = new MessageBroker({
createLogger: () =>
{
return {
debug: console.log,
info: console.log,
debug: console.log,
info: console.log,
status: console.log,
warn: console.log,
error: console.error
warn: console.log,
error: console.error
};
}
}, credentials);

34
tests/testClusterStatus.js
View File

@ -8,21 +8,21 @@ const maria = new MariaDB({
createLogger: () =>
{
return {
debug: console.log,
info: console.log,
debug: console.log,
info: console.log,
status: console.log,
warn: console.log,
error: console.error
warn: console.log,
error: console.error
};
}
}, {
load: true,
load: true,
credentials,
cluster: {
canRetry: true,
canRetry: true,
removeNodeErrorCount: 5,
restoreNodeTimeout: 60000,
defaultSelector: 'RR'
restoreNodeTimeout: 60000,
defaultSelector: 'RR'
}
});
@ -30,20 +30,20 @@ const broker = new MessageBroker({
createLogger: () =>
{
return {
debug: console.log,
info: console.log,
debug: console.log,
info: console.log,
status: console.log,
warn: console.log,
error: console.error
warn: console.log,
error: console.error
};
}
}, {
load: true,
host: 'rabbitmq-01.stylis.local',
user: 'stylis',
pass: 'RrwJyrfeXFMimDH3hjZ5xSreMAmXtQJj',
load: true,
host: 'rabbitmq-01.stylis.local',
user: 'stylis',
pass: 'RrwJyrfeXFMimDH3hjZ5xSreMAmXtQJj',
vhost: 'development',
port: 5672
port: 5672
});
await broker.init();

16
tests/testMaria.js
View File

@ -8,21 +8,21 @@ const maria = new MariaDB({
createLogger: () =>
{
return {
debug: console.log,
info: console.log,
debug: console.log,
info: console.log,
status: console.log,
warn: console.log,
error: console.error
warn: console.log,
error: console.error
};
}
}, {
load: true,
load: true,
credentials,
cluster: {
canRetry: true,
canRetry: true,
removeNodeErrorCount: 5,
restoreNodeTimeout: 60000,
defaultSelector: 'RR'
restoreNodeTimeout: 60000,
defaultSelector: 'RR'
}
});

10
tests/testMongo.js
View File

@ -4,16 +4,16 @@ const mongo = new MongoDB({
createLogger: () =>
{
return {
debug: console.log,
info: console.log,
debug: console.log,
info: console.log,
status: console.log,
warn: console.log,
error: console.error
warn: console.log,
error: console.error
};
}
}, {
credentials: {
URI: 'mongodb://127.0.0.1:27017',
URI: 'mongodb://127.0.0.1:27017',
database: 'framework-proto'
},
load: true