diff --git a/src/localization/en_gb/observers/en_gb_errorLog.lang b/src/localization/en_gb/observers/en_gb_errorLog.lang index 04a0386..a06139d 100644 --- a/src/localization/en_gb/observers/en_gb_errorLog.lang +++ b/src/localization/en_gb/observers/en_gb_errorLog.lang @@ -44,4 +44,9 @@ Missing target member for action **{actionType}**. [LINKFILTER_WARN] Failed to validate domain `{domain}`. +The domain has a valid top level domain (e.g. .com, .net) but failed a DNS check. + +{emoji_warning} **Warning** +Exercise caution when clicking on unknown links. Adviced to run the link through a link checker such as urlscan.io or similar. +Full link: ||{link}|| If this is a valid domain, add it to the greylist to ensure it is filtered. \ No newline at end of file diff --git a/src/middleware/BaseClient.js b/src/middleware/BaseClient.js index bcbad8a..f128a92 100644 --- a/src/middleware/BaseClient.js +++ b/src/middleware/BaseClient.js @@ -1,4 +1,5 @@ const { EventEmitter } = require('events'); +const { inspect } = require('util'); const SlashCommandManager = require('./rest/SlashCommandManager.js'); const { ShardingManager } = require('./shard/'); @@ -6,6 +7,10 @@ const Logger = require('./Logger.js'); const Metrics = require('./Metrics.js'); const ApiClientUtil = require('./ApiClientUtil.js'); +// Available for evals +// eslint-disable-next-line no-unused-vars +const ClientUtils = require('./ClientUtils.js'); + class BaseClient extends EventEmitter { constructor(options, version) { @@ -85,7 +90,7 @@ class BaseClient extends EventEmitter { * @memberof Manager * @private */ - async eval(shard, { script }) { + async eval(shard, { script, debug }) { this.logger.info(`Incoming manager eval from shard ${shard.id}:\n${script}`); let result = null, @@ -95,6 +100,7 @@ class BaseClient extends EventEmitter { // eslint-disable-next-line no-eval result = await eval(script); //if(typeof result !== 'string') result = inspect(result); + if(debug) this.logger.debug(`Eval result: ${inspect(result)}`); } catch (err) { error = err.stack || err; } diff --git a/src/middleware/ClientUtils.js b/src/middleware/ClientUtils.js new file mode 100644 index 0000000..683c94c --- /dev/null +++ b/src/middleware/ClientUtils.js @@ -0,0 +1,23 @@ +const { Util } = require("../utilities"); + +// Class to house various functions the clients use on the main process +class ClientUtils { + + static cache = {}; + + constructor() { + throw new Error('Static class, may not be instantiated.'); + } + + static async fetchTlds(force = false) { + if (this.cache.tlds && !force) return this.cache.tlds; + const buffer = await Util.downloadAsBuffer('https://data.iana.org/TLD/tlds-alpha-by-domain.txt'); + const list = buffer.toString().split('\n').filter((str) => !str.startsWith('#')).map((str) => str.toLowerCase()); + + this.cache.tlds = list; + return list; + } + +} + +module.exports = ClientUtils; \ No newline at end of file diff --git a/src/structure/client/Registry.js b/src/structure/client/Registry.js index f033821..5edf554 100644 --- a/src/structure/client/Registry.js +++ b/src/structure/client/Registry.js @@ -85,6 +85,7 @@ class Registry { module.components.set(component.resolveable, component); } + if (component.initialise && typeof component.initialise === 'function') await component.initialise(); this.components.set(component.resolveable, component); if(!silent) this.client.emit('componentUpdate', { component, type: 'LOAD' }); return component; diff --git a/src/structure/components/observers/Automoderation.js b/src/structure/components/observers/Automoderation.js index e5865d0..2169d80 100644 --- a/src/structure/components/observers/Automoderation.js +++ b/src/structure/components/observers/Automoderation.js @@ -29,9 +29,6 @@ const CONSTANTS = { } }; -const linkRegG = /(https?:\/\/(www\.)?)?(?([a-z0-9-]{1,63}\.)?([a-z0-9-]{2,63})(\.[a-z0-9-]{2,63})(\.[a-z0-9-]{2,63})?)(\/\S*)?/iug; -const linkReg = /(https?:\/\/(www\.)?)?(?([a-z0-9-]{1,63}\.)?([a-z0-9-]{2,63})(\.[a-z0-9-]{2,63})(\.[a-z0-9-]{2,63})?)(\/\S*)?/iu; - // TODO: // Clean up commented out code once testing of new code is done // Implement missing automod features -- done @@ -60,17 +57,38 @@ module.exports = class AutoModeration extends Observer { ['interactionCreate', this.flagAction.bind(this)] ]; - this.whitelist = new BinaryTree(this.client, FilterPresets.whitelist); + this.topLevelDomains = null; + // Moved to FilterUtil + // this.whitelist = new BinaryTree(this.client, FilterPresets.whitelist); this.executing = {}; this.regex = { invite: /((discord)?\s*\.?\s*gg\s*|discord(app)?\.com\/invite)\/\s?(?[a-z0-9]+)/i, + linkRegG: /(https?:\/\/(www\.)?)?(?([a-z0-9-]{1,63}\.)?([a-z0-9-]{2,63})(\.[a-z0-9-]{2,63})(\.[a-z0-9-]{2,63})?)(\/\S*)?/iug, + linkReg: /(https?:\/\/(www\.)?)?(?([a-z0-9-]{1,63}\.)?([a-z0-9-]{2,63})(\.[a-z0-9-]{2,63})(\.[a-z0-9-]{2,63})?)(\/\S*)?/iu, mention: /<@!?(?[0-9]{18,22})>/, mentionG: /<@!?(?[0-9]{18,22})>/g, }; } + async initialise() { + + // Fetch a list of TLDs from iana + const tldList = await this.client.managerEval(` + (() => { + return ClientUtils.fetchTlds() + })() + `).catch(this.logger.error.bind(this.logger)); + + const middlePoint = Math.floor(tldList.length / 2); + const [midEntry] = tldList.splice(middlePoint, 1); + tldList.splice(0, 0, midEntry); + this.topLevelDomains = new BinaryTree(this.client, tldList); + this.topLevelDomains.add('onion'); + + } + async _moderate(action, wrapper, channel, member, reason, filterResult, moderator) { // Prevent simultaneous execution of the same filter on the same user when spamming @@ -150,29 +168,11 @@ module.exports = class AutoModeration extends Observer { // 2. Filter explicit - no bypass checking (unless you count normalising the text, i.e. emoji letters => normal letters) if (explicit.length && !filterResult.matched) { - //filterResult = FilterUtil.filterExplicit(words, explicit); - // if(filterResult) - const result = FilterUtil.filterExplicit(words, explicit); if (result) { log += `\nMessage matched with "${result.match}" in the explicit list.\nFull content: ${content}`; filterResult = result; } - // for (const word of explicit) { - // //Do it like this instead of regex so it doesn't match stuff like Scunthorpe with cunt - // if (words.some((_word) => _word === word)) { - // log += `\nMessage matched with "${word}" in the explicit list.\nFull content: ${content}`; - // filterResult = { - // match: word, - // matched: true, - // matcher: 'explicit', - // _matcher: word, - // type: 'explicit' - // }; - // break; - // } - - // } } @@ -184,40 +184,7 @@ module.exports = class AutoModeration extends Observer { log += `\nMessage matched with "${result.matcher}" in the regex list.\nMatch: ${result._raw}, Full word: ${result.match}\nFull content: ${content}`; filterResult = result; } - // for (const reg of regex) { - - // const match = content.toLowerCase().match(new RegExp(`(?:^|\\s)(${reg})`, 'iu')); // (?:^|\\s) |un - - // if (match) { - // //log += `\next reg: ${tmp}`; - // const fullWord = words.find((word) => word.includes(match[1])); - - // let inWL = false; - // try { // This is for debugging only - // inWL = this.whitelist.find(fullWord); - // } catch (err) { - // this.logger.error(`Whitelist.find error: ${fullWord} ${match[1]} ${words}`); - // } - // if (inWL || whitelist.some((word) => word === fullWord)) continue; - - // const diff = Math.abs(fullWord.length - match[0].length); - // if (diff > 3) { - // this.logger.debug(`Match length diff: ${diff} MATCH: ${match[0]} FULL: ${fullWord}`); - // continue; - // } - // log += `\nMessage matched with "${reg}" in the regex list.\nMatch: ${match[0]}, Full word: ${fullWord}\nFull content: ${content}`; - // filterResult = { - // match: fullWord, - // matched: true, - // _matcher: match[1].toLowerCase(), - // matcher: `Regex: __${reg}__`, - // type: 'regex' - // }; - // break; - // } - - // } - + } // 4. Filter fuzzy @@ -228,52 +195,7 @@ module.exports = class AutoModeration extends Observer { filterResult = result; log += `\nMessage matched with "${result._matcher}" in fuzzy.\nMatched word: ${result.match}\nFull content: ${content}\nSimilarity: ${result._sim}\nThreshold: ${result._threshold}`; } - // const text = words.join('').replace(/\s/u, ''); - // let threshold = (0.93 - 0.133 * Math.log(text.length)).toFixed(3); - // if (threshold < 0.6) threshold = 0.6; - // else if (threshold > 0.9) threshold = 0.9; - - // outer: - // for (const _word of fuzzy) { - - // for (const word of words) { - // const sim = similarity(word, _word); - // let threshold = (0.93 - 0.133 * Math.log(word.length)).toFixed(3); - // if (threshold < 0.6) threshold = 0.6; - // else if (threshold > 0.9) threshold = 0.9; - // if (sim >= threshold && Math.abs(_word.length - word.length) <= 2) { - // if (this.whitelist.find(word) || whitelist.some((w) => w === word) && sim < 1) continue; - // log += `\nMessage matched with "${_word}" in fuzzy.\nMatched word: ${word}\nFull content: ${content}\nSimilarity: ${sim}\nThreshold: ${threshold}`; - // filterResult = { - // match: word, - // matched: true, - // _matcher: _word, - // matcher: `fuzzy [\`${_word}\`, \`${sim}\`, \`${threshold}\`]`, - // type: 'fuzzy' - // }; - // break outer; - // } - - // } - - // const sim = similarity(text, _word); - // if (sim >= threshold && Math.abs(_word.length - text.length) <= 2) { - // if (this.whitelist.find(text) || whitelist.some((w) => w === text) && sim < 1) continue; - // log += `\nMessage matched with "${_word}" in fuzzy.\nMatched word: ${text}\nFull content: ${content}\nSimilarity: ${sim}\nThreshold: ${threshold}`; - // filterResult = { - // match: text, - // matched: true, - // _matcher: _word, - // matcher: `fuzzy [\`${_word}\`, \`${sim}\`, \`${threshold}\`]`, - // type: 'fuzzy' - // }; - // break; - // } - - // //this.client.logger.debug(`Message did not match with "${_word}" in fuzzy.\nFull content: ${content}\nSimilarity: ${sim}\nThreshold: ${threshold}`); - - // } - + } // 5. Remove message, inline response and add a reason to msg object @@ -510,15 +432,15 @@ module.exports = class AutoModeration extends Observer { const msg = edited || message; if (!msg.content) return; const content = msg.content.split('').join(''); //Copy the string... - let matches = content.match(linkRegG); - if (!matches) matches = content.replace(/\s/u, '').match(linkRegG); + let matches = content.match(this.regex.linkRegG); + if (!matches) matches = content.replace(/\s/u, '').match(this.regex.linkRegG); if (!matches) return; let remove = false; const filterResult = {}; let log = `${guild.name} Link filter debug:`; for (const match of matches) { - const { domain } = match.match(linkReg).groups; + const { domain } = match.match(this.regex.linkReg).groups; log += `\nMatched link ${match}: `; const predicate = (dom) => { @@ -543,9 +465,13 @@ module.exports = class AutoModeration extends Observer { continue; } + const parts = domain.split('.'); + const validTld = this.topLevelDomains.find(parts[parts.length - 1]); + // console.log(parts, validTld); + if (!validTld) continue; const valid = await resolver.validateDomain(domain); if (!valid) { - this.client.emit('linkFilterWarn', { guild: wrapper, message: wrapper.format('LINKFILTER_WARN', { domain }) }); + this.client.emit('linkFilterWarn', { guild: wrapper, message: wrapper.format('LINKFILTER_WARN', { domain, link: match }) }); continue; }