From ee9d5a304f8af98c3fdda02005b59a9cbc9bd107 Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 26 Aug 2020 10:30:50 +0000 Subject: [PATCH 01/89] fix func_queue typehint --- src/IRCBot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/IRCBot.py b/src/IRCBot.py index b2c2861f..843fb6e6 100644 --- a/src/IRCBot.py +++ b/src/IRCBot.py @@ -116,7 +116,8 @@ def trigger(self, self._trigger_both() return returned - func_queue = queue.Queue(1) # type: queue.Queue[str] + func_queue: queue.Queue[typing.Tuple[TriggerResult, str] + ] = queue.Queue(1) def _action(): try: From eec8d1a6a64664c8f279edb843641fd03eeab255 Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 26 Aug 2020 10:32:31 +0000 Subject: [PATCH 02/89] actually fix typehint and you can only throw Exception inheritors --- src/IRCBot.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/IRCBot.py b/src/IRCBot.py index 843fb6e6..8c7bdf47 100644 --- a/src/IRCBot.py +++ b/src/IRCBot.py @@ -116,7 +116,7 @@ def trigger(self, self._trigger_both() return returned - func_queue: queue.Queue[typing.Tuple[TriggerResult, str] + func_queue: queue.Queue[typing.Tuple[TriggerResult, typing.Any] ] = queue.Queue(1) def _action(): @@ -135,7 +135,8 @@ def _action(): if trigger_threads: self._trigger_both() - if type == TriggerResult.Exception: + if (type == TriggerResult.Exception and + isinstance(returned, Exception)): raise returned elif type == TriggerResult.Return: return returned From e51aeb1ca6bd59a534262631e0d43942862e0eee Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 26 Aug 2020 10:35:10 +0000 Subject: [PATCH 03/89] "unpacking a string is disallowed" --- src/utils/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/http.py b/src/utils/http.py index 045d9641..34d6a576 100644 --- a/src/utils/http.py +++ b/src/utils/http.py @@ -7,7 +7,7 @@ REGEX_URL = re.compile("https?://\S+", re.I) -PAIRED_CHARACTERS = ["<>", "()"] +PAIRED_CHARACTERS = [("<", ">"), ("(", ")")] # best-effort tidying up of URLs def url_sanitise(url: str): From c32e073c35b3824c739042b7b6927d3b3f860931 Mon Sep 17 00:00:00 2001 From: jesopo Date: Tue, 8 Sep 2020 13:55:45 +0000 Subject: [PATCH 04/89] explicit support for dronebl type 19 (abused vpn) --- modules/dnsbl/lists.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/dnsbl/lists.py b/modules/dnsbl/lists.py index b84628ea..6ae19e2f 100644 --- a/modules/dnsbl/lists.py +++ b/modules/dnsbl/lists.py @@ -39,6 +39,8 @@ def process(self, result): return "flooding" elif result in ["12", "13", "15", "16"]: return "exploits" + elif result == "19": + return "abused vpn" class AbuseAtCBL(DNSBL): hostname = "cbl.abuseat.org" From b6e8f668c49061c9f55e83700ded287da1c64379 Mon Sep 17 00:00:00 2001 From: jesopo Date: Tue, 8 Sep 2020 15:48:14 +0000 Subject: [PATCH 05/89] better dronebl descriptions, show category in all list descriptions --- modules/dnsbl/lists.py | 50 ++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/modules/dnsbl/lists.py b/modules/dnsbl/lists.py index 6ae19e2f..1e4e1ac9 100644 --- a/modules/dnsbl/lists.py +++ b/modules/dnsbl/lists.py @@ -13,41 +13,59 @@ class ZenSpamhaus(DNSBL): def process(self, result): result = result.rsplit(".", 1)[1] if result in ["2", "3", "9"]: - return "spam" + desc = "spam" elif result in ["4", "5", "6", "7"]: - return "exploits" + desc = "exploits" + return f"{result} - {desc}" + class EFNetRBL(DNSBL): hostname = "rbl.efnetrbl.org" def process(self, result): result = result.rsplit(".", 1)[1] if result == "1": - return "proxy" + desc = "proxy" elif result in ["2", "3"]: - return "spamtap" + desc = "spamtap" elif result == "4": - return "tor" + desc = "tor" elif result == "5": - return "flooding" + desc = "flooding" + return f"{result} - {desc}" +DRONEBL_CATEGORIES = { + 3: "IRC drone", + 5: "bottler", + 6: "unknown spambot or drone", + 7: "DDoS drone", + 8: "open SOCKS proxy", + 9: "open HTTP proxy", + 10: "proxychain", + 11: "web page proxy", + 12: "open DNS resolver", + 13: "brute force attacker", + 14: "open WINGATE proxy", + 15: "compromised router/gateway", + 16: "autorooting malware", + 17: "detected botnet IP", + 18: "DNS/MX on IRC", + 19: "abused VPN service" +} class DroneBL(DNSBL): hostname = "dnsbl.dronebl.org" def process(self, result): - result = result.rsplit(".", 1)[1] - if result in ["8", "9", "10", "11", "14"]: - return "proxy" - elif result in ["3", "6", "7"]: - return "flooding" - elif result in ["12", "13", "15", "16"]: - return "exploits" - elif result == "19": - return "abused vpn" + result = int(result.rsplit(".", 1)[1]) + desc = DRONEBL_CATEGORIES.get(result, "unknown") + return f"{result} - {desc}" class AbuseAtCBL(DNSBL): hostname = "cbl.abuseat.org" def process(self, result): result = result.rsplit(".", 1)[1] if result == "2": - return "abuse" + desc = "abuse" + else: + desc = "unknown" + return f"{result} - {desc}" DEFAULT_LISTS = [ ZenSpamhaus(), From 6d99a9fae65e10cff63014f300dce7485e97bf93 Mon Sep 17 00:00:00 2001 From: jesopo Date: Mon, 14 Sep 2020 13:32:59 +0000 Subject: [PATCH 06/89] support dnsbl TXT records --- modules/dnsbl/__init__.py | 14 ++++++++++--- modules/dnsbl/lists.py | 41 +++++++++++---------------------------- 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/modules/dnsbl/__init__.py b/modules/dnsbl/__init__.py index 2b3daf35..dd64a98d 100644 --- a/modules/dnsbl/__init__.py +++ b/modules/dnsbl/__init__.py @@ -43,14 +43,22 @@ def _check_lists(self, lists, address): failed = [] for list in lists: record = self._check_list(list.hostname, address) - if not record == None: - reason = list.process(record) or "unknown" + if record is not None: + a_record, txt_record = record + reason = list.process(a_record, txt_record) or "unknown" failed.append((list.hostname, reason)) return failed def _check_list(self, list, address): list_address = "%s.%s" % (address, list) try: - return dns.resolver.query(list_address, "A")[0].to_text() + a_record = dns.resolver.query(list_address, "A")[0].to_text() except dns.resolver.NXDOMAIN: return None + + try: + txt_record = dns.resolver.query(list_address, "TXT")[0].to_text() + except: + txt_record = None + + return (a_record, txt_record) diff --git a/modules/dnsbl/lists.py b/modules/dnsbl/lists.py index 1e4e1ac9..44975467 100644 --- a/modules/dnsbl/lists.py +++ b/modules/dnsbl/lists.py @@ -5,13 +5,16 @@ def __init__(self, hostname=None): if not hostname == None: self.hostname = hostname - def process(self, result: str): - return result + def process(self, a_record, txt_record): + out = a_record + if txt_record is not None: + out += f" - {txt_record}" + return out class ZenSpamhaus(DNSBL): hostname = "zen.spamhaus.org" - def process(self, result): - result = result.rsplit(".", 1)[1] + def process(self, a_record, txt_record): + result = a_record.rsplit(".", 1)[1] if result in ["2", "3", "9"]: desc = "spam" elif result in ["4", "5", "6", "7"]: @@ -20,8 +23,8 @@ def process(self, result): class EFNetRBL(DNSBL): hostname = "rbl.efnetrbl.org" - def process(self, result): - result = result.rsplit(".", 1)[1] + def process(self, a_record, txt_record): + result = a_record.rsplit(".", 1)[1] if result == "1": desc = "proxy" elif result in ["2", "3"]: @@ -32,35 +35,13 @@ def process(self, result): desc = "flooding" return f"{result} - {desc}" -DRONEBL_CATEGORIES = { - 3: "IRC drone", - 5: "bottler", - 6: "unknown spambot or drone", - 7: "DDoS drone", - 8: "open SOCKS proxy", - 9: "open HTTP proxy", - 10: "proxychain", - 11: "web page proxy", - 12: "open DNS resolver", - 13: "brute force attacker", - 14: "open WINGATE proxy", - 15: "compromised router/gateway", - 16: "autorooting malware", - 17: "detected botnet IP", - 18: "DNS/MX on IRC", - 19: "abused VPN service" -} class DroneBL(DNSBL): hostname = "dnsbl.dronebl.org" - def process(self, result): - result = int(result.rsplit(".", 1)[1]) - desc = DRONEBL_CATEGORIES.get(result, "unknown") - return f"{result} - {desc}" class AbuseAtCBL(DNSBL): hostname = "cbl.abuseat.org" - def process(self, result): - result = result.rsplit(".", 1)[1] + def process(self, a_record, txt_record): + result = a_record.rsplit(".", 1)[1] if result == "2": desc = "abuse" else: From 68939b7ee0174364041f4288346fb677150007ef Mon Sep 17 00:00:00 2001 From: jesopo Date: Mon, 14 Sep 2020 13:52:54 +0000 Subject: [PATCH 07/89] update dnspython lib, use new .resolve --- modules/dnsbl/__init__.py | 4 ++-- modules/ip_addresses.py | 2 +- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/dnsbl/__init__.py b/modules/dnsbl/__init__.py index dd64a98d..96a21ee0 100644 --- a/modules/dnsbl/__init__.py +++ b/modules/dnsbl/__init__.py @@ -52,12 +52,12 @@ def _check_lists(self, lists, address): def _check_list(self, list, address): list_address = "%s.%s" % (address, list) try: - a_record = dns.resolver.query(list_address, "A")[0].to_text() + a_record = dns.resolver.resolve(list_address, "A")[0].to_text() except dns.resolver.NXDOMAIN: return None try: - txt_record = dns.resolver.query(list_address, "TXT")[0].to_text() + txt_record = dns.resolver.resolve(list_address, "TXT")[0].to_text() except: txt_record = None diff --git a/modules/ip_addresses.py b/modules/ip_addresses.py index 73ec061f..f2bce5cc 100644 --- a/modules/ip_addresses.py +++ b/modules/ip_addresses.py @@ -55,7 +55,7 @@ def dns(self, event): for record_type in record_types: record_type_strip = record_type.rstrip("?").upper() try: - query_result = resolver.query(hostname, record_type_strip, + query_result = resolver.resolve(hostname, record_type_strip, lifetime=4) query_results = [q.to_text() for q in query_result] results.append([record_type_strip, query_result.rrset.ttl, diff --git a/requirements.txt b/requirements.txt index 94ee9290..4bbf04c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ beautifulsoup4 ==4.8.0 cryptography ==2.7 dataclasses ==0.6;python_version<'3.7' -dnspython ==1.16.0 +dnspython ==2.0.0 feedparser ==5.2.1 html5lib ==1.0.1 isodate ==0.6.0 From 5c1942a35a224bcc23b4ba65f7e25ecf8c3fd91a Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 17 Sep 2020 14:23:11 +0000 Subject: [PATCH 08/89] handle unknown Zen Spamhaus results --- modules/dnsbl/lists.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/dnsbl/lists.py b/modules/dnsbl/lists.py index 44975467..d6dc1af4 100644 --- a/modules/dnsbl/lists.py +++ b/modules/dnsbl/lists.py @@ -19,6 +19,8 @@ def process(self, a_record, txt_record): desc = "spam" elif result in ["4", "5", "6", "7"]: desc = "exploits" + else: + desc = "unknown" return f"{result} - {desc}" class EFNetRBL(DNSBL): From f7a1c12cfa14e51d823c1f4686c67befc3a6577e Mon Sep 17 00:00:00 2001 From: jesopo Date: Fri, 18 Sep 2020 01:02:31 +0000 Subject: [PATCH 09/89] add torexit.dan.me.uk to dnsbls --- modules/dnsbl/lists.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/dnsbl/lists.py b/modules/dnsbl/lists.py index d6dc1af4..e497b063 100644 --- a/modules/dnsbl/lists.py +++ b/modules/dnsbl/lists.py @@ -50,11 +50,17 @@ def process(self, a_record, txt_record): desc = "unknown" return f"{result} - {desc}" +class TorExitDan(DNSBL): + hostname = "torexit.dan.me.uk" + def process(self, a_record, txt_record): + return "tor exit" + DEFAULT_LISTS = [ ZenSpamhaus(), EFNetRBL(), DroneBL(), - AbuseAtCBL() + AbuseAtCBL(), + TorExitDan() ] def default_lists(): From 027b9d75f827c1add3c0beac45651a7c07b523cb Mon Sep 17 00:00:00 2001 From: Alyx Wolcott Date: Wed, 23 Sep 2020 09:25:43 -0500 Subject: [PATCH 10/89] Add parameter checking so bitbot doesn't add a None webhook and break webhooks until restart --- modules/git_webhooks/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/git_webhooks/__init__.py b/modules/git_webhooks/__init__.py index 44d9b5dc..bf412860 100644 --- a/modules/git_webhooks/__init__.py +++ b/modules/git_webhooks/__init__.py @@ -228,6 +228,9 @@ def github_webhook(self, event): if existing_hook: raise utils.EventError("There's already a hook for %s" % hook_name) + if hook_name == None: + command = "%s%s" % (event["command_prefix"], event["command"]) + raise utils.EventError("Not enough arguments (Usage: %s add )" % command) all_hooks[hook_name] = { "events": DEFAULT_EVENT_CATEGORIES.copy(), From 09fc00b5da8ace1b787bf260243c32d0a1b34f4f Mon Sep 17 00:00:00 2001 From: jesopo Date: Fri, 25 Sep 2020 18:09:10 +0000 Subject: [PATCH 11/89] fix !cmute +time --- modules/channel_op.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/channel_op.py b/modules/channel_op.py index d37e27b4..410b0236 100644 --- a/modules/channel_op.py +++ b/modules/channel_op.py @@ -83,7 +83,7 @@ def unmode(self, timer): channel = server.channels.get(channel_name) args = timer.kwargs.get("args", [timer.kwargs.get("arg", None)]) - if args: + if any(args): channel.send_modes(args, False) else: channel.send_mode(timer.kwargs["mode"], False) @@ -238,7 +238,7 @@ def cmute(self, event): if event["spec"][1]: self.timers.add_persistent("unmode", event["spec"][1], - channel=event["spec"][0].id, mode="m") + channel=event["spec"][0].id, mode="-m") @utils.hook("received.command.cunmute") @utils.kwarg("require_mode", "o") @utils.kwarg("require_access", "high,cmute") From e50c4ecbe2436bb09fe4be268f78e0b897acfbd3 Mon Sep 17 00:00:00 2001 From: jesopo Date: Tue, 29 Sep 2020 15:06:37 +0000 Subject: [PATCH 12/89] add !karmawho to see who gave karma to --- modules/karma.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/modules/karma.py b/modules/karma.py index 33359123..104d7308 100644 --- a/modules/karma.py +++ b/modules/karma.py @@ -66,7 +66,8 @@ def _change_karma(self, server, sender, target, positive): self._set_throttle(sender, positive) karma_str = self._karma_str(karma) - karma_total = self._karma_str(self._get_karma(server, target)) + karma_total = sum(self._get_karma(server, target).values()) + karma_total = self._karma_str(karma_total) return True, "%s now has %s karma (%s from %s)" % ( target, karma_total, karma_str, sender.nickname) @@ -118,18 +119,30 @@ def karma(self, event): target = event["user"].nickname target = self._get_target(event["server"], target) - karma = self._karma_str(self._get_karma(event["server"], target)) + karma = sum(self._get_karma(event["server"], target).values()) + karma = self._karma_str(karma) event["stdout"].write("%s has %s karma" % (target, karma)) - def _get_karma(self, server, target): + @utils.hook("received.command.karmawho") + @utils.spec("!string") + def karmawho(self, event): + target = event["spec"][0] + karma = self._get_karma(event["server"], target, True) + karma = sorted(list(karma.items()), key=lambda k: k[1]) + + parts = ["%s (%d)" % (n, v) for n, v in karma] + event["stdout"].write("%s has karma from: %s" % + (target, ", ".join(parts))) + + def _get_karma(self, server, target, own=False): settings = dict(server.get_all_user_settings("karma-%s" % target)) target_lower = server.irc_lower(target) - if target_lower in settings: + if target_lower in settings and not own: del settings[target_lower] - return sum(settings.values()) + return settings @utils.hook("received.command.resetkarma") @utils.kwarg("min_args", 2) From 777c14b6800354cc55b347946b8f306644abcc35 Mon Sep 17 00:00:00 2001 From: jesopo Date: Tue, 29 Sep 2020 15:36:43 +0000 Subject: [PATCH 13/89] sort karma reversed and by abs() --- modules/karma.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/karma.py b/modules/karma.py index 104d7308..bf21b3b5 100644 --- a/modules/karma.py +++ b/modules/karma.py @@ -129,7 +129,9 @@ def karma(self, event): def karmawho(self, event): target = event["spec"][0] karma = self._get_karma(event["server"], target, True) - karma = sorted(list(karma.items()), key=lambda k: k[1]) + karma = sorted(list(karma.items()), + key=lambda k: abs(k[1]), + reverse=True) parts = ["%s (%d)" % (n, v) for n, v in karma] event["stdout"].write("%s has karma from: %s" % From 84aa7d1bd51f8b7ab5548911e178a36bd19516f8 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Sun, 4 Oct 2020 17:44:17 -0500 Subject: [PATCH 14/89] add ban-enforce-max config option --- modules/ban_enforce.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/ban_enforce.py b/modules/ban_enforce.py index 4e40bb84..42d60130 100644 --- a/modules/ban_enforce.py +++ b/modules/ban_enforce.py @@ -4,6 +4,8 @@ @utils.export("channelset", utils.BoolSetting("ban-enforce", "Whether or not to parse new bans and kick who they affect")) +@utils.export("channelset", utils.IntSetting("ban-enforce-max", + "Do not enforce ban if the ban effects more than this many users. Default is half of total channel users.")) class Module(ModuleManager.BaseModule): @utils.hook("received.mode.channel") def on_mode(self, event): @@ -15,13 +17,19 @@ def on_mode(self, event): bans.append(arg) if bans: + affected = 0 umasks = {u.hostmask(): u for u in event["channel"].users} + defaultmax = len(event["channel"].users) // 2 + realmax = event["channel"].get_setting("ban-enforce-max", defaultmax) for ban in bans: mask = utils.irc.hostmask_parse(ban) matches = list(utils.irc.hostmask_match_many( umasks.keys(), mask)) for match in matches: + affected = affected + 1 kicks.add(umasks[match]) if kicks: + if affected > realmax: + return nicks = [u.nickname for u in kicks] event["channel"].send_kicks(sorted(nicks), REASON) From 2d76365214408d466d125357b10d47c72208adce Mon Sep 17 00:00:00 2001 From: David Schultz Date: Sun, 4 Oct 2020 19:24:45 -0500 Subject: [PATCH 15/89] Update ban_enforce.py --- modules/ban_enforce.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/ban_enforce.py b/modules/ban_enforce.py index 42d60130..b504c9fc 100644 --- a/modules/ban_enforce.py +++ b/modules/ban_enforce.py @@ -16,11 +16,12 @@ def on_mode(self, event): if mode[0] == "+" and mode[1] == "b": bans.append(arg) + affected = 0 + defaultmax = len(event["channel"].users) // 2 + realmax = event["channel"].get_setting("ban-enforce-max", defaultmax) + if bans: - affected = 0 umasks = {u.hostmask(): u for u in event["channel"].users} - defaultmax = len(event["channel"].users) // 2 - realmax = event["channel"].get_setting("ban-enforce-max", defaultmax) for ban in bans: mask = utils.irc.hostmask_parse(ban) matches = list(utils.irc.hostmask_match_many( From 09cfae75b8b4b590407fa3c9d710ed0d8863133c Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 17 Oct 2020 14:07:02 +0000 Subject: [PATCH 16/89] github.py needs exports from git_webhooks --- modules/git_webhooks/__init__.py | 2 +- modules/git_webhooks/github.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/git_webhooks/__init__.py b/modules/git_webhooks/__init__.py index bf412860..776c0212 100644 --- a/modules/git_webhooks/__init__.py +++ b/modules/git_webhooks/__init__.py @@ -31,7 +31,7 @@ class Module(ModuleManager.BaseModule): _name = "Webhooks" def on_load(self): - self._github = github.GitHub(self.log) + self._github = github.GitHub(self.log, self.exports) self._gitea = gitea.Gitea() self._gitlab = gitlab.GitLab() diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py index 8bf61114..a208d2fc 100644 --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -88,8 +88,9 @@ CHECK_RUN_FAILURES = ["failure", "cancelled", "timed_out", "action_required"] class GitHub(object): - def __init__(self, log): + def __init__(self, log, exports): self.log = log + self.exports = exports def is_private(self, data, headers): if "repository" in data: From 2f5d001a7993d38b65e013cd3c9ebefdf3398703 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 17 Oct 2020 16:46:37 +0000 Subject: [PATCH 17/89] shorturl-any shouldn't need a server --- modules/shorturl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shorturl.py b/modules/shorturl.py index 345d2183..6b881454 100644 --- a/modules/shorturl.py +++ b/modules/shorturl.py @@ -41,7 +41,7 @@ def _call_shortener(self, server, context, shortener_name, url): @utils.export("shorturl-any") def _shorturl_any(self, url): - return self._call_shortener(server, None, "bitly", url) or url + return self._call_shortener(None, None, "bitly", url) or url @utils.export("shorturl") def _shorturl(self, server, url, context=None): From c4c6fdde1c7f584309d7c8b3394112154cac03be Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 17 Oct 2020 17:13:52 +0000 Subject: [PATCH 18/89] support check_run.status as a category+[status] --- modules/git_webhooks/github.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py index a208d2fc..ac8f3091 100644 --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -126,6 +126,8 @@ def event(self, data, headers): category_action = None if "review" in data and "state" in data["review"]: category = "%s+%s" % (event, data["review"]["state"]) + elif "check_run" in data and "status" in data["check_run"]: + category = "%s+%s" % (event, data["check_run"]["status"]) if action: if category: From a91c03421f7fb83b2c1c7a5cffd26b5d69ee197f Mon Sep 17 00:00:00 2001 From: jesopo Date: Sun, 18 Oct 2020 00:37:05 +0000 Subject: [PATCH 19/89] show PRs on check_run output where possible --- modules/git_webhooks/github.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py index ac8f3091..59e33783 100644 --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -441,6 +441,12 @@ def check_run(self, data): commit = self._short_hash(data["check_run"]["head_sha"]) commit = utils.irc.color(commit, utils.consts.LIGHTBLUE) + pr = "" + if ("pull_requests" in data["check_run"] and + data["check_run"]["pull_requests"]): + pr_num = data["check_run"]["pull_requests"][0]["number"] + pr = "/PR%s" % utils.irc.color("#%s" % pr_num, colors.COLOR_ID) + url = "" if data["check_run"]["details_url"]: url = data["check_run"]["details_url"] @@ -472,8 +478,8 @@ def check_run(self, data): status_str = utils.irc.color( CHECK_RUN_CONCLUSION[conclusion], conclusion_color) - return ["[build @%s] %s: %s%s%s" % ( - commit, name, status_str, duration, url)] + return ["[build @%s%s] %s: %s%s%s" % ( + commit, pr, name, status_str, duration, url)] def fork(self, full_name, data): forker = utils.irc.bold(data["sender"]["login"]) From aa4b5d91eea988262281bb48d48fcc8c52e9f115 Mon Sep 17 00:00:00 2001 From: Alma Attwater Date: Sun, 1 Nov 2020 20:26:44 +0000 Subject: [PATCH 20/89] Change example pronouns to neutral pronouns --- modules/pronouns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pronouns.py b/modules/pronouns.py index f800dad5..55b0a8d8 100644 --- a/modules/pronouns.py +++ b/modules/pronouns.py @@ -4,7 +4,7 @@ from src import ModuleManager, utils @utils.export("set", utils.Setting("pronouns", "Set your pronouns", - example="she/her")) + example="they/them")) class Module(ModuleManager.BaseModule): @utils.hook("received.command.pronouns") def pronouns(self, event): From fcbeaf3114cd8cada39b4e0d86ba9aa17b068ad7 Mon Sep 17 00:00:00 2001 From: Dax <23448388+fndax@users.noreply.github.com> Date: Wed, 4 Nov 2020 21:45:40 -0800 Subject: [PATCH 21/89] [Tweets] Fix tweet age calc for TZ!=UTC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dt is a naive datetime object, so its timezone is assumed to be the system timezone. However, the actual timezone from the API is UTC. Therefore, we need to set tzinfo before doing the calculation. See the note at https://docs.python.org/3/library/datetime.html#datetime.datetime.timestamp for more info. Ideally this would be fixed in tweepy, but there's a report of this on forums from 7 years ago so let's just fix it in BitBot. This bug found by an anonymous contributor. Thank you 😺! --- modules/tweets/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tweets/format.py b/modules/tweets/format.py index 9648dc51..0817370a 100644 --- a/modules/tweets/format.py +++ b/modules/tweets/format.py @@ -2,7 +2,7 @@ from src import utils def _timestamp(dt): - seconds_since = time.time()-dt.timestamp() + seconds_since = time.time()-dt.replace(tzinfo=datetime.timezone.utc).timestamp() timestamp = utils.datetime.format.to_pretty_since( seconds_since, max_units=2) return "%s ago" % timestamp From 8cc47a9321fab3ef5036b25ebd174633b5b041c9 Mon Sep 17 00:00:00 2001 From: jesopo Date: Mon, 9 Nov 2020 23:32:44 +0000 Subject: [PATCH 22/89] refuse setting location to timezones we can't understand --- modules/location.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/location.py b/modules/location.py index 08c8b1fd..33811d8e 100644 --- a/modules/location.py +++ b/modules/location.py @@ -2,6 +2,7 @@ #--require-config opencagedata-api-key import typing +import pytz from src import ModuleManager, utils URL_OPENCAGE = "https://api.opencagedata.com/geocode/v1/json" @@ -19,6 +20,11 @@ def _get_location(self, s): if page and page["results"]: result = page["results"][0] timezone = result["annotations"]["timezone"]["name"] + try: + pytz.timezone(timezone) + except pytz.exceptions.UnknownTimeZoneError: + return None + lat = result["geometry"]["lat"] lon = result["geometry"]["lng"] From 86520b31f988d4f50aeda3eb7eb23e2cafa0d6d2 Mon Sep 17 00:00:00 2001 From: Shreyas Minocha Date: Fri, 27 Nov 2020 02:32:40 +0530 Subject: [PATCH 23/89] Improve weather formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Example: `14.8km/h/9.2mi/h` → `14.8km/h (9.2mi/h)` --- modules/weather.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/weather.py b/modules/weather.py index bd6b5e8b..fcc81d15 100644 --- a/modules/weather.py +++ b/modules/weather.py @@ -81,7 +81,7 @@ def weather(self, event): location_str = "(%s) %s" % (nickname, location_str) event["stdout"].write( - "%s | %s/%s | %s | Humidity: %s | Wind: %s/%s" % ( + "%s | %s/%s | %s | Humidity: %s | Wind: %s (%s)" % ( location_str, celsius, fahrenheit, description, humidity, wind_speed_k, wind_speed_m)) else: From cb43a6ae2b9d6d2f435a3dccc05c250ccb8fe0ad Mon Sep 17 00:00:00 2001 From: David Schultz Date: Sat, 28 Nov 2020 16:45:26 -0600 Subject: [PATCH 24/89] RSS custom format (#286) * Update rss.py * add even more customization options --- modules/rss.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/modules/rss.py b/modules/rss.py index 9ba23db4..0a881d57 100644 --- a/modules/rss.py +++ b/modules/rss.py @@ -9,11 +9,11 @@ SETTING_BIND = utils.Setting("rss-bindhost", "Which local address to bind to for RSS requests", example="127.0.0.1") - @utils.export("botset", utils.IntSetting("rss-interval", "Interval (in seconds) between RSS polls", example="120")) @utils.export("channelset", utils.BoolSetting("rss-shorten", "Whether or not to shorten RSS urls")) +@utils.export("channelset", utils.Setting("rss-format", "Format of RSS announcements", example="$longtitle: $title - $link [$author]")) @utils.export("serverset", SETTING_BIND) @utils.export("channelset", SETTING_BIND) class Module(ModuleManager.BaseModule): @@ -22,12 +22,12 @@ def on_load(self): self.timers.add("rss-feeds", self._timer, self.bot.get_setting("rss-interval", RSS_INTERVAL)) - def _format_entry(self, server, feed_title, entry, shorten): + def _format_entry(self, server, channel, feed_title, entry, shorten): title = utils.parse.line_normalise(utils.http.strip_html( entry["title"])) - author = entry.get("author", None) - author = " by %s" % author if author else "" + author = entry.get("author", "unknown author") + author = "%s" % author if author else "" link = entry.get("link", None) if shorten: @@ -35,11 +35,18 @@ def _format_entry(self, server, feed_title, entry, shorten): link = self.exports.get("shorturl")(server, link) except: pass - link = " - %s" % link if link else "" + link = "%s" % link if link else "" - feed_title_str = "%s: " % feed_title if feed_title else "" + feed_title_str = "%s" % feed_title if feed_title else "" + # just in case the format starts keyerroring and you're not sure why + self.log.trace("RSS Entry: " + str(entry)) + try: + format = channel.get_setting("rss-format", "$longtitle: $title by $author - $link").replace("$longtitle", feed_title_str).replace("$title", title).replace("$link", link).replace("$author", author).format(**entry) + except KeyError: + self.log.warn(f"Failed to format RSS entry for {channel}. Falling back to default format.") + format = f"{feed_title_str}: {title} by {author} - {link}" - return "%s%s%s%s" % (feed_title_str, title, author, link) + return format def _timer(self, timer): start_time = time.monotonic() @@ -106,7 +113,7 @@ def _timer(self, timer): valid += 1 shorten = channel.get_setting("rss-shorten", False) - output = self._format_entry(server, feed_title, entry, + output = self._format_entry(server, channel, feed_title, entry, shorten) self.events.on("send.stdout").call(target=channel, @@ -203,7 +210,7 @@ def rss(self, event): raise utils.EventError("Failed to get RSS entries") shorten = event["target"].get_setting("rss-shorten", False) - out = self._format_entry(event["server"], title, entries[0], + out = self._format_entry(event["server"], event["target"], title, entries[0], shorten) event["stdout"].write(out) else: From b046c36052540e554e4f8947ed3a533466a697bc Mon Sep 17 00:00:00 2001 From: David Schultz Date: Mon, 11 Jan 2021 11:04:57 -0600 Subject: [PATCH 25/89] make karmawho work better --- modules/karma.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/karma.py b/modules/karma.py index bf21b3b5..b5df23f3 100644 --- a/modules/karma.py +++ b/modules/karma.py @@ -14,6 +14,11 @@ @utils.export("channelset", utils.BoolSetting("karma-pattern", "Enable/disable parsing ++/-- karma format")) class Module(ModuleManager.BaseModule): + def listify(self, items): + if type(items) != list: + items = list(items) + return len(items) > 2 and ', '.join(items[:-1]) + ', and ' + items[-1] or len(items) > 1 and items[0] + ' and ' + items[1] or items and items[0] or '' + def _karma_str(self, karma): karma_str = str(karma) if karma < 0: @@ -134,8 +139,12 @@ def karmawho(self, event): reverse=True) parts = ["%s (%d)" % (n, v) for n, v in karma] + print(parts) + if len(parts) == 0: + event["stdout"].write("%s has no karma." % target) + return event["stdout"].write("%s has karma from: %s" % - (target, ", ".join(parts))) + (target, self.listify(parts))) def _get_karma(self, server, target, own=False): settings = dict(server.get_all_user_settings("karma-%s" % target)) From 1fe8cb677e0cf78a7da590185a9ba7378651d29b Mon Sep 17 00:00:00 2001 From: David Schultz Date: Mon, 11 Jan 2021 11:12:46 -0600 Subject: [PATCH 26/89] make karmawho case insensitive --- modules/karma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/karma.py b/modules/karma.py index b5df23f3..ae5d0279 100644 --- a/modules/karma.py +++ b/modules/karma.py @@ -132,7 +132,7 @@ def karma(self, event): @utils.hook("received.command.karmawho") @utils.spec("!string") def karmawho(self, event): - target = event["spec"][0] + target = event["server"].irc_lower(event["spec"][0]) karma = self._get_karma(event["server"], target, True) karma = sorted(list(karma.items()), key=lambda k: abs(k[1]), From a0d6e515895a68f3929319650de5c09b7aa84c4a Mon Sep 17 00:00:00 2001 From: David Schultz Date: Mon, 11 Jan 2021 11:13:43 -0600 Subject: [PATCH 27/89] rm debug line --- modules/karma.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/karma.py b/modules/karma.py index ae5d0279..8e635d28 100644 --- a/modules/karma.py +++ b/modules/karma.py @@ -139,7 +139,6 @@ def karmawho(self, event): reverse=True) parts = ["%s (%d)" % (n, v) for n, v in karma] - print(parts) if len(parts) == 0: event["stdout"].write("%s has no karma." % target) return From 7283a266e33a610cea91c8fd66c459917e37cb10 Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 14 Jan 2021 21:50:45 +0000 Subject: [PATCH 28/89] update lxml --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b1476d60..b0bb9dd8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ dnspython ==2.0.0 feedparser ==5.2.1 html5lib ==1.0.1 isodate ==0.6.0 -lxml ==4.4.1 +lxml ==4.6.2 netifaces ==0.10.9 PySocks ==1.7.1 python-dateutil ==2.8.1 From 97693aa784b83900953fb0ed0dfe29c07b97aac8 Mon Sep 17 00:00:00 2001 From: owen Date: Sun, 17 Jan 2021 17:37:03 -0500 Subject: [PATCH 29/89] casefold nickname so sed-sender-only works with capital letters in nick (#299) --- modules/sed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sed.py b/modules/sed.py index af053ec3..df880e22 100644 --- a/modules/sed.py +++ b/modules/sed.py @@ -35,7 +35,7 @@ def channel_message(self, event): sed.replace = utils.irc.bold(sed.replace) if self._closest_setting(event, "sed-sender-only", False): - for_user = event["user"].nickname + for_user = event["user"].nickname_lower match_line = None match_message = None From 3fa2034c25cb45b9f1d302ab71d739cae3daa1aa Mon Sep 17 00:00:00 2001 From: David Schultz Date: Fri, 5 Feb 2021 18:32:00 -0600 Subject: [PATCH 30/89] add 'delserver' command (#297) --- src/core_modules/admin.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/core_modules/admin.py b/src/core_modules/admin.py index 5008952d..dca8938c 100644 --- a/src/core_modules/admin.py +++ b/src/core_modules/admin.py @@ -151,6 +151,27 @@ def add_server(self, event): return event["stdout"].write("Added server '%s'" % alias) + @utils.hook("received.command.delserver") + @utils.kwarg("help", "Delete a server") + @utils.kwarg("pemission", "delserver") + @utils.spec("!word") + def del_server(self, event): + alias = event["spec"][0] + sid = self.bot.database.servers.by_alias(alias) + if sid == None: + event["stderr"].write("Server '%s' does not exist" % alias) + return + if self._server_from_alias(alias): + event["stderr"].write("You must disconnect from %s before deleting it" % alias) + return + try: + self.bot.database.servers.delete(sid) + except Exception as e: + event["stderr"].write("Failed to delete server") + self.log.error("failed to add server \"%s\"", [alias], exc_info=True) + return + event["stderr"].write("Server '%s' has been deleted" % alias) + @utils.hook("received.command.editserver") @utils.kwarg("help", "Edit server details") @utils.kwarg("permission", "editserver") From 5d2a3865a9e7e70937766fc05cbe131b09ee84b6 Mon Sep 17 00:00:00 2001 From: Dax <23448388+fndax@users.noreply.github.com> Date: Fri, 5 Feb 2021 16:34:22 -0800 Subject: [PATCH 31/89] Clarify which bitbotd -a options are optional (#263) Also disambiguate alias vs. hostname a little, just in case. --- src/utils/cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/utils/cli.py b/src/utils/cli.py index 2c2b3e3b..40c1bf0b 100644 --- a/src/utils/cli.py +++ b/src/utils/cli.py @@ -5,15 +5,15 @@ def bool_input(s: str): return not result or result[0].lower() in ["", "y"] def add_server(): - alias = input("alias: ") - hostname = input("hostname: ") + alias = input("alias (display name): ") + hostname = input("hostname (address of server): ") port = int(input("port: ")) tls = bool_input("tls?") - password = input("password?: ") + password = input("password (optional, leave blank to skip): ") nickname = input("nickname: ") - username = input("username: ") - realname = input("realname: ") - bindhost = input("bindhost?: ") + username = input("username (optional): ") + realname = input("realname (optional): ") + bindhost = input("bindhost (optional): ") return irc.IRCConnectionParameters(-1, alias, hostname, port, password, tls, bindhost, nickname, username, realname) From 37523c7a09d248233825e712073c7e3e49ef3596 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Fri, 5 Feb 2021 18:38:27 -0600 Subject: [PATCH 32/89] make that easier on the eyes --- modules/karma.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/karma.py b/modules/karma.py index 8e635d28..e21d9130 100644 --- a/modules/karma.py +++ b/modules/karma.py @@ -17,7 +17,14 @@ class Module(ModuleManager.BaseModule): def listify(self, items): if type(items) != list: items = list(items) - return len(items) > 2 and ', '.join(items[:-1]) + ', and ' + items[-1] or len(items) > 1 and items[0] + ' and ' + items[1] or items and items[0] or '' + listified = "" + if len(items) > 2: + listified = ', '.join(items[:-1]) + ', and ' + items[-1] + elif len(items) > 1: + listified = items[0] + ' and ' + items[1] + elif items: + listified = items[0] + return listified def _karma_str(self, karma): karma_str = str(karma) From a5f29ce4c07c9e7c9c264119b5f16872a7b08c21 Mon Sep 17 00:00:00 2001 From: PeGaSuS Date: Fri, 12 Feb 2021 17:31:13 +0100 Subject: [PATCH 33/89] Update config.md Add an example of how to create a self-signed cert and key to use with bitbot --- docs/help/config.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/help/config.md b/docs/help/config.md index 4db2b78a..845c6f8c 100644 --- a/docs/help/config.md +++ b/docs/help/config.md @@ -14,6 +14,9 @@ Generate a TLS keypair and point `bot.conf`'s `tls-key` to the private key and `tls-certificate` to the public key. +Below is an OpenSSL command example that will create a `bitbot-cert.pem` and `bitbot-key.pem` with `10y` validity (self-signed): +> openssl req -x509 -nodes -sha512 -newkey rsa:4096 -keyout bitbot-key.pem -out bitbot-cert.pem -days 3650 -subj "/CN=YourBotNick" + ### Configure SASL Configure the bot to use SASL to authenticate (usually used for `NickServ` identification) From 47965f5fad133fac7cd9a5142403938e18bf719b Mon Sep 17 00:00:00 2001 From: jesopo Date: Fri, 12 Feb 2021 16:58:08 +0000 Subject: [PATCH 34/89] support {DATA} in tls-certificate and tls-key --- src/IRCServer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/IRCServer.py b/src/IRCServer.py index bb22adab..0399c6da 100644 --- a/src/IRCServer.py +++ b/src/IRCServer.py @@ -95,8 +95,12 @@ def connect(self): self.connection_params.bindhost, self.connection_params.tls, tls_verify=self.get_setting("ssl-verify", True), - cert=self.bot.config.get("tls-certificate", None), - key=self.bot.config.get("tls-key", None)) + cert=self.bot.config.get("tls-certificate", '').format( + DATA=self.bot.data_directory + ) or None, + key=self.bot.config.get("tls-key", '').format( + DATA=self.bot.data_directory + )) self.events.on("preprocess.connect").call(server=self) self.socket.connect() From c0810f80f5bc1547f8db60e435e23e4e13211340 Mon Sep 17 00:00:00 2001 From: jesopo Date: Fri, 12 Feb 2021 16:58:55 +0000 Subject: [PATCH 35/89] update py cryptography lib to 3.3.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b0bb9dd8..bbb1a9a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ beautifulsoup4 ==4.8.0 -cryptography ==3.2 +cryptography >=3.3.2 dataclasses ==0.6;python_version<'3.7' dnspython ==2.0.0 feedparser ==5.2.1 From 422d30978720e7a858be9e896df7910762d78f6b Mon Sep 17 00:00:00 2001 From: PeGaSuS Date: Fri, 12 Feb 2021 18:21:09 +0100 Subject: [PATCH 36/89] Update bitbot_user.service Always update the systemd service daemon --- docs/systemd/bitbot_user.service | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/systemd/bitbot_user.service b/docs/systemd/bitbot_user.service index eb397307..a3984f54 100644 --- a/docs/systemd/bitbot_user.service +++ b/docs/systemd/bitbot_user.service @@ -10,6 +10,8 @@ # which can be disabled with: systemctl --user disable systemd-tmpfiles-clean.timer # # After placing this script in the correct location, and with bitbot stopped, type: +# systemcl --user daemon-reload +# Afert that start bitbot with: # systemctl --user enable bitbot_user.service --now # This will enable the systemd script and launch bitbot From 8c8d362884ed54eec1583cea802445afe93c8058 Mon Sep 17 00:00:00 2001 From: PeGaSuS Date: Fri, 12 Feb 2021 18:28:16 +0100 Subject: [PATCH 37/89] Update rest_api.md Missing one ` :P --- docs/help/rest_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/help/rest_api.md b/docs/help/rest_api.md index feef26da..c4a2652a 100644 --- a/docs/help/rest_api.md +++ b/docs/help/rest_api.md @@ -15,7 +15,7 @@ Either set up a reverse proxy (with persisted Host header) with your favourite H #### Apache2 * Run `$ a2enmod ssl proxy proxy_http` as root * Copy example config file from [/docs/rest_api/apache2](/docs/rest_api/apache2) to `/etc/apache2/sites-enabled/` -* Edit `ServerName`, `SSLCertificateFile and `SSLCertificateKeyFile` +* Edit `ServerName`, `SSLCertificateFile` and `SSLCertificateKeyFile` * `$ service apache2 restart` as root #### Lighttpd From bfb34a4eb9058bb788d089c698f4122409ed14f7 Mon Sep 17 00:00:00 2001 From: jesopo Date: Mon, 15 Feb 2021 21:44:32 +0000 Subject: [PATCH 38/89] switch from check_run to check_suite for github webhooks --- modules/git_webhooks/github.py | 84 +++++++++++++--------------------- 1 file changed, 33 insertions(+), 51 deletions(-) diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py index 59e33783..f02f10eb 100644 --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -5,6 +5,7 @@ COMMIT_RANGE_URL = "https://github.com/%s/compare/%s...%s" CREATE_URL = "https://github.com/%s/tree/%s" +PR_URL = "https://github.com/%s/pull/%s" PR_COMMIT_RANGE_URL = "https://github.com/%s/pull/%s/files/%s..%s" PR_COMMIT_URL = "https://github.com/%s/pull/%s/commits/%s" @@ -77,15 +78,14 @@ } COMMENT_MAX = 100 -CHECK_RUN_CONCLUSION = { - "success": "passed", - "failure": "failed", - "neutral": "finished", - "cancelled": "was cancelled", - "timed_out": "timed out", - "action_required": "requires action" +CHECK_SUITE_CONCLUSION = { + "success": ("passed", colors.COLOR_POSITIVE), + "failure": ("failed", colors.COLOR_NEGATIVE), + "neutral": ("finished", colors.COLOR_NEUTRAL), + "cancelled": ("was cancelled", colors.COLOR_NEGATIVE), + "timed_out": ("timed out", colors.COLOR_NEGATIVE), + "action_required": ("requires action", colors.COLOR_NEUTRAL) } -CHECK_RUN_FAILURES = ["failure", "cancelled", "timed_out", "action_required"] class GitHub(object): def __init__(self, log, exports): @@ -126,8 +126,8 @@ def event(self, data, headers): category_action = None if "review" in data and "state" in data["review"]: category = "%s+%s" % (event, data["review"]["state"]) - elif "check_run" in data and "status" in data["check_run"]: - category = "%s+%s" % (event, data["check_run"]["status"]) + elif "check_suite" in data and "conclusion" in data["check_suite"]: + category = "%s+%s" % (event, data["check_suite"]["conclusion"]) if action: if category: @@ -162,8 +162,8 @@ def webhook(self, full_name, event, data, headers): out = self.delete(full_name, data) elif event == "release": out = self.release(full_name, data) - elif event == "check_run": - out = self.check_run(data) + elif event == "check_suite": + out = self.check_suite(full_name, data) elif event == "fork": out = self.fork(full_name, data) elif event == "ping": @@ -436,50 +436,32 @@ def release(self, full_name, data): url = self._short_url(data["release"]["html_url"]) return ["%s %s a release%s - %s" % (author, action, name, url)] - def check_run(self, data): - name = data["check_run"]["name"] - commit = self._short_hash(data["check_run"]["head_sha"]) + def check_suite(self, full_name, data): + suite = data["check_suite"] + + commit = self._short_hash(suite["head_sha"]) commit = utils.irc.color(commit, utils.consts.LIGHTBLUE) pr = "" - if ("pull_requests" in data["check_run"] and - data["check_run"]["pull_requests"]): - pr_num = data["check_run"]["pull_requests"][0]["number"] + url = "" + if suite["pull_requests"]: + pr_num = suite["pull_requests"][0]["number"] pr = "/PR%s" % utils.irc.color("#%s" % pr_num, colors.COLOR_ID) + url = self._short_url(PR_URL % (full_name, pr_num)) + url = " - %s" % url - url = "" - if data["check_run"]["details_url"]: - url = data["check_run"]["details_url"] - url = " - %s" % self.exports.get("shorturl-any")(url) - - duration = "" - if data["check_run"]["completed_at"]: - started_at = self._iso8601(data["check_run"]["started_at"]) - completed_at = self._iso8601(data["check_run"]["completed_at"]) - if completed_at > started_at: - seconds = (completed_at-started_at).total_seconds() - duration = " in %s" % utils.datetime.format.to_pretty_time( - seconds) - - status = data["check_run"]["status"] - status_str = "" - if status == "queued": - status_str = utils.irc.bold("queued") - elif status == "in_progress": - status_str = utils.irc.bold("started") - elif status == "completed": - conclusion = data["check_run"]["conclusion"] - conclusion_color = colors.COLOR_POSITIVE - if conclusion in CHECK_RUN_FAILURES: - conclusion_color = colors.COLOR_NEGATIVE - if conclusion == "neutral": - conclusion_color = colors.COLOR_NEUTRAL - - status_str = utils.irc.color( - CHECK_RUN_CONCLUSION[conclusion], conclusion_color) - - return ["[build @%s%s] %s: %s%s%s" % ( - commit, pr, name, status_str, duration, url)] + name = suite["app"]["name"] + conclusion = suite["conclusion"] + conclusion, conclusion_color = CHECK_SUITE_CONCLUSION[conclusion] + conclusion = utils.irc.color(conclusion, conclusion_color) + + created_at = self._iso8601(suite["created_at"]) + updated_at = self._iso8601(suite["updated_at"]) + seconds = (updated_at-created_at).total_seconds() + duration = utils.datetime.format.to_pretty_time(seconds) + + return ["[build @%s%s] %s: %s in %s%s" % ( + commit, pr, name, conclusion, duration, url)] def fork(self, full_name, data): forker = utils.irc.bold(data["sender"]["login"]) From e645a32f93a73c732b9046b8126cd89c536a0d1f Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 20 Feb 2021 18:41:40 +0000 Subject: [PATCH 39/89] handle /[/,`#]/ as sed delimeters --- modules/sed.py | 2 +- src/utils/parse/sed.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/sed.py b/modules/sed.py index df880e22..79f2fcb9 100644 --- a/modules/sed.py +++ b/modules/sed.py @@ -4,7 +4,7 @@ import re, traceback from src import ModuleManager, utils -REGEX_SED = re.compile("^(?:(\\S+)[:,] )?s/") +REGEX_SED = re.compile(r"^(?:(\S+)[:,] )?s([/,`#]).*\2") @utils.export("channelset", utils.BoolSetting("sed","Disable/Enable sed in a channel")) diff --git a/src/utils/parse/sed.py b/src/utils/parse/sed.py index 8a1c895d..087b9a41 100644 --- a/src/utils/parse/sed.py +++ b/src/utils/parse/sed.py @@ -44,7 +44,7 @@ def match(self, s): return None def _sed_split(s: str) -> typing.List[str]: - tokens = _tokens(s, "/") + tokens = _tokens(s, s[1]) if tokens and (not tokens[-1] == (len(s)-1)): tokens.append(len(s)) From e2fbbc24067173bc43ccc6b1ca9cbf4ff0fe2360 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sun, 7 Mar 2021 15:36:17 +0000 Subject: [PATCH 40/89] remove scrypt requirement, use hashlib.scrypt instead --- requirements.txt | 1 - src/utils/security.py | 11 +++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index bbb1a9a4..35e1b8b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,6 @@ PySocks ==1.7.1 python-dateutil ==2.8.1 pytz ==2019.2 requests ==2.22.0 -scrypt ==0.8.13 suds-jurko ==0.6 tornado ==6.0.3 tweepy ==3.8.0 diff --git a/src/utils/security.py b/src/utils/security.py index 085826d5..70a28ff8 100644 --- a/src/utils/security.py +++ b/src/utils/security.py @@ -25,13 +25,20 @@ def ssl_wrap(sock: socket.socket, cert: str=None, key: str=None, def constant_time_compare(a: typing.AnyStr, b: typing.AnyStr) -> bool: return hmac.compare_digest(a, b) -import scrypt +import hashlib def password(byte_n: int=32) -> str: return binascii.hexlify(os.urandom(byte_n)).decode("utf8") def salt(byte_n: int=64) -> str: return base64.b64encode(os.urandom(byte_n)).decode("utf8") def hash(given_salt: str, data: str): - return base64.b64encode(scrypt.hash(data, given_salt)).decode("utf8") + hash = hashlib.scrypt( + data.encode("utf8"), + salt=given_salt.encode("utf8"), + n=1<<14, + r=8, + p=1 + ) + return base64.b64encode(hash).decode("ascii") def hash_verify(salt: str, data: str, compare: str): given_hash = hash(salt, data) return constant_time_compare(given_hash, compare) From fabb6d85af1fbbcb6c4190f91ba6e96dfc6da477 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Tue, 9 Mar 2021 17:24:55 -0600 Subject: [PATCH 41/89] make no entries message more specific --- modules/rss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/rss.py b/modules/rss.py index 0a881d57..4a23addc 100644 --- a/modules/rss.py +++ b/modules/rss.py @@ -207,7 +207,7 @@ def rss(self, event): title, entries = self._get_entries(url) if not entries: - raise utils.EventError("Failed to get RSS entries") + raise utils.EventError("%s has no entries" % url) shorten = event["target"].get_setting("rss-shorten", False) out = self._format_entry(event["server"], event["target"], title, entries[0], From e5e94501ebd55dab801d661abdbc5c6bcbd90cb5 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Sun, 9 May 2021 17:23:50 -0500 Subject: [PATCH 42/89] move highlight prevention before urls --- modules/git_webhooks/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/git_webhooks/__init__.py b/modules/git_webhooks/__init__.py index 776c0212..8b44e198 100644 --- a/modules/git_webhooks/__init__.py +++ b/modules/git_webhooks/__init__.py @@ -135,6 +135,10 @@ def _webhook(self, webhook_type, webhook_name, handler, payload_str, for output, url in outputs: output = "(%s) %s" % ( utils.irc.color(source, colors.COLOR_REPO), output) + + if channel.get_setting("git-prevent-highlight", False): + output = self._prevent_highlight(server, channel, + output) if url: if channel.get_setting("git-shorten-urls", False): @@ -142,10 +146,6 @@ def _webhook(self, webhook_type, webhook_name, handler, payload_str, context=channel) or url output = "%s - %s" % (output, url) - if channel.get_setting("git-prevent-highlight", False): - output = self._prevent_highlight(server, channel, - output) - hide_prefix = channel.get_setting("git-hide-prefix", False) self.events.on("send.stdout").call(target=channel, module_name=webhook_name, server=server, message=output, From 2951bfc18f5c74dea424f1fc2d815f97b776e13e Mon Sep 17 00:00:00 2001 From: jesopo Date: Sun, 16 May 2021 16:04:02 +0000 Subject: [PATCH 43/89] add !dig as alias of !dns --- modules/ip_addresses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ip_addresses.py b/modules/ip_addresses.py index f2bce5cc..ca32bf6b 100644 --- a/modules/ip_addresses.py +++ b/modules/ip_addresses.py @@ -21,6 +21,7 @@ def _parse(value): @utils.export("channelset", utils.FunctionSetting(_parse, "dns-nameserver", "Set DNS nameserver", example="8.8.8.8")) class Module(ModuleManager.BaseModule): + @utils.hook("received.command.dig", alias_of="dns") @utils.hook("received.command.dns", min_args=1) def dns(self, event): """ From 8f6799b78167b0879b4faaf365ab115dc97f1cf0 Mon Sep 17 00:00:00 2001 From: jesopo Date: Tue, 25 May 2021 17:43:17 +0000 Subject: [PATCH 44/89] freenode is dead long live libera.chat --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 09a119e5..56979110 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ If you wish to create backups of your BitBot instance (which you should, [borgba I run BitBot as-a-service on most popular networks (willing to add more networks!) and offer github/gitea/gitlab webhook to IRC notifications for free to FOSS projects. Contact me for more information! ## Contact/Support -Come say hi at [#bitbot on freenode](https://webchat.freenode.net/?channels=#bitbot) +Come say hi at `#bitbot` on irc.libera.chat ## License This project is licensed under GNU General Public License v2.0 - see [LICENSE](LICENSE) for details. From df331bbd927fb6da68a04318f5a9c164e30fd19e Mon Sep 17 00:00:00 2001 From: David Schultz Date: Tue, 25 May 2021 16:57:13 -0500 Subject: [PATCH 45/89] weather.py: add kelvin unit --- modules/weather.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/weather.py b/modules/weather.py index fcc81d15..3af09be3 100644 --- a/modules/weather.py +++ b/modules/weather.py @@ -69,6 +69,7 @@ def weather(self, event): celsius = "%dC" % page["main"]["temp"] fahrenheit = "%dF" % ((page["main"]["temp"]*(9/5))+32) + kelvin = "%dK" % ((page["main"]["temp"])+273.15) description = page["weather"][0]["description"].title() humidity = "%s%%" % page["main"]["humidity"] @@ -81,10 +82,11 @@ def weather(self, event): location_str = "(%s) %s" % (nickname, location_str) event["stdout"].write( - "%s | %s/%s | %s | Humidity: %s | Wind: %s (%s)" % ( - location_str, celsius, fahrenheit, description, humidity, - wind_speed_k, wind_speed_m)) + "%s | %s/%s/%s | %s | Humidity: %s | Wind: %s/%s" % ( + location_str, celsius, fahrenheit, kelvin, description, + humidity, wind_speed_k, wind_speed_m)) else: event["stderr"].write("No weather information for this location") else: raise utils.EventResultsError() + From 81dd0d2bd572975688c5adf09a345170278d45b3 Mon Sep 17 00:00:00 2001 From: Aaron Jones Date: Sun, 30 May 2021 15:55:44 +0000 Subject: [PATCH 46/89] GitHub PRs: Correctly attribute PR authors Untested; just judging by the JSON contents of a successfully develivered webhook payload that triggered a misattribution. --- modules/git_webhooks/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py index f02f10eb..6532fa96 100644 --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -275,7 +275,7 @@ def pull_request(self, full_name, data): colored_branch = utils.irc.color(branch, colors.COLOR_BRANCH) sender = utils.irc.bold(data["sender"]["login"]) - author = utils.irc.bold(data["sender"]["login"]) + author = utils.irc.bold(data["pull_request"]["user"]["login"]) number = utils.irc.color("#%s" % data["pull_request"]["number"], colors.COLOR_ID) identifier = "%s by %s" % (number, author) From 6637a79c89807e52b39cbb8817b423ea09353c99 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 May 2021 16:29:22 +0100 Subject: [PATCH 47/89] Bump lxml from 4.6.2 to 4.6.3 (#309) * update lxml * update py cryptography lib to 3.3.2 * Bump lxml from 4.6.2 to 4.6.3 Bumps [lxml](https://github.com/lxml/lxml) from 4.6.2 to 4.6.3. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.2...lxml-4.6.3) Signed-off-by: dependabot[bot] Co-authored-by: jesopo Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 35e1b8b6..f35760b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ dnspython ==2.0.0 feedparser ==5.2.1 html5lib ==1.0.1 isodate ==0.6.0 -lxml ==4.6.2 +lxml ==4.6.3 netifaces ==0.10.9 PySocks ==1.7.1 python-dateutil ==2.8.1 From 219f126230a7a350eaa7400e2af3d5ff322289d4 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Fri, 4 Jun 2021 11:38:49 -0500 Subject: [PATCH 48/89] bump feedparser to 6.0.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f35760b8..4d3883e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ beautifulsoup4 ==4.8.0 cryptography >=3.3.2 dataclasses ==0.6;python_version<'3.7' dnspython ==2.0.0 -feedparser ==5.2.1 +feedparser ==6.0.2 html5lib ==1.0.1 isodate ==0.6.0 lxml ==4.6.3 From 07fcbd6c9e61caaeffb073b759b33b769b43621a Mon Sep 17 00:00:00 2001 From: David Schultz Date: Fri, 11 Jun 2021 23:52:13 -0500 Subject: [PATCH 49/89] fully support draft/bot spec --- modules/ircv3_botignore.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/modules/ircv3_botignore.py b/modules/ircv3_botignore.py index 968e6fb1..42ece9c9 100644 --- a/modules/ircv3_botignore.py +++ b/modules/ircv3_botignore.py @@ -1,11 +1,22 @@ from src import EventManager, ModuleManager, utils -TAG = utils.irc.MessageTag(None, "inspircd.org/bot") +TAGS = { + utils.irc.MessageTag(None, "inspircd.org/bot"), + utils.irc.MessageTag(None, "draft/bot") +} class Module(ModuleManager.BaseModule): + @utils.hook("received.376") + @utils.hook("received.422") + def botmode(self, event): + if "BOT" in event["server"].isupport: + botmode = event["server"].isupport["BOT"] + event["server"].send_raw("MODE %s +%s" % (event["server"].nickname, botmode)) + @utils.hook("received.message.private") @utils.hook("received.message.channel") @utils.kwarg("priority", EventManager.PRIORITY_HIGH) def message(self, event): - if TAG.present(event["tags"]): - event.eat() + for tag in TAGS: + if tag.present(event["tags"]): + event.eat() From 3ef21e0477ac3295d4488fdb20c58679eae20ff9 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Sat, 12 Jun 2021 00:34:00 -0500 Subject: [PATCH 50/89] support `draft/react` spec --- modules/ircv3_react.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 modules/ircv3_react.py diff --git a/modules/ircv3_react.py b/modules/ircv3_react.py new file mode 100644 index 00000000..8f4cd046 --- /dev/null +++ b/modules/ircv3_react.py @@ -0,0 +1,26 @@ +from src import IRCLine, ModuleManager, utils + +TAG = utils.irc.MessageTag("msgid", "draft/msgid") +CAP = utils.irc.Capability("message-tags", "draft/message-tags-0.2") + +class Module(ModuleManager.BaseModule): + def _tagmsg(self, target, msgid, reaction): + return IRCLine.ParsedLine("TAGMSG", [target], + tags={ + "+draft/reply": msgid, + "+draft/react": reaction + }) + def _has_tags(self, server): + return server.has_capability(CAP) + + def _expect_output(self, event): + kwarg = event["hook"].get_kwarg("expect_output", None) + return kwarg if not kwarg is None else event["expect_output"] + + @utils.hook("preprocess.command") + def preprocess(self, event): + if self._has_tags(event["server"]) and self._expect_output(event): + msgid = TAG.get_value(event["line"].tags) + if msgid: + event["server"].send(self._tagmsg(event["target_str"], msgid, "👍"), + immediate=True) From b7e1cc96f19e9e951f78be02248e1cb0f0bc1f34 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Thu, 24 Jun 2021 20:57:14 -0500 Subject: [PATCH 51/89] quotes.py: allow opting out of quotes --- modules/quotes.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/modules/quotes.py b/modules/quotes.py index e323cc9b..727f6fd6 100644 --- a/modules/quotes.py +++ b/modules/quotes.py @@ -5,6 +5,9 @@ @utils.export("channelset", utils.BoolSetting("channel-quotes", "Whether or not quotes added from this channel are kept in this channel")) +@utils.export("set", utils.BoolSetting("quotable", + "Whether or not you wish to be quoted")) + class Module(ModuleManager.BaseModule): def category_and_quote(self, s): category, sep, quote = s.partition("=") @@ -31,6 +34,11 @@ def quote_add(self, event): "channel-quotes", False): target = event["target"] + if not event["server"].get_user(category).get_setting( + "quotable", True): + event["stderr"].write("%s does not wish to be quoted" % category) + return + quotes = self._get_quotes(target, category) quotes.append([event["user"].name, int(time.time()), quote]) self._set_quotes(target, category, quotes) @@ -148,6 +156,10 @@ def quote_grab(self, event): text = " ".join(lines_str) quote_category = line.sender + if not event["server"].get_user(quote_category).get_setting( + "quotable", True): + event["stderr"].write("%s does not wish to be quoted" % quote_category) + return if event["server"].has_user(quote_category): quote_category = event["server"].get_user_nickname( event["server"].get_user(quote_category).get_id()) From 4311d86a7112cde11c3830ea0a0550d81df2feba Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 10 Jul 2021 16:57:59 +0000 Subject: [PATCH 52/89] handle lastfm tracks only having 1 tag --- modules/lastfm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/lastfm.py b/modules/lastfm.py index 6931e0c0..a11c0d93 100644 --- a/modules/lastfm.py +++ b/modules/lastfm.py @@ -82,7 +82,10 @@ def np(self, event): tags_str = "" if "toptags" in track and track["toptags"]["tag"]: - tags = [t["name"] for t in track["toptags"]["tag"]] + tags_list = track["toptags"]["tag"] + if not type(tags_list) == list: + tags_list = [tags_list] + tags = [t["name"] for t in tags_list] tags_str = " [%s]" % ", ".join(tags) play_count_str = "" From f3c8d86b372182044d841a56eda4dd4851029d68 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Thu, 22 Jul 2021 13:07:09 -0500 Subject: [PATCH 53/89] ignore.py: fix permissions --- src/core_modules/ignore.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core_modules/ignore.py b/src/core_modules/ignore.py index 4592ef6e..c7c2dff2 100644 --- a/src/core_modules/ignore.py +++ b/src/core_modules/ignore.py @@ -43,7 +43,7 @@ def preprocess_command(self, event): return utils.consts.PERMISSION_HARD_FAIL, None @utils.hook("received.command.ignore", min_args=1) - @utils.kwarg("permission", "ignore") + @utils.kwarg("permissions", "ignore") @utils.kwarg("help", "Ignore commands from a given user") @utils.spec("?duration !ouser ?wordlower") def ignore(self, event): @@ -73,7 +73,7 @@ def _timer_unignore(self, event): @utils.hook("received.command.unignore") @utils.kwarg("help", "Unignore commands from a given user") - @utils.kwarg("permission", "unignore") + @utils.kwarg("permissions", "unignore") @utils.spec("!ouser ?wordlower") def unignore(self, event): setting = "ignore" @@ -99,7 +99,7 @@ def unignore(self, event): @utils.kwarg("channel_only", True) @utils.kwarg("min_args", 1) @utils.kwarg("usage", "") - @utils.kwarg("permission", "cignore") + @utils.kwarg("permissions", "cignore") @utils.kwarg("require_mode", "o") @utils.kwarg("require_access", "high,cignore") def cignore(self, event): @@ -154,4 +154,3 @@ def server_unignore(self, event): event["server"].del_setting(setting) event["stdout"].write("No longer ignoring '%s' for %s" % (command, str(event["server"]))) - From b71afea8c4722708b2329c7cb7f38a034d7f454b Mon Sep 17 00:00:00 2001 From: David Schultz Date: Thu, 22 Jul 2021 13:15:12 -0500 Subject: [PATCH 54/89] ignore.py: should actually be `permission` --- src/core_modules/ignore.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core_modules/ignore.py b/src/core_modules/ignore.py index c7c2dff2..dd51a655 100644 --- a/src/core_modules/ignore.py +++ b/src/core_modules/ignore.py @@ -43,7 +43,7 @@ def preprocess_command(self, event): return utils.consts.PERMISSION_HARD_FAIL, None @utils.hook("received.command.ignore", min_args=1) - @utils.kwarg("permissions", "ignore") + @utils.kwarg("permission", "ignore") @utils.kwarg("help", "Ignore commands from a given user") @utils.spec("?duration !ouser ?wordlower") def ignore(self, event): @@ -73,7 +73,7 @@ def _timer_unignore(self, event): @utils.hook("received.command.unignore") @utils.kwarg("help", "Unignore commands from a given user") - @utils.kwarg("permissions", "unignore") + @utils.kwarg("permission", "unignore") @utils.spec("!ouser ?wordlower") def unignore(self, event): setting = "ignore" @@ -99,7 +99,7 @@ def unignore(self, event): @utils.kwarg("channel_only", True) @utils.kwarg("min_args", 1) @utils.kwarg("usage", "") - @utils.kwarg("permissions", "cignore") + @utils.kwarg("permission", "cignore") @utils.kwarg("require_mode", "o") @utils.kwarg("require_access", "high,cignore") def cignore(self, event): @@ -125,7 +125,7 @@ def cignore(self, event): @utils.hook("received.command.serverignore") @utils.kwarg("help", "Ignore a command on the current server") - @utils.kwarg("permissions", "serverignore") + @utils.kwarg("permission", "serverignore") @utils.spec("!wordlower") def server_ignore(self, event): command = event["spec"][0] @@ -141,7 +141,7 @@ def server_ignore(self, event): @utils.hook("received.command.serverunignore") @utils.kwarg("help", "Unignore a command on the current server") - @utils.kwarg("permissions", "serverunignore") + @utils.kwarg("permission", "serverunignore") @utils.spec("!wordlower") def server_unignore(self, event): command = event["spec"][0] From 027e27d121d55b440ec73f3ac8dea500715cfcaa Mon Sep 17 00:00:00 2001 From: Shell Turner Date: Wed, 6 Oct 2021 18:33:45 +0200 Subject: [PATCH 55/89] Delete ircv3_react.py (#330) This is an incredibly annoying module that has bothered everyone with a client which displays IRCv3 reactions. --- modules/ircv3_react.py | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 modules/ircv3_react.py diff --git a/modules/ircv3_react.py b/modules/ircv3_react.py deleted file mode 100644 index 8f4cd046..00000000 --- a/modules/ircv3_react.py +++ /dev/null @@ -1,26 +0,0 @@ -from src import IRCLine, ModuleManager, utils - -TAG = utils.irc.MessageTag("msgid", "draft/msgid") -CAP = utils.irc.Capability("message-tags", "draft/message-tags-0.2") - -class Module(ModuleManager.BaseModule): - def _tagmsg(self, target, msgid, reaction): - return IRCLine.ParsedLine("TAGMSG", [target], - tags={ - "+draft/reply": msgid, - "+draft/react": reaction - }) - def _has_tags(self, server): - return server.has_capability(CAP) - - def _expect_output(self, event): - kwarg = event["hook"].get_kwarg("expect_output", None) - return kwarg if not kwarg is None else event["expect_output"] - - @utils.hook("preprocess.command") - def preprocess(self, event): - if self._has_tags(event["server"]) and self._expect_output(event): - msgid = TAG.get_value(event["line"].tags) - if msgid: - event["server"].send(self._tagmsg(event["target_str"], msgid, "👍"), - immediate=True) From e68af22d6093ecd81030c7603ff55a95316b7134 Mon Sep 17 00:00:00 2001 From: JeDaYoshi Date: Sun, 17 Oct 2021 21:27:46 -0400 Subject: [PATCH 56/89] Add AS and hostname to geoip --- modules/ip_addresses.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/ip_addresses.py b/modules/ip_addresses.py index ca32bf6b..be1971aa 100644 --- a/modules/ip_addresses.py +++ b/modules/ip_addresses.py @@ -89,17 +89,24 @@ def geoip(self, event): page = utils.http.request(URL_GEOIP % event["args_split"][0]).json() if page: if page["status"] == "success": + try: + hostname, alias, ips = socket.gethostbyaddr(page["query"]) + except (socket.herror, socket.gaierror): + pass + data = page["query"] + if hostname: + data += " (%s)" % hostname data += " | Organisation: %s" % page["org"] data += " | City: %s" % page["city"] data += " | Region: %s (%s)" % ( page["regionName"], page["countryCode"]) - data += " | ISP: %s" % page["isp"] + data += " | ISP: %s (%s)" % (page["isp"], page["as"]) data += " | Lon/Lat: %s/%s" % (page["lon"], page["lat"]) data += " | Timezone: %s" % page["timezone"] event["stdout"].write(data) else: - event["stderr"].write("No geoip data found") + event["stderr"].write("No GeoIP data found") else: raise utils.EventResultsError() From 2e5836ae2218dd340837eb4074adfbaff9c3cc7e Mon Sep 17 00:00:00 2001 From: JeDaYoshi Date: Sun, 17 Oct 2021 21:42:15 -0400 Subject: [PATCH 57/89] Get IP from buffer on geoip too --- modules/ip_addresses.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/modules/ip_addresses.py b/modules/ip_addresses.py index be1971aa..24c87cd4 100644 --- a/modules/ip_addresses.py +++ b/modules/ip_addresses.py @@ -21,6 +21,17 @@ def _parse(value): @utils.export("channelset", utils.FunctionSetting(_parse, "dns-nameserver", "Set DNS nameserver", example="8.8.8.8")) class Module(ModuleManager.BaseModule): + def _get_ip(self, event): + ip = event["args_split"][0] if event["args"] else "" + if not ip: + line = event["target"].buffer.find(REGEX_IP) + if line: + ip = line.match + if not ip: + raise utils.EventError("No IP provided") + + return ip + @utils.hook("received.command.dig", alias_of="dns") @utils.hook("received.command.dns", min_args=1) def dns(self, event): @@ -79,14 +90,16 @@ def dns(self, event): (t, ttl, ", ".join(r)) for t, ttl, r in results] event["stdout"].write("(%s) %s" % (hostname, " | ".join(results_str))) - @utils.hook("received.command.geoip", min_args=1) + @utils.hook("received.command.geoip") def geoip(self, event): """ - :help: Get geoip data on a given IPv4/IPv6 address + :help: Get GeoIP data on a given IPv4/IPv6 address :usage: :prefix: GeoIP """ - page = utils.http.request(URL_GEOIP % event["args_split"][0]).json() + ip = self._get_ip(event) + + page = utils.http.request(URL_GEOIP % ip).json() if page: if page["status"] == "success": try: @@ -117,13 +130,7 @@ def rdns(self, event): :usage: :prefix: rDNS """ - ip = event["args_split"][0] if event["args"] else "" - if not ip: - line = event["target"].buffer.find(REGEX_IP) - if line: - ip = line.match - if not ip: - raise utils.EventError("No IP provided") + ip = self._get_ip(event) try: hostname, alias, ips = socket.gethostbyaddr(ip) From 37aa85e601ccf4a8981c9a8a5792a1433c743e04 Mon Sep 17 00:00:00 2001 From: JeDaYoshi Date: Sun, 17 Oct 2021 21:53:25 -0400 Subject: [PATCH 58/89] Add ipinfo command --- docs/bot.conf.example | 3 +++ modules/ip_addresses.py | 45 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/docs/bot.conf.example b/docs/bot.conf.example index fdb7a971..da22a1ee 100644 --- a/docs/bot.conf.example +++ b/docs/bot.conf.example @@ -94,3 +94,6 @@ bitly-api-key = # https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line github-token = + +# https://ipinfo.io/account/token +ipinfo-token = diff --git a/modules/ip_addresses.py b/modules/ip_addresses.py index 24c87cd4..e2cacbac 100644 --- a/modules/ip_addresses.py +++ b/modules/ip_addresses.py @@ -5,6 +5,7 @@ import dns.resolver URL_GEOIP = "http://ip-api.com/json/%s" +URL_IPINFO = "https://ipinfo.io/%s/json" REGEX_IPv6 = r"(?:(?:[a-f0-9]{1,4}:){2,}|[a-f0-9:]*::)[a-f0-9:]*" REGEX_IPv4 = r"(?:\d{1,3}\.){3}\d{1,3}" REGEX_IP = re.compile("%s|%s" % (REGEX_IPv4, REGEX_IPv6), re.I) @@ -32,6 +33,14 @@ def _get_ip(self, event): return ip + def _ipinfo_get(self, url): + access_token = self.bot.config.get("ipinfo-token", None) + headers = {} + if not access_token == None: + headers["Authorization"] = "Bearer %s" % access_token + request = utils.http.Request(url, headers=headers) + return utils.http.request(request) + @utils.hook("received.command.dig", alias_of="dns") @utils.hook("received.command.dns", min_args=1) def dns(self, event): @@ -108,8 +117,7 @@ def geoip(self, event): pass data = page["query"] - if hostname: - data += " (%s)" % hostname + data += " (%s)" % hostname if hostname else "" data += " | Organisation: %s" % page["org"] data += " | City: %s" % page["city"] data += " | Region: %s (%s)" % ( @@ -123,6 +131,39 @@ def geoip(self, event): else: raise utils.EventResultsError() + @utils.hook("received.command.ipinfo") + def ipinfo(self, event): + """ + :help: Get IPinfo.io data on a given IPv4/IPv6 address + :usage: + :prefix: IPinfo + """ + ip = self._get_ip(event) + + page = self._ipinfo_get(URL_IPINFO % ip).json() + if page: + if not page.get("error", None): + hostname = page.get("hostname", None) + if not hostname: + try: + hostname, alias, ips = socket.gethostbyaddr(page["ip"]) + except (socket.herror, socket.gaierror): + pass + + data = page["ip"] + data += " (%s)" % hostname if hostname else "" + data += " | ISP: %s" % page["org"] + data += " | Location: %s, %s, %s" % ( + page["city"], page["region"], page["country"]) + data += " (Anycast)" if page.get("anycast", False) == True else "" + data += " | Lon/Lat: %s" % page["loc"] + data += " | Timezone: %s" % page["timezone"] + event["stdout"].write(data) + else: + event["stderr"].write(page["error"]["message"]) + else: + raise utils.EventResultsError() + @utils.hook("received.command.rdns") def rdns(self, event): """ From 858b3dbe62eaf79c44e14906b9226f9107c76779 Mon Sep 17 00:00:00 2001 From: JeDaYoshi Date: Sun, 17 Oct 2021 21:56:24 -0400 Subject: [PATCH 59/89] Make ipinfo command more similar to geoip --- modules/ip_addresses.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/ip_addresses.py b/modules/ip_addresses.py index e2cacbac..a06fecc8 100644 --- a/modules/ip_addresses.py +++ b/modules/ip_addresses.py @@ -152,10 +152,10 @@ def ipinfo(self, event): data = page["ip"] data += " (%s)" % hostname if hostname else "" - data += " | ISP: %s" % page["org"] - data += " | Location: %s, %s, %s" % ( - page["city"], page["region"], page["country"]) data += " (Anycast)" if page.get("anycast", False) == True else "" + data += " | City: %s" % page["city"] + data += " | Region: %s (%s)" % (page["region"], page["country"]) + data += " | ISP: %s" % page["org"] data += " | Lon/Lat: %s" % page["loc"] data += " | Timezone: %s" % page["timezone"] event["stdout"].write(data) From d8ba18a2dcc6416e5ea95a7512774a5fc787a0fe Mon Sep 17 00:00:00 2001 From: JeDaYoshi Date: Sun, 17 Oct 2021 22:12:19 -0400 Subject: [PATCH 60/89] Add support for multiple endpoints in ipinfo, fixes --- modules/ip_addresses.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/modules/ip_addresses.py b/modules/ip_addresses.py index a06fecc8..1102b64d 100644 --- a/modules/ip_addresses.py +++ b/modules/ip_addresses.py @@ -142,25 +142,34 @@ def ipinfo(self, event): page = self._ipinfo_get(URL_IPINFO % ip).json() if page: - if not page.get("error", None): + if page.get("error", False): + if isinstance(page["error"], (list, dict)): + event["stderr"].write(page["error"]["message"]) + else: + event["stderr"].write(page["error"]) + elif page.get("ip", False): + bogon = page.get("bogon", False) hostname = page.get("hostname", None) - if not hostname: + if not hostname and not bogon: try: hostname, alias, ips = socket.gethostbyaddr(page["ip"]) except (socket.herror, socket.gaierror): pass data = page["ip"] - data += " (%s)" % hostname if hostname else "" - data += " (Anycast)" if page.get("anycast", False) == True else "" - data += " | City: %s" % page["city"] - data += " | Region: %s (%s)" % (page["region"], page["country"]) - data += " | ISP: %s" % page["org"] - data += " | Lon/Lat: %s" % page["loc"] - data += " | Timezone: %s" % page["timezone"] + if bogon: + data += " (Bogon)" + else: + data += " (%s)" % hostname if hostname else "" + data += " (Anycast)" if page.get("anycast", False) == True else "" + data += " | City: %s" % page["city"] + data += " | Region: %s (%s)" % (page["region"], page["country"]) + data += " | ISP: %s" % page["org"] + data += " | Lon/Lat: %s" % page["loc"] + data += " | Timezone: %s" % page["timezone"] event["stdout"].write(data) else: - event["stderr"].write(page["error"]["message"]) + event["stderr"].write("Unsupported endpoint") else: raise utils.EventResultsError() From 24e073313c1fdeb52df6a79d1dd6223aa9c792f7 Mon Sep 17 00:00:00 2001 From: JeDaYoshi Date: Mon, 18 Oct 2021 18:18:05 -0400 Subject: [PATCH 61/89] additional fixes to ipinfo/geoip --- modules/ip_addresses.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/ip_addresses.py b/modules/ip_addresses.py index 1102b64d..09d46b5b 100644 --- a/modules/ip_addresses.py +++ b/modules/ip_addresses.py @@ -30,7 +30,6 @@ def _get_ip(self, event): ip = line.match if not ip: raise utils.EventError("No IP provided") - return ip def _ipinfo_get(self, url): @@ -111,6 +110,7 @@ def geoip(self, event): page = utils.http.request(URL_GEOIP % ip).json() if page: if page["status"] == "success": + hostname = None try: hostname, alias, ips = socket.gethostbyaddr(page["query"]) except (socket.herror, socket.gaierror): @@ -156,17 +156,18 @@ def ipinfo(self, event): except (socket.herror, socket.gaierror): pass - data = page["ip"] + data = page["ip"] if bogon: data += " (Bogon)" else: data += " (%s)" % hostname if hostname else "" data += " (Anycast)" if page.get("anycast", False) == True else "" - data += " | City: %s" % page["city"] - data += " | Region: %s (%s)" % (page["region"], page["country"]) - data += " | ISP: %s" % page["org"] - data += " | Lon/Lat: %s" % page["loc"] - data += " | Timezone: %s" % page["timezone"] + if page.get("country", False): + data += " | City: %s" % page["city"] + data += " | Region: %s (%s)" % (page["region"], page["country"]) + data += " | ISP: %s" % page.get("org", "Unknown") + data += " | Lon/Lat: %s" % page["loc"] + data += " | Timezone: %s" % page["timezone"] event["stdout"].write(data) else: event["stderr"].write("Unsupported endpoint") From 9ff4c23759cc5c241395cb2eeaeb53f70f017ac5 Mon Sep 17 00:00:00 2001 From: PeGaSuS Date: Sun, 12 Dec 2021 13:29:23 +0100 Subject: [PATCH 62/89] Update apache2 `ProxyPassReverse` should also end with a slash (`/`) --- docs/rest_api/apache2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rest_api/apache2 b/docs/rest_api/apache2 index e6fb6759..9aadfd24 100644 --- a/docs/rest_api/apache2 +++ b/docs/rest_api/apache2 @@ -15,6 +15,6 @@ Listen 5000 ProxyRequests off ProxyPass / http://[::1]:5001/ - ProxyPassReverse / http://[::1]:5001 + ProxyPassReverse / http://[::1]:5001/ ProxyPreserveHost on From fc219553deee09d03c1c2975a86b4c71a8285a75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 20:01:47 +0000 Subject: [PATCH 63/89] Bump lxml from 4.6.3 to 4.6.5 Bumps [lxml](https://github.com/lxml/lxml) from 4.6.3 to 4.6.5. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.3...lxml-4.6.5) --- updated-dependencies: - dependency-name: lxml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d13bb9e..37aeb196 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ dnspython ==2.0.0 feedparser ==6.0.2 html5lib ==1.0.1 isodate ==0.6.0 -lxml ==4.6.3 +lxml ==4.6.5 netifaces ==0.10.9 PySocks ==1.7.1 python-dateutil ==2.8.1 From ada778515584234bc7db8745d6ef1efa8c7394ac Mon Sep 17 00:00:00 2001 From: jesopo Date: Sun, 19 Dec 2021 00:36:20 +0000 Subject: [PATCH 64/89] don't run filters/replaces on assured lines --- modules/message_filter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/message_filter.py b/modules/message_filter.py index f773ad36..11cdcde6 100644 --- a/modules/message_filter.py +++ b/modules/message_filter.py @@ -15,6 +15,10 @@ def _get_filters(self, server, target): @utils.hook("preprocess.send.privmsg") @utils.hook("preprocess.send.notice") def channel_message(self, event): + if event["line"].assured(): + # don't run filters/replaces against assured lines + return + message = event["line"].args[1] original_message = message message_plain = utils.irc.strip_font(message) From 904cb2d94cb7c7a7ebc5769b05cecbdbef32f6c7 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Wed, 19 Jan 2022 21:32:50 -0600 Subject: [PATCH 65/89] git_webhooks/github.py: remove git url shortening --- modules/git_webhooks/github.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py index 8bf61114..8ead7b9c 100644 --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -172,15 +172,10 @@ def webhook(self, full_name, event, data, headers): return list(zip(out, [None]*len(out))) def _short_url(self, url): - self.log.debug("git.io shortening: %s" % url) - try: - page = utils.http.request("https://git.io", method="POST", - post_data={"url": url}) - return page.headers["Location"] - except utils.http.HTTPTimeoutException: - self.log.warn( - "HTTPTimeoutException while waiting for github short URL", []) - return url + # TODO: find an alternative to git.io + # see https://github.com/jesopo/bitbot/issues/338 + # ~ examknow 1/19/2022 + return url def _iso8601(self, s): return utils.datetime.parse.iso8601(s) From 046aa1b2cf901c440aa37dcef874583d8bfa3452 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Wed, 19 Jan 2022 21:33:45 -0600 Subject: [PATCH 66/89] github.py: remove git url shortening --- modules/git_webhooks/github.py | 85 +++++++++++++++------------------- modules/github.py | 12 ++--- 2 files changed, 42 insertions(+), 55 deletions(-) diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py index 8ead7b9c..035e0809 100644 --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -5,6 +5,7 @@ COMMIT_RANGE_URL = "https://github.com/%s/compare/%s...%s" CREATE_URL = "https://github.com/%s/tree/%s" +PR_URL = "https://github.com/%s/pull/%s" PR_COMMIT_RANGE_URL = "https://github.com/%s/pull/%s/files/%s..%s" PR_COMMIT_URL = "https://github.com/%s/pull/%s/commits/%s" @@ -77,19 +78,19 @@ } COMMENT_MAX = 100 -CHECK_RUN_CONCLUSION = { - "success": "passed", - "failure": "failed", - "neutral": "finished", - "cancelled": "was cancelled", - "timed_out": "timed out", - "action_required": "requires action" +CHECK_SUITE_CONCLUSION = { + "success": ("passed", colors.COLOR_POSITIVE), + "failure": ("failed", colors.COLOR_NEGATIVE), + "neutral": ("finished", colors.COLOR_NEUTRAL), + "cancelled": ("was cancelled", colors.COLOR_NEGATIVE), + "timed_out": ("timed out", colors.COLOR_NEGATIVE), + "action_required": ("requires action", colors.COLOR_NEUTRAL) } -CHECK_RUN_FAILURES = ["failure", "cancelled", "timed_out", "action_required"] class GitHub(object): - def __init__(self, log): + def __init__(self, log, exports): self.log = log + self.exports = exports def is_private(self, data, headers): if "repository" in data: @@ -125,6 +126,8 @@ def event(self, data, headers): category_action = None if "review" in data and "state" in data["review"]: category = "%s+%s" % (event, data["review"]["state"]) + elif "check_suite" in data and "conclusion" in data["check_suite"]: + category = "%s+%s" % (event, data["check_suite"]["conclusion"]) if action: if category: @@ -159,8 +162,8 @@ def webhook(self, full_name, event, data, headers): out = self.delete(full_name, data) elif event == "release": out = self.release(full_name, data) - elif event == "check_run": - out = self.check_run(data) + elif event == "check_suite": + out = self.check_suite(full_name, data) elif event == "fork": out = self.fork(full_name, data) elif event == "ping": @@ -267,7 +270,7 @@ def pull_request(self, full_name, data): colored_branch = utils.irc.color(branch, colors.COLOR_BRANCH) sender = utils.irc.bold(data["sender"]["login"]) - author = utils.irc.bold(data["sender"]["login"]) + author = utils.irc.bold(data["pull_request"]["user"]["login"]) number = utils.irc.color("#%s" % data["pull_request"]["number"], colors.COLOR_ID) identifier = "%s by %s" % (number, author) @@ -428,44 +431,32 @@ def release(self, full_name, data): url = self._short_url(data["release"]["html_url"]) return ["%s %s a release%s - %s" % (author, action, name, url)] - def check_run(self, data): - name = data["check_run"]["name"] - commit = self._short_hash(data["check_run"]["head_sha"]) + def check_suite(self, full_name, data): + suite = data["check_suite"] + + commit = self._short_hash(suite["head_sha"]) commit = utils.irc.color(commit, utils.consts.LIGHTBLUE) + pr = "" url = "" - if data["check_run"]["details_url"]: - url = data["check_run"]["details_url"] - url = " - %s" % self.exports.get("shorturl-any")(url) - - duration = "" - if data["check_run"]["completed_at"]: - started_at = self._iso8601(data["check_run"]["started_at"]) - completed_at = self._iso8601(data["check_run"]["completed_at"]) - if completed_at > started_at: - seconds = (completed_at-started_at).total_seconds() - duration = " in %s" % utils.datetime.format.to_pretty_time( - seconds) - - status = data["check_run"]["status"] - status_str = "" - if status == "queued": - status_str = utils.irc.bold("queued") - elif status == "in_progress": - status_str = utils.irc.bold("started") - elif status == "completed": - conclusion = data["check_run"]["conclusion"] - conclusion_color = colors.COLOR_POSITIVE - if conclusion in CHECK_RUN_FAILURES: - conclusion_color = colors.COLOR_NEGATIVE - if conclusion == "neutral": - conclusion_color = colors.COLOR_NEUTRAL - - status_str = utils.irc.color( - CHECK_RUN_CONCLUSION[conclusion], conclusion_color) - - return ["[build @%s] %s: %s%s%s" % ( - commit, name, status_str, duration, url)] + if suite["pull_requests"]: + pr_num = suite["pull_requests"][0]["number"] + pr = "/PR%s" % utils.irc.color("#%s" % pr_num, colors.COLOR_ID) + url = self._short_url(PR_URL % (full_name, pr_num)) + url = " - %s" % url + + name = suite["app"]["name"] + conclusion = suite["conclusion"] + conclusion, conclusion_color = CHECK_SUITE_CONCLUSION[conclusion] + conclusion = utils.irc.color(conclusion, conclusion_color) + + created_at = self._iso8601(suite["created_at"]) + updated_at = self._iso8601(suite["updated_at"]) + seconds = (updated_at-created_at).total_seconds() + duration = utils.datetime.format.to_pretty_time(seconds) + + return ["[build @%s%s] %s: %s in %s%s" % ( + commit, pr, name, conclusion, duration, url)] def fork(self, full_name, data): forker = utils.irc.bold(data["sender"]["login"]) diff --git a/modules/github.py b/modules/github.py index b528ad2b..9393cd49 100644 --- a/modules/github.py +++ b/modules/github.py @@ -47,14 +47,10 @@ def _parse_ref(self, channel, ref, sep): return org, repo, number def _short_url(self, url): - try: - page = utils.http.request("https://git.io", method="POST", - post_data={"url": url}) - return page.headers["Location"] - except utils.http.HTTPTimeoutException: - self.log.warn( - "HTTPTimeoutException while waiting for github short URL", []) - return url + # TODO: find an alternative to git.io + # see https://github.com/jesopo/bitbot/issues/338 + # ~ examknow 1/19/2022 + return url def _change_count(self, n, symbol, color): return utils.irc.color("%s%d" % (symbol, n), color)+utils.irc.bold("") From 9e4e4925c038d6fd4f8138c6884a7d4a7072f7d1 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Thu, 20 Jan 2022 13:43:26 -0600 Subject: [PATCH 67/89] git_webhooks/github.py: handle url shortening like everyone else --- modules/git_webhooks/github.py | 73 +++++++++++++++------------------- 1 file changed, 32 insertions(+), 41 deletions(-) diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py index 035e0809..6c9d4f10 100644 --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -172,13 +172,7 @@ def webhook(self, full_name, event, data, headers): out = self.membership(organisation, data) elif event == "watch": out = self.watch(data) - return list(zip(out, [None]*len(out))) - - def _short_url(self, url): - # TODO: find an alternative to git.io - # see https://github.com/jesopo/bitbot/issues/338 - # ~ examknow 1/19/2022 - return url + return out def _iso8601(self, s): return utils.datetime.parse.iso8601(s) @@ -234,15 +228,14 @@ def _format_push(self, branch, author, commits, forced, single_url, hash = commit["id"] hash_colored = utils.irc.color(self._short_hash(hash), colors.COLOR_ID) message = commit["message"].split("\n")[0].strip() - url = self._short_url(single_url % hash) + url = single_url % hash outputs.append( "%s %spushed %s to %s: %s - %s" % (author, forced_str, hash_colored, branch, message, url)) else: - outputs.append("%s %spushed %d commits to %s - %s" - % (author, forced_str, len(commits), branch, - self._short_url(range_url))) + outputs.append(("%s %spushed %d commits to %s" + % (author, forced_str, len(commits), branch), url)) return outputs @@ -260,9 +253,9 @@ def commit_comment(self, full_name, data): action = data["action"] commit = self._short_hash(data["comment"]["commit_id"]) commenter = utils.irc.bold(data["comment"]["user"]["login"]) - url = self._short_url(data["comment"]["html_url"]) - return ["[commit/%s] %s %s a comment - %s" % (commit, commenter, - action, url)] + url = data["comment"]["html_url"] + return [("[commit/%s] %s %s a comment" % (commit, commenter, + action), url)] def pull_request(self, full_name, data): raw_number = data["pull_request"]["number"] @@ -326,9 +319,9 @@ def pull_request(self, full_name, data): action_desc = "renamed %s" % identifier pr_title = data["pull_request"]["title"] - url = self._short_url(data["pull_request"]["html_url"]) - return ["[PR] %s %s: %s - %s" % ( - sender, action_desc, pr_title, url)] + url = data["pull_request"]["html_url"] + return ["[PR] %s %s: %s" % ( + sender, action_desc, pr_title), url] def pull_request_review(self, full_name, data): if not data["action"] == "submitted": @@ -346,7 +339,7 @@ def pull_request_review(self, full_name, data): action = data["action"] pr_title = data["pull_request"]["title"] reviewer = utils.irc.bold(data["sender"]["login"]) - url = self._short_url(data["review"]["html_url"]) + url = data["review"]["html_url"] state_desc = state if state == "approved": @@ -356,8 +349,8 @@ def pull_request_review(self, full_name, data): elif state == "dismissed": state_desc = "dismissed a review" - return ["[PR] %s %s on %s: %s - %s" % - (reviewer, state_desc, number, pr_title, url)] + return [("[PR] %s %s on %s: %s" % + (reviewer, state_desc, number, pr_title), url)] def pull_request_review_comment(self, full_name, data): number = utils.irc.color("#%s" % data["pull_request"]["number"], @@ -365,9 +358,9 @@ def pull_request_review_comment(self, full_name, data): action = data["action"] pr_title = data["pull_request"]["title"] sender = utils.irc.bold(data["sender"]["login"]) - url = self._short_url(data["comment"]["html_url"]) - return ["[PR] %s %s on a review on %s: %s - %s" % - (sender, COMMENT_ACTIONS[action], number, pr_title, url)] + url = data["comment"]["html_url"] + return [("[PR] %s %s on a review on %s: %s" % + (sender, COMMENT_ACTIONS[action], number, pr_title), url)] def issues(self, full_name, data): number = utils.irc.color("#%s" % data["issue"]["number"], @@ -381,9 +374,9 @@ def issues(self, full_name, data): issue_title = data["issue"]["title"] author = utils.irc.bold(data["sender"]["login"]) - url = self._short_url(data["issue"]["html_url"]) - return ["[issue] %s %s: %s - %s" % - (author, action_str, issue_title, url)] + url = data["issue"]["html_url"] + return [("[issue] %s %s: %s" % + (author, action_str, issue_title), url)] def issue_comment(self, full_name, data): if "changes" in data: # don't show this event when nothing has actually changed @@ -396,23 +389,22 @@ def issue_comment(self, full_name, data): type = "PR" if "pull_request" in data["issue"] else "issue" title = data["issue"]["title"] commenter = utils.irc.bold(data["sender"]["login"]) - url = self._short_url(data["comment"]["html_url"]) + url = data["comment"]["html_url"] body = "" if not action == "deleted": body = ": %s" % self._comment(data["comment"]["body"]) - return ["[%s] %s %s on %s (%s)%s - %s" % - (type, commenter, COMMENT_ACTIONS[action], number, title, body, - url)] + return [("[%s] %s %s on %s (%s)%s" % + (type, commenter, COMMENT_ACTIONS[action], number, title, body), url)] def create(self, full_name, data): ref = data["ref"] ref_color = utils.irc.color(ref, colors.COLOR_BRANCH) type = data["ref_type"] sender = utils.irc.bold(data["sender"]["login"]) - url = self._short_url(CREATE_URL % (full_name, ref)) - return ["%s created a %s: %s - %s" % (sender, type, ref_color, url)] + url = CREATE_URL % (full_name, ref) + return [("%s created a %s: %s" % (sender, type, ref_color), url)] def delete(self, full_name, data): ref = data["ref"] @@ -428,8 +420,8 @@ def release(self, full_name, data): if name: name = ": %s" % name author = utils.irc.bold(data["release"]["author"]["login"]) - url = self._short_url(data["release"]["html_url"]) - return ["%s %s a release%s - %s" % (author, action, name, url)] + url = data["release"]["html_url"] + return [("%s %s a release%s" % (author, action, name), url)] def check_suite(self, full_name, data): suite = data["check_suite"] @@ -442,8 +434,7 @@ def check_suite(self, full_name, data): if suite["pull_requests"]: pr_num = suite["pull_requests"][0]["number"] pr = "/PR%s" % utils.irc.color("#%s" % pr_num, colors.COLOR_ID) - url = self._short_url(PR_URL % (full_name, pr_num)) - url = " - %s" % url + url = PR_URL % (full_name, pr_num) name = suite["app"]["name"] conclusion = suite["conclusion"] @@ -455,16 +446,16 @@ def check_suite(self, full_name, data): seconds = (updated_at-created_at).total_seconds() duration = utils.datetime.format.to_pretty_time(seconds) - return ["[build @%s%s] %s: %s in %s%s" % ( - commit, pr, name, conclusion, duration, url)] + return [("[build @%s%s] %s: %s in %s" % ( + commit, pr, name, conclusion, duration), url)] def fork(self, full_name, data): forker = utils.irc.bold(data["sender"]["login"]) fork_full_name = utils.irc.color(data["forkee"]["full_name"], utils.consts.LIGHTBLUE) - url = self._short_url(data["forkee"]["html_url"]) - return ["%s forked into %s - %s" % - (forker, fork_full_name, url)] + url = data["forkee"]["html_url"] + return [("%s forked into %s" % + (forker, fork_full_name), url)] def membership(self, organisation, data): return ["%s %s %s to team %s" % From bf76b41485504d9a8e2e3abb5955da3e8c93da1a Mon Sep 17 00:00:00 2001 From: David Schultz Date: Thu, 20 Jan 2022 13:10:15 -0600 Subject: [PATCH 68/89] github.py: use default shorteners --- modules/github.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/github.py b/modules/github.py index 9393cd49..4945b57c 100644 --- a/modules/github.py +++ b/modules/github.py @@ -47,10 +47,7 @@ def _parse_ref(self, channel, ref, sep): return org, repo, number def _short_url(self, url): - # TODO: find an alternative to git.io - # see https://github.com/jesopo/bitbot/issues/338 - # ~ examknow 1/19/2022 - return url + return self.exports.get("shorturl")(self.bot, url) or url def _change_count(self, n, symbol, color): return utils.irc.color("%s%d" % (symbol, n), color)+utils.irc.bold("") From a4f41cdfd7872c55c7b39bd69d324fc2c937cd23 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Sun, 13 Feb 2022 17:22:14 -0600 Subject: [PATCH 69/89] ducks.py: do not accept `,bef` or `,trap` if ducks are disabled --- modules/ducks.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/ducks.py b/modules/ducks.py index eb6d7ef4..22a9bb99 100644 --- a/modules/ducks.py +++ b/modules/ducks.py @@ -98,6 +98,10 @@ def _no_duck(self, channel, user, stderr): @utils.kwarg("help", "Befriend a duck") @utils.spec("!-channelonly") def befriend(self, event): + if not event["target"].get_setting("ducks-enabled", False): + return event["stderr"].write( + "Ducks are not enabled in this channel" + ) if event["target"].duck_active: action = self._duck_action(event["target"], event["user"], "befriended", "ducks-befriended") @@ -109,6 +113,10 @@ def befriend(self, event): @utils.kwarg("help", "Trap a duck") @utils.spec("!-channelonly") def trap(self, event): + if not event["target"].get_setting("ducks-enabled", False): + return event["stderr"].write( + "Ducks are not enabled in this channel" + ) if event["target"].duck_active: action = self._duck_action(event["target"], event["user"], "trapped", "ducks-shot") From 42fccbaec7e5dbeda6568b2022c22e28e03c7ef0 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Sun, 13 Feb 2022 18:51:55 -0600 Subject: [PATCH 70/89] git_webhooks/github.py: fix some rough edges --- modules/git_webhooks/github.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) mode change 100644 => 100755 modules/git_webhooks/github.py diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py old mode 100644 new mode 100755 index 6c9d4f10..d516dbca --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -222,7 +222,7 @@ def _format_push(self, branch, author, commits, forced, single_url, if len(commits) == 0 and forced: outputs.append( - "%s %spushed to %s" % (author, forced_str, branch)) + "%s %spushed to %s" % (author, forced_str, branch), None) elif len(commits) <= 3: for commit in commits: hash = commit["id"] @@ -230,9 +230,9 @@ def _format_push(self, branch, author, commits, forced, single_url, message = commit["message"].split("\n")[0].strip() url = single_url % hash - outputs.append( - "%s %spushed %s to %s: %s - %s" - % (author, forced_str, hash_colored, branch, message, url)) + outputs.append(( + "%s %spushed %s to %s: %s" + % (author, forced_str, hash_colored, branch, message), url)) else: outputs.append(("%s %spushed %d commits to %s" % (author, forced_str, len(commits), branch), url)) @@ -320,8 +320,8 @@ def pull_request(self, full_name, data): pr_title = data["pull_request"]["title"] url = data["pull_request"]["html_url"] - return ["[PR] %s %s: %s" % ( - sender, action_desc, pr_title), url] + return [("[PR] %s %s: %s" % ( + sender, action_desc, pr_title), url)] def pull_request_review(self, full_name, data): if not data["action"] == "submitted": @@ -411,7 +411,7 @@ def delete(self, full_name, data): ref_color = utils.irc.color(ref, colors.COLOR_BRANCH) type = data["ref_type"] sender = utils.irc.bold(data["sender"]["login"]) - return ["%s deleted a %s: %s" % (sender, type, ref_color)] + return [("%s deleted a %s: %s" % (sender, type, ref_color), None)] def release(self, full_name, data): action = data["action"] @@ -458,9 +458,9 @@ def fork(self, full_name, data): (forker, fork_full_name), url)] def membership(self, organisation, data): - return ["%s %s %s to team %s" % + return [("%s %s %s to team %s" % (data["sender"]["login"], data["action"], data["member"]["login"], - data["team"]["name"])] + data["team"]["name"]), None)] def watch(self, data): - return ["%s starred the repository" % data["sender"]["login"]] + return [("%s starred the repository" % data["sender"]["login"], None)] From 8edf89da53bdf17eb09d1d029863ee5eff8148c7 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Tue, 1 Mar 2022 09:59:59 -0600 Subject: [PATCH 71/89] git_webhooks/github.py: fix `ping()` --- modules/git_webhooks/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py index d516dbca..2e94f991 100755 --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -178,7 +178,7 @@ def _iso8601(self, s): return utils.datetime.parse.iso8601(s) def ping(self, data): - return ["Received new webhook"] + return [("Received new webhook", None)] def _change_count(self, n, symbol, color): return utils.irc.color("%s%d" % (symbol, n), color)+utils.irc.bold("") From 9f33fb438134085e29e69f37056d42ff1c48dd46 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Mon, 11 Jul 2022 16:36:32 -0500 Subject: [PATCH 72/89] modules/youtube.py: add api exception handling --- modules/youtube.py | 134 ++++++++++++++++++++++++--------------------- 1 file changed, 72 insertions(+), 62 deletions(-) diff --git a/modules/youtube.py b/modules/youtube.py index 99c7ba98..4e5e3bc8 100644 --- a/modules/youtube.py +++ b/modules/youtube.py @@ -36,46 +36,49 @@ def _number(self, n): def video_details(self, video_id): page = self.get_video_page(video_id) - if page["items"]: - item = page["items"][0] - snippet = item["snippet"] - statistics = item["statistics"] - content = item["contentDetails"] - - video_uploaded_at = utils.datetime.parse.iso8601( - snippet["publishedAt"]) - video_uploaded_at = utils.datetime.format.date_human( - video_uploaded_at) - - video_uploader = snippet["channelTitle"] - video_title = utils.irc.bold(snippet["title"]) - - video_views = self._number(statistics.get("viewCount")) - video_likes = self._number(statistics.get("likeCount")) - video_dislikes = self._number(statistics.get("dislikeCount")) - - video_opinions = "" - if video_likes and video_dislikes: - likes = utils.irc.color("%s%s" % (video_likes, ARROW_UP), - utils.consts.GREEN) - dislikes = utils.irc.color("%s%s" % - (ARROW_DOWN, video_dislikes), utils.consts.RED) - video_opinions = " (%s%s)" % (likes, dislikes) - - video_views_str = "" - if video_views: - video_views_str = ", %s views" % video_views - - td = utils.datetime.parse.iso8601_duration(content["duration"]) - video_duration = utils.datetime.format.to_pretty_time( - td.total_seconds()) - - url = URL_YOUTUBESHORT % video_id - - return "%s (%s) uploaded by %s on %s%s%s" % ( - video_title, video_duration, video_uploader, video_uploaded_at, - video_views_str, video_opinions), url - return None + try: + if page["items"]: + item = page["items"][0] + snippet = item["snippet"] + statistics = item["statistics"] + content = item["contentDetails"] + + video_uploaded_at = utils.datetime.parse.iso8601( + snippet["publishedAt"]) + video_uploaded_at = utils.datetime.format.date_human( + video_uploaded_at) + + video_uploader = snippet["channelTitle"] + video_title = utils.irc.bold(snippet["title"]) + + video_views = self._number(statistics.get("viewCount")) + video_likes = self._number(statistics.get("likeCount")) + video_dislikes = self._number(statistics.get("dislikeCount")) + + video_opinions = "" + if video_likes and video_dislikes: + likes = utils.irc.color("%s%s" % (video_likes, ARROW_UP), + utils.consts.GREEN) + dislikes = utils.irc.color("%s%s" % + (ARROW_DOWN, video_dislikes), utils.consts.RED) + video_opinions = " (%s%s)" % (likes, dislikes) + + video_views_str = "" + if video_views: + video_views_str = ", %s views" % video_views + + td = utils.datetime.parse.iso8601_duration(content["duration"]) + video_duration = utils.datetime.format.to_pretty_time( + td.total_seconds()) + + url = URL_YOUTUBESHORT % video_id + + return "%s (%s) uploaded by %s on %s%s%s" % ( + video_title, video_duration, video_uploader, video_uploaded_at, + video_views_str, video_opinions), url + return None + except KeyError: + return None def get_playlist_page(self, playlist_id): self.log.debug("youtube API request: " @@ -86,16 +89,19 @@ def get_playlist_page(self, playlist_id): "key": self.bot.config["google-api-key"]}).json() def playlist_details(self, playlist_id): page = self.get_playlist_page(playlist_id) - if page["items"]: - item = page["items"][0] - snippet = item["snippet"] - content = item["contentDetails"] + try: + if page["items"]: + item = page["items"][0] + snippet = item["snippet"] + content = item["contentDetails"] - count = content["itemCount"] + count = content["itemCount"] - return "%s - %s (%s %s)" % (snippet["channelTitle"], - snippet["title"], count, "video" if count == 1 else "videos" - ), URL_PLAYLIST % playlist_id + return "%s - %s (%s %s)" % (snippet["channelTitle"], + snippet["title"], count, "video" if count == 1 else "videos" + ), URL_PLAYLIST % playlist_id + except KeyError: + return None def _from_url(self, url): parsed = urllib.parse.urlparse(url) @@ -148,23 +154,27 @@ def yt(self, event): from_url = not url == None - if not url: - safe_setting = event["target"].get_setting("youtube-safesearch", True) - safe = "moderate" if safe_setting else "none" - self.log.debug("youtube API request: search.list (B) [snippet]") + try: + if not url: + safe_setting = event["target"].get_setting("youtube-safesearch", True) + safe = "moderate" if safe_setting else "none" - search_page = utils.http.request(URL_YOUTUBESEARCH, - get_params={"q": search, "part": "snippet", "maxResults": "1", - "type": "video", "key": self.bot.config["google-api-key"], - "safeSearch": safe}).json() - if search_page: - if search_page["pageInfo"]["totalResults"] > 0: - url = URL_VIDEO % search_page["items"][0]["id"]["videoId"] + self.log.debug("youtube API request: search.list (B) [snippet]") + + search_page = utils.http.request(URL_YOUTUBESEARCH, + get_params={"q": search, "part": "snippet", "maxResults": "1", + "type": "video", "key": self.bot.config["google-api-key"], + "safeSearch": safe}).json() + if search_page: + if search_page["pageInfo"]["totalResults"] > 0: + url = URL_VIDEO % search_page["items"][0]["id"]["videoId"] + else: + raise utils.EventError("No videos found") else: - raise utils.EventError("No videos found") - else: - raise utils.EventResultsError() + raise utils.EventResultsError() + except KeyError: + raise utils.EventError("API error") if url: out = self._from_url(url) From 55cc01e61b1f08bd2a343d18bfd39153456c9e6d Mon Sep 17 00:00:00 2001 From: DarkFeather Date: Tue, 25 Oct 2022 13:19:42 -0500 Subject: [PATCH 73/89] Switching from html5lib to lxml for tree builder issue in https://gist.github.com/Dark-Feather/025d9ff32487fa76457d52119dc0ff24 --- src/utils/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/http.py b/src/utils/http.py index 045d9641..0bbe92be 100644 --- a/src/utils/http.py +++ b/src/utils/http.py @@ -316,7 +316,7 @@ class Client(object): request_many = request_many def strip_html(s: str) -> str: - return bs4.BeautifulSoup(s, "html5lib").get_text() + return bs4.BeautifulSoup(s, "lxml").get_text() def resolve_hostname(hostname: str) -> typing.List[str]: try: From 757a763d50dfbe3b9d0288b00154fcc926c45d67 Mon Sep 17 00:00:00 2001 From: DarkFeather Date: Tue, 25 Oct 2022 13:22:16 -0500 Subject: [PATCH 74/89] Removing deprecated loop=loop removed in 3.10 https://docs.python.org/3/library/asyncio-task.html --- src/utils/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/http.py b/src/utils/http.py index 0bbe92be..f465ab0d 100644 --- a/src/utils/http.py +++ b/src/utils/http.py @@ -305,7 +305,7 @@ async def _request(request): awaits = [] for request in requests: awaits.append(_request(request)) - task = asyncio.wait(awaits, loop=loop, timeout=5) + task = asyncio.wait(awaits, timeout=5) loop.run_until_complete(task) loop.close() From 398aca20fa17ca9a4ec6f4ba8e3f07a9aff60372 Mon Sep 17 00:00:00 2001 From: deepend-tildeclub <58404188+deepend-tildeclub@users.noreply.github.com> Date: Mon, 2 Jan 2023 21:53:12 -0700 Subject: [PATCH 75/89] config.md: add information on `-c` option (#352) * Update config.md added -c option to the how to run the bot. Without it I'm sure a ton of people had troubles with module loading and just gave up. * Update config.md making -c a clearer option? * Update docs/help/config.md Co-authored-by: David Schultz Co-authored-by: David Schultz --- docs/help/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/help/config.md b/docs/help/config.md index 845c6f8c..b6542828 100644 --- a/docs/help/config.md +++ b/docs/help/config.md @@ -2,7 +2,7 @@ * Move `docs/bot.conf.example` to `~/.bitbot/bot.conf` and fill in the config options you care about. Ones blank or removed will disable relevant functionality. * Run `./bitbotd -a` to add a server. -* Run `./bitbotd` to start the bot. +* Run `./bitbotd` to start the bot or `./bitbotd -c /path/to/bot.conf` for non-standard config location (outside of `~/.bitbot`). * Run `./bitbotctl command master-password` to get the master admin password (needed to add regular admin accounts) * Join `#bitbot` on a server with the bot (or invite it to another channel) * `/msg register ` to register your nickname with the bot From 4e37f7cb357d4bebce853f5213cbc8c60f79883f Mon Sep 17 00:00:00 2001 From: David Schultz Date: Tue, 17 Jan 2023 13:46:52 -0600 Subject: [PATCH 76/89] shorturl.py: use bitly v4 api (#355) --- modules/shorturl.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/modules/shorturl.py b/modules/shorturl.py index 6b881454..23ed4571 100644 --- a/modules/shorturl.py +++ b/modules/shorturl.py @@ -4,7 +4,7 @@ import re from src import ModuleManager, utils -URL_BITLYSHORTEN = "https://api-ssl.bitly.com/v3/shorten" +URL_BITLYSHORTEN = "https://api-ssl.bitly.com/v4/shorten" class Module(ModuleManager.BaseModule): def on_load(self): @@ -66,11 +66,16 @@ def _bitly(self, url): access_token = self.bot.config.get("bitly-api-key", None) if access_token: - page = utils.http.request(URL_BITLYSHORTEN, get_params={ - "access_token": access_token, "longUrl": url}).json() - - if page["data"]: - return page["data"]["url"] + resp = utils.http.request( + URL_BITLYSHORTEN, + method="POST", + post_data={"long_url": url}, + json_body=True, + headers={"Authorization": f"Bearer {access_token}"} + ) + + if resp.code == 200: + return resp.json()["link"] return None def _find_url(self, target, args): @@ -112,4 +117,4 @@ def unshorten(self, event): event["stdout"].write("Unshortened: %s" % response.headers["location"]) else: - event["stderr"].write("Failed to unshorten URL") + event["stderr"].write("Failed to unshorten URL") \ No newline at end of file From 765689f9dc6c311cff45393e5000f1c2eee942ca Mon Sep 17 00:00:00 2001 From: David Schultz Date: Tue, 17 Jan 2023 19:33:28 -0600 Subject: [PATCH 77/89] urbandictionary: use https (#357) --- modules/urbandictionary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/urbandictionary.py b/modules/urbandictionary.py index a5098c3f..d7b43332 100644 --- a/modules/urbandictionary.py +++ b/modules/urbandictionary.py @@ -2,7 +2,7 @@ from src import ModuleManager, utils -URL_URBANDICTIONARY = "http://api.urbandictionary.com/v0/define" +URL_URBANDICTIONARY = "https://api.urbandictionary.com/v0/define" class Module(ModuleManager.BaseModule): _name = "UrbanDictionary" From a02bd2ca8b32507f9bf8c367dbc50366ed86ae09 Mon Sep 17 00:00:00 2001 From: PeGaSuS Date: Thu, 2 Mar 2023 04:05:22 +0100 Subject: [PATCH 78/89] Update requirements.txt (#358) - suds-jurko seems to be stalled, with the last release in Jan 24, 2014 - suds seems to actively mantained again, with the last release in Jun 28, 2022 - I'm putting version 1.0.0 as the minimum, which was released on Dec 05, 2021 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 87ce27d6..87613c42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ PySocks ==1.7.1 python-dateutil ==2.8.1 pytz ==2019.2 requests ==2.22.0 -suds-jurko ==0.6 +suds ==1.0.0 tornado ==6.0.3 tweepy ==3.8.0 requests-toolbelt ==0.9.1 From 6e808e2510577d89ad14588c93e27215c8c75fb4 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Sun, 19 Mar 2023 19:19:44 -0500 Subject: [PATCH 79/89] Delete nr.py (#360) To be moved to `bitbot-modules` --- modules/nr.py | 518 -------------------------------------------------- 1 file changed, 518 deletions(-) delete mode 100644 modules/nr.py diff --git a/modules/nr.py b/modules/nr.py deleted file mode 100644 index 7c7303d0..00000000 --- a/modules/nr.py +++ /dev/null @@ -1,518 +0,0 @@ -#--depends-on commands -#--require-config nre-api-key - -import collections, re, time -from datetime import datetime, date -from collections import Counter - -from src import ModuleManager, utils - -from suds.client import Client -from suds import WebFault - -# Note that this module requires the open *Staff Version* of the Darwin API -# You can register for an API key here: http://openldbsv.nationalrail.co.uk/ -# We use this instead of the 'regular' version because it offers a *lot* more -# information. - -URL = 'https://lite.realtime.nationalrail.co.uk/OpenLDBSVWS/wsdl.aspx?ver=2016-02-16' - -class Module(ModuleManager.BaseModule): - _name = "NR" - _client = None - - PASSENGER_ACTIVITIES = ["U", "P", "R"] - COLOURS = [utils.consts.LIGHTBLUE, utils.consts.GREEN, - utils.consts.RED, utils.consts.CYAN, utils.consts.LIGHTGREY, - utils.consts.ORANGE] - - @property - def client(self): - if self._client: return self._client - try: - token = self.bot.config["nre-api-key"] - client = Client(URL) - header_token = client.factory.create('ns2:AccessToken') - header_token.TokenValue = token - client.set_options(soapheaders=header_token) - self._client = client - except Exception as e: - pass - return self._client - - def filter(self, args, defaults): - args = re.findall(r"[^\s,]+", args) - params = {} - - for arg in args: - if ":" in arg: - params[arg.split(":", 1)[0]] = arg.split(":", 1)[1] - elif "=" in arg: - params[arg.split("=", 1)[0]] = arg.split("=", 1)[1] - else: - params[arg.replace("!", "")] = '!' not in arg - - ret = {k: v[0] for k,v in defaults.items()} - ret["default"] = True - ret["errors"] = [] - - for k,v in params.items(): - if not k in defaults.keys(): - ret["errors"].append((k, "Invalid parameter")) - continue - if not defaults[k][1](v): - ret["errors"].append((v, 'Invalid value for "%s"' % k)) - continue - ret["default"] = False - ret[k] = v if len(defaults[k]) == 2 else defaults[k][2](v) - ret["errors_summary"] = ", ".join(['"%s": %s' % (a[0], a[1]) for a in ret["errors"]]) - return ret - - def process(self, service): - ut_now = datetime.now().timestamp() - nonetime = {"orig": None, "datetime": None, "ut": 0, - "short": ' ', "prefix": '', "on_time": False, - "estimate": False, "status": 4, "schedule": False} - times = {} - a_types = ["eta", "ata", "sta"] - d_types = ["etd", "atd", "std"] - - for a in a_types + d_types: - if a in service and service[a]: - times[a] = {"orig": service[a]} - - if len(service[a]) > 5: - times[a]["datetime"] = datetime.strptime(service[a], "%Y-%m-%dT%H:%M:%S") - else: - times[a]["datetime"] = datetime.strptime( - datetime.now().date().isoformat() + "T" + service[a][:4], - "%Y-%m-%dT%H%M" - ) - times[a]["ut"] = times[a]["datetime"].timestamp() - else: - times[a] = nonetime - - for k, a in times.items(): - if not a["orig"]: continue - a["short"] = a["datetime"].strftime("%H%M") if len(a["orig"]) > 5 else a["orig"] - a["shortest"] = "%02d" % a["datetime"].minute if -300 < a["ut"]-ut_now < 1800 else a["short"] - a["prefix"] = k[2] + ("s" if k[0] == "s" else "") - a["estimate"] = k[0] == "e" - a["schedule"] = k[0] == "s" - a["on_time"] = a["ut"] - times["s"+ k[1:]]["ut"] < 300 - a["status"] = 1 if a["on_time"] else 2 - if "a" + k[1:] in service: a["status"] = {"d": 0, "a": 3}[k[2]] - if k[0] == "s": a["status"] = 4 - - arr, dep = [times[a] for a in a_types if times[a]["ut"]], [times[a] for a in d_types if times[a]["ut"]] - times["arrival"] = (arr + dep + [nonetime])[0] - times["departure"] = (dep + arr + [nonetime])[0] - times["a"], times["d"] = (arr + [nonetime])[0], (dep + [nonetime])[0] - times["both"] = times["departure"] - times["max_sched"] = {"ut": max(times["sta"]["ut"], times["std"]["ut"])} - return times - - def activities(self, string): return [a+b.strip() for a,b in list(zip(*[iter(string)]*2)) if (a+b).strip()] - - def reduced_activities(self, string): return [a for a in self.activities(string) if a in self.PASSENGER_ACTIVITIES] - - @utils.hook("received.command.nrtrains", min_args=1) - def trains(self, event): - """ - :help: Get train/bus services for a station (Powered by NRE) - :usage: - """ - - client = self.client - colours = self.COLOURS - - schedule = {} - - location_code = event["args_split"][0].upper() - filter = self.filter(' '.join(event["args_split"][1:]) if len(event["args_split"]) > 1 else "", { - "dest": ('', lambda x: x.isalpha() and len(x)==3), - "origin":('', lambda x: x.isalpha() and len(x)==3), - "inter": ('', lambda x: x.isalpha() and len(x)==3, lambda x: x.upper()), - "toc": ('', lambda x: x.isalpha() and len(x) == 2), - "dedup": (False, lambda x: type(x)==type(True)), - "plat": ('', lambda x: len(x) <= 3), - "type": ("departure", lambda x: x in ["departure", "arrival", "both"]), - "terminating": (False, lambda x: type(x)==type(True)), - "period": (120, lambda x: x.isdigit() and 1 <= int(x) <= 480, lambda x: int(x)), - "nonpassenger": (False, lambda x: type(x)==type(True)), - "time": ("", lambda x: len(x)==4 and x.isdigit()), - "date": ("", lambda x: len(x)==10), - "tops": (None, lambda x: len(x)<4 and x.isdigit()), - "power": (None, lambda x: x.upper() in ["EMU", "DMU", "HST", "D", "E", "DEM"], lambda x: x.upper()), - "crs": (False, lambda x: type(x)==type(True)), - "st": (False, lambda x: type(x)==type(True)) - }) - - if filter["errors"]: - raise utils.EventError("Filter: " + filter["errors_summary"]) - - if filter["inter"] and filter["type"]!="departure": - raise utils.EventError("Filtering by intermediate stations is only " - "supported for departures.") - - nr_filterlist = client.factory.create("filterList") - if filter["inter"]: nr_filterlist.crs.append(filter["inter"]) - - now = datetime.now() - if filter["time"]: - now = now.replace(hour=int(filter["time"][:2])) - now = now.replace(minute=int(filter["time"][2:])) - if filter["date"]: - newdate = datetime.strptime(filter["date"], "%Y-%m-%d").date() - now = now.replace(day=newdate.day, month=newdate.month, year=newdate.year) - - method = client.service.GetArrivalDepartureBoardByCRS if len(location_code) == 3 else client.service.GetArrivalDepartureBoardByTIPLOC - try: - query = method(100, location_code, now.isoformat().split(".")[0], filter["period"], - nr_filterlist, "to", '', "PBS", filter["nonpassenger"]) - except WebFault as detail: - if str(detail) == "Server raised fault: 'Invalid crs code supplied'": - raise utils.EventError("Invalid CRS code.") - else: - raise utils.EventError("An error occurred.") - - nrcc_severe = len([a for a in query["nrccMessages"][0] if a["severity"] == "Major"]) if "nrccMessages" in query else 0 - if event.get("external"): - station_summary = "%s (%s) - %s (%s):\n" % (query["locationName"], query["crs"], query["stationManager"], - query["stationManagerCode"]) - else: - severe_summary = "" - if nrcc_severe: - severe_summary += ", " - severe_summary += utils.irc.bold(utils.irc.color("%s severe messages" % nrcc_severe, utils.consts.RED)) - station_summary = "%s (%s, %s%s)" % (query["locationName"], query["crs"], query["stationManagerCode"], severe_summary) - - if not "trainServices" in query and not "busServices" in query and not "ferryServices" in query: - return event["stdout"].write("%s: No services for the next %s minutes" % ( - station_summary, filter["period"])) - - trains = [] - services = [] - if "trainServices" in query: services += query["trainServices"][0] - if "busServices" in query: services += query["busServices"][0] - if "ferryServices" in query: services += query["ferryServices"][0] - for t in services: - parsed = { - "rid" : t["rid"], - "uid" : t["uid"], - "head" : t["trainid"], - "platform": '?' if not "platform" in t else t["platform"], - "platform_hidden": "platformIsHidden" in t and t["platformIsHidden"], - "platform_prefix": "", - "toc": t["operatorCode"], - "cancelled" : t["isCancelled"] if "isCancelled" in t else False, - "delayed" : t["departureType"]=="Delayed" if "departureType" in t else None, - "cancel_reason" : t["cancelReason"]["value"] if "cancelReason" in t else "", - "delay_reason" : t["delayReason"]["value"] if "delayReason" in t else "", - "terminating" : not "std" in t and not "etd" in t and not "atd" in t, - "bus" : t["trainid"]=="0B00", - "times" : self.process(t), - "activity" : self.reduced_activities(t["activities"]), - } - parsed["destinations"] = [{"name": a["locationName"], "tiploc": a["tiploc"], - "crs": a["crs"] if "crs" in a else '', "code": a["crs"] if "crs" - in a else a["tiploc"], "via": a["via"] if "via" in a else ''} - for a in t["destination"][0]] - - parsed["origins"] = [{"name": a["locationName"], "tiploc": a["tiploc"], - "crs": a["crs"] if "crs" in a else '', "code": a["crs"] if "crs" - in a else a["tiploc"], "via": a["via"] if "via" in a else ''} - for a in t["origin"][0]] - - parsed["departure_only"] = location_code in [a["code"] for a in parsed["origins"]] - - if parsed["cancelled"] or parsed["delayed"]: - for k, time in parsed["times"].items(): - time["short"], time["on_time"], time["status"], time["prefix"] = ( - "%s:%s" % ("C" if parsed["cancel_reason"] else "D", parsed["cancel_reason"] or parsed["delay_reason"] or "?"), - False, 2, "" - ) - - trains.append(parsed) - - - for t in trains: - t["dest_summary"] = "/".join(["%s%s" %(a["code"]*filter["crs"] or a["name"], " " + a["via"] - if a["via"] else '') for a in t["destinations"]]) - t["origin_summary"] = "/".join(["%s%s" %(a["code"]*filter["crs"] or a["name"], " " + a["via"] - if a["via"] else '') for a in t["origins"]]) - - trains = sorted(trains, key=lambda t: t["times"]["max_sched"]["ut"] if filter["type"]=="both" else t["times"]["st" + filter["type"][0]]["ut"]) - - trains_filtered = [] - train_locs_toc = [] - - for train in trains: - if not True in [ - (train["destinations"], train["toc"]) in train_locs_toc and (filter["dedup"] or filter["default"]), - filter["dest"] and not filter["dest"].upper() in [a["code"] for a in train["destinations"]], - filter["origin"] and not filter["origin"].upper() in [a["code"] for a in train["origins"]], - filter["toc"] and not filter["toc"].upper() == train["toc"], - filter["plat"] and not filter["plat"] == train["platform"], - filter["type"] == "departure" and train["terminating"], - filter["type"] == "arrival" and train["departure_only"], - filter["terminating"] and not train["terminating"], - filter["tops"] and not filter["tops"] in train.get("tops_possible", []), - filter["power"] and not filter["power"]==train.get("power_type", None), - ]: - train_locs_toc.append((train["destinations"], train["toc"])) - trains_filtered.append(train) - if event.get("external"): - trains_string = "\n".join(["%-6s %-4s %-2s %-3s %1s%-6s %1s %s" % ( - t["uid"], t["head"], t["toc"], "bus" if t["bus"] else t["platform"], - "~" if t["times"]["both"]["estimate"] else '', - t["times"]["both"]["prefix"] + t["times"]["both"]["short"], - "←" if t["terminating"] or filter["type"]=="arrival" else "→", - t["origin_summary"] if t["terminating"] or filter["type"]=="arrival" else t["dest_summary"] - ) for t in trains_filtered]) - else: - trains_string = ", ".join(["%s%s (%s, %s%s%s%s, %s%s%s)" % ( - "from " if not filter["type"][0] in "ad" and t["terminating"] else '', - t["origin_summary"] if t["terminating"] or filter["type"]=="arrival" else t["dest_summary"], - t["uid"], - t["platform_prefix"], - "bus" if t["bus"] else t["platform"], - "*" if t["platform_hidden"] else '', - "?" if "platformsAreUnreliable" in query and query["platformsAreUnreliable"] else '', - t["times"][filter["type"]]["prefix"].replace(filter["type"][0], '') if not t["cancelled"] else "", - utils.irc.bold(utils.irc.color(t["times"][filter["type"]]["shortest"*filter["st"] or "short"], colours[t["times"][filter["type"]]["status"]])), - bool(t["activity"])*", " + "+".join(t["activity"]), - ) for t in trains_filtered]) - if event.get("external"): - event["stdout"].write("%s%s\n%s" % ( - station_summary, "\n calling at %s" % filter["inter"] if filter["inter"] else '', trains_string)) - else: - event["stdout"].write("%s%s: %s" % (station_summary, " departures calling at %s" % filter["inter"] if filter["inter"] else '', trains_string)) - - @utils.hook("received.command.nrservice", min_args=1) - def service(self, event): - """ - :help: Get train service information for a UID, headcode or RID - (Powered by NRE) - :usage: - """ - client = self.client - colours = self.COLOURS - external = event.get("external", False) - - SCHEDULE_STATUS = {"B": "perm bus", "F": "freight train", "P": "train", - "S": "ship", "T": "trip", "1": "train", "2": "freight", - "3": "trip", "4": "ship", "5": "bus"} - - schedule = {} - sources = [] - - service_id = event["args_split"][0] - - filter = self.filter(' '.join(event["args_split"][1:]) if len(event["args_split"]) > 1 else "", { - "passing": (False, lambda x: type(x)==type(True)), - "associations": (False, lambda x: type(x)==type(True)), - "type": ("arrival", lambda x: x in ["arrival", "departure"]) - }) - - if filter["errors"]: - raise utils.EventError("Filter: " + filter["errors_summary"]) - - rid = service_id - if len(service_id) <= 8: - query = client.service.QueryServices(service_id, datetime.utcnow().date().isoformat(), - datetime.utcnow().time().strftime("%H:%M:%S+0000")) - if not query and not schedule: - return event["stdout"].write("No service information is available for this identifier.") - - if query and len(query["serviceList"][0]) > 1: - return event["stdout"].write("Identifier refers to multiple services: " + - ", ".join(["%s (%s->%s)" % (a["uid"], a["originCrs"], a["destinationCrs"]) for a in query["serviceList"][0]])) - if query: rid = query["serviceList"][0][0]["rid"] - - if query: - sources.append("LDBSVWS") - query = client.service.GetServiceDetailsByRID(rid) - if schedule: - sources.append("Eagle/SCHEDULE") - if not query: query = {"trainid": schedule["signalling_id"] or "0000", "operator": schedule["operator_name"] or schedule["atoc_code"]} - stype = "%s %s" % (schedule_query.data["tops_inferred"], schedule["power_type"]) if schedule_query.data["tops_inferred"] else schedule["power_type"] - for k,v in { - "operatorCode": schedule["atoc_code"], - "serviceType": stype if stype else SCHEDULE_STATUS[schedule["status"]], - }.items(): - query[k] = v - - disruptions = [] - if "cancelReason" in query: - disruptions.append("Cancelled (%s%s)" % (query["cancelReason"]["value"], " at " + query["cancelReason"]["_tiploc"] if query["cancelReason"]["_tiploc"] else "")) - if "delayReason" in query: - disruptions.append("Delayed (%s%s)" % (query["delayReason"]["value"], " at " + query["delayReason"]["_tiploc"] if query["delayReason"]["_tiploc"] else "")) - if disruptions and not external: - disruptions = utils.irc.color(", ".join(disruptions), utils.consts.RED) + " " - elif disruptions and external: - disruptions = ", ".join(disruptions) - else: disruptions = "" - - stations = [] - for station in query["locations"][0] if "locations" in query else schedule["locations"]: - if "locations" in query: - parsed = {"name": station["locationName"], - "crs": (station["crs"] if "crs" in station else station["tiploc"]).rstrip(), - "tiploc": station["tiploc"].rstrip(), - "called": "atd" in station, - "passing": station["isPass"] if "isPass" in station else False, - "first": len(stations) == 0, - "last" : False, - "cancelled" : station["isCancelled"] if "isCancelled" in station else False, - "associations": [], - "length": station["length"] if "length" in station else None, - "times": self.process(station), - "platform": station["platform"] if "platform" in station else None, - "activity": self.activities(station["activities"]) if "activities" in station else [], - "activity_p": self.reduced_activities(station["activities"]) if "activities" in station else [], - } - - if parsed["cancelled"]: - parsed["times"]["arrival"].update({"short": "Cancelled", "on_time": False, "status": 2}) - parsed["times"]["departure"].update({"short": "Cancelled", "on_time": False, "status": 2}) - - associations = station["associations"][0] if "associations" in station else [] - for assoc in associations: - parsed_assoc = { - "uid_assoc": assoc.uid, - "category": {"divide": "VV", "join": "JJ", "next": "NP"}[assoc["category"]], - "from": parsed["first"], "direction": assoc["destTiploc"].rstrip()==parsed["tiploc"], - "origin_name": assoc["origin"], "origin_tiploc": assoc["originTiploc"], - "origin_crs": assoc["originCRS"] if "originCRS" in assoc else None, - - "dest_name": assoc["destination"], "dest_tiploc": assoc["destTiploc"], - "dest_crs": assoc["destCRS"] if "destCRS" in assoc else None, - - "far_name": assoc["destination"], "far_tiploc": assoc["destTiploc"], - "far_crs": assoc["destCRS"] if "destCRS" in assoc else None, - } - if parsed_assoc["direction"]: - parsed_assoc.update({"far_name": parsed_assoc["origin_name"], - "far_tiploc": parsed_assoc["origin_tiploc"], "far_crs": parsed_assoc["origin_crs"]}) - parsed["associations"].append(parsed_assoc) - else: - parsed = {"name": (station["name"] or "none"), - "crs": station["crs"] if station["crs"] else station["tiploc"], - "tiploc": station["tiploc"], - "called": False, - "passing": bool(station.get("pass")), - "first": len(stations) == 0, - "last" : False, - "cancelled" : False, - "length": None, - "times": self.process(station["dolphin_times"]), - "platform": station["platform"], - "associations": station["associations"] or [], - "activity": self.activities(station["activity"]), - "activity_p": self.reduced_activities(station["activity"]), - } - stations.append(parsed) - - [a for a in stations if a["called"] or a["first"]][-1]["last"] = True - - for station in stations[0:[k for k,v in enumerate(stations) if v["last"]][0]]: - if not station["first"]: station["called"] = True - - for station in stations: - for assoc in station["associations"]: - assoc["summary"] = "{arrow} {assoc[category]} {assoc[uid_assoc]} {dir_arrow} {assoc[far_name]} ({code})".format(assoc=assoc, arrow=assoc["from"]*"<-" or "->", dir_arrow=(assoc["direction"])*"<-" or "->", code=assoc["far_crs"] or assoc["far_tiploc"]) - - if station["passing"]: - station["times"]["arrival"]["status"], station["times"]["departure"]["status"] = 5, 5 - elif station["called"]: - station["times"]["arrival"]["status"], station["times"]["departure"]["status"] = 0, 0 - - station["summary"] = "%s%s (%s%s%s%s%s)%s" % ( - "*" * station["passing"], - station["name"], - station["crs"] + ", " if station["name"] != station["crs"] else '', - station["length"] + " car, " if station["length"] and (station["first"] or station["associations"]) else '', - ("~" if station["times"][filter["type"]]["estimate"] else '') + - station["times"][filter["type"]]["prefix"].replace(filter["type"][0], ""), - utils.irc.color(station["times"][filter["type"]]["short"], colours[station["times"][filter["type"]]["status"]]), - ", "*bool(station["activity_p"]) + "+".join(station["activity_p"]), - ", ".join([a["summary"] for a in station["associations"]] if filter["associations"] else ""), - ) - station["summary_external"] = "%1s%-5s %1s%-5s %-3s %-3s %-3s %s%s" % ( - "~"*station["times"]["a"]["estimate"] + "s"*(station["times"]["a"]["schedule"]), - station["times"]["a"]["short"], - "~"*station["times"]["d"]["estimate"] + "s"*(station["times"]["d"]["schedule"]), - station["times"]["d"]["short"], - station["platform"] or '', - ",".join(station["activity"]) or '', - station["crs"] or station["tiploc"], - station["name"], - "\n" + "\n".join([a["summary"] for a in station["associations"]]) if station["associations"] else "", - ) - - stations_filtered = [] - for station in stations: - if station["passing"] and not filter["passing"]: continue - if station["called"] and filter["default"] and not external: - if not station["first"] and not station["last"]: - continue - - stations_filtered.append(station) - if station["first"] and not station["last"] and filter["default"] and not external: - stations_filtered.append({"summary": "(...)", "summary_external": "(...)"}) - - done_count = len([s for s in stations if s["called"]]) - total_count = len(stations) - if external: - event["stdout"].write("%s: %s\n%s%s (%s) %s %s\n\n%s" % ( - service_id, ", ".join(sources), - disruptions + "\n" if disruptions else '', - query["operator"], query["operatorCode"], query["trainid"], query["serviceType"], - "\n".join([s["summary_external"] for s in stations_filtered]) - )) - else: - event["stdout"].write("%s%s %s %s (%s/%s): %s" % (disruptions, query["operatorCode"], - query["trainid"], query["serviceType"], - done_count, total_count, - ", ".join([s["summary"] for s in stations_filtered]))) - - @utils.hook("received.command.nrhead", min_args=1) - def head(self, event): - """ - :help: Get information for a given headcode/UID/RID (Powered by NRE) - :usage: - """ - client = self.client - service_id = event["args_split"][0] - - query = client.service.QueryServices(service_id, datetime.utcnow().date().isoformat(), - datetime.utcnow().time().strftime("%H:%M:%S+0000")) - - if not query: - raise utils.EventError("No currently running services match this " - "identifier") - - services = query["serviceList"][0] - if event.get("external"): - event["stdout"].write("\n".join(["{a.uid:6} {a.trainid:4} {a.originName} ({a.originCrs}) → {a.destinationName} ({a.destinationCrs})".format(a=a) for a in services])) - else: - event["stdout"].write(", ".join(["h/%s r/%s u/%s rs/%s %s (%s) -> %s (%s)" % (a["trainid"], a["rid"], a["uid"], a["rsid"], a["originName"], a["originCrs"], a["destinationName"], a["destinationCrs"]) for a in services])) - - @utils.hook("received.command.nrcode", min_args=1) - def service_code(self, event): - """ - :help: Get the text for a given delay/cancellation code (Powered by NRE) - :usage: - """ - - client = self.client - - if not event["args"].isnumeric(): - raise utils.EventError("The delay/cancellation code must be a " - "number") - reasons = {a["code"]:(a["lateReason"], a["cancReason"]) for a in client.service.GetReasonCodeList()[0]} - if event["args"] in reasons: - event["stdout"].write("%s: %s" % (event["args"], " / ".join(reasons[event["args"]]))) - else: - event["stdout"].write("This doesn't seem to be a valid reason code") From c4a30430d435e0fde2b7f660d261f7482cbf2395 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Sun, 19 Mar 2023 19:20:47 -0500 Subject: [PATCH 80/89] requirement no longer necessary per #360 --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 87613c42..a3155c0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,6 @@ PySocks ==1.7.1 python-dateutil ==2.8.1 pytz ==2019.2 requests ==2.22.0 -suds ==1.0.0 tornado ==6.0.3 tweepy ==3.8.0 requests-toolbelt ==0.9.1 From 25fde5b7c16c4d8151355fb486ddeaece005c9f9 Mon Sep 17 00:00:00 2001 From: musk <64192316+muskIRC@users.noreply.github.com> Date: Mon, 20 Mar 2023 01:31:38 +0100 Subject: [PATCH 81/89] Update nginx (#359) Fix proxy_set_header Host $port to $server_port --- docs/rest_api/nginx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rest_api/nginx b/docs/rest_api/nginx index 9f9af494..186965d7 100644 --- a/docs/rest_api/nginx +++ b/docs/rest_api/nginx @@ -8,7 +8,7 @@ server { location / { proxy_pass http://[::1]:5001; - proxy_set_header Host $host:$port; + proxy_set_header Host $host:$server_port; proxy_set_header X-Forwarded-For $remote_addr; } } From a3578664800840692f3c6a9ff7d16f1d42a7b8ff Mon Sep 17 00:00:00 2001 From: David Schultz Date: Sun, 19 Mar 2023 19:33:36 -0500 Subject: [PATCH 82/89] ignore.py: allow ignoring commands in channel (#356) --- src/core_modules/ignore.py | 52 ++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/src/core_modules/ignore.py b/src/core_modules/ignore.py index dd51a655..52aaa2bb 100644 --- a/src/core_modules/ignore.py +++ b/src/core_modules/ignore.py @@ -12,13 +12,24 @@ def _user_channel_ignored(self, channel, user): return channel.get_user_setting(user.get_id(), "ignore", False) def _server_command_ignored(self, server, command): return server.get_setting("ignore-%s" % command, False) + def _channel_command_ignored(self, channel, command): + return channel.get_setting("ignore-command-%s" % command, False) - def _is_command_ignored(self, server, user, command): - if self._user_command_ignored(user, command): + def _is_command_ignored(self, event): + if self._user_command_ignored(event["user"], event["command"]): return True - elif self._server_command_ignored(server, command): + elif self._server_command_ignored(event["server"], event["command"]): + return True + elif event["is_channel"] and self._channel_command_ignored(event["target"], event["command"]): return True + def _is_valid_command(self, command): + hooks = self.events.on("received.command").on(command).get_hooks() + if hooks: + return True + else: + return False + @utils.hook("received.message.private") @utils.hook("received.message.channel") @utils.hook("received.notice.private") @@ -38,8 +49,7 @@ def preprocess_command(self, event): elif event["is_channel"] and self._user_channel_ignored(event["target"], event["user"]): return utils.consts.PERMISSION_HARD_FAIL, None - elif self._is_command_ignored(event["server"], event["user"], - event["command"]): + elif self._is_command_ignored(event): return utils.consts.PERMISSION_HARD_FAIL, None @utils.hook("received.command.ignore", min_args=1) @@ -123,6 +133,38 @@ def cignore(self, event): True) event["stdout"].write("Ignoring %s" % target_user.nickname) + @utils.hook("received.command.ignorecommand", + help="Ignore a command in this channel") + @utils.hook("received.command.unignorecommand", + help="Unignore a command in this channel") + @utils.kwarg("channel_only", True) + @utils.kwarg("min_args", 1) + @utils.kwarg("usage", "") + @utils.kwarg("permission", "cignore") + @utils.kwarg("require_mode", "o") + @utils.kwarg("require_access", "high,cignore") + def cignore_command(self, event): + remove = event["command"] == "unignorecommand" + + command = event["args_split"][0] + if not self._is_valid_command(command): + raise utils.EventError("Unknown command '%s'" % command) + is_ignored = self._channel_command_ignored(event["target"], command) + + if remove: + if not is_ignored: + raise utils.EventError("I'm not ignoring '%s' in this channel" % + target_user.nickname) + event["target"].del_setting("ignore-command-%s" % command) + event["stdout"].write("Unignored '%s' command" % command) + else: + if is_ignored: + raise utils.EventError("I'm already ignoring '%s' in this channel" + % command) + event["target"].set_setting("ignore-command-%s" % command, True) + event["stdout"].write("Ignoring '%s' command" % command) + + @utils.hook("received.command.serverignore") @utils.kwarg("help", "Ignore a command on the current server") @utils.kwarg("permission", "serverignore") From e68b773f952e22337e112ec0d0350997ddd523fc Mon Sep 17 00:00:00 2001 From: vulpine Date: Tue, 18 Apr 2023 20:57:32 -0400 Subject: [PATCH 83/89] wolfram|alpha: squash newlines put in returned input (#361) oddly, wolfram|alpha's returned input string can sometimes contain newlines, especially when asking for distances between locations. previously this caused bitbot's output to get cut off, sending the remaining section to ,more --- modules/wolframalpha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/wolframalpha.py b/modules/wolframalpha.py index 72a649b0..fe765837 100644 --- a/modules/wolframalpha.py +++ b/modules/wolframalpha.py @@ -32,7 +32,7 @@ def wa(self, event): for pod in page["queryresult"]["pods"]: text = pod["subpods"][0]["plaintext"] if pod["id"] == "Input" and text: - input = text + input = text.replace("\n", " | ") elif pod.get("primary", False): primaries.append(text) From 0addf135ce212dc1445d4ab07a826ac65a895827 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 May 2023 10:53:27 -0500 Subject: [PATCH 84/89] Bump tornado from 6.0.3 to 6.3.2 (#366) * Bump requests from 2.22.0 to 2.31.0 (#365) Bumps [requests](https://github.com/psf/requests) from 2.22.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.22.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tornado from 6.0.3 to 6.3.2 Bumps [tornado](https://github.com/tornadoweb/tornado) from 6.0.3 to 6.3.2. - [Changelog](https://github.com/tornadoweb/tornado/blob/master/docs/releases.rst) - [Commits](https://github.com/tornadoweb/tornado/compare/v6.0.3...v6.3.2) --- updated-dependencies: - dependency-name: tornado dependency-type: direct:production ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Schultz --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a3155c0d..4b2bcad4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ netifaces ==0.10.9 PySocks ==1.7.1 python-dateutil ==2.8.1 pytz ==2019.2 -requests ==2.22.0 -tornado ==6.0.3 +requests ==2.31.0 +tornado ==6.3.2 tweepy ==3.8.0 requests-toolbelt ==0.9.1 From ababe8428a710501de68917f9a725f75f6ddf633 Mon Sep 17 00:00:00 2001 From: Val Lorentz Date: Wed, 14 Jun 2023 04:12:18 +0200 Subject: [PATCH 85/89] rss: Replace crashy double-formatting with standard format_token_replace (#370) * rss: Simplify entry formatting * Use format_token_replace * Apply suggestions * add `rss-format` config migration --------- Co-authored-by: David Schultz --- modules/rss.py | 52 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/modules/rss.py b/modules/rss.py index 4a23addc..af482555 100644 --- a/modules/rss.py +++ b/modules/rss.py @@ -1,7 +1,7 @@ #--depends-on config #--depends-on shorturl -import difflib, hashlib, time +import difflib, hashlib, time, re from src import ModuleManager, utils import feedparser @@ -13,22 +13,39 @@ "Interval (in seconds) between RSS polls", example="120")) @utils.export("channelset", utils.BoolSetting("rss-shorten", "Whether or not to shorten RSS urls")) -@utils.export("channelset", utils.Setting("rss-format", "Format of RSS announcements", example="$longtitle: $title - $link [$author]")) +@utils.export("channelset", utils.Setting("rss-format", "Format of RSS announcements", example="${longtitle}: ${title} - ${link} [${author}]")) @utils.export("serverset", SETTING_BIND) @utils.export("channelset", SETTING_BIND) class Module(ModuleManager.BaseModule): _name = "RSS" + def _migrate_formats(self): + count = 0 + migration_sub = re.compile(r"(?:\$|{)+(?P[^}\s]+)(?:})?") + old_formats = self.bot.database.execute_fetchall(""" + SELECT channel_id, value FROM channel_settings + WHERE setting = 'rss-format' + """) + + for channel_id, format in old_formats: + new_format = migration_re.sub(r"${\1}", format) + self.bot.database.execute(""" + UPDATE channel_settings SET value = ? + WHERE setting = 'rss-format' + AND channel_id = ? + """, [new_format, channel_id]) + count += 1 + + self.log.info("Successfully migrated %d rss-format settings" % count) + def on_load(self): + if self.bot.get_setting("rss-fmt-migration", False): + self.log.info("Attempting to migrate old rss-format settings") + self._migrate_formats() + self.bot.set_setting("rss-fmt-migration", True) self.timers.add("rss-feeds", self._timer, self.bot.get_setting("rss-interval", RSS_INTERVAL)) def _format_entry(self, server, channel, feed_title, entry, shorten): - title = utils.parse.line_normalise(utils.http.strip_html( - entry["title"])) - - author = entry.get("author", "unknown author") - author = "%s" % author if author else "" - link = entry.get("link", None) if shorten: try: @@ -37,16 +54,21 @@ def _format_entry(self, server, channel, feed_title, entry, shorten): pass link = "%s" % link if link else "" - feed_title_str = "%s" % feed_title if feed_title else "" + variables = dict( + longtitle=feed_title or "", + title=utils.parse.line_normalise(utils.http.strip_html( + entry["title"])), + link=link or "", + author=entry.get("author", "unknown author") or "", + ) + variables.update(entry) + # just in case the format starts keyerroring and you're not sure why self.log.trace("RSS Entry: " + str(entry)) - try: - format = channel.get_setting("rss-format", "$longtitle: $title by $author - $link").replace("$longtitle", feed_title_str).replace("$title", title).replace("$link", link).replace("$author", author).format(**entry) - except KeyError: - self.log.warn(f"Failed to format RSS entry for {channel}. Falling back to default format.") - format = f"{feed_title_str}: {title} by {author} - {link}" + template = channel.get_setting("rss-format", "${longtitle}: ${title} by ${author} - ${link}") + _, formatted = utils.parse.format_token_replace(template, variables) + return formatted - return format def _timer(self, timer): start_time = time.monotonic() From 59ed31f5d9be8779cb1f1288d26a5c2b6c870045 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Fri, 16 Jun 2023 21:47:00 -0500 Subject: [PATCH 86/89] rss: make format migration actually work --- modules/rss.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/rss.py b/modules/rss.py index af482555..677ca063 100644 --- a/modules/rss.py +++ b/modules/rss.py @@ -20,7 +20,7 @@ class Module(ModuleManager.BaseModule): _name = "RSS" def _migrate_formats(self): count = 0 - migration_sub = re.compile(r"(?:\$|{)+(?P[^}\s]+)(?:})?") + migration_re = re.compile(r"(?:\$|{)+(?P[^}\s]+)(?:})?") old_formats = self.bot.database.execute_fetchall(""" SELECT channel_id, value FROM channel_settings WHERE setting = 'rss-format' @@ -38,7 +38,7 @@ def _migrate_formats(self): self.log.info("Successfully migrated %d rss-format settings" % count) def on_load(self): - if self.bot.get_setting("rss-fmt-migration", False): + if not self.bot.get_setting("rss-fmt-migration", False): self.log.info("Attempting to migrate old rss-format settings") self._migrate_formats() self.bot.set_setting("rss-fmt-migration", True) From af2ff08c3ca5e6d8beebdcf3d428b2b174d1c4ef Mon Sep 17 00:00:00 2001 From: David Schultz Date: Fri, 16 Jun 2023 21:57:32 -0500 Subject: [PATCH 87/89] rss: tweak migration regex This pattern best reflects the custom formats currently in use --- modules/rss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/rss.py b/modules/rss.py index 677ca063..3a341dff 100644 --- a/modules/rss.py +++ b/modules/rss.py @@ -20,7 +20,7 @@ class Module(ModuleManager.BaseModule): _name = "RSS" def _migrate_formats(self): count = 0 - migration_re = re.compile(r"(?:\$|{)+(?P[^}\s]+)(?:})?") + migration_re = re.compile(r"(?:\$|{)+(?P[^}:\s]+)(?:})?") old_formats = self.bot.database.execute_fetchall(""" SELECT channel_id, value FROM channel_settings WHERE setting = 'rss-format' From 03ce25643151ca0dbedd58ecf85927c903704d0d Mon Sep 17 00:00:00 2001 From: David Schultz Date: Sun, 5 Jan 2025 04:59:40 -0600 Subject: [PATCH 88/89] some fixes for python 3.11 --- src/ModuleManager.py | 2 +- src/utils/http.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ModuleManager.py b/src/ModuleManager.py index a43181fe..034be660 100644 --- a/src/ModuleManager.py +++ b/src/ModuleManager.py @@ -235,7 +235,7 @@ def _load_module(self, bot: "IRCBot.Bot", definition: ModuleDefinition, definition.filename) module = importlib.util.module_from_spec(import_spec) sys.modules[import_name] = module - loader = typing.cast(importlib.abc.Loader, import_spec.loader) + loader = typing.cast(importlib._abc.Loader, import_spec.loader) loader.exec_module(module) module_object_pointer = getattr(module, "Module", None) diff --git a/src/utils/http.py b/src/utils/http.py index b07afbeb..822bc6a1 100644 --- a/src/utils/http.py +++ b/src/utils/http.py @@ -304,7 +304,7 @@ async def _request(request): loop = asyncio.new_event_loop() awaits = [] for request in requests: - awaits.append(_request(request)) + awaits.append(loop.create_task(_request(request))) task = asyncio.wait(awaits, timeout=5) loop.run_until_complete(task) loop.close() From 64a1153e9d5af2ed189fc4405d3d696884530a7e Mon Sep 17 00:00:00 2001 From: David Schultz Date: Thu, 9 Jan 2025 22:48:23 -0600 Subject: [PATCH 89/89] http.py: updates for latest python (`html5lib` -> `html.parser`) --- src/utils/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/http.py b/src/utils/http.py index 822bc6a1..2fb50cd0 100644 --- a/src/utils/http.py +++ b/src/utils/http.py @@ -131,7 +131,7 @@ def decode(self, encoding: typing.Optional[str]=None) -> str: return self.data.decode(encoding or self.encoding) def json(self) -> typing.Any: return _json.loads(self.data) - def soup(self, parser: str="html5lib") -> bs4.BeautifulSoup: + def soup(self, parser: str="html.parser") -> bs4.BeautifulSoup: return bs4.BeautifulSoup(self.decode(), parser) def _split_content(s: str) -> typing.Dict[str, str]: @@ -148,7 +148,7 @@ def _find_encoding(headers: typing.Dict[str, str], data: bytes if "charset" in content_header: return content_header["charset"] - soup = bs4.BeautifulSoup(data, "html5lib") + soup = bs4.BeautifulSoup(data, "html.parser") if not soup.meta == None: meta_charset = soup.meta.get("charset") if not meta_charset == None: