diff --git a/docs/bot.conf.example b/docs/bot.conf.example index fdb7a971..da22a1ee 100644 --- a/docs/bot.conf.example +++ b/docs/bot.conf.example @@ -94,3 +94,6 @@ bitly-api-key = # https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line github-token = + +# https://ipinfo.io/account/token +ipinfo-token = diff --git a/docs/help/config.md b/docs/help/config.md index 4db2b78a..b6542828 100644 --- a/docs/help/config.md +++ b/docs/help/config.md @@ -2,7 +2,7 @@ * Move `docs/bot.conf.example` to `~/.bitbot/bot.conf` and fill in the config options you care about. Ones blank or removed will disable relevant functionality. * Run `./bitbotd -a` to add a server. -* Run `./bitbotd` to start the bot. +* Run `./bitbotd` to start the bot or `./bitbotd -c /path/to/bot.conf` for non-standard config location (outside of `~/.bitbot`). * Run `./bitbotctl command master-password` to get the master admin password (needed to add regular admin accounts) * Join `#bitbot` on a server with the bot (or invite it to another channel) * `/msg register ` to register your nickname with the bot @@ -14,6 +14,9 @@ Generate a TLS keypair and point `bot.conf`'s `tls-key` to the private key and `tls-certificate` to the public key. +Below is an OpenSSL command example that will create a `bitbot-cert.pem` and `bitbot-key.pem` with `10y` validity (self-signed): +> openssl req -x509 -nodes -sha512 -newkey rsa:4096 -keyout bitbot-key.pem -out bitbot-cert.pem -days 3650 -subj "/CN=YourBotNick" + ### Configure SASL Configure the bot to use SASL to authenticate (usually used for `NickServ` identification) diff --git a/docs/help/rest_api.md b/docs/help/rest_api.md index feef26da..c4a2652a 100644 --- a/docs/help/rest_api.md +++ b/docs/help/rest_api.md @@ -15,7 +15,7 @@ Either set up a reverse proxy (with persisted Host header) with your favourite H #### Apache2 * Run `$ a2enmod ssl proxy proxy_http` as root * Copy example config file from [/docs/rest_api/apache2](/docs/rest_api/apache2) to `/etc/apache2/sites-enabled/` -* Edit `ServerName`, `SSLCertificateFile and `SSLCertificateKeyFile` +* Edit `ServerName`, `SSLCertificateFile` and `SSLCertificateKeyFile` * `$ service apache2 restart` as root #### Lighttpd diff --git a/docs/rest_api/apache2 b/docs/rest_api/apache2 index e6fb6759..9aadfd24 100644 --- a/docs/rest_api/apache2 +++ b/docs/rest_api/apache2 @@ -15,6 +15,6 @@ Listen 5000 ProxyRequests off ProxyPass / http://[::1]:5001/ - ProxyPassReverse / http://[::1]:5001 + ProxyPassReverse / http://[::1]:5001/ ProxyPreserveHost on diff --git a/docs/rest_api/nginx b/docs/rest_api/nginx index 9f9af494..186965d7 100644 --- a/docs/rest_api/nginx +++ b/docs/rest_api/nginx @@ -8,7 +8,7 @@ server { location / { proxy_pass http://[::1]:5001; - proxy_set_header Host $host:$port; + proxy_set_header Host $host:$server_port; proxy_set_header X-Forwarded-For $remote_addr; } } diff --git a/docs/systemd/bitbot_user.service b/docs/systemd/bitbot_user.service index eb397307..a3984f54 100644 --- a/docs/systemd/bitbot_user.service +++ b/docs/systemd/bitbot_user.service @@ -10,6 +10,8 @@ # which can be disabled with: systemctl --user disable systemd-tmpfiles-clean.timer # # After placing this script in the correct location, and with bitbot stopped, type: +# systemcl --user daemon-reload +# Afert that start bitbot with: # systemctl --user enable bitbot_user.service --now # This will enable the systemd script and launch bitbot diff --git a/modules/ban_enforce.py b/modules/ban_enforce.py index 4e40bb84..b504c9fc 100644 --- a/modules/ban_enforce.py +++ b/modules/ban_enforce.py @@ -4,6 +4,8 @@ @utils.export("channelset", utils.BoolSetting("ban-enforce", "Whether or not to parse new bans and kick who they affect")) +@utils.export("channelset", utils.IntSetting("ban-enforce-max", + "Do not enforce ban if the ban effects more than this many users. Default is half of total channel users.")) class Module(ModuleManager.BaseModule): @utils.hook("received.mode.channel") def on_mode(self, event): @@ -14,6 +16,10 @@ def on_mode(self, event): if mode[0] == "+" and mode[1] == "b": bans.append(arg) + affected = 0 + defaultmax = len(event["channel"].users) // 2 + realmax = event["channel"].get_setting("ban-enforce-max", defaultmax) + if bans: umasks = {u.hostmask(): u for u in event["channel"].users} for ban in bans: @@ -21,7 +27,10 @@ def on_mode(self, event): matches = list(utils.irc.hostmask_match_many( umasks.keys(), mask)) for match in matches: + affected = affected + 1 kicks.add(umasks[match]) if kicks: + if affected > realmax: + return nicks = [u.nickname for u in kicks] event["channel"].send_kicks(sorted(nicks), REASON) diff --git a/modules/channel_op.py b/modules/channel_op.py index d37e27b4..410b0236 100644 --- a/modules/channel_op.py +++ b/modules/channel_op.py @@ -83,7 +83,7 @@ def unmode(self, timer): channel = server.channels.get(channel_name) args = timer.kwargs.get("args", [timer.kwargs.get("arg", None)]) - if args: + if any(args): channel.send_modes(args, False) else: channel.send_mode(timer.kwargs["mode"], False) @@ -238,7 +238,7 @@ def cmute(self, event): if event["spec"][1]: self.timers.add_persistent("unmode", event["spec"][1], - channel=event["spec"][0].id, mode="m") + channel=event["spec"][0].id, mode="-m") @utils.hook("received.command.cunmute") @utils.kwarg("require_mode", "o") @utils.kwarg("require_access", "high,cmute") diff --git a/modules/dnsbl/__init__.py b/modules/dnsbl/__init__.py index 2b3daf35..96a21ee0 100644 --- a/modules/dnsbl/__init__.py +++ b/modules/dnsbl/__init__.py @@ -43,14 +43,22 @@ def _check_lists(self, lists, address): failed = [] for list in lists: record = self._check_list(list.hostname, address) - if not record == None: - reason = list.process(record) or "unknown" + if record is not None: + a_record, txt_record = record + reason = list.process(a_record, txt_record) or "unknown" failed.append((list.hostname, reason)) return failed def _check_list(self, list, address): list_address = "%s.%s" % (address, list) try: - return dns.resolver.query(list_address, "A")[0].to_text() + a_record = dns.resolver.resolve(list_address, "A")[0].to_text() except dns.resolver.NXDOMAIN: return None + + try: + txt_record = dns.resolver.resolve(list_address, "TXT")[0].to_text() + except: + txt_record = None + + return (a_record, txt_record) diff --git a/modules/dnsbl/lists.py b/modules/dnsbl/lists.py index b84628ea..e497b063 100644 --- a/modules/dnsbl/lists.py +++ b/modules/dnsbl/lists.py @@ -5,53 +5,62 @@ def __init__(self, hostname=None): if not hostname == None: self.hostname = hostname - def process(self, result: str): - return result + def process(self, a_record, txt_record): + out = a_record + if txt_record is not None: + out += f" - {txt_record}" + return out class ZenSpamhaus(DNSBL): hostname = "zen.spamhaus.org" - def process(self, result): - result = result.rsplit(".", 1)[1] + def process(self, a_record, txt_record): + result = a_record.rsplit(".", 1)[1] if result in ["2", "3", "9"]: - return "spam" + desc = "spam" elif result in ["4", "5", "6", "7"]: - return "exploits" + desc = "exploits" + else: + desc = "unknown" + return f"{result} - {desc}" + class EFNetRBL(DNSBL): hostname = "rbl.efnetrbl.org" - def process(self, result): - result = result.rsplit(".", 1)[1] + def process(self, a_record, txt_record): + result = a_record.rsplit(".", 1)[1] if result == "1": - return "proxy" + desc = "proxy" elif result in ["2", "3"]: - return "spamtap" + desc = "spamtap" elif result == "4": - return "tor" + desc = "tor" elif result == "5": - return "flooding" + desc = "flooding" + return f"{result} - {desc}" class DroneBL(DNSBL): hostname = "dnsbl.dronebl.org" - def process(self, result): - result = result.rsplit(".", 1)[1] - if result in ["8", "9", "10", "11", "14"]: - return "proxy" - elif result in ["3", "6", "7"]: - return "flooding" - elif result in ["12", "13", "15", "16"]: - return "exploits" class AbuseAtCBL(DNSBL): hostname = "cbl.abuseat.org" - def process(self, result): - result = result.rsplit(".", 1)[1] + def process(self, a_record, txt_record): + result = a_record.rsplit(".", 1)[1] if result == "2": - return "abuse" + desc = "abuse" + else: + desc = "unknown" + return f"{result} - {desc}" + +class TorExitDan(DNSBL): + hostname = "torexit.dan.me.uk" + def process(self, a_record, txt_record): + return "tor exit" DEFAULT_LISTS = [ ZenSpamhaus(), EFNetRBL(), DroneBL(), - AbuseAtCBL() + AbuseAtCBL(), + TorExitDan() ] def default_lists(): diff --git a/modules/ducks.py b/modules/ducks.py index eb6d7ef4..22a9bb99 100644 --- a/modules/ducks.py +++ b/modules/ducks.py @@ -98,6 +98,10 @@ def _no_duck(self, channel, user, stderr): @utils.kwarg("help", "Befriend a duck") @utils.spec("!-channelonly") def befriend(self, event): + if not event["target"].get_setting("ducks-enabled", False): + return event["stderr"].write( + "Ducks are not enabled in this channel" + ) if event["target"].duck_active: action = self._duck_action(event["target"], event["user"], "befriended", "ducks-befriended") @@ -109,6 +113,10 @@ def befriend(self, event): @utils.kwarg("help", "Trap a duck") @utils.spec("!-channelonly") def trap(self, event): + if not event["target"].get_setting("ducks-enabled", False): + return event["stderr"].write( + "Ducks are not enabled in this channel" + ) if event["target"].duck_active: action = self._duck_action(event["target"], event["user"], "trapped", "ducks-shot") diff --git a/modules/git_webhooks/__init__.py b/modules/git_webhooks/__init__.py index 44d9b5dc..8b44e198 100644 --- a/modules/git_webhooks/__init__.py +++ b/modules/git_webhooks/__init__.py @@ -31,7 +31,7 @@ class Module(ModuleManager.BaseModule): _name = "Webhooks" def on_load(self): - self._github = github.GitHub(self.log) + self._github = github.GitHub(self.log, self.exports) self._gitea = gitea.Gitea() self._gitlab = gitlab.GitLab() @@ -135,6 +135,10 @@ def _webhook(self, webhook_type, webhook_name, handler, payload_str, for output, url in outputs: output = "(%s) %s" % ( utils.irc.color(source, colors.COLOR_REPO), output) + + if channel.get_setting("git-prevent-highlight", False): + output = self._prevent_highlight(server, channel, + output) if url: if channel.get_setting("git-shorten-urls", False): @@ -142,10 +146,6 @@ def _webhook(self, webhook_type, webhook_name, handler, payload_str, context=channel) or url output = "%s - %s" % (output, url) - if channel.get_setting("git-prevent-highlight", False): - output = self._prevent_highlight(server, channel, - output) - hide_prefix = channel.get_setting("git-hide-prefix", False) self.events.on("send.stdout").call(target=channel, module_name=webhook_name, server=server, message=output, @@ -228,6 +228,9 @@ def github_webhook(self, event): if existing_hook: raise utils.EventError("There's already a hook for %s" % hook_name) + if hook_name == None: + command = "%s%s" % (event["command_prefix"], event["command"]) + raise utils.EventError("Not enough arguments (Usage: %s add )" % command) all_hooks[hook_name] = { "events": DEFAULT_EVENT_CATEGORIES.copy(), diff --git a/modules/git_webhooks/github.py b/modules/git_webhooks/github.py old mode 100644 new mode 100755 index 8bf61114..2e94f991 --- a/modules/git_webhooks/github.py +++ b/modules/git_webhooks/github.py @@ -5,6 +5,7 @@ COMMIT_RANGE_URL = "https://github.com/%s/compare/%s...%s" CREATE_URL = "https://github.com/%s/tree/%s" +PR_URL = "https://github.com/%s/pull/%s" PR_COMMIT_RANGE_URL = "https://github.com/%s/pull/%s/files/%s..%s" PR_COMMIT_URL = "https://github.com/%s/pull/%s/commits/%s" @@ -77,19 +78,19 @@ } COMMENT_MAX = 100 -CHECK_RUN_CONCLUSION = { - "success": "passed", - "failure": "failed", - "neutral": "finished", - "cancelled": "was cancelled", - "timed_out": "timed out", - "action_required": "requires action" +CHECK_SUITE_CONCLUSION = { + "success": ("passed", colors.COLOR_POSITIVE), + "failure": ("failed", colors.COLOR_NEGATIVE), + "neutral": ("finished", colors.COLOR_NEUTRAL), + "cancelled": ("was cancelled", colors.COLOR_NEGATIVE), + "timed_out": ("timed out", colors.COLOR_NEGATIVE), + "action_required": ("requires action", colors.COLOR_NEUTRAL) } -CHECK_RUN_FAILURES = ["failure", "cancelled", "timed_out", "action_required"] class GitHub(object): - def __init__(self, log): + def __init__(self, log, exports): self.log = log + self.exports = exports def is_private(self, data, headers): if "repository" in data: @@ -125,6 +126,8 @@ def event(self, data, headers): category_action = None if "review" in data and "state" in data["review"]: category = "%s+%s" % (event, data["review"]["state"]) + elif "check_suite" in data and "conclusion" in data["check_suite"]: + category = "%s+%s" % (event, data["check_suite"]["conclusion"]) if action: if category: @@ -159,8 +162,8 @@ def webhook(self, full_name, event, data, headers): out = self.delete(full_name, data) elif event == "release": out = self.release(full_name, data) - elif event == "check_run": - out = self.check_run(data) + elif event == "check_suite": + out = self.check_suite(full_name, data) elif event == "fork": out = self.fork(full_name, data) elif event == "ping": @@ -169,24 +172,13 @@ def webhook(self, full_name, event, data, headers): out = self.membership(organisation, data) elif event == "watch": out = self.watch(data) - return list(zip(out, [None]*len(out))) - - def _short_url(self, url): - self.log.debug("git.io shortening: %s" % url) - try: - page = utils.http.request("https://git.io", method="POST", - post_data={"url": url}) - return page.headers["Location"] - except utils.http.HTTPTimeoutException: - self.log.warn( - "HTTPTimeoutException while waiting for github short URL", []) - return url + return out def _iso8601(self, s): return utils.datetime.parse.iso8601(s) def ping(self, data): - return ["Received new webhook"] + return [("Received new webhook", None)] def _change_count(self, n, symbol, color): return utils.irc.color("%s%d" % (symbol, n), color)+utils.irc.bold("") @@ -230,21 +222,20 @@ def _format_push(self, branch, author, commits, forced, single_url, if len(commits) == 0 and forced: outputs.append( - "%s %spushed to %s" % (author, forced_str, branch)) + "%s %spushed to %s" % (author, forced_str, branch), None) elif len(commits) <= 3: for commit in commits: hash = commit["id"] hash_colored = utils.irc.color(self._short_hash(hash), colors.COLOR_ID) message = commit["message"].split("\n")[0].strip() - url = self._short_url(single_url % hash) + url = single_url % hash - outputs.append( - "%s %spushed %s to %s: %s - %s" - % (author, forced_str, hash_colored, branch, message, url)) + outputs.append(( + "%s %spushed %s to %s: %s" + % (author, forced_str, hash_colored, branch, message), url)) else: - outputs.append("%s %spushed %d commits to %s - %s" - % (author, forced_str, len(commits), branch, - self._short_url(range_url))) + outputs.append(("%s %spushed %d commits to %s" + % (author, forced_str, len(commits), branch), url)) return outputs @@ -262,9 +253,9 @@ def commit_comment(self, full_name, data): action = data["action"] commit = self._short_hash(data["comment"]["commit_id"]) commenter = utils.irc.bold(data["comment"]["user"]["login"]) - url = self._short_url(data["comment"]["html_url"]) - return ["[commit/%s] %s %s a comment - %s" % (commit, commenter, - action, url)] + url = data["comment"]["html_url"] + return [("[commit/%s] %s %s a comment" % (commit, commenter, + action), url)] def pull_request(self, full_name, data): raw_number = data["pull_request"]["number"] @@ -272,7 +263,7 @@ def pull_request(self, full_name, data): colored_branch = utils.irc.color(branch, colors.COLOR_BRANCH) sender = utils.irc.bold(data["sender"]["login"]) - author = utils.irc.bold(data["sender"]["login"]) + author = utils.irc.bold(data["pull_request"]["user"]["login"]) number = utils.irc.color("#%s" % data["pull_request"]["number"], colors.COLOR_ID) identifier = "%s by %s" % (number, author) @@ -328,9 +319,9 @@ def pull_request(self, full_name, data): action_desc = "renamed %s" % identifier pr_title = data["pull_request"]["title"] - url = self._short_url(data["pull_request"]["html_url"]) - return ["[PR] %s %s: %s - %s" % ( - sender, action_desc, pr_title, url)] + url = data["pull_request"]["html_url"] + return [("[PR] %s %s: %s" % ( + sender, action_desc, pr_title), url)] def pull_request_review(self, full_name, data): if not data["action"] == "submitted": @@ -348,7 +339,7 @@ def pull_request_review(self, full_name, data): action = data["action"] pr_title = data["pull_request"]["title"] reviewer = utils.irc.bold(data["sender"]["login"]) - url = self._short_url(data["review"]["html_url"]) + url = data["review"]["html_url"] state_desc = state if state == "approved": @@ -358,8 +349,8 @@ def pull_request_review(self, full_name, data): elif state == "dismissed": state_desc = "dismissed a review" - return ["[PR] %s %s on %s: %s - %s" % - (reviewer, state_desc, number, pr_title, url)] + return [("[PR] %s %s on %s: %s" % + (reviewer, state_desc, number, pr_title), url)] def pull_request_review_comment(self, full_name, data): number = utils.irc.color("#%s" % data["pull_request"]["number"], @@ -367,9 +358,9 @@ def pull_request_review_comment(self, full_name, data): action = data["action"] pr_title = data["pull_request"]["title"] sender = utils.irc.bold(data["sender"]["login"]) - url = self._short_url(data["comment"]["html_url"]) - return ["[PR] %s %s on a review on %s: %s - %s" % - (sender, COMMENT_ACTIONS[action], number, pr_title, url)] + url = data["comment"]["html_url"] + return [("[PR] %s %s on a review on %s: %s" % + (sender, COMMENT_ACTIONS[action], number, pr_title), url)] def issues(self, full_name, data): number = utils.irc.color("#%s" % data["issue"]["number"], @@ -383,9 +374,9 @@ def issues(self, full_name, data): issue_title = data["issue"]["title"] author = utils.irc.bold(data["sender"]["login"]) - url = self._short_url(data["issue"]["html_url"]) - return ["[issue] %s %s: %s - %s" % - (author, action_str, issue_title, url)] + url = data["issue"]["html_url"] + return [("[issue] %s %s: %s" % + (author, action_str, issue_title), url)] def issue_comment(self, full_name, data): if "changes" in data: # don't show this event when nothing has actually changed @@ -398,30 +389,29 @@ def issue_comment(self, full_name, data): type = "PR" if "pull_request" in data["issue"] else "issue" title = data["issue"]["title"] commenter = utils.irc.bold(data["sender"]["login"]) - url = self._short_url(data["comment"]["html_url"]) + url = data["comment"]["html_url"] body = "" if not action == "deleted": body = ": %s" % self._comment(data["comment"]["body"]) - return ["[%s] %s %s on %s (%s)%s - %s" % - (type, commenter, COMMENT_ACTIONS[action], number, title, body, - url)] + return [("[%s] %s %s on %s (%s)%s" % + (type, commenter, COMMENT_ACTIONS[action], number, title, body), url)] def create(self, full_name, data): ref = data["ref"] ref_color = utils.irc.color(ref, colors.COLOR_BRANCH) type = data["ref_type"] sender = utils.irc.bold(data["sender"]["login"]) - url = self._short_url(CREATE_URL % (full_name, ref)) - return ["%s created a %s: %s - %s" % (sender, type, ref_color, url)] + url = CREATE_URL % (full_name, ref) + return [("%s created a %s: %s" % (sender, type, ref_color), url)] def delete(self, full_name, data): ref = data["ref"] ref_color = utils.irc.color(ref, colors.COLOR_BRANCH) type = data["ref_type"] sender = utils.irc.bold(data["sender"]["login"]) - return ["%s deleted a %s: %s" % (sender, type, ref_color)] + return [("%s deleted a %s: %s" % (sender, type, ref_color), None)] def release(self, full_name, data): action = data["action"] @@ -430,60 +420,47 @@ def release(self, full_name, data): if name: name = ": %s" % name author = utils.irc.bold(data["release"]["author"]["login"]) - url = self._short_url(data["release"]["html_url"]) - return ["%s %s a release%s - %s" % (author, action, name, url)] + url = data["release"]["html_url"] + return [("%s %s a release%s" % (author, action, name), url)] + + def check_suite(self, full_name, data): + suite = data["check_suite"] - def check_run(self, data): - name = data["check_run"]["name"] - commit = self._short_hash(data["check_run"]["head_sha"]) + commit = self._short_hash(suite["head_sha"]) commit = utils.irc.color(commit, utils.consts.LIGHTBLUE) + pr = "" url = "" - if data["check_run"]["details_url"]: - url = data["check_run"]["details_url"] - url = " - %s" % self.exports.get("shorturl-any")(url) - - duration = "" - if data["check_run"]["completed_at"]: - started_at = self._iso8601(data["check_run"]["started_at"]) - completed_at = self._iso8601(data["check_run"]["completed_at"]) - if completed_at > started_at: - seconds = (completed_at-started_at).total_seconds() - duration = " in %s" % utils.datetime.format.to_pretty_time( - seconds) - - status = data["check_run"]["status"] - status_str = "" - if status == "queued": - status_str = utils.irc.bold("queued") - elif status == "in_progress": - status_str = utils.irc.bold("started") - elif status == "completed": - conclusion = data["check_run"]["conclusion"] - conclusion_color = colors.COLOR_POSITIVE - if conclusion in CHECK_RUN_FAILURES: - conclusion_color = colors.COLOR_NEGATIVE - if conclusion == "neutral": - conclusion_color = colors.COLOR_NEUTRAL - - status_str = utils.irc.color( - CHECK_RUN_CONCLUSION[conclusion], conclusion_color) - - return ["[build @%s] %s: %s%s%s" % ( - commit, name, status_str, duration, url)] + if suite["pull_requests"]: + pr_num = suite["pull_requests"][0]["number"] + pr = "/PR%s" % utils.irc.color("#%s" % pr_num, colors.COLOR_ID) + url = PR_URL % (full_name, pr_num) + + name = suite["app"]["name"] + conclusion = suite["conclusion"] + conclusion, conclusion_color = CHECK_SUITE_CONCLUSION[conclusion] + conclusion = utils.irc.color(conclusion, conclusion_color) + + created_at = self._iso8601(suite["created_at"]) + updated_at = self._iso8601(suite["updated_at"]) + seconds = (updated_at-created_at).total_seconds() + duration = utils.datetime.format.to_pretty_time(seconds) + + return [("[build @%s%s] %s: %s in %s" % ( + commit, pr, name, conclusion, duration), url)] def fork(self, full_name, data): forker = utils.irc.bold(data["sender"]["login"]) fork_full_name = utils.irc.color(data["forkee"]["full_name"], utils.consts.LIGHTBLUE) - url = self._short_url(data["forkee"]["html_url"]) - return ["%s forked into %s - %s" % - (forker, fork_full_name, url)] + url = data["forkee"]["html_url"] + return [("%s forked into %s" % + (forker, fork_full_name), url)] def membership(self, organisation, data): - return ["%s %s %s to team %s" % + return [("%s %s %s to team %s" % (data["sender"]["login"], data["action"], data["member"]["login"], - data["team"]["name"])] + data["team"]["name"]), None)] def watch(self, data): - return ["%s starred the repository" % data["sender"]["login"]] + return [("%s starred the repository" % data["sender"]["login"], None)] diff --git a/modules/github.py b/modules/github.py index b528ad2b..4945b57c 100644 --- a/modules/github.py +++ b/modules/github.py @@ -47,14 +47,7 @@ def _parse_ref(self, channel, ref, sep): return org, repo, number def _short_url(self, url): - try: - page = utils.http.request("https://git.io", method="POST", - post_data={"url": url}) - return page.headers["Location"] - except utils.http.HTTPTimeoutException: - self.log.warn( - "HTTPTimeoutException while waiting for github short URL", []) - return url + return self.exports.get("shorturl")(self.bot, url) or url def _change_count(self, n, symbol, color): return utils.irc.color("%s%d" % (symbol, n), color)+utils.irc.bold("") diff --git a/modules/ip_addresses.py b/modules/ip_addresses.py index 73ec061f..09d46b5b 100644 --- a/modules/ip_addresses.py +++ b/modules/ip_addresses.py @@ -5,6 +5,7 @@ import dns.resolver URL_GEOIP = "http://ip-api.com/json/%s" +URL_IPINFO = "https://ipinfo.io/%s/json" REGEX_IPv6 = r"(?:(?:[a-f0-9]{1,4}:){2,}|[a-f0-9:]*::)[a-f0-9:]*" REGEX_IPv4 = r"(?:\d{1,3}\.){3}\d{1,3}" REGEX_IP = re.compile("%s|%s" % (REGEX_IPv4, REGEX_IPv6), re.I) @@ -21,6 +22,25 @@ def _parse(value): @utils.export("channelset", utils.FunctionSetting(_parse, "dns-nameserver", "Set DNS nameserver", example="8.8.8.8")) class Module(ModuleManager.BaseModule): + def _get_ip(self, event): + ip = event["args_split"][0] if event["args"] else "" + if not ip: + line = event["target"].buffer.find(REGEX_IP) + if line: + ip = line.match + if not ip: + raise utils.EventError("No IP provided") + return ip + + def _ipinfo_get(self, url): + access_token = self.bot.config.get("ipinfo-token", None) + headers = {} + if not access_token == None: + headers["Authorization"] = "Bearer %s" % access_token + request = utils.http.Request(url, headers=headers) + return utils.http.request(request) + + @utils.hook("received.command.dig", alias_of="dns") @utils.hook("received.command.dns", min_args=1) def dns(self, event): """ @@ -55,7 +75,7 @@ def dns(self, event): for record_type in record_types: record_type_strip = record_type.rstrip("?").upper() try: - query_result = resolver.query(hostname, record_type_strip, + query_result = resolver.resolve(hostname, record_type_strip, lifetime=4) query_results = [q.to_text() for q in query_result] results.append([record_type_strip, query_result.rrset.ttl, @@ -78,27 +98,79 @@ def dns(self, event): (t, ttl, ", ".join(r)) for t, ttl, r in results] event["stdout"].write("(%s) %s" % (hostname, " | ".join(results_str))) - @utils.hook("received.command.geoip", min_args=1) + @utils.hook("received.command.geoip") def geoip(self, event): """ - :help: Get geoip data on a given IPv4/IPv6 address + :help: Get GeoIP data on a given IPv4/IPv6 address :usage: :prefix: GeoIP """ - page = utils.http.request(URL_GEOIP % event["args_split"][0]).json() + ip = self._get_ip(event) + + page = utils.http.request(URL_GEOIP % ip).json() if page: if page["status"] == "success": + hostname = None + try: + hostname, alias, ips = socket.gethostbyaddr(page["query"]) + except (socket.herror, socket.gaierror): + pass + data = page["query"] + data += " (%s)" % hostname if hostname else "" data += " | Organisation: %s" % page["org"] data += " | City: %s" % page["city"] data += " | Region: %s (%s)" % ( page["regionName"], page["countryCode"]) - data += " | ISP: %s" % page["isp"] + data += " | ISP: %s (%s)" % (page["isp"], page["as"]) data += " | Lon/Lat: %s/%s" % (page["lon"], page["lat"]) data += " | Timezone: %s" % page["timezone"] event["stdout"].write(data) else: - event["stderr"].write("No geoip data found") + event["stderr"].write("No GeoIP data found") + else: + raise utils.EventResultsError() + + @utils.hook("received.command.ipinfo") + def ipinfo(self, event): + """ + :help: Get IPinfo.io data on a given IPv4/IPv6 address + :usage: + :prefix: IPinfo + """ + ip = self._get_ip(event) + + page = self._ipinfo_get(URL_IPINFO % ip).json() + if page: + if page.get("error", False): + if isinstance(page["error"], (list, dict)): + event["stderr"].write(page["error"]["message"]) + else: + event["stderr"].write(page["error"]) + elif page.get("ip", False): + bogon = page.get("bogon", False) + hostname = page.get("hostname", None) + if not hostname and not bogon: + try: + hostname, alias, ips = socket.gethostbyaddr(page["ip"]) + except (socket.herror, socket.gaierror): + pass + + data = page["ip"] + if bogon: + data += " (Bogon)" + else: + data += " (%s)" % hostname if hostname else "" + data += " (Anycast)" if page.get("anycast", False) == True else "" + if page.get("country", False): + data += " | City: %s" % page["city"] + data += " | Region: %s (%s)" % (page["region"], page["country"]) + data += " | ISP: %s" % page.get("org", "Unknown") + data += " | Lon/Lat: %s" % page["loc"] + data += " | Timezone: %s" % page["timezone"] + event["stdout"].write(data) + else: + event["stderr"].write("Unsupported endpoint") else: raise utils.EventResultsError() @@ -109,13 +181,7 @@ def rdns(self, event): :usage: :prefix: rDNS """ - ip = event["args_split"][0] if event["args"] else "" - if not ip: - line = event["target"].buffer.find(REGEX_IP) - if line: - ip = line.match - if not ip: - raise utils.EventError("No IP provided") + ip = self._get_ip(event) try: hostname, alias, ips = socket.gethostbyaddr(ip) diff --git a/modules/ircv3_botignore.py b/modules/ircv3_botignore.py index 968e6fb1..42ece9c9 100644 --- a/modules/ircv3_botignore.py +++ b/modules/ircv3_botignore.py @@ -1,11 +1,22 @@ from src import EventManager, ModuleManager, utils -TAG = utils.irc.MessageTag(None, "inspircd.org/bot") +TAGS = { + utils.irc.MessageTag(None, "inspircd.org/bot"), + utils.irc.MessageTag(None, "draft/bot") +} class Module(ModuleManager.BaseModule): + @utils.hook("received.376") + @utils.hook("received.422") + def botmode(self, event): + if "BOT" in event["server"].isupport: + botmode = event["server"].isupport["BOT"] + event["server"].send_raw("MODE %s +%s" % (event["server"].nickname, botmode)) + @utils.hook("received.message.private") @utils.hook("received.message.channel") @utils.kwarg("priority", EventManager.PRIORITY_HIGH) def message(self, event): - if TAG.present(event["tags"]): - event.eat() + for tag in TAGS: + if tag.present(event["tags"]): + event.eat() diff --git a/modules/karma.py b/modules/karma.py index 33359123..e21d9130 100644 --- a/modules/karma.py +++ b/modules/karma.py @@ -14,6 +14,18 @@ @utils.export("channelset", utils.BoolSetting("karma-pattern", "Enable/disable parsing ++/-- karma format")) class Module(ModuleManager.BaseModule): + def listify(self, items): + if type(items) != list: + items = list(items) + listified = "" + if len(items) > 2: + listified = ', '.join(items[:-1]) + ', and ' + items[-1] + elif len(items) > 1: + listified = items[0] + ' and ' + items[1] + elif items: + listified = items[0] + return listified + def _karma_str(self, karma): karma_str = str(karma) if karma < 0: @@ -66,7 +78,8 @@ def _change_karma(self, server, sender, target, positive): self._set_throttle(sender, positive) karma_str = self._karma_str(karma) - karma_total = self._karma_str(self._get_karma(server, target)) + karma_total = sum(self._get_karma(server, target).values()) + karma_total = self._karma_str(karma_total) return True, "%s now has %s karma (%s from %s)" % ( target, karma_total, karma_str, sender.nickname) @@ -118,18 +131,35 @@ def karma(self, event): target = event["user"].nickname target = self._get_target(event["server"], target) - karma = self._karma_str(self._get_karma(event["server"], target)) + karma = sum(self._get_karma(event["server"], target).values()) + karma = self._karma_str(karma) event["stdout"].write("%s has %s karma" % (target, karma)) - def _get_karma(self, server, target): + @utils.hook("received.command.karmawho") + @utils.spec("!string") + def karmawho(self, event): + target = event["server"].irc_lower(event["spec"][0]) + karma = self._get_karma(event["server"], target, True) + karma = sorted(list(karma.items()), + key=lambda k: abs(k[1]), + reverse=True) + + parts = ["%s (%d)" % (n, v) for n, v in karma] + if len(parts) == 0: + event["stdout"].write("%s has no karma." % target) + return + event["stdout"].write("%s has karma from: %s" % + (target, self.listify(parts))) + + def _get_karma(self, server, target, own=False): settings = dict(server.get_all_user_settings("karma-%s" % target)) target_lower = server.irc_lower(target) - if target_lower in settings: + if target_lower in settings and not own: del settings[target_lower] - return sum(settings.values()) + return settings @utils.hook("received.command.resetkarma") @utils.kwarg("min_args", 2) diff --git a/modules/lastfm.py b/modules/lastfm.py index 6931e0c0..a11c0d93 100644 --- a/modules/lastfm.py +++ b/modules/lastfm.py @@ -82,7 +82,10 @@ def np(self, event): tags_str = "" if "toptags" in track and track["toptags"]["tag"]: - tags = [t["name"] for t in track["toptags"]["tag"]] + tags_list = track["toptags"]["tag"] + if not type(tags_list) == list: + tags_list = [tags_list] + tags = [t["name"] for t in tags_list] tags_str = " [%s]" % ", ".join(tags) play_count_str = "" diff --git a/modules/location.py b/modules/location.py index 08c8b1fd..33811d8e 100644 --- a/modules/location.py +++ b/modules/location.py @@ -2,6 +2,7 @@ #--require-config opencagedata-api-key import typing +import pytz from src import ModuleManager, utils URL_OPENCAGE = "https://api.opencagedata.com/geocode/v1/json" @@ -19,6 +20,11 @@ def _get_location(self, s): if page and page["results"]: result = page["results"][0] timezone = result["annotations"]["timezone"]["name"] + try: + pytz.timezone(timezone) + except pytz.exceptions.UnknownTimeZoneError: + return None + lat = result["geometry"]["lat"] lon = result["geometry"]["lng"] diff --git a/modules/message_filter.py b/modules/message_filter.py index f773ad36..11cdcde6 100644 --- a/modules/message_filter.py +++ b/modules/message_filter.py @@ -15,6 +15,10 @@ def _get_filters(self, server, target): @utils.hook("preprocess.send.privmsg") @utils.hook("preprocess.send.notice") def channel_message(self, event): + if event["line"].assured(): + # don't run filters/replaces against assured lines + return + message = event["line"].args[1] original_message = message message_plain = utils.irc.strip_font(message) diff --git a/modules/nr.py b/modules/nr.py deleted file mode 100644 index 7c7303d0..00000000 --- a/modules/nr.py +++ /dev/null @@ -1,518 +0,0 @@ -#--depends-on commands -#--require-config nre-api-key - -import collections, re, time -from datetime import datetime, date -from collections import Counter - -from src import ModuleManager, utils - -from suds.client import Client -from suds import WebFault - -# Note that this module requires the open *Staff Version* of the Darwin API -# You can register for an API key here: http://openldbsv.nationalrail.co.uk/ -# We use this instead of the 'regular' version because it offers a *lot* more -# information. - -URL = 'https://lite.realtime.nationalrail.co.uk/OpenLDBSVWS/wsdl.aspx?ver=2016-02-16' - -class Module(ModuleManager.BaseModule): - _name = "NR" - _client = None - - PASSENGER_ACTIVITIES = ["U", "P", "R"] - COLOURS = [utils.consts.LIGHTBLUE, utils.consts.GREEN, - utils.consts.RED, utils.consts.CYAN, utils.consts.LIGHTGREY, - utils.consts.ORANGE] - - @property - def client(self): - if self._client: return self._client - try: - token = self.bot.config["nre-api-key"] - client = Client(URL) - header_token = client.factory.create('ns2:AccessToken') - header_token.TokenValue = token - client.set_options(soapheaders=header_token) - self._client = client - except Exception as e: - pass - return self._client - - def filter(self, args, defaults): - args = re.findall(r"[^\s,]+", args) - params = {} - - for arg in args: - if ":" in arg: - params[arg.split(":", 1)[0]] = arg.split(":", 1)[1] - elif "=" in arg: - params[arg.split("=", 1)[0]] = arg.split("=", 1)[1] - else: - params[arg.replace("!", "")] = '!' not in arg - - ret = {k: v[0] for k,v in defaults.items()} - ret["default"] = True - ret["errors"] = [] - - for k,v in params.items(): - if not k in defaults.keys(): - ret["errors"].append((k, "Invalid parameter")) - continue - if not defaults[k][1](v): - ret["errors"].append((v, 'Invalid value for "%s"' % k)) - continue - ret["default"] = False - ret[k] = v if len(defaults[k]) == 2 else defaults[k][2](v) - ret["errors_summary"] = ", ".join(['"%s": %s' % (a[0], a[1]) for a in ret["errors"]]) - return ret - - def process(self, service): - ut_now = datetime.now().timestamp() - nonetime = {"orig": None, "datetime": None, "ut": 0, - "short": ' ', "prefix": '', "on_time": False, - "estimate": False, "status": 4, "schedule": False} - times = {} - a_types = ["eta", "ata", "sta"] - d_types = ["etd", "atd", "std"] - - for a in a_types + d_types: - if a in service and service[a]: - times[a] = {"orig": service[a]} - - if len(service[a]) > 5: - times[a]["datetime"] = datetime.strptime(service[a], "%Y-%m-%dT%H:%M:%S") - else: - times[a]["datetime"] = datetime.strptime( - datetime.now().date().isoformat() + "T" + service[a][:4], - "%Y-%m-%dT%H%M" - ) - times[a]["ut"] = times[a]["datetime"].timestamp() - else: - times[a] = nonetime - - for k, a in times.items(): - if not a["orig"]: continue - a["short"] = a["datetime"].strftime("%H%M") if len(a["orig"]) > 5 else a["orig"] - a["shortest"] = "%02d" % a["datetime"].minute if -300 < a["ut"]-ut_now < 1800 else a["short"] - a["prefix"] = k[2] + ("s" if k[0] == "s" else "") - a["estimate"] = k[0] == "e" - a["schedule"] = k[0] == "s" - a["on_time"] = a["ut"] - times["s"+ k[1:]]["ut"] < 300 - a["status"] = 1 if a["on_time"] else 2 - if "a" + k[1:] in service: a["status"] = {"d": 0, "a": 3}[k[2]] - if k[0] == "s": a["status"] = 4 - - arr, dep = [times[a] for a in a_types if times[a]["ut"]], [times[a] for a in d_types if times[a]["ut"]] - times["arrival"] = (arr + dep + [nonetime])[0] - times["departure"] = (dep + arr + [nonetime])[0] - times["a"], times["d"] = (arr + [nonetime])[0], (dep + [nonetime])[0] - times["both"] = times["departure"] - times["max_sched"] = {"ut": max(times["sta"]["ut"], times["std"]["ut"])} - return times - - def activities(self, string): return [a+b.strip() for a,b in list(zip(*[iter(string)]*2)) if (a+b).strip()] - - def reduced_activities(self, string): return [a for a in self.activities(string) if a in self.PASSENGER_ACTIVITIES] - - @utils.hook("received.command.nrtrains", min_args=1) - def trains(self, event): - """ - :help: Get train/bus services for a station (Powered by NRE) - :usage: - """ - - client = self.client - colours = self.COLOURS - - schedule = {} - - location_code = event["args_split"][0].upper() - filter = self.filter(' '.join(event["args_split"][1:]) if len(event["args_split"]) > 1 else "", { - "dest": ('', lambda x: x.isalpha() and len(x)==3), - "origin":('', lambda x: x.isalpha() and len(x)==3), - "inter": ('', lambda x: x.isalpha() and len(x)==3, lambda x: x.upper()), - "toc": ('', lambda x: x.isalpha() and len(x) == 2), - "dedup": (False, lambda x: type(x)==type(True)), - "plat": ('', lambda x: len(x) <= 3), - "type": ("departure", lambda x: x in ["departure", "arrival", "both"]), - "terminating": (False, lambda x: type(x)==type(True)), - "period": (120, lambda x: x.isdigit() and 1 <= int(x) <= 480, lambda x: int(x)), - "nonpassenger": (False, lambda x: type(x)==type(True)), - "time": ("", lambda x: len(x)==4 and x.isdigit()), - "date": ("", lambda x: len(x)==10), - "tops": (None, lambda x: len(x)<4 and x.isdigit()), - "power": (None, lambda x: x.upper() in ["EMU", "DMU", "HST", "D", "E", "DEM"], lambda x: x.upper()), - "crs": (False, lambda x: type(x)==type(True)), - "st": (False, lambda x: type(x)==type(True)) - }) - - if filter["errors"]: - raise utils.EventError("Filter: " + filter["errors_summary"]) - - if filter["inter"] and filter["type"]!="departure": - raise utils.EventError("Filtering by intermediate stations is only " - "supported for departures.") - - nr_filterlist = client.factory.create("filterList") - if filter["inter"]: nr_filterlist.crs.append(filter["inter"]) - - now = datetime.now() - if filter["time"]: - now = now.replace(hour=int(filter["time"][:2])) - now = now.replace(minute=int(filter["time"][2:])) - if filter["date"]: - newdate = datetime.strptime(filter["date"], "%Y-%m-%d").date() - now = now.replace(day=newdate.day, month=newdate.month, year=newdate.year) - - method = client.service.GetArrivalDepartureBoardByCRS if len(location_code) == 3 else client.service.GetArrivalDepartureBoardByTIPLOC - try: - query = method(100, location_code, now.isoformat().split(".")[0], filter["period"], - nr_filterlist, "to", '', "PBS", filter["nonpassenger"]) - except WebFault as detail: - if str(detail) == "Server raised fault: 'Invalid crs code supplied'": - raise utils.EventError("Invalid CRS code.") - else: - raise utils.EventError("An error occurred.") - - nrcc_severe = len([a for a in query["nrccMessages"][0] if a["severity"] == "Major"]) if "nrccMessages" in query else 0 - if event.get("external"): - station_summary = "%s (%s) - %s (%s):\n" % (query["locationName"], query["crs"], query["stationManager"], - query["stationManagerCode"]) - else: - severe_summary = "" - if nrcc_severe: - severe_summary += ", " - severe_summary += utils.irc.bold(utils.irc.color("%s severe messages" % nrcc_severe, utils.consts.RED)) - station_summary = "%s (%s, %s%s)" % (query["locationName"], query["crs"], query["stationManagerCode"], severe_summary) - - if not "trainServices" in query and not "busServices" in query and not "ferryServices" in query: - return event["stdout"].write("%s: No services for the next %s minutes" % ( - station_summary, filter["period"])) - - trains = [] - services = [] - if "trainServices" in query: services += query["trainServices"][0] - if "busServices" in query: services += query["busServices"][0] - if "ferryServices" in query: services += query["ferryServices"][0] - for t in services: - parsed = { - "rid" : t["rid"], - "uid" : t["uid"], - "head" : t["trainid"], - "platform": '?' if not "platform" in t else t["platform"], - "platform_hidden": "platformIsHidden" in t and t["platformIsHidden"], - "platform_prefix": "", - "toc": t["operatorCode"], - "cancelled" : t["isCancelled"] if "isCancelled" in t else False, - "delayed" : t["departureType"]=="Delayed" if "departureType" in t else None, - "cancel_reason" : t["cancelReason"]["value"] if "cancelReason" in t else "", - "delay_reason" : t["delayReason"]["value"] if "delayReason" in t else "", - "terminating" : not "std" in t and not "etd" in t and not "atd" in t, - "bus" : t["trainid"]=="0B00", - "times" : self.process(t), - "activity" : self.reduced_activities(t["activities"]), - } - parsed["destinations"] = [{"name": a["locationName"], "tiploc": a["tiploc"], - "crs": a["crs"] if "crs" in a else '', "code": a["crs"] if "crs" - in a else a["tiploc"], "via": a["via"] if "via" in a else ''} - for a in t["destination"][0]] - - parsed["origins"] = [{"name": a["locationName"], "tiploc": a["tiploc"], - "crs": a["crs"] if "crs" in a else '', "code": a["crs"] if "crs" - in a else a["tiploc"], "via": a["via"] if "via" in a else ''} - for a in t["origin"][0]] - - parsed["departure_only"] = location_code in [a["code"] for a in parsed["origins"]] - - if parsed["cancelled"] or parsed["delayed"]: - for k, time in parsed["times"].items(): - time["short"], time["on_time"], time["status"], time["prefix"] = ( - "%s:%s" % ("C" if parsed["cancel_reason"] else "D", parsed["cancel_reason"] or parsed["delay_reason"] or "?"), - False, 2, "" - ) - - trains.append(parsed) - - - for t in trains: - t["dest_summary"] = "/".join(["%s%s" %(a["code"]*filter["crs"] or a["name"], " " + a["via"] - if a["via"] else '') for a in t["destinations"]]) - t["origin_summary"] = "/".join(["%s%s" %(a["code"]*filter["crs"] or a["name"], " " + a["via"] - if a["via"] else '') for a in t["origins"]]) - - trains = sorted(trains, key=lambda t: t["times"]["max_sched"]["ut"] if filter["type"]=="both" else t["times"]["st" + filter["type"][0]]["ut"]) - - trains_filtered = [] - train_locs_toc = [] - - for train in trains: - if not True in [ - (train["destinations"], train["toc"]) in train_locs_toc and (filter["dedup"] or filter["default"]), - filter["dest"] and not filter["dest"].upper() in [a["code"] for a in train["destinations"]], - filter["origin"] and not filter["origin"].upper() in [a["code"] for a in train["origins"]], - filter["toc"] and not filter["toc"].upper() == train["toc"], - filter["plat"] and not filter["plat"] == train["platform"], - filter["type"] == "departure" and train["terminating"], - filter["type"] == "arrival" and train["departure_only"], - filter["terminating"] and not train["terminating"], - filter["tops"] and not filter["tops"] in train.get("tops_possible", []), - filter["power"] and not filter["power"]==train.get("power_type", None), - ]: - train_locs_toc.append((train["destinations"], train["toc"])) - trains_filtered.append(train) - if event.get("external"): - trains_string = "\n".join(["%-6s %-4s %-2s %-3s %1s%-6s %1s %s" % ( - t["uid"], t["head"], t["toc"], "bus" if t["bus"] else t["platform"], - "~" if t["times"]["both"]["estimate"] else '', - t["times"]["both"]["prefix"] + t["times"]["both"]["short"], - "←" if t["terminating"] or filter["type"]=="arrival" else "→", - t["origin_summary"] if t["terminating"] or filter["type"]=="arrival" else t["dest_summary"] - ) for t in trains_filtered]) - else: - trains_string = ", ".join(["%s%s (%s, %s%s%s%s, %s%s%s)" % ( - "from " if not filter["type"][0] in "ad" and t["terminating"] else '', - t["origin_summary"] if t["terminating"] or filter["type"]=="arrival" else t["dest_summary"], - t["uid"], - t["platform_prefix"], - "bus" if t["bus"] else t["platform"], - "*" if t["platform_hidden"] else '', - "?" if "platformsAreUnreliable" in query and query["platformsAreUnreliable"] else '', - t["times"][filter["type"]]["prefix"].replace(filter["type"][0], '') if not t["cancelled"] else "", - utils.irc.bold(utils.irc.color(t["times"][filter["type"]]["shortest"*filter["st"] or "short"], colours[t["times"][filter["type"]]["status"]])), - bool(t["activity"])*", " + "+".join(t["activity"]), - ) for t in trains_filtered]) - if event.get("external"): - event["stdout"].write("%s%s\n%s" % ( - station_summary, "\n calling at %s" % filter["inter"] if filter["inter"] else '', trains_string)) - else: - event["stdout"].write("%s%s: %s" % (station_summary, " departures calling at %s" % filter["inter"] if filter["inter"] else '', trains_string)) - - @utils.hook("received.command.nrservice", min_args=1) - def service(self, event): - """ - :help: Get train service information for a UID, headcode or RID - (Powered by NRE) - :usage: - """ - client = self.client - colours = self.COLOURS - external = event.get("external", False) - - SCHEDULE_STATUS = {"B": "perm bus", "F": "freight train", "P": "train", - "S": "ship", "T": "trip", "1": "train", "2": "freight", - "3": "trip", "4": "ship", "5": "bus"} - - schedule = {} - sources = [] - - service_id = event["args_split"][0] - - filter = self.filter(' '.join(event["args_split"][1:]) if len(event["args_split"]) > 1 else "", { - "passing": (False, lambda x: type(x)==type(True)), - "associations": (False, lambda x: type(x)==type(True)), - "type": ("arrival", lambda x: x in ["arrival", "departure"]) - }) - - if filter["errors"]: - raise utils.EventError("Filter: " + filter["errors_summary"]) - - rid = service_id - if len(service_id) <= 8: - query = client.service.QueryServices(service_id, datetime.utcnow().date().isoformat(), - datetime.utcnow().time().strftime("%H:%M:%S+0000")) - if not query and not schedule: - return event["stdout"].write("No service information is available for this identifier.") - - if query and len(query["serviceList"][0]) > 1: - return event["stdout"].write("Identifier refers to multiple services: " + - ", ".join(["%s (%s->%s)" % (a["uid"], a["originCrs"], a["destinationCrs"]) for a in query["serviceList"][0]])) - if query: rid = query["serviceList"][0][0]["rid"] - - if query: - sources.append("LDBSVWS") - query = client.service.GetServiceDetailsByRID(rid) - if schedule: - sources.append("Eagle/SCHEDULE") - if not query: query = {"trainid": schedule["signalling_id"] or "0000", "operator": schedule["operator_name"] or schedule["atoc_code"]} - stype = "%s %s" % (schedule_query.data["tops_inferred"], schedule["power_type"]) if schedule_query.data["tops_inferred"] else schedule["power_type"] - for k,v in { - "operatorCode": schedule["atoc_code"], - "serviceType": stype if stype else SCHEDULE_STATUS[schedule["status"]], - }.items(): - query[k] = v - - disruptions = [] - if "cancelReason" in query: - disruptions.append("Cancelled (%s%s)" % (query["cancelReason"]["value"], " at " + query["cancelReason"]["_tiploc"] if query["cancelReason"]["_tiploc"] else "")) - if "delayReason" in query: - disruptions.append("Delayed (%s%s)" % (query["delayReason"]["value"], " at " + query["delayReason"]["_tiploc"] if query["delayReason"]["_tiploc"] else "")) - if disruptions and not external: - disruptions = utils.irc.color(", ".join(disruptions), utils.consts.RED) + " " - elif disruptions and external: - disruptions = ", ".join(disruptions) - else: disruptions = "" - - stations = [] - for station in query["locations"][0] if "locations" in query else schedule["locations"]: - if "locations" in query: - parsed = {"name": station["locationName"], - "crs": (station["crs"] if "crs" in station else station["tiploc"]).rstrip(), - "tiploc": station["tiploc"].rstrip(), - "called": "atd" in station, - "passing": station["isPass"] if "isPass" in station else False, - "first": len(stations) == 0, - "last" : False, - "cancelled" : station["isCancelled"] if "isCancelled" in station else False, - "associations": [], - "length": station["length"] if "length" in station else None, - "times": self.process(station), - "platform": station["platform"] if "platform" in station else None, - "activity": self.activities(station["activities"]) if "activities" in station else [], - "activity_p": self.reduced_activities(station["activities"]) if "activities" in station else [], - } - - if parsed["cancelled"]: - parsed["times"]["arrival"].update({"short": "Cancelled", "on_time": False, "status": 2}) - parsed["times"]["departure"].update({"short": "Cancelled", "on_time": False, "status": 2}) - - associations = station["associations"][0] if "associations" in station else [] - for assoc in associations: - parsed_assoc = { - "uid_assoc": assoc.uid, - "category": {"divide": "VV", "join": "JJ", "next": "NP"}[assoc["category"]], - "from": parsed["first"], "direction": assoc["destTiploc"].rstrip()==parsed["tiploc"], - "origin_name": assoc["origin"], "origin_tiploc": assoc["originTiploc"], - "origin_crs": assoc["originCRS"] if "originCRS" in assoc else None, - - "dest_name": assoc["destination"], "dest_tiploc": assoc["destTiploc"], - "dest_crs": assoc["destCRS"] if "destCRS" in assoc else None, - - "far_name": assoc["destination"], "far_tiploc": assoc["destTiploc"], - "far_crs": assoc["destCRS"] if "destCRS" in assoc else None, - } - if parsed_assoc["direction"]: - parsed_assoc.update({"far_name": parsed_assoc["origin_name"], - "far_tiploc": parsed_assoc["origin_tiploc"], "far_crs": parsed_assoc["origin_crs"]}) - parsed["associations"].append(parsed_assoc) - else: - parsed = {"name": (station["name"] or "none"), - "crs": station["crs"] if station["crs"] else station["tiploc"], - "tiploc": station["tiploc"], - "called": False, - "passing": bool(station.get("pass")), - "first": len(stations) == 0, - "last" : False, - "cancelled" : False, - "length": None, - "times": self.process(station["dolphin_times"]), - "platform": station["platform"], - "associations": station["associations"] or [], - "activity": self.activities(station["activity"]), - "activity_p": self.reduced_activities(station["activity"]), - } - stations.append(parsed) - - [a for a in stations if a["called"] or a["first"]][-1]["last"] = True - - for station in stations[0:[k for k,v in enumerate(stations) if v["last"]][0]]: - if not station["first"]: station["called"] = True - - for station in stations: - for assoc in station["associations"]: - assoc["summary"] = "{arrow} {assoc[category]} {assoc[uid_assoc]} {dir_arrow} {assoc[far_name]} ({code})".format(assoc=assoc, arrow=assoc["from"]*"<-" or "->", dir_arrow=(assoc["direction"])*"<-" or "->", code=assoc["far_crs"] or assoc["far_tiploc"]) - - if station["passing"]: - station["times"]["arrival"]["status"], station["times"]["departure"]["status"] = 5, 5 - elif station["called"]: - station["times"]["arrival"]["status"], station["times"]["departure"]["status"] = 0, 0 - - station["summary"] = "%s%s (%s%s%s%s%s)%s" % ( - "*" * station["passing"], - station["name"], - station["crs"] + ", " if station["name"] != station["crs"] else '', - station["length"] + " car, " if station["length"] and (station["first"] or station["associations"]) else '', - ("~" if station["times"][filter["type"]]["estimate"] else '') + - station["times"][filter["type"]]["prefix"].replace(filter["type"][0], ""), - utils.irc.color(station["times"][filter["type"]]["short"], colours[station["times"][filter["type"]]["status"]]), - ", "*bool(station["activity_p"]) + "+".join(station["activity_p"]), - ", ".join([a["summary"] for a in station["associations"]] if filter["associations"] else ""), - ) - station["summary_external"] = "%1s%-5s %1s%-5s %-3s %-3s %-3s %s%s" % ( - "~"*station["times"]["a"]["estimate"] + "s"*(station["times"]["a"]["schedule"]), - station["times"]["a"]["short"], - "~"*station["times"]["d"]["estimate"] + "s"*(station["times"]["d"]["schedule"]), - station["times"]["d"]["short"], - station["platform"] or '', - ",".join(station["activity"]) or '', - station["crs"] or station["tiploc"], - station["name"], - "\n" + "\n".join([a["summary"] for a in station["associations"]]) if station["associations"] else "", - ) - - stations_filtered = [] - for station in stations: - if station["passing"] and not filter["passing"]: continue - if station["called"] and filter["default"] and not external: - if not station["first"] and not station["last"]: - continue - - stations_filtered.append(station) - if station["first"] and not station["last"] and filter["default"] and not external: - stations_filtered.append({"summary": "(...)", "summary_external": "(...)"}) - - done_count = len([s for s in stations if s["called"]]) - total_count = len(stations) - if external: - event["stdout"].write("%s: %s\n%s%s (%s) %s %s\n\n%s" % ( - service_id, ", ".join(sources), - disruptions + "\n" if disruptions else '', - query["operator"], query["operatorCode"], query["trainid"], query["serviceType"], - "\n".join([s["summary_external"] for s in stations_filtered]) - )) - else: - event["stdout"].write("%s%s %s %s (%s/%s): %s" % (disruptions, query["operatorCode"], - query["trainid"], query["serviceType"], - done_count, total_count, - ", ".join([s["summary"] for s in stations_filtered]))) - - @utils.hook("received.command.nrhead", min_args=1) - def head(self, event): - """ - :help: Get information for a given headcode/UID/RID (Powered by NRE) - :usage: - """ - client = self.client - service_id = event["args_split"][0] - - query = client.service.QueryServices(service_id, datetime.utcnow().date().isoformat(), - datetime.utcnow().time().strftime("%H:%M:%S+0000")) - - if not query: - raise utils.EventError("No currently running services match this " - "identifier") - - services = query["serviceList"][0] - if event.get("external"): - event["stdout"].write("\n".join(["{a.uid:6} {a.trainid:4} {a.originName} ({a.originCrs}) → {a.destinationName} ({a.destinationCrs})".format(a=a) for a in services])) - else: - event["stdout"].write(", ".join(["h/%s r/%s u/%s rs/%s %s (%s) -> %s (%s)" % (a["trainid"], a["rid"], a["uid"], a["rsid"], a["originName"], a["originCrs"], a["destinationName"], a["destinationCrs"]) for a in services])) - - @utils.hook("received.command.nrcode", min_args=1) - def service_code(self, event): - """ - :help: Get the text for a given delay/cancellation code (Powered by NRE) - :usage: - """ - - client = self.client - - if not event["args"].isnumeric(): - raise utils.EventError("The delay/cancellation code must be a " - "number") - reasons = {a["code"]:(a["lateReason"], a["cancReason"]) for a in client.service.GetReasonCodeList()[0]} - if event["args"] in reasons: - event["stdout"].write("%s: %s" % (event["args"], " / ".join(reasons[event["args"]]))) - else: - event["stdout"].write("This doesn't seem to be a valid reason code") diff --git a/modules/pronouns.py b/modules/pronouns.py index f800dad5..55b0a8d8 100644 --- a/modules/pronouns.py +++ b/modules/pronouns.py @@ -4,7 +4,7 @@ from src import ModuleManager, utils @utils.export("set", utils.Setting("pronouns", "Set your pronouns", - example="she/her")) + example="they/them")) class Module(ModuleManager.BaseModule): @utils.hook("received.command.pronouns") def pronouns(self, event): diff --git a/modules/quotes.py b/modules/quotes.py index e323cc9b..727f6fd6 100644 --- a/modules/quotes.py +++ b/modules/quotes.py @@ -5,6 +5,9 @@ @utils.export("channelset", utils.BoolSetting("channel-quotes", "Whether or not quotes added from this channel are kept in this channel")) +@utils.export("set", utils.BoolSetting("quotable", + "Whether or not you wish to be quoted")) + class Module(ModuleManager.BaseModule): def category_and_quote(self, s): category, sep, quote = s.partition("=") @@ -31,6 +34,11 @@ def quote_add(self, event): "channel-quotes", False): target = event["target"] + if not event["server"].get_user(category).get_setting( + "quotable", True): + event["stderr"].write("%s does not wish to be quoted" % category) + return + quotes = self._get_quotes(target, category) quotes.append([event["user"].name, int(time.time()), quote]) self._set_quotes(target, category, quotes) @@ -148,6 +156,10 @@ def quote_grab(self, event): text = " ".join(lines_str) quote_category = line.sender + if not event["server"].get_user(quote_category).get_setting( + "quotable", True): + event["stderr"].write("%s does not wish to be quoted" % quote_category) + return if event["server"].has_user(quote_category): quote_category = event["server"].get_user_nickname( event["server"].get_user(quote_category).get_id()) diff --git a/modules/rss.py b/modules/rss.py index 9ba23db4..3a341dff 100644 --- a/modules/rss.py +++ b/modules/rss.py @@ -1,7 +1,7 @@ #--depends-on config #--depends-on shorturl -import difflib, hashlib, time +import difflib, hashlib, time, re from src import ModuleManager, utils import feedparser @@ -9,37 +9,66 @@ SETTING_BIND = utils.Setting("rss-bindhost", "Which local address to bind to for RSS requests", example="127.0.0.1") - @utils.export("botset", utils.IntSetting("rss-interval", "Interval (in seconds) between RSS polls", example="120")) @utils.export("channelset", utils.BoolSetting("rss-shorten", "Whether or not to shorten RSS urls")) +@utils.export("channelset", utils.Setting("rss-format", "Format of RSS announcements", example="${longtitle}: ${title} - ${link} [${author}]")) @utils.export("serverset", SETTING_BIND) @utils.export("channelset", SETTING_BIND) class Module(ModuleManager.BaseModule): _name = "RSS" + def _migrate_formats(self): + count = 0 + migration_re = re.compile(r"(?:\$|{)+(?P[^}:\s]+)(?:})?") + old_formats = self.bot.database.execute_fetchall(""" + SELECT channel_id, value FROM channel_settings + WHERE setting = 'rss-format' + """) + + for channel_id, format in old_formats: + new_format = migration_re.sub(r"${\1}", format) + self.bot.database.execute(""" + UPDATE channel_settings SET value = ? + WHERE setting = 'rss-format' + AND channel_id = ? + """, [new_format, channel_id]) + count += 1 + + self.log.info("Successfully migrated %d rss-format settings" % count) + def on_load(self): + if not self.bot.get_setting("rss-fmt-migration", False): + self.log.info("Attempting to migrate old rss-format settings") + self._migrate_formats() + self.bot.set_setting("rss-fmt-migration", True) self.timers.add("rss-feeds", self._timer, self.bot.get_setting("rss-interval", RSS_INTERVAL)) - def _format_entry(self, server, feed_title, entry, shorten): - title = utils.parse.line_normalise(utils.http.strip_html( - entry["title"])) - - author = entry.get("author", None) - author = " by %s" % author if author else "" - + def _format_entry(self, server, channel, feed_title, entry, shorten): link = entry.get("link", None) if shorten: try: link = self.exports.get("shorturl")(server, link) except: pass - link = " - %s" % link if link else "" + link = "%s" % link if link else "" + + variables = dict( + longtitle=feed_title or "", + title=utils.parse.line_normalise(utils.http.strip_html( + entry["title"])), + link=link or "", + author=entry.get("author", "unknown author") or "", + ) + variables.update(entry) - feed_title_str = "%s: " % feed_title if feed_title else "" + # just in case the format starts keyerroring and you're not sure why + self.log.trace("RSS Entry: " + str(entry)) + template = channel.get_setting("rss-format", "${longtitle}: ${title} by ${author} - ${link}") + _, formatted = utils.parse.format_token_replace(template, variables) + return formatted - return "%s%s%s%s" % (feed_title_str, title, author, link) def _timer(self, timer): start_time = time.monotonic() @@ -106,7 +135,7 @@ def _timer(self, timer): valid += 1 shorten = channel.get_setting("rss-shorten", False) - output = self._format_entry(server, feed_title, entry, + output = self._format_entry(server, channel, feed_title, entry, shorten) self.events.on("send.stdout").call(target=channel, @@ -200,10 +229,10 @@ def rss(self, event): title, entries = self._get_entries(url) if not entries: - raise utils.EventError("Failed to get RSS entries") + raise utils.EventError("%s has no entries" % url) shorten = event["target"].get_setting("rss-shorten", False) - out = self._format_entry(event["server"], title, entries[0], + out = self._format_entry(event["server"], event["target"], title, entries[0], shorten) event["stdout"].write(out) else: diff --git a/modules/sed.py b/modules/sed.py index af053ec3..79f2fcb9 100644 --- a/modules/sed.py +++ b/modules/sed.py @@ -4,7 +4,7 @@ import re, traceback from src import ModuleManager, utils -REGEX_SED = re.compile("^(?:(\\S+)[:,] )?s/") +REGEX_SED = re.compile(r"^(?:(\S+)[:,] )?s([/,`#]).*\2") @utils.export("channelset", utils.BoolSetting("sed","Disable/Enable sed in a channel")) @@ -35,7 +35,7 @@ def channel_message(self, event): sed.replace = utils.irc.bold(sed.replace) if self._closest_setting(event, "sed-sender-only", False): - for_user = event["user"].nickname + for_user = event["user"].nickname_lower match_line = None match_message = None diff --git a/modules/shorturl.py b/modules/shorturl.py index 345d2183..23ed4571 100644 --- a/modules/shorturl.py +++ b/modules/shorturl.py @@ -4,7 +4,7 @@ import re from src import ModuleManager, utils -URL_BITLYSHORTEN = "https://api-ssl.bitly.com/v3/shorten" +URL_BITLYSHORTEN = "https://api-ssl.bitly.com/v4/shorten" class Module(ModuleManager.BaseModule): def on_load(self): @@ -41,7 +41,7 @@ def _call_shortener(self, server, context, shortener_name, url): @utils.export("shorturl-any") def _shorturl_any(self, url): - return self._call_shortener(server, None, "bitly", url) or url + return self._call_shortener(None, None, "bitly", url) or url @utils.export("shorturl") def _shorturl(self, server, url, context=None): @@ -66,11 +66,16 @@ def _bitly(self, url): access_token = self.bot.config.get("bitly-api-key", None) if access_token: - page = utils.http.request(URL_BITLYSHORTEN, get_params={ - "access_token": access_token, "longUrl": url}).json() - - if page["data"]: - return page["data"]["url"] + resp = utils.http.request( + URL_BITLYSHORTEN, + method="POST", + post_data={"long_url": url}, + json_body=True, + headers={"Authorization": f"Bearer {access_token}"} + ) + + if resp.code == 200: + return resp.json()["link"] return None def _find_url(self, target, args): @@ -112,4 +117,4 @@ def unshorten(self, event): event["stdout"].write("Unshortened: %s" % response.headers["location"]) else: - event["stderr"].write("Failed to unshorten URL") + event["stderr"].write("Failed to unshorten URL") \ No newline at end of file diff --git a/modules/tweets/format.py b/modules/tweets/format.py index 9648dc51..0817370a 100644 --- a/modules/tweets/format.py +++ b/modules/tweets/format.py @@ -2,7 +2,7 @@ from src import utils def _timestamp(dt): - seconds_since = time.time()-dt.timestamp() + seconds_since = time.time()-dt.replace(tzinfo=datetime.timezone.utc).timestamp() timestamp = utils.datetime.format.to_pretty_since( seconds_since, max_units=2) return "%s ago" % timestamp diff --git a/modules/urbandictionary.py b/modules/urbandictionary.py index a5098c3f..d7b43332 100644 --- a/modules/urbandictionary.py +++ b/modules/urbandictionary.py @@ -2,7 +2,7 @@ from src import ModuleManager, utils -URL_URBANDICTIONARY = "http://api.urbandictionary.com/v0/define" +URL_URBANDICTIONARY = "https://api.urbandictionary.com/v0/define" class Module(ModuleManager.BaseModule): _name = "UrbanDictionary" diff --git a/modules/weather.py b/modules/weather.py index bd6b5e8b..3af09be3 100644 --- a/modules/weather.py +++ b/modules/weather.py @@ -69,6 +69,7 @@ def weather(self, event): celsius = "%dC" % page["main"]["temp"] fahrenheit = "%dF" % ((page["main"]["temp"]*(9/5))+32) + kelvin = "%dK" % ((page["main"]["temp"])+273.15) description = page["weather"][0]["description"].title() humidity = "%s%%" % page["main"]["humidity"] @@ -81,10 +82,11 @@ def weather(self, event): location_str = "(%s) %s" % (nickname, location_str) event["stdout"].write( - "%s | %s/%s | %s | Humidity: %s | Wind: %s/%s" % ( - location_str, celsius, fahrenheit, description, humidity, - wind_speed_k, wind_speed_m)) + "%s | %s/%s/%s | %s | Humidity: %s | Wind: %s/%s" % ( + location_str, celsius, fahrenheit, kelvin, description, + humidity, wind_speed_k, wind_speed_m)) else: event["stderr"].write("No weather information for this location") else: raise utils.EventResultsError() + diff --git a/modules/wolframalpha.py b/modules/wolframalpha.py index 72a649b0..fe765837 100644 --- a/modules/wolframalpha.py +++ b/modules/wolframalpha.py @@ -32,7 +32,7 @@ def wa(self, event): for pod in page["queryresult"]["pods"]: text = pod["subpods"][0]["plaintext"] if pod["id"] == "Input" and text: - input = text + input = text.replace("\n", " | ") elif pod.get("primary", False): primaries.append(text) diff --git a/modules/youtube.py b/modules/youtube.py index 99c7ba98..4e5e3bc8 100644 --- a/modules/youtube.py +++ b/modules/youtube.py @@ -36,46 +36,49 @@ def _number(self, n): def video_details(self, video_id): page = self.get_video_page(video_id) - if page["items"]: - item = page["items"][0] - snippet = item["snippet"] - statistics = item["statistics"] - content = item["contentDetails"] - - video_uploaded_at = utils.datetime.parse.iso8601( - snippet["publishedAt"]) - video_uploaded_at = utils.datetime.format.date_human( - video_uploaded_at) - - video_uploader = snippet["channelTitle"] - video_title = utils.irc.bold(snippet["title"]) - - video_views = self._number(statistics.get("viewCount")) - video_likes = self._number(statistics.get("likeCount")) - video_dislikes = self._number(statistics.get("dislikeCount")) - - video_opinions = "" - if video_likes and video_dislikes: - likes = utils.irc.color("%s%s" % (video_likes, ARROW_UP), - utils.consts.GREEN) - dislikes = utils.irc.color("%s%s" % - (ARROW_DOWN, video_dislikes), utils.consts.RED) - video_opinions = " (%s%s)" % (likes, dislikes) - - video_views_str = "" - if video_views: - video_views_str = ", %s views" % video_views - - td = utils.datetime.parse.iso8601_duration(content["duration"]) - video_duration = utils.datetime.format.to_pretty_time( - td.total_seconds()) - - url = URL_YOUTUBESHORT % video_id - - return "%s (%s) uploaded by %s on %s%s%s" % ( - video_title, video_duration, video_uploader, video_uploaded_at, - video_views_str, video_opinions), url - return None + try: + if page["items"]: + item = page["items"][0] + snippet = item["snippet"] + statistics = item["statistics"] + content = item["contentDetails"] + + video_uploaded_at = utils.datetime.parse.iso8601( + snippet["publishedAt"]) + video_uploaded_at = utils.datetime.format.date_human( + video_uploaded_at) + + video_uploader = snippet["channelTitle"] + video_title = utils.irc.bold(snippet["title"]) + + video_views = self._number(statistics.get("viewCount")) + video_likes = self._number(statistics.get("likeCount")) + video_dislikes = self._number(statistics.get("dislikeCount")) + + video_opinions = "" + if video_likes and video_dislikes: + likes = utils.irc.color("%s%s" % (video_likes, ARROW_UP), + utils.consts.GREEN) + dislikes = utils.irc.color("%s%s" % + (ARROW_DOWN, video_dislikes), utils.consts.RED) + video_opinions = " (%s%s)" % (likes, dislikes) + + video_views_str = "" + if video_views: + video_views_str = ", %s views" % video_views + + td = utils.datetime.parse.iso8601_duration(content["duration"]) + video_duration = utils.datetime.format.to_pretty_time( + td.total_seconds()) + + url = URL_YOUTUBESHORT % video_id + + return "%s (%s) uploaded by %s on %s%s%s" % ( + video_title, video_duration, video_uploader, video_uploaded_at, + video_views_str, video_opinions), url + return None + except KeyError: + return None def get_playlist_page(self, playlist_id): self.log.debug("youtube API request: " @@ -86,16 +89,19 @@ def get_playlist_page(self, playlist_id): "key": self.bot.config["google-api-key"]}).json() def playlist_details(self, playlist_id): page = self.get_playlist_page(playlist_id) - if page["items"]: - item = page["items"][0] - snippet = item["snippet"] - content = item["contentDetails"] + try: + if page["items"]: + item = page["items"][0] + snippet = item["snippet"] + content = item["contentDetails"] - count = content["itemCount"] + count = content["itemCount"] - return "%s - %s (%s %s)" % (snippet["channelTitle"], - snippet["title"], count, "video" if count == 1 else "videos" - ), URL_PLAYLIST % playlist_id + return "%s - %s (%s %s)" % (snippet["channelTitle"], + snippet["title"], count, "video" if count == 1 else "videos" + ), URL_PLAYLIST % playlist_id + except KeyError: + return None def _from_url(self, url): parsed = urllib.parse.urlparse(url) @@ -148,23 +154,27 @@ def yt(self, event): from_url = not url == None - if not url: - safe_setting = event["target"].get_setting("youtube-safesearch", True) - safe = "moderate" if safe_setting else "none" - self.log.debug("youtube API request: search.list (B) [snippet]") + try: + if not url: + safe_setting = event["target"].get_setting("youtube-safesearch", True) + safe = "moderate" if safe_setting else "none" - search_page = utils.http.request(URL_YOUTUBESEARCH, - get_params={"q": search, "part": "snippet", "maxResults": "1", - "type": "video", "key": self.bot.config["google-api-key"], - "safeSearch": safe}).json() - if search_page: - if search_page["pageInfo"]["totalResults"] > 0: - url = URL_VIDEO % search_page["items"][0]["id"]["videoId"] + self.log.debug("youtube API request: search.list (B) [snippet]") + + search_page = utils.http.request(URL_YOUTUBESEARCH, + get_params={"q": search, "part": "snippet", "maxResults": "1", + "type": "video", "key": self.bot.config["google-api-key"], + "safeSearch": safe}).json() + if search_page: + if search_page["pageInfo"]["totalResults"] > 0: + url = URL_VIDEO % search_page["items"][0]["id"]["videoId"] + else: + raise utils.EventError("No videos found") else: - raise utils.EventError("No videos found") - else: - raise utils.EventResultsError() + raise utils.EventResultsError() + except KeyError: + raise utils.EventError("API error") if url: out = self._from_url(url) diff --git a/requirements.txt b/requirements.txt index 7a160547..4b2bcad4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,9 +10,7 @@ netifaces ==0.10.9 PySocks ==1.7.1 python-dateutil ==2.8.1 pytz ==2019.2 -requests ==2.22.0 -scrypt ==0.8.13 -suds-jurko ==0.6 -tornado ==6.0.3 +requests ==2.31.0 +tornado ==6.3.2 tweepy ==3.8.0 requests-toolbelt ==0.9.1 diff --git a/src/IRCBot.py b/src/IRCBot.py index b2c2861f..8c7bdf47 100644 --- a/src/IRCBot.py +++ b/src/IRCBot.py @@ -116,7 +116,8 @@ def trigger(self, self._trigger_both() return returned - func_queue = queue.Queue(1) # type: queue.Queue[str] + func_queue: queue.Queue[typing.Tuple[TriggerResult, typing.Any] + ] = queue.Queue(1) def _action(): try: @@ -134,7 +135,8 @@ def _action(): if trigger_threads: self._trigger_both() - if type == TriggerResult.Exception: + if (type == TriggerResult.Exception and + isinstance(returned, Exception)): raise returned elif type == TriggerResult.Return: return returned diff --git a/src/IRCServer.py b/src/IRCServer.py index bb22adab..0399c6da 100644 --- a/src/IRCServer.py +++ b/src/IRCServer.py @@ -95,8 +95,12 @@ def connect(self): self.connection_params.bindhost, self.connection_params.tls, tls_verify=self.get_setting("ssl-verify", True), - cert=self.bot.config.get("tls-certificate", None), - key=self.bot.config.get("tls-key", None)) + cert=self.bot.config.get("tls-certificate", '').format( + DATA=self.bot.data_directory + ) or None, + key=self.bot.config.get("tls-key", '').format( + DATA=self.bot.data_directory + )) self.events.on("preprocess.connect").call(server=self) self.socket.connect() diff --git a/src/ModuleManager.py b/src/ModuleManager.py index a43181fe..034be660 100644 --- a/src/ModuleManager.py +++ b/src/ModuleManager.py @@ -235,7 +235,7 @@ def _load_module(self, bot: "IRCBot.Bot", definition: ModuleDefinition, definition.filename) module = importlib.util.module_from_spec(import_spec) sys.modules[import_name] = module - loader = typing.cast(importlib.abc.Loader, import_spec.loader) + loader = typing.cast(importlib._abc.Loader, import_spec.loader) loader.exec_module(module) module_object_pointer = getattr(module, "Module", None) diff --git a/src/core_modules/admin.py b/src/core_modules/admin.py index 5008952d..dca8938c 100644 --- a/src/core_modules/admin.py +++ b/src/core_modules/admin.py @@ -151,6 +151,27 @@ def add_server(self, event): return event["stdout"].write("Added server '%s'" % alias) + @utils.hook("received.command.delserver") + @utils.kwarg("help", "Delete a server") + @utils.kwarg("pemission", "delserver") + @utils.spec("!word") + def del_server(self, event): + alias = event["spec"][0] + sid = self.bot.database.servers.by_alias(alias) + if sid == None: + event["stderr"].write("Server '%s' does not exist" % alias) + return + if self._server_from_alias(alias): + event["stderr"].write("You must disconnect from %s before deleting it" % alias) + return + try: + self.bot.database.servers.delete(sid) + except Exception as e: + event["stderr"].write("Failed to delete server") + self.log.error("failed to add server \"%s\"", [alias], exc_info=True) + return + event["stderr"].write("Server '%s' has been deleted" % alias) + @utils.hook("received.command.editserver") @utils.kwarg("help", "Edit server details") @utils.kwarg("permission", "editserver") diff --git a/src/core_modules/ignore.py b/src/core_modules/ignore.py index dd51a655..52aaa2bb 100644 --- a/src/core_modules/ignore.py +++ b/src/core_modules/ignore.py @@ -12,13 +12,24 @@ def _user_channel_ignored(self, channel, user): return channel.get_user_setting(user.get_id(), "ignore", False) def _server_command_ignored(self, server, command): return server.get_setting("ignore-%s" % command, False) + def _channel_command_ignored(self, channel, command): + return channel.get_setting("ignore-command-%s" % command, False) - def _is_command_ignored(self, server, user, command): - if self._user_command_ignored(user, command): + def _is_command_ignored(self, event): + if self._user_command_ignored(event["user"], event["command"]): return True - elif self._server_command_ignored(server, command): + elif self._server_command_ignored(event["server"], event["command"]): + return True + elif event["is_channel"] and self._channel_command_ignored(event["target"], event["command"]): return True + def _is_valid_command(self, command): + hooks = self.events.on("received.command").on(command).get_hooks() + if hooks: + return True + else: + return False + @utils.hook("received.message.private") @utils.hook("received.message.channel") @utils.hook("received.notice.private") @@ -38,8 +49,7 @@ def preprocess_command(self, event): elif event["is_channel"] and self._user_channel_ignored(event["target"], event["user"]): return utils.consts.PERMISSION_HARD_FAIL, None - elif self._is_command_ignored(event["server"], event["user"], - event["command"]): + elif self._is_command_ignored(event): return utils.consts.PERMISSION_HARD_FAIL, None @utils.hook("received.command.ignore", min_args=1) @@ -123,6 +133,38 @@ def cignore(self, event): True) event["stdout"].write("Ignoring %s" % target_user.nickname) + @utils.hook("received.command.ignorecommand", + help="Ignore a command in this channel") + @utils.hook("received.command.unignorecommand", + help="Unignore a command in this channel") + @utils.kwarg("channel_only", True) + @utils.kwarg("min_args", 1) + @utils.kwarg("usage", "") + @utils.kwarg("permission", "cignore") + @utils.kwarg("require_mode", "o") + @utils.kwarg("require_access", "high,cignore") + def cignore_command(self, event): + remove = event["command"] == "unignorecommand" + + command = event["args_split"][0] + if not self._is_valid_command(command): + raise utils.EventError("Unknown command '%s'" % command) + is_ignored = self._channel_command_ignored(event["target"], command) + + if remove: + if not is_ignored: + raise utils.EventError("I'm not ignoring '%s' in this channel" % + target_user.nickname) + event["target"].del_setting("ignore-command-%s" % command) + event["stdout"].write("Unignored '%s' command" % command) + else: + if is_ignored: + raise utils.EventError("I'm already ignoring '%s' in this channel" + % command) + event["target"].set_setting("ignore-command-%s" % command, True) + event["stdout"].write("Ignoring '%s' command" % command) + + @utils.hook("received.command.serverignore") @utils.kwarg("help", "Ignore a command on the current server") @utils.kwarg("permission", "serverignore") diff --git a/src/utils/cli.py b/src/utils/cli.py index 2c2b3e3b..40c1bf0b 100644 --- a/src/utils/cli.py +++ b/src/utils/cli.py @@ -5,15 +5,15 @@ def bool_input(s: str): return not result or result[0].lower() in ["", "y"] def add_server(): - alias = input("alias: ") - hostname = input("hostname: ") + alias = input("alias (display name): ") + hostname = input("hostname (address of server): ") port = int(input("port: ")) tls = bool_input("tls?") - password = input("password?: ") + password = input("password (optional, leave blank to skip): ") nickname = input("nickname: ") - username = input("username: ") - realname = input("realname: ") - bindhost = input("bindhost?: ") + username = input("username (optional): ") + realname = input("realname (optional): ") + bindhost = input("bindhost (optional): ") return irc.IRCConnectionParameters(-1, alias, hostname, port, password, tls, bindhost, nickname, username, realname) diff --git a/src/utils/http.py b/src/utils/http.py index 045d9641..2fb50cd0 100644 --- a/src/utils/http.py +++ b/src/utils/http.py @@ -7,7 +7,7 @@ REGEX_URL = re.compile("https?://\S+", re.I) -PAIRED_CHARACTERS = ["<>", "()"] +PAIRED_CHARACTERS = [("<", ">"), ("(", ")")] # best-effort tidying up of URLs def url_sanitise(url: str): @@ -131,7 +131,7 @@ def decode(self, encoding: typing.Optional[str]=None) -> str: return self.data.decode(encoding or self.encoding) def json(self) -> typing.Any: return _json.loads(self.data) - def soup(self, parser: str="html5lib") -> bs4.BeautifulSoup: + def soup(self, parser: str="html.parser") -> bs4.BeautifulSoup: return bs4.BeautifulSoup(self.decode(), parser) def _split_content(s: str) -> typing.Dict[str, str]: @@ -148,7 +148,7 @@ def _find_encoding(headers: typing.Dict[str, str], data: bytes if "charset" in content_header: return content_header["charset"] - soup = bs4.BeautifulSoup(data, "html5lib") + soup = bs4.BeautifulSoup(data, "html.parser") if not soup.meta == None: meta_charset = soup.meta.get("charset") if not meta_charset == None: @@ -304,8 +304,8 @@ async def _request(request): loop = asyncio.new_event_loop() awaits = [] for request in requests: - awaits.append(_request(request)) - task = asyncio.wait(awaits, loop=loop, timeout=5) + awaits.append(loop.create_task(_request(request))) + task = asyncio.wait(awaits, timeout=5) loop.run_until_complete(task) loop.close() @@ -316,7 +316,7 @@ class Client(object): request_many = request_many def strip_html(s: str) -> str: - return bs4.BeautifulSoup(s, "html5lib").get_text() + return bs4.BeautifulSoup(s, "lxml").get_text() def resolve_hostname(hostname: str) -> typing.List[str]: try: diff --git a/src/utils/parse/sed.py b/src/utils/parse/sed.py index 8a1c895d..087b9a41 100644 --- a/src/utils/parse/sed.py +++ b/src/utils/parse/sed.py @@ -44,7 +44,7 @@ def match(self, s): return None def _sed_split(s: str) -> typing.List[str]: - tokens = _tokens(s, "/") + tokens = _tokens(s, s[1]) if tokens and (not tokens[-1] == (len(s)-1)): tokens.append(len(s)) diff --git a/src/utils/security.py b/src/utils/security.py index 085826d5..70a28ff8 100644 --- a/src/utils/security.py +++ b/src/utils/security.py @@ -25,13 +25,20 @@ def ssl_wrap(sock: socket.socket, cert: str=None, key: str=None, def constant_time_compare(a: typing.AnyStr, b: typing.AnyStr) -> bool: return hmac.compare_digest(a, b) -import scrypt +import hashlib def password(byte_n: int=32) -> str: return binascii.hexlify(os.urandom(byte_n)).decode("utf8") def salt(byte_n: int=64) -> str: return base64.b64encode(os.urandom(byte_n)).decode("utf8") def hash(given_salt: str, data: str): - return base64.b64encode(scrypt.hash(data, given_salt)).decode("utf8") + hash = hashlib.scrypt( + data.encode("utf8"), + salt=given_salt.encode("utf8"), + n=1<<14, + r=8, + p=1 + ) + return base64.b64encode(hash).decode("ascii") def hash_verify(salt: str, data: str, compare: str): given_hash = hash(salt, data) return constant_time_compare(given_hash, compare)