From 46192de8561a0186c1e1896c620ecbf819d73c4d Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Tue, 19 Mar 2019 12:47:02 +0100 Subject: [PATCH 01/12] don't log access token --- bot/github.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bot/github.py b/bot/github.py index 103b280..aacf760 100644 --- a/bot/github.py +++ b/bot/github.py @@ -34,8 +34,6 @@ def handle_auth_update(self, update: GithubAuthUpdate, context: CallbackContext) access_token = github_api.get_oauth_access_token(update.code, update.raw_state) - self.logger.debug('Access token for user %s: %s', user_id, access_token) - context.user_data['access_token'] = access_token from bot.settings import login_menu From f9f096930ffb381c7dd54cb4e89e6b84a9a3165e Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Tue, 19 Mar 2019 12:47:50 +0100 Subject: [PATCH 02/12] switch check icon --- bot/menu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/menu.py b/bot/menu.py index 81fdf17..c1fa9da 100644 --- a/bot/menu.py +++ b/bot/menu.py @@ -180,7 +180,7 @@ def __init__(self, key, value, text=None, states=None, default=None): if (text is None and states is None) or (text is not None and states is not None): raise RuntimeError if text is not None: - states = ((False, text), (True, '\u2714' + text)) + states = ((False, text), (True, '☑️' + text)) if default is None: default = states[0][0] self.default = default From 66b168f916f1600cebc142f86226511d1d9ea173 Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Tue, 19 Mar 2019 12:51:42 +0100 Subject: [PATCH 03/12] filter out registered repos --- bot/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/settings.py b/bot/settings.py index c035260..0224061 100644 --- a/bot/settings.py +++ b/bot/settings.py @@ -226,6 +226,7 @@ def inline_add_repo(update, context): results = [] if access_token: filtered_repositories = [] + registered_repos = context.chat_data.setdefault('repos', {}) search = context.match.group(1).strip() installations = github_api.get_installations_for_user(access_token) for installation_index, installation in enumerate(installations): @@ -234,7 +235,7 @@ def inline_add_repo(update, context): repositories = github_api.get_repositories_for_installation(installation['id'], access_token) for repo_index, repo in enumerate(repositories): - if repo_index <= repo_offset: + if repo_index <= repo_offset or repo['id'] in registered_repos: continue if repo['full_name'].startswith(search) or repo['name'].startswith(search): filtered_repositories.append(repo) From 6d3dcf015b46a3da6cfb611a523f1285e8f4e41d Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Tue, 19 Mar 2019 13:33:27 +0100 Subject: [PATCH 04/12] start with docker setup wrapper --- Dockerfile | 18 ++++++++++++++++++ bot/{main.py => __main__.py} | 0 docker-compose.yml | 13 +++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 Dockerfile rename bot/{main.py => __main__.py} (100%) create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a5f269e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.7 + +WORKDIR /usr/src/app + +VOLUME [ "/data" ] +VOLUME [ "/config" ] + +ENV DATABASE_FILE /data/db.pickle +ENV SERVER_PORT 80 +ENV GITHUB_PRIVATE_KEY_PATH /config/key + + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD [ "python", "-m", "bot" ] \ No newline at end of file diff --git a/bot/main.py b/bot/__main__.py similarity index 100% rename from bot/main.py rename to bot/__main__.py diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b856e98 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.0' + +services: + bot: + build: + context: . + volumes: + - './:/usr/src/app' + - 'bot_data:/data' + +volumes: + bot_data: null + From 80445879071ed6f7b6338e4fbeec5a0fb080cb11 Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Tue, 19 Mar 2019 13:34:10 +0100 Subject: [PATCH 05/12] satisfy tslint --- bot/webhookupdater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/webhookupdater.py b/bot/webhookupdater.py index 6739226..003d463 100644 --- a/bot/webhookupdater.py +++ b/bot/webhookupdater.py @@ -34,7 +34,7 @@ def post(self): self.process_data(data) def process_data(self, data: Dict): - raise NotImplemented + raise NotImplementedError def validate(self): ct_header = self.request.headers.get("Content-Type", None) From caee4d16700d65b821cc828f02a6c0e48b5567e6 Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Tue, 19 Mar 2019 13:41:41 +0100 Subject: [PATCH 06/12] reorder + default values --- bot/const.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bot/const.py b/bot/const.py index 9c5ddb3..f910886 100644 --- a/bot/const.py +++ b/bot/const.py @@ -2,16 +2,20 @@ GITHUB_WEBHOOK_SECRET = os.getenv('GITHUB_WEBHOOK_SECRET').encode() TELEGRAM_BOT_TOKEN = os.getenv('TELEGRAM_BOT_TOKEN') -SERVER_PORT = int(os.getenv('SERVER_PORT')) SERVER_URL_BASE = os.getenv('SERVER_URL_BASE').rstrip('/') -TELEGRAM_WEBHOOK_URL = SERVER_URL_BASE + '/' + TELEGRAM_BOT_TOKEN +SERVER_PORT = int(os.getenv('SERVER_PORT', 80)) SERVER_HOSTNAME_PATTERN = os.getenv('SERVER_HOSTNAME_PATTERN') + HMAC_SECRET = TELEGRAM_BOT_TOKEN.encode('ascii') GITHUB_PRIVATE_KEY_PATH = os.getenv('GITHUB_PRIVATE_KEY_PATH') GITHUB_APP_ID = os.getenv('GITHUB_APP_ID') -DATABASE_FILE = os.getenv('DATABASE_FILE') +DATABASE_FILE = os.getenv('DATABASE_FILE', '/data/db.pickle') GITHUB_OAUTH_CLIENT_ID = os.getenv('GITHUB_OAUTH_CLIENT_ID') GITHUB_OAUTH_CLIENT_SECRET = os.getenv('GITHUB_OAUTH_CLIENT_SECRET') -GITHUB_OAUTH_REDIRECT_URI = SERVER_URL_BASE + '/github/auth' DEBUG = os.getenv('DEBUG', False) + + DEFAULT_TRUNCATION_LIMIT = 4096 + +TELEGRAM_WEBHOOK_URL = SERVER_URL_BASE + '/' + TELEGRAM_BOT_TOKEN +GITHUB_OAUTH_REDIRECT_URI = SERVER_URL_BASE + '/github/auth' From d3347f6eb6547bffce4f18a2a57c7b6ac18412bb Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Wed, 27 Mar 2019 15:55:08 +0100 Subject: [PATCH 07/12] small changes --- bot/github.py | 2 -- bot/menu.py | 2 +- bot/webhookupdater.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bot/github.py b/bot/github.py index 103b280..aacf760 100644 --- a/bot/github.py +++ b/bot/github.py @@ -34,8 +34,6 @@ def handle_auth_update(self, update: GithubAuthUpdate, context: CallbackContext) access_token = github_api.get_oauth_access_token(update.code, update.raw_state) - self.logger.debug('Access token for user %s: %s', user_id, access_token) - context.user_data['access_token'] = access_token from bot.settings import login_menu diff --git a/bot/menu.py b/bot/menu.py index 81fdf17..c1fa9da 100644 --- a/bot/menu.py +++ b/bot/menu.py @@ -180,7 +180,7 @@ def __init__(self, key, value, text=None, states=None, default=None): if (text is None and states is None) or (text is not None and states is not None): raise RuntimeError if text is not None: - states = ((False, text), (True, '\u2714' + text)) + states = ((False, text), (True, '☑️' + text)) if default is None: default = states[0][0] self.default = default diff --git a/bot/webhookupdater.py b/bot/webhookupdater.py index 6739226..003d463 100644 --- a/bot/webhookupdater.py +++ b/bot/webhookupdater.py @@ -34,7 +34,7 @@ def post(self): self.process_data(data) def process_data(self, data: Dict): - raise NotImplemented + raise NotImplementedError def validate(self): ct_header = self.request.headers.get("Content-Type", None) From bae250354b9c1648305680f8163748937fe445fb Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Wed, 27 Mar 2019 17:49:13 +0100 Subject: [PATCH 08/12] proper setup using localtunnel --- Dockerfile | 2 +- docker-compose.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a5f269e..8aceb06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ VOLUME [ "/config" ] ENV DATABASE_FILE /data/db.pickle ENV SERVER_PORT 80 -ENV GITHUB_PRIVATE_KEY_PATH /config/key +ENV GITHUB_PRIVATE_KEY_PATH /config/key.pem COPY requirements.txt ./ diff --git a/docker-compose.yml b/docker-compose.yml index b856e98..42f8e3d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,18 @@ services: volumes: - './:/usr/src/app' - 'bot_data:/data' + - './data/private-key.pem:/config/key' + env_file: + - .env + localtunnel: + image: kaixhin/localtunnel + command: + - '80' + - '--local-host' + - 'bot' + - '--subdomain' + - '${SERVER_SUBDOMAIN}' + restart: always volumes: bot_data: null From 4eb4d49d1cd5c2e4d7b8201ef309fe2203795e4f Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Wed, 27 Mar 2019 18:01:27 +0100 Subject: [PATCH 09/12] better default settings --- Dockerfile | 1 + docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8aceb06..b6f0938 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ VOLUME [ "/data" ] VOLUME [ "/config" ] ENV DATABASE_FILE /data/db.pickle +ENV SERVER_HOSTNAME_PATTERN=.* ENV SERVER_PORT 80 ENV GITHUB_PRIVATE_KEY_PATH /config/key.pem diff --git a/docker-compose.yml b/docker-compose.yml index 42f8e3d..9554e59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: volumes: - './:/usr/src/app' - 'bot_data:/data' - - './data/private-key.pem:/config/key' + - './private-key.pem:/config/key.pem' env_file: - .env localtunnel: From 2fc7741e142e9e14836256f0c716166851c783bb Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Wed, 27 Mar 2019 18:01:42 +0100 Subject: [PATCH 10/12] customize github app name --- bot/__main__.py | 4 ++-- bot/const.py | 3 ++- bot/webhookupdater.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bot/__main__.py b/bot/__main__.py index 349a0ed..bc2ee92 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -5,7 +5,7 @@ from telegram.ext import TypeHandler, CallbackContext, CommandHandler, MessageHandler, Filters from bot import settings -from bot.const import TELEGRAM_BOT_TOKEN, DATABASE_FILE, DEBUG +from bot.const import TELEGRAM_BOT_TOKEN, DATABASE_FILE, DEBUG, GITHUB_APP_NAME from bot.github import GithubHandler from bot.githubapi import github_api from bot.githubupdates import GithubUpdate, GithubAuthUpdate @@ -50,7 +50,7 @@ def help_handler(update: Update, context: CallbackContext): msg = update.effective_message private = update.effective_chat.type == Chat.PRIVATE steps = [ - f'First you must allow me access to the repositories in question. To do this, install my GitHub App on your account or organisation, and make sure that it has access to the desired repositories.', + f'First you must allow me access to the repositories in question. To do this, install my GitHub App on your account or organisation, and make sure that it has access to the desired repositories.', f'Use the command /settings to open my settings interface and press the login button. This way I will know who you are.', f'Add me ({context.bot.name}) to the chat/group in which you would like to receive notifications.', f'In that chat use /settings to add the repositories you would like to receive notifications for.' diff --git a/bot/const.py b/bot/const.py index f910886..26adde1 100644 --- a/bot/const.py +++ b/bot/const.py @@ -7,12 +7,13 @@ SERVER_HOSTNAME_PATTERN = os.getenv('SERVER_HOSTNAME_PATTERN') HMAC_SECRET = TELEGRAM_BOT_TOKEN.encode('ascii') +GITHUB_APP_NAME = os.getenv('GITHUB_APP_NAME', 'telegram-githubbot-revised') GITHUB_PRIVATE_KEY_PATH = os.getenv('GITHUB_PRIVATE_KEY_PATH') GITHUB_APP_ID = os.getenv('GITHUB_APP_ID') DATABASE_FILE = os.getenv('DATABASE_FILE', '/data/db.pickle') GITHUB_OAUTH_CLIENT_ID = os.getenv('GITHUB_OAUTH_CLIENT_ID') GITHUB_OAUTH_CLIENT_SECRET = os.getenv('GITHUB_OAUTH_CLIENT_SECRET') -DEBUG = os.getenv('DEBUG', False) +DEBUG = bool(os.getenv('DEBUG', False)) DEFAULT_TRUNCATION_LIMIT = 4096 diff --git a/bot/webhookupdater.py b/bot/webhookupdater.py index 003d463..5edc065 100644 --- a/bot/webhookupdater.py +++ b/bot/webhookupdater.py @@ -11,7 +11,7 @@ from tornado.ioloop import IOLoop from tornado.web import Application, RequestHandler, HTTPError -from bot.const import GITHUB_WEBHOOK_SECRET, SERVER_HOSTNAME_PATTERN, SERVER_PORT, TELEGRAM_WEBHOOK_URL, HMAC_SECRET +from bot.const import GITHUB_WEBHOOK_SECRET, SERVER_HOSTNAME_PATTERN, SERVER_PORT, TELEGRAM_WEBHOOK_URL, HMAC_SECRET, DEBUG from bot.githubupdates import GithubUpdate, GithubAuthUpdate from bot.utils import secure_decode_64, HMACException @@ -145,7 +145,7 @@ def __init__(self, token, updater_kwargs=None): self.dispatcher = self.updater.dispatcher self.update_queue = self.updater.update_queue - self.app = Application() + self.app = Application(debug=DEBUG) self.app.add_handlers(SERVER_HOSTNAME_PATTERN, [ ( r'/{}/?'.format(token), From 84b40aca23c5b7a69f2e47b493c041b373b61cad Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Wed, 27 Mar 2019 18:24:51 +0100 Subject: [PATCH 11/12] add setup dev instructions --- setup_dev.md | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 setup_dev.md diff --git a/setup_dev.md b/setup_dev.md new file mode 100644 index 0000000..2e1326e --- /dev/null +++ b/setup_dev.md @@ -0,0 +1,119 @@ +# Create Dev Setup + +## Define some names + +will be later referred using the right notation + + * Github app name: `` e.g., GithubBot Revised + * Localtunnel custom domain: `` e.g., githubbot-revised + * telegram bot name: `` e.g., githubrevised_bot + * random webhook url secret: `` e.g., abcde + +create an `.env` file in the repo directory and enter the following values: + +``` +DEBUG=True + +SERVER_SUBDOMAIN= +SERVER_URL_BASE=https://.localtunnel.me +``` + +## Register new Github app + +### Define App + +Go to: https://github.com/settings/apps/new + +enter following values while replacing the corresponding chosen names: + +* name: `` +* url: `https://t.me/` +* webhook url: `https://.localtunnel.me` +* user callback url: `https://.localtunnel.me/github/auth` +* setup url: `https://t.me/` + +permissions: + * repo admin -> read + * repository contents -> read + * deployments -> read + * issues -> read+write + * repo meta -> read + * pages -> read + * pr -> read + * repo projects -> read + * security vulneratibly alerts -> read + * commit status -> read + * organizatin projects -> read + * team discussions -> read + +events + * all + +This will results in values for the following ids: + +``` +App ID: e.g., 12345 +(OAuth) Client ID: e.g., Iv1.abcd... +(OAuth) Client secret: e.g., 123... +``` + +### Generate private key +create a new private key and store it in the repo directory as `private-key.pem` + +### Configure repo +extend the `.env` file with: + +``` +GITHUB_APP_NAME=telegramgithubbot-sam +GITHUB_APP_ID= +GITHUB_WEBHOOK_SECRET= +GITHUB_OAUTH_CLIENT_ID= +GITHUB_OAUTH_CLIENT_SECRET= +``` + +## Create Telegram bot + +### Create via BotFather +Use bot father to create a new bot named `` + +``` +/newbot +``` + +e.g., results in `https://t.me/githubrevised_sam_bot` along with a secret token + +``` +token: 34334:adff3f... +``` + +### Advanced BotFather settings + +enable inline mode +``` +/setinline ... to inline query enable +``` + +add available commands for better autocompletion + +``` +start - Start the bot +help - Show help +login - Login to Github +privacy - Privacy Policy +settings - Settings +``` + +### Configure bot +extend the `.env` file with the received token +``` +TELEGRAM_BOT_TOKEN= +``` + +## Launch docker-compose + + +``` +docker-compose up +``` + +now you should be able to chat with the bot. \ No newline at end of file From 5a3d5e2606a2c090d5b00ff56b74e86af0bd51fb Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Wed, 27 Mar 2019 18:39:48 +0100 Subject: [PATCH 12/12] info not available in inline queries --- bot/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bot/settings.py b/bot/settings.py index 0224061..c035260 100644 --- a/bot/settings.py +++ b/bot/settings.py @@ -226,7 +226,6 @@ def inline_add_repo(update, context): results = [] if access_token: filtered_repositories = [] - registered_repos = context.chat_data.setdefault('repos', {}) search = context.match.group(1).strip() installations = github_api.get_installations_for_user(access_token) for installation_index, installation in enumerate(installations): @@ -235,7 +234,7 @@ def inline_add_repo(update, context): repositories = github_api.get_repositories_for_installation(installation['id'], access_token) for repo_index, repo in enumerate(repositories): - if repo_index <= repo_offset or repo['id'] in registered_repos: + if repo_index <= repo_offset: continue if repo['full_name'].startswith(search) or repo['name'].startswith(search): filtered_repositories.append(repo)