feat: improving logging (#38)

* Add logging to service

* Add log_level to readme

Author: jackwotherspoon

README.md

@@ -203,7 +203,9 @@ Where:
 - **sql_instances**: List of all Cloud SQL instances to configure.
 - **private_ip** (optional): Boolean flag for private or public IP addresses.
 
-**Note:** These are placeholder values and should be replaced with proper IAM groups and Cloud SQL instance connection names.
+**Note:** These are placeholder values and should be replaced with proper IAM groups and Cloud SQL instance connection names. 
+
+There is an additional optional parameter `"log_level"` for the JSON payload which can be set to one of `"INFO"`, `"DEBUG"`, `"WARNING"`, or `"ERROR"` to change severity of outputted logs. Defaults to `"INFO"` when not specified.
 
 It is recommended to save your JSON payload as a `.json` file (ex. "config.json").
 

app.py

@@ -16,6 +16,8 @@
 from quart import Quart, request
 from google.auth import default
 from google.cloud.sql.connector.instance_connection_manager import IPTypes
+import logging
+import google.cloud.logging
 from iam_groups_authn.sync import (
     get_credentials,
     get_users_with_roles,
@@ -35,6 +37,16 @@
 
 app = Quart(__name__)
 
+# start logging client
+client = google.cloud.logging.Client()
+client.setup_logging()
+log_levels = {
+    "INFO": logging.INFO,
+    "DEBUG": logging.DEBUG,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+}
+
 
 @app.route("/", methods=["GET"])
 def health_check():
@@ -67,6 +79,15 @@ async def run_groups_authn():
             400,
         )
 
+    # optional param to change log level
+    log_level = body.get("log_level")
+    if (
+        log_level is not None
+        and type(log_level) is str
+        and log_level.upper() in log_levels
+    ):
+        logging.getLogger().setLevel(log_levels[log_level.upper()])
+
     # set ip_type to proper type for connector
     ip_type = IPTypes.PRIVATE if private_ip else IPTypes.PUBLIC
 
@@ -123,6 +144,13 @@ async def run_groups_authn():
                 if issubclass(type(result), Exception):
                     raise result
 
+            # log IAM users added as database users
+            added_users = results[0]
+            logging.debug(
+                "[%s][%s] Users added to database: %s."
+                % (instance, group, list(added_users))
+            )
+
             # revoke group role from users no longer in IAM group
             revoke_role_task = asyncio.create_task(
                 revoke_iam_group_role(
@@ -150,4 +178,17 @@ async def run_groups_authn():
                 if issubclass(type(result), Exception):
                     raise result
 
+            # log sync info
+            revoked_users, granted_users = results
+            logging.info(
+                "[%s][%s] Sync successful: %s users were revoked group role, %s users were granted group role."
+                % (instance, group, len(revoked_users), len(granted_users))
+            )
+            logging.debug(
+                "[%s][%s] Users revoked role: %s." % (instance, group, revoked_users)
+            )
+            logging.debug(
+                "[%s][%s] Users granted role: %s." % (instance, group, granted_users)
+            )
+
     return "Sync successful.", 200

iam_groups_authn/mysql.py

@@ -68,20 +68,19 @@ def fetch_role_grants(self, group_name):
         return results
 
     @async_wrap
-    def create_group_role(self, group):
+    def create_group_role(self, role):
         """Verify or create DB role.
 
-        Given a group name, verify existance of DB role or create new DB role matching
-        name of group to manage DB users.
+        Given a group role, verify existance of role on DB or create new role
+        to manage DB users.
 
         Args:
-            db: Database connection pool instance.
-            group: Name of group to be verified as role or created as new role.
+            role: Name of group role to be verified or created as new role.
         """
         # create connection to db instance
         with self.db.connect() as db_connection:
             stmt = sqlalchemy.text("CREATE ROLE IF NOT EXISTS :role")
-            db_connection.execute(stmt, {"role": group})
+            db_connection.execute(stmt, {"role": role})
 
     @async_wrap
     def grant_group_role(self, role, users):
@@ -90,7 +89,6 @@ def grant_group_role(self, role, users):
         Given a DB group role and a list of DB users, grant the DB role to each user.
 
         Args:
-            db: Database connection pool instance.
             role: Name of DB role to grant to users.
             users: List of DB users' usernames.
         """
@@ -107,7 +105,6 @@ def revoke_group_role(self, role, users):
         Given a DB group role and a list of DB users, revoke the DB role from each user.
 
         Args:
-            db: Database connection pool instance.
             role: Name of DB role to revoke from users.
             users: List of DB users' usernames.
         """

iam_groups_authn/sql_admin.py

@@ -76,3 +76,4 @@ async def add_missing_db_users(
         user_service.insert_db_user(
             user, InstanceConnectionName(*instance_connection_name.split(":"))
         )
+    return missing_db_users

iam_groups_authn/sync.py

@@ -60,8 +60,9 @@ def get_group_members(self, group):
             return members
         # handle errors if IAM group does not exist etc.
         except HttpError as e:
-            print(f"Could not get IAM group `{group}`. Error: {e}")
-            raise
+            raise HttpError(
+                f"Error: Failed to get IAM members of IAM group `{group}`. Verify group exists and is configured correctly."
+            ) from e
 
     @async_wrap
     def get_db_users(self, instance_connection_name):
@@ -80,16 +81,21 @@ def get_db_users(self, instance_connection_name):
         """
         # build service to call SQL Admin API
         service = build("sqladmin", "v1beta4", credentials=self.creds)
-        results = (
-            service.users()
-            .list(
-                project=instance_connection_name.project,
-                instance=instance_connection_name.instance,
+        try:
+            results = (
+                service.users()
+                .list(
+                    project=instance_connection_name.project,
+                    instance=instance_connection_name.instance,
+                )
+                .execute()
             )
-            .execute()
-        )
-        users = results.get("items", [])
-        return users
+            users = results.get("items", [])
+            return users
+        except Exception as e:
+            raise Exception(
+                f"Error: Failed to get the database users for instance `{instance_connection_name}`. Verify instance connection name and instance details."
+            ) from e
 
     def insert_db_user(self, user_email, instance_connection_name):
         """Create DB user from IAM user.
@@ -117,10 +123,9 @@ def insert_db_user(self, user_email, instance_connection_name):
             )
             return
         except Exception as e:
-            print(
-                f"Could not add IAM user `{user_email}` to DB Instance `{instance_connection_name.instance}`. Error: {e}"
-            )
-            raise
+            raise Exception(
+                f"Error: Failed to add IAM user `{user_email}` to Cloud SQL database instance `{instance_connection_name.instance}`."
+            ) from e
 
 
 async def get_users_with_roles(role_service, role):
@@ -173,8 +178,10 @@ def get_credentials(creds, scopes):
         # if not valid, refresh credentials
         if not updated_credentials.valid:
             updated_credentials.refresh(request)
-    except Exception:
-        raise
+    except Exception as e:
+        raise Exception(
+            "Error: Failed to get proper credentials for service. Verify service account used to run service."
+        ) from e
 
     return updated_credentials
 
@@ -227,6 +234,8 @@ async def revoke_iam_group_role(
     # revoke group role from users no longer in IAM group
     await role_service.revoke_group_role(role, users_to_revoke)
 
+    return users_to_revoke
+
 
 async def grant_iam_group_role(
     role_service,
@@ -253,3 +262,5 @@ async def grant_iam_group_role(
         username for username in mysql_usernames if username not in users_with_roles
     ]
     await role_service.grant_group_role(role, users_to_grant)
+
+    return users_to_grant

requirements.txt

@@ -5,3 +5,4 @@ google-api-python-client==2.19.1
 google-auth==2.0.0
 PyMySQL==1.0.2
 cloud-sql-python-connector[pymysql]==0.4.1
+google-cloud-logging==2.6.0