Merge pull request #172 from datafold/redacted_log

Print configuration during debug, but with passwords redacted

Author: erezsh

data_diff/__main__.py

@@ -5,6 +5,8 @@
 import logging
 from itertools import islice
 
+from .utils import remove_password_from_url
+
 from .diff_tables import (
     TableSegment,
     TableDiffer,
@@ -33,6 +35,8 @@ def _remove_passwords_in_dict(d: dict):
             d[k] = "*" * len(v)
         elif isinstance(v, dict):
             _remove_passwords_in_dict(v)
+        elif k.startswith("database"):
+            d[k] = remove_password_from_url(v)
 
 
 @click.command()
@@ -120,11 +124,10 @@ def _main(
 
     if debug:
         logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT)
-        # XXX Temporarily commented out, until we remove the passwords from URIs as well. See issue #150.
-        # if __conf__:
-        #     __conf__ = deepcopy(__conf__)
-        #     _remove_passwords_in_dict(__conf__)
-        #     logging.debug(f"Applied run configuration: {__conf__}")
+        if __conf__:
+            __conf__ = deepcopy(__conf__)
+            _remove_passwords_in_dict(__conf__)
+            logging.debug(f"Applied run configuration: {__conf__}")
     elif verbose:
         logging.basicConfig(level=logging.INFO, format=LOG_FORMAT, datefmt=DATE_FORMAT)
 

data_diff/utils.py

@@ -1,6 +1,7 @@
 import math
+from urllib.parse import urlparse
 
-from typing import Sequence, Optional, Tuple, Union, Dict, Any
+from typing import Union, Any
 from uuid import UUID
 
 
@@ -53,9 +54,26 @@ def number_to_human(n):
     return "{:.0f}{}".format(n / 10 ** (3 * millidx), millnames[millidx])
 
 
+def _join_if_any(sym, args):
+    args = list(args)
+    if not args:
+        return ''
+    return sym.join(str(a) for a in args if a)
+
+def remove_password_from_url(url: str, replace_with: str="***") -> str:
+    parsed = urlparse(url)
+    account = parsed.username or ''
+    if parsed.password:
+        account += ':' + replace_with
+    host = _join_if_any(":", filter(None, [parsed.hostname, parsed.port]))
+    netloc = _join_if_any("@", filter(None, [account, host]))
+    replaced = parsed._replace(netloc=netloc)
+    return replaced.geturl()
+
 def join_iter(joiner: Any, iterable: iter) -> iter:
     it = iter(iterable)
     yield next(it)
     for i in it:
         yield joiner
         yield i
+

tests/test_config.py

@@ -1,6 +1,7 @@
 import unittest
 
 from data_diff.config import apply_config_from_string, ConfigParseError
+from data_diff.utils import remove_password_from_url
 
 
 class TestConfig(unittest.TestCase):
@@ -41,3 +42,20 @@ def test_basic(self):
         res = apply_config_from_string(config, "pg_pg", {"update_column": "foo", "table2": "bar"})
         assert res["update_column"] == "foo"
         assert res["table2"] == "bar"
+
+    def test_remove_password(self):
+        replace_with = "*****"
+        urls = [
+            'd://host/',
+            'd://host:123/',
+            'd://user@host:123/',
+            'd://user:PASS@host:123/',
+            'd://:PASS@host:123/',
+            'd://:PASS@host:123/path',
+            'd://:PASS@host:123/path?whatever#blabla',
+        ]
+        for url in urls:
+            removed = remove_password_from_url(url, replace_with)
+            expected = url.replace('PASS', replace_with)
+            removed = remove_password_from_url(url, replace_with)
+            self.assertEqual(removed, expected)