Merge pull request #1186 from dimitri-yatsenko/logger

Loglevel validation in settings (reimplement #1044)

Author: ethho

datajoint/dependencies.py

@@ -56,11 +56,11 @@ def topo_sort(graph):
             pos -= 1
         else:
             placed.add(part)
-            j = sorted_nodes.index(master)
-            if pos > j + 1:
-                # move the part to its master
+            insert_pos = sorted_nodes.index(master) + 1
+            if pos > insert_pos:
+                # move the part to the position immediately after its master
                 del sorted_nodes[pos]
-                sorted_nodes.insert(j + 1, part)
+                sorted_nodes.insert(insert_pos, part)
 
     return sorted_nodes
 

datajoint/external.py

@@ -8,7 +8,7 @@
 from .table import Table, FreeTable
 from .heading import Heading
 from .declare import EXTERNAL_TABLE_ROOT
-from . import s3
+from . import s3, errors
 from .utils import safe_write, safe_copy
 
 logger = logging.getLogger(__name__.split(".")[0])
@@ -141,7 +141,12 @@ def _download_buffer(self, external_path):
         if self.spec["protocol"] == "s3":
             return self.s3.get(external_path)
         if self.spec["protocol"] == "file":
-            return Path(external_path).read_bytes()
+            try:
+                return Path(external_path).read_bytes()
+            except FileNotFoundError:
+                raise errors.MissingExternalFile(
+                    f"Missing external file {external_path}"
+                ) from None
         assert False
 
     def _remove_external_file(self, external_path):

datajoint/settings.py

@@ -246,6 +246,11 @@ def __setitem__(self, key, value):
                 self._conf[key] = value
             else:
                 raise DataJointError("Validator for {0:s} did not pass".format(key))
+            valid_logging_levels = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}
+            if key == "loglevel":
+                if value not in valid_logging_levels:
+                    raise ValueError(f"{'value'} is not a valid logging value")
+                logger.setLevel(value)
 
 
 # Load configuration from file
@@ -270,6 +275,7 @@ def __setitem__(self, key, value):
             "database.password",
             "external.aws_access_key_id",
             "external.aws_secret_access_key",
+            "loglevel",
         ),
         map(
             os.getenv,
@@ -279,6 +285,7 @@ def __setitem__(self, key, value):
                 "DJ_PASS",
                 "DJ_AWS_ACCESS_KEY_ID",
                 "DJ_AWS_SECRET_ACCESS_KEY",
+                "DJ_LOG_LEVEL",
             ),
         ),
     )

tests/test_dependencies.py

@@ -19,6 +19,26 @@ def test_nullable_dependency(thing_tables):
     assert len(c) == len(c.fetch()) == 5
 
 
+def test_topo_sort():
+    import networkx as nx
+    import datajoint as dj
+
+    graph = nx.DiGraph(
+        [
+            ("`a`.`a`", "`a`.`m`"),
+            ("`a`.`a`", "`a`.`z`"),
+            ("`a`.`m`", "`a`.`m__part`"),
+            ("`a`.`z`", "`a`.`m__part`"),
+        ]
+    )
+    assert dj.dependencies.topo_sort(graph) == [
+        "`a`.`a`",
+        "`a`.`z`",
+        "`a`.`m`",
+        "`a`.`m__part`",
+    ]
+
+
 def test_unique_dependency(thing_tables):
     """test nullable unique foreign key"""
     # Thing C has a nullable dependency on B whose primary key is composite