Merge branch 'master' into key_populate

Author: dimitri-yatsenko

CHANGELOG.md

@@ -6,6 +6,8 @@
 - Added - Datajoint python CLI ([#940](https://github.com/datajoint/datajoint-python/issues/940)) PR [#1095](https://github.com/datajoint/datajoint-python/pull/1095)
 - Added - Ability to set hidden attributes on a table - PR [#1091](https://github.com/datajoint/datajoint-python/pull/1091)
 - Added - Ability to specify a list of keys to populate - PR [#989](https://github.com/datajoint/datajoint-python/pull/989)
+- Fixed - fixed topological sort [#1057](https://github.com/datajoint/datajoint-python/issues/1057)- PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184)
+- Fixed - .parts() not always returning parts [#1103](https://github.com/datajoint/datajoint-python/issues/1103)- PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184)
 
 ### 0.14.2 -- Aug 19, 2024
 - Added - Migrate nosetests to pytest - PR [#1142](https://github.com/datajoint/datajoint-python/pull/1142)

datajoint/dependencies.py

@@ -1,34 +1,9 @@
 import networkx as nx
 import itertools
-import re
 from collections import defaultdict
 from .errors import DataJointError
 
 
-def unite_master_parts(lst):
-    """
-    re-order a list of table names so that part tables immediately follow their master tables without breaking
-    the topological order.
-    Without this correction, a simple topological sort may insert other descendants between master and parts.
-    The input list must be topologically sorted.
-    :example:
-    unite_master_parts(
-        ['`s`.`a`', '`s`.`a__q`', '`s`.`b`', '`s`.`c`', '`s`.`c__q`', '`s`.`b__q`', '`s`.`d`', '`s`.`a__r`']) ->
-        ['`s`.`a`', '`s`.`a__q`', '`s`.`a__r`', '`s`.`b`', '`s`.`b__q`', '`s`.`c`', '`s`.`c__q`', '`s`.`d`']
-    """
-    for i in range(2, len(lst)):
-        name = lst[i]
-        match = re.match(r"(?P<master>`\w+`.`#?\w+)__\w+`", name)
-        if match:  # name is a part table
-            master = match.group("master")
-            for j in range(i - 1, -1, -1):
-                if lst[j] == master + "`" or lst[j].startswith(master + "__"):
-                    # move from the ith position to the (j+1)th position
-                    lst[j + 1 : i + 1] = [name] + lst[j + 1 : i]
-                    break
-    return lst
-
-
 class Dependencies(nx.DiGraph):
     """
     The graph of dependencies (foreign keys) between loaded tables.
@@ -168,9 +143,7 @@ def descendants(self, full_table_name):
         """
         self.load(force=False)
         nodes = self.subgraph(nx.algorithms.dag.descendants(self, full_table_name))
-        return unite_master_parts(
-            [full_table_name] + list(nx.algorithms.dag.topological_sort(nodes))
-        )
+        return [full_table_name] + list(nx.algorithms.dag.topological_sort(nodes))
 
     def ancestors(self, full_table_name):
         """
@@ -181,8 +154,6 @@ def ancestors(self, full_table_name):
         nodes = self.subgraph(nx.algorithms.dag.ancestors(self, full_table_name))
         return list(
             reversed(
-                unite_master_parts(
-                    list(nx.algorithms.dag.topological_sort(nodes)) + [full_table_name]
-                )
+                list(nx.algorithms.dag.topological_sort(nodes)) + [full_table_name]
             )
         )

datajoint/diagram.py

@@ -5,7 +5,6 @@
 import logging
 import inspect
 from .table import Table
-from .dependencies import unite_master_parts
 from .user_tables import Manual, Imported, Computed, Lookup, Part
 from .errors import DataJointError
 from .table import lookup_class_name
@@ -59,8 +58,7 @@ class Diagram:
         Entity relationship diagram, currently disabled due to the lack of required packages: matplotlib and pygraphviz.
 
         To enable Diagram feature, please install both matplotlib and pygraphviz. For instructions on how to install
-        these two packages, refer to http://docs.datajoint.io/setup/Install-and-connect.html#python and
-        http://tutorials.datajoint.io/setting-up/datajoint-python.html
+        these two packages, refer to https://datajoint.com/docs/core/datajoint-python/0.14/client/install/
         """
 
         def __init__(self, *args, **kwargs):
@@ -181,11 +179,9 @@ def is_part(part, master):
 
         def topological_sort(self):
             """:return:  list of nodes in topological order"""
-            return unite_master_parts(
-                list(
-                    nx.algorithms.dag.topological_sort(
-                        nx.DiGraph(self).subgraph(self.nodes_to_show)
-                    )
+            return list(
+                nx.algorithms.dag.topological_sort(
+                    nx.DiGraph(self).subgraph(self.nodes_to_show)
                 )
             )
 

datajoint/schemas.py

@@ -2,18 +2,16 @@
 import logging
 import inspect
 import re
-import itertools
-import collections
 from .connection import conn
-from .diagram import Diagram, _get_tier
+from .diagram import Diagram
 from .settings import config
 from .errors import DataJointError, AccessError
 from .jobs import JobTable
 from .external import ExternalMapping
 from .heading import Heading
 from .utils import user_choice, to_camel_case
 from .user_tables import Part, Computed, Imported, Manual, Lookup
-from .table import lookup_class_name, Log, FreeTable
+from .table import lookup_class_name, Log
 import types
 
 logger = logging.getLogger(__name__.split(".")[0])
@@ -401,78 +399,6 @@ def jobs(self):
             self._jobs = JobTable(self.connection, self.database)
         return self._jobs
 
-    @property
-    def code(self):
-        self._assert_exists()
-        return self.save()
-
-    def save(self, python_filename=None):
-        """
-        Generate the code for a module that recreates the schema.
-        This method is in preparation for a future release and is not officially supported.
-
-        :return: a string containing the body of a complete Python module defining this schema.
-        """
-        self._assert_exists()
-        module_count = itertools.count()
-        # add virtual modules for referenced modules with names vmod0, vmod1, ...
-        module_lookup = collections.defaultdict(
-            lambda: "vmod" + str(next(module_count))
-        )
-        db = self.database
-
-        def make_class_definition(table):
-            tier = _get_tier(table).__name__
-            class_name = table.split(".")[1].strip("`")
-            indent = ""
-            if tier == "Part":
-                class_name = class_name.split("__")[-1]
-                indent += "    "
-            class_name = to_camel_case(class_name)
-
-            def replace(s):
-                d, tabs = s.group(1), s.group(2)
-                return ("" if d == db else (module_lookup[d] + ".")) + ".".join(
-                    to_camel_case(tab) for tab in tabs.lstrip("__").split("__")
-                )
-
-            return ("" if tier == "Part" else "\n@schema\n") + (
-                "{indent}class {class_name}(dj.{tier}):\n"
-                '{indent}    definition = """\n'
-                '{indent}    {defi}"""'
-            ).format(
-                class_name=class_name,
-                indent=indent,
-                tier=tier,
-                defi=re.sub(
-                    r"`([^`]+)`.`([^`]+)`",
-                    replace,
-                    FreeTable(self.connection, table).describe(),
-                ).replace("\n", "\n    " + indent),
-            )
-
-        diagram = Diagram(self)
-        body = "\n\n".join(
-            make_class_definition(table) for table in diagram.topological_sort()
-        )
-        python_code = "\n\n".join(
-            (
-                '"""This module was auto-generated by datajoint from an existing schema"""',
-                "import datajoint as dj\n\nschema = dj.Schema('{db}')".format(db=db),
-                "\n".join(
-                    "{module} = dj.VirtualModule('{module}', '{schema_name}')".format(
-                        module=v, schema_name=k
-                    )
-                    for k, v in module_lookup.items()
-                ),
-                body,
-            )
-        )
-        if python_filename is None:
-            return python_code
-        with open(python_filename, "wt") as f:
-            f.write(python_code)
-
     def list_tables(self):
         """
         Return a list of all tables in the schema except tables with ~ in first character such

datajoint/table.py

@@ -196,7 +196,6 @@ def parents(self, primary=None, as_objects=False, foreign_key_info=False):
 
     def children(self, primary=None, as_objects=False, foreign_key_info=False):
         """
-
         :param primary: if None, then all children are returned. If True, then only foreign keys composed of
             primary key attributes are considered.  If False, return foreign keys including at least one
             secondary attribute.
@@ -230,7 +229,6 @@ def descendants(self, as_objects=False):
 
     def ancestors(self, as_objects=False):
         """
-
         :param as_objects: False - a list of table names; True - a list of table objects.
         :return: list of tables ancestors in topological order.
         """
@@ -246,6 +244,7 @@ def parts(self, as_objects=False):
 
         :param as_objects: if False (default), the output is a dict describing the foreign keys. If True, return table objects.
         """
+        self.connection.dependencies.load(force=False)
         nodes = [
             node
             for node in self.connection.dependencies.nodes
@@ -427,7 +426,8 @@ def insert(
             self.connection.query(query)
             return
 
-        field_list = []  # collects the field list from first row (passed by reference)
+        # collects the field list from first row (passed by reference)
+        field_list = []
         rows = list(
             self.__make_row_to_insert(row, field_list, ignore_extra_fields)
             for row in rows
@@ -520,7 +520,8 @@ def cascade(table):
                     delete_count = table.delete_quick(get_count=True)
                 except IntegrityError as error:
                     match = foreign_key_error_regexp.match(error.args[0]).groupdict()
-                    if "`.`" not in match["child"]:  # if schema name missing, use table
+                    # if schema name missing, use table
+                    if "`.`" not in match["child"]:
                         match["child"] = "{}.{}".format(
                             table.full_table_name.split(".")[0], match["child"]
                         )
@@ -964,7 +965,8 @@ def lookup_class_name(name, context, depth=3):
     while nodes:
         node = nodes.pop(0)
         for member_name, member in node["context"].items():
-            if not member_name.startswith("_"):  # skip IPython's implicit variables
+            # skip IPython's implicit variables
+            if not member_name.startswith("_"):
                 if inspect.isclass(member) and issubclass(member, Table):
                     if member.full_table_name == name:  # found it!
                         return ".".join([node["context_name"], member_name]).lstrip(".")

tests/test_blob.py

@@ -185,7 +185,7 @@ def test_insert_longblob(schema_any):
 
     query_mym_blob = {"id": 1, "data": np.array([1, 2, 3])}
     Longblob.insert1(query_mym_blob)
-    assert (Longblob & "id=1").fetch1()["data"].all() == query_mym_blob["data"].all()
+    assert_array_equal((Longblob & "id=1").fetch1()["data"], query_mym_blob["data"])
     (Longblob & "id=1").delete()
 
 
@@ -218,7 +218,8 @@ def test_insert_longblob_32bit(schema_any, enable_feature_32bit_dims):
         ),
     }
     assert fetched["id"] == expected["id"]
-    assert np.array_equal(fetched["data"], expected["data"])
+    for name in expected["data"][0][0].dtype.names:
+        assert_array_equal(expected["data"][0][0][name], fetched["data"][0][0][name])
     (Longblob & "id=1").delete()
 
 
@@ -248,4 +249,5 @@ def test_datetime_serialization_speed():
     )
     print(f"python time {baseline_exe_time}")
 
-    assert optimized_exe_time * 900 < baseline_exe_time
+    # The time savings were much greater (x1000) but use x10 for testing
+    assert optimized_exe_time * 10 < baseline_exe_time

tests/test_dependencies.py

@@ -1,51 +1,5 @@
 from datajoint import errors
 from pytest import raises
-from datajoint.dependencies import unite_master_parts
-
-
-def test_unite_master_parts():
-    assert unite_master_parts(
-        [
-            "`s`.`a`",
-            "`s`.`a__q`",
-            "`s`.`b`",
-            "`s`.`c`",
-            "`s`.`c__q`",
-            "`s`.`b__q`",
-            "`s`.`d`",
-            "`s`.`a__r`",
-        ]
-    ) == [
-        "`s`.`a`",
-        "`s`.`a__q`",
-        "`s`.`a__r`",
-        "`s`.`b`",
-        "`s`.`b__q`",
-        "`s`.`c`",
-        "`s`.`c__q`",
-        "`s`.`d`",
-    ]
-    assert unite_master_parts(
-        [
-            "`lab`.`#equipment`",
-            "`cells`.`cell_analysis_method`",
-            "`cells`.`cell_analysis_method_task_type`",
-            "`cells`.`cell_analysis_method_users`",
-            "`cells`.`favorite_selection`",
-            "`cells`.`cell_analysis_method__cell_selection_params`",
-            "`lab`.`#equipment__config`",
-            "`cells`.`cell_analysis_method__field_detect_params`",
-        ]
-    ) == [
-        "`lab`.`#equipment`",
-        "`lab`.`#equipment__config`",
-        "`cells`.`cell_analysis_method`",
-        "`cells`.`cell_analysis_method__cell_selection_params`",
-        "`cells`.`cell_analysis_method__field_detect_params`",
-        "`cells`.`cell_analysis_method_task_type`",
-        "`cells`.`cell_analysis_method_users`",
-        "`cells`.`favorite_selection`",
-    ]
 
 
 def test_nullable_dependency(thing_tables):

tests/test_schema.py

@@ -218,14 +218,6 @@ def test_list_tables(schema_simp):
     assert actual == expected, f"Missing from list_tables(): {expected - actual}"
 
 
-def test_schema_save_any(schema_any):
-    assert "class Experiment(dj.Imported)" in schema_any.code
-
-
-def test_schema_save_empty(schema_empty):
-    assert "class Experiment(dj.Imported)" in schema_empty.code
-
-
 def test_uppercase_schema(db_creds_root):
     """
     https://github.com/datajoint/datajoint-python/issues/564