Merge pull request #2628 from jsiirola/deepcopy-improvements

Deepcopy improvements

Author: blnicho

pyomo/common/autoslots.py

@@ -20,14 +20,26 @@
 )
 
 def _deepcopy_tuple(obj, memo, _id):
-    _unchanged.append(True)
-    ans = tuple(fast_deepcopy(x, memo) for x in obj)
-    if _unchanged.pop():
-        # It appears to be faster *not* to cache the fact that this
-        # particular tuple was unchanged by the deepcopy
-        #   memo[_id] = obj
+    ans = []
+    unchanged = True
+    for item in obj:
+        new_item = fast_deepcopy(item, memo)
+        ans.append(new_item)
+        if new_item is not item:
+            unchanged = False
+    if unchanged:
+        # Python does not duplicate "unchanged" tuples (i.e. allows the
+        # original objecct to be returned from deepcopy()).  We will
+        # preserve that behavior here.
+        #
+        # It also appears to be faster *not* to cache the fact that this
+        # particular tuple was unchanged by the deepcopy (Note: the
+        # standard library also does not cache the unchanged tuples in
+        # the memo)
+        #
+        #  memo[_id] = obj
         return obj
-    memo[_id] = ans
+    memo[_id] = ans = tuple(ans)
     return ans
 
 def _deepcopy_list(obj, memo, _id):
@@ -55,7 +67,6 @@ def _deepcopier(obj, memo, _id):
     dict: _deepcopy_dict,
 }
 
-_unchanged = [None]
 
 def fast_deepcopy(obj, memo):
     """A faster implementation of copy.deepcopy()
@@ -70,12 +81,9 @@ def fast_deepcopy(obj, memo):
         return obj
     _id = id(obj)
     if _id in memo:
-        ans = memo[_id]
+        return memo[_id]
     else:
-        ans = _deepcopy_mapper.get(obj.__class__, _deepcopier)(obj, memo, _id)
-    if ans is not obj:
-        _unchanged[-1] = False
-    return ans
+        return _deepcopy_mapper.get(obj.__class__, _deepcopier)(obj, memo, _id)
 
 
 class AutoSlots(type):

pyomo/core/base/component.py

@@ -107,32 +107,35 @@ def __deepcopy__(self, memo):
         # we need to override __deepcopy__ for both Component and
         # ComponentData.
         #
-        # Note: there is an edge case when cloning a block: the initial
-        # call to deepcopy (on the target block) has __block_scope__
-        # defined, however, the parent block of self is either None, or
-        # is (by definition) out of scope.  So we will check that
-        # id(self) is not in __block_scope__: if it is, then this is the
-        # top-level block and we need to do the normal deepcopy.
-        if '__block_scope__' in memo and \
-                id(self) not in memo['__block_scope__']:
-            _known = memo['__block_scope__']
-            _new = []
+        if '__block_scope__' in memo:
+            _scope = memo['__block_scope__']
+            _new = None
             tmp = self.parent_block()
-            tmpId = id(tmp)
             # Note: normally we would need to check that tmp does not
             # end up being None.  However, since clone() inserts
             # id(None) into the __block_scope__ dictionary, we are safe
-            while tmpId not in _known:
-                _new.append(tmpId)
+            while id(tmp) not in _scope:
+                _new = (_new, id(tmp))
                 tmp = tmp.parent_block()
-                tmpId = id(tmp)
+            _in_scope = _scope[id(tmp)]
 
             # Remember whether all newly-encountered blocks are in or
             # out of scope (prevent duplicate work)
-            for _id in _new:
-                _known[_id] = _known[tmpId]
-
-            if not _known[tmpId]:
+            while _new is not None:
+                _new, _id = _new
+                _scope[_id] = _in_scope
+
+            # Note: there is an edge case when cloning a block: the
+            # initial call to deepcopy (on the target block) has
+            # __block_scope__ defined, however, the parent block of self
+            # is either None, or is (by definition) out of scope.  So we
+            # will check that id(self) is not in __block_scope__: if it
+            # is, then this is the top-level block and we need to do the
+            # normal deepcopy.  We defer this check until now for
+            # efficiency reasons beause we expect that (for sane models)
+            # the bulk of the ccomponents we will encounter will be *in*
+            # scope.
+            if not _in_scope and id(self) not in _scope:
                 # component is out-of-scope.  shallow copy only
                 memo[id(self)] = self
                 return self
@@ -258,7 +261,7 @@ def _create_objects_for_deepcopy(self, memo, component_list):
     def _deepcopy_field(self, memo, slot_name, value):
         saved_memo = len(memo)
         try:
-            return deepcopy(value, memo)
+            return fast_deepcopy(value, memo)
         except CloneError:
             raise
         except:

pyomo/core/base/indexed_component.py

@@ -27,6 +27,7 @@
 from pyomo.core.base.config import PyomoOptions
 from pyomo.core.base.global_set import UnindexedComponent_set
 from pyomo.common import DeveloperError
+from pyomo.common.autoslots import fast_deepcopy
 from pyomo.common.dependencies import numpy as np, numpy_available
 from pyomo.common.deprecation import deprecated
 from pyomo.common.modeling import NOTSET
@@ -330,19 +331,12 @@ def _create_objects_for_deepcopy(self, memo, component_list):
                 # Because we are already checking / updating the memo
                 # for the _data dict, we can effectively "deepcopy" it
                 # right now (almost for free!)
-                memo[id(self._data)] = _new._data = _data = {}
-                for idx, obj in self._data.items():
-                    # We need to deepcopy the index, but deepcopying
-                    # tuples in Python is SLOW.  We will only deepcopy
-                    # if necessary.
-                    if idx.__class__ is tuple:
-                        if any(x.__class__ not in native_types for x in idx):
-                            idx = deepcopy(idx, memo)
-                    elif idx.__class__ not in native_types:
-                        idx = deepcopy(idx, memo)
-
-                    _data[idx] = obj._create_objects_for_deepcopy(
-                        memo, component_list)
+                _src = self._data
+                memo[id(_src)] = _new._data = _data = _src.__class__()
+                for idx, obj in _src.items():
+                    _data[fast_deepcopy(idx, memo)] \
+                        = obj._create_objects_for_deepcopy(memo, component_list)
+
         return _ans
 
     def to_dense_data(self):