Merge pull request #3633 from jsiirola/default-pyomo-log-handler

mrmundt · web-flow · commit 339d1e763456 · 2025-06-17T13:05:15.000-06:00
Fix `capture_output` management of logger file descriptors
diff --git a/doc/OnlineDocs/_templates/recursive-base.rst b/doc/OnlineDocs/_templates/recursive-base.rst
@@ -7,6 +7,11 @@
    # import everything from the module containing this class so that
    # doctests for the class docstrings see the correct environment
    from {{ module }} import *
+   try:
+       from {{ module }} import _autosummary_doctest_setup
+       _autosummary_doctest_setup()
+   except ImportError:
+       pass
 
 .. currentmodule:: {{ module }}
 
diff --git a/doc/OnlineDocs/_templates/recursive-class.rst b/doc/OnlineDocs/_templates/recursive-class.rst
@@ -7,6 +7,11 @@
    # import everything from the module containing this class so that
    # doctests for the class docstrings see the correct environment
    from {{ module }} import *
+   try:
+       from {{ module }} import _autosummary_doctest_setup
+       _autosummary_doctest_setup()
+   except ImportError:
+       pass
 
 .. currentmodule:: {{ module }}
 
diff --git a/doc/OnlineDocs/_templates/recursive-enum.rst b/doc/OnlineDocs/_templates/recursive-enum.rst
@@ -7,6 +7,11 @@
    # import everything from the module containing this class so that
    # doctests for the class docstrings see the correct environment
    from {{ module }} import *
+   try:
+       from {{ module }} import _autosummary_doctest_setup
+       _autosummary_doctest_setup()
+   except ImportError:
+       pass
 
 .. currentmodule:: {{ module }}
 
diff --git a/doc/OnlineDocs/_templates/recursive-module.rst b/doc/OnlineDocs/_templates/recursive-module.rst
@@ -5,6 +5,19 @@ Library Reference
 {{ name | escape | underline}}
 {% endif %}
 
+.. testsetup:: *
+
+   # import everything from the module containing this class so that
+   # doctests for the class docstrings see the correct environment
+   from {{ module }} import *
+   try:
+       from {{ module }} import _autosummary_doctest_setup
+       _autosummary_doctest_setup()
+   except ImportError:
+       pass
+
+.. currentmodule:: {{ module }}
+
 .. automodule:: {{ fullname }}
    :undoc-members:
 
diff --git a/doc/OnlineDocs/errors.rst b/doc/OnlineDocs/errors.rst
@@ -8,10 +8,13 @@ Common Warnings/Errors
    for backwards compatibility, DO NOT recycle old ID (no longer used)
    numbers.
 
-.. doctest::
-   :hide:
+.. testsetup::
 
-   >>> import pyomo.environ as pyo
+   import pyomo.environ as pyo
+   # Ensure that all logged messages are sent to stdout
+   # (so they show up in the doctest output and can be tested)
+   import pyomo.common.log as _log
+   _log.pyomo_handler.__class__ = _log.StdoutHandler
 
 .. py:currentmodule:: pyomo.environ
 
diff --git a/doc/OnlineDocs/explanation/developer_utils/deprecation.rst b/doc/OnlineDocs/explanation/developer_utils/deprecation.rst
@@ -9,40 +9,29 @@ Deprecation
 -----------
 
 We offer a set of tools to help with deprecation in
-``pyomo.common.deprecation``.
+:py:mod:`pyomo.common.deprecation`.
 
 By policy, when deprecating or moving an existing capability, one of the
 following utilities should be leveraged.  Each has a required
 ``version`` argument that should be set to current development version (e.g.,
 ``"6.6.2.dev0"``).  This version will be updated to the next actual
 release as part of the Pyomo release process.  The current development version
-can be found by running ``pyomo --version`` on your local fork/branch.
+can be found by running
+
+   ``pyomo --version``
+
+on your local fork/branch.
 
 .. currentmodule:: pyomo.common.deprecation
 
 .. autosummary::
 
    deprecated
    deprecation_warning
-   relocated_module
+   moved_module
    relocated_module_attribute
    RenamedClass
 
-.. autodecorator:: pyomo.common.deprecation.deprecated
-   :noindex:
-
-.. autofunction:: pyomo.common.deprecation.deprecation_warning
-   :noindex:
-
-.. autofunction:: pyomo.common.deprecation.relocated_module
-   :noindex:
-
-.. autofunction:: pyomo.common.deprecation.relocated_module_attribute
-   :noindex:
-
-.. autoclass:: pyomo.common.deprecation.RenamedClass
-   :noindex:
-
 
 Removal
 -------
@@ -52,10 +41,8 @@ warning, pending extenuating circumstances. The functionality should
 be deprecated, following the information above.
 
 If the functionality is documented in the most recent
-edition of [`Pyomo - Optimization Modeling in Python`_], it may not be removed
-until the next major version release.
-
-.. _Pyomo - Optimization Modeling in Python: https://doi.org/10.1007/978-3-030-68928-5
+edition of :ref:`Pyomo - Optimization Modeling in Python <pyomobookiii>`,
+it may not be removed until the next major version release.
 
 For other functionality, it is preferred that ample time is given
 before removing the functionality. At minimum, significant functionality
diff --git a/doc/OnlineDocs/explanation/solvers/pynumero/tutorial.linear_solver_interfaces.rst b/doc/OnlineDocs/explanation/solvers/pynumero/tutorial.linear_solver_interfaces.rst
@@ -42,7 +42,7 @@ Interface to MA27
    >>> status = solver.do_numeric_factorization(A)
    >>> x, status = solver.do_back_solve(rhs)
    >>> np.max(np.abs(A*x - rhs)) <= 1e-15
-   True
+   np.True_
 
 
 Interface to MUMPS
diff --git a/doc/OnlineDocs/reference/bibliography.rst b/doc/OnlineDocs/reference/bibliography.rst
@@ -29,6 +29,10 @@ These publications describe various Pyomo capabilitites or subpackages:
    Vol. 67. Springer. 2021.  DOI `10.1007/978-3-030-68928-5
    <https://doi.org/10.1007/978-3-030-68928-5>`_
 
+..
+   NOTE: When adding a new edition of the Pyomo Book, search the codebase
+   both for citation references "[PyomoBookIII]" and references "pyomobookiii"
+
 .. [PyomoDAE-paper] Bethany Nicholson, John D. Siirola, Jean-Paul Watson,
    Victor M. Zavala, and Lorenz T. Biegler. "pyomo.dae: a modeling and
    automatic discretization framework for optimization with differential
@@ -126,7 +130,7 @@ Bibliography
 
 .. [RB01] W. C. Rooney and L. T. Biegler. "Design for model parameter
    uncertainty using nonlinear confidence regions", *AIChE Journal*,
-   47(8). 2001.
+   47(8). 2001.  DOI `10.1002/aic.690470811 <https://doi.org/10.1002/aic.690470811>`_
 
 .. [RG94] R. Raman and I. E. Grossmann. "Modelling and computational
    techniques for logic based integer programming", *Computers and
diff --git a/pyomo/common/dependencies.py b/pyomo/common/dependencies.py
@@ -856,7 +856,6 @@ def declare_deferred_modules_as_importable(globals_dict):
        ...     'scipy', callback=_finalize_scipy,
        ...     deferred_submodules=['stats', 'sparse', 'spatial', 'integrate'])
        >>> declare_deferred_modules_as_importable(globals())
-       WARNING: DEPRECATED: ...
 
     Which enables users to use:
 
diff --git a/pyomo/common/deprecation.py b/pyomo/common/deprecation.py
@@ -36,6 +36,18 @@
 _doc_flag = '.. deprecated::'
 
 
+def _autosummary_doctest_setup():
+    """This function gets called to setup the doctest environment before
+    running any autosummary doctests in this module.
+    """
+
+    # Ensure that all logged messages are sent to stdout
+    # (so they show up in the doctest output and can be tested)
+    import pyomo.common.log as _log
+
+    _log.pyomo_handler.__class__ = _log.StdoutHandler
+
+
 def default_deprecation_msg(obj, user_msg, version, remove_in):
     """Generate the default deprecation message.
 
@@ -157,7 +169,7 @@ def _find_calling_frame(module_offset):
 def deprecation_warning(
     msg, logger=None, version=None, remove_in=None, calling_frame=None
 ):
-    """Standardized formatter for deprecation warnings
+    """Standardized function for formatting and emitting deprecation warnings.
 
     This is a standardized routine for formatting deprecation warnings
     so that things look consistent and "nice".
diff --git a/pyomo/common/log.py b/pyomo/common/log.py
@@ -208,13 +208,25 @@ def format(self, record):
 class StdoutHandler(logging.StreamHandler):
     """A logging handler that emits to the current value of sys.stdout"""
 
+    def __init__(self):
+        super().__init__()
+        self.stream = None
+
     def flush(self):
-        self.stream = sys.stdout
-        super(StdoutHandler, self).flush()
+        try:
+            orig = self.stream
+            self.stream = sys.stdout
+            super(StdoutHandler, self).flush()
+        finally:
+            self.stream = orig
 
     def emit(self, record):
-        self.stream = sys.stdout
-        super(StdoutHandler, self).emit(record)
+        try:
+            orig = self.stream
+            self.stream = sys.stdout
+            super(StdoutHandler, self).emit(record)
+        finally:
+            self.stream = orig
 
 
 class Preformatted(object):
@@ -248,7 +260,7 @@ def filter(self, record):
 # debugging.  It has been updated to suppress output if any handlers
 # have been defined at the root level.
 pyomo_logger = logging.getLogger('pyomo')
-pyomo_handler = StdoutHandler()
+pyomo_handler = logging.StreamHandler(sys.stdout)
 pyomo_formatter = LegacyPyomoFormatter(
     base=PYOMO_ROOT_DIR, verbosity=lambda: pyomo_logger.isEnabledFor(logging.DEBUG)
 )
@@ -445,46 +457,50 @@ class _StreamRedirector(object):
     def __init__(self, handler, fd):
         self.handler = handler
         self.fd = fd
+        self.local_fd = None
         self.orig_stream = None
 
     def __enter__(self):
+        assert self.local_fd is None
         self.orig_stream = self.handler.stream
         # Note: ideally, we would use closefd=True and let Python handle
         # closing the local file descriptor that we are about to create.
         # However, it appears that closefd is ignored on Windows (see
         # #3587), so we will just handle it explicitly ourselves.
+        self.local_fd = os.dup(self.fd)
         self.handler.stream = os.fdopen(
-            os.dup(self.fd), mode="w", closefd=False
+            self.local_fd, mode="a", closefd=False
         ).__enter__()
 
     def __exit__(self, et, ev, tb):
         try:
-            fd = self.handler.stream.fileno()
             self.handler.stream.__exit__(et, ev, tb)
-            os.close(fd)
+            os.close(self.local_fd)
         finally:
             self.handler.stream = self.orig_stream
 
 
 class _LastResortRedirector(object):
     def __init__(self, fd):
         self.fd = fd
+        self.local_fd = None
         self.orig_stream = None
 
     def __enter__(self):
+        assert self.local_fd is None
         self.orig = logging.lastResort
         # Note: ideally, we would use closefd=True and let Python handle
         # closing the local file descriptor that we are about to create.
         # However, it appears that closefd is ignored on Windows (see
         # #3587), so we will just handle it explicitly ourselves.
+        self.local_fd = os.dup(self.fd)
         logging.lastResort = logging.StreamHandler(
-            os.fdopen(os.dup(self.fd), mode="w", closefd=False).__enter__()
+            os.fdopen(self.local_fd, mode="a", closefd=False).__enter__()
         )
 
     def __exit__(self, et, ev, tb):
         try:
-            fd = logging.lastResort.stream.fileno()
             logging.lastResort.stream.close()
-            os.close(fd)
+            os.close(self.local_fd)
         finally:
             logging.lastResort = self.orig
diff --git a/pyomo/common/tee.py b/pyomo/common/tee.py
@@ -169,7 +169,7 @@ def __enter__(self):
 
         if self.synchronize and self.std:
             # Cause Python's stdout to point to our new file
-            self.target_file = os.fdopen(self.fd, 'w', closefd=False)
+            self.target_file = os.fdopen(self.fd, 'a', closefd=False)
             setattr(sys, self.std, self.target_file)
 
         return self
@@ -349,7 +349,7 @@ def _enter_impl(self):
                 log_stream = self._enter_context(
                     os.fdopen(
                         self._enter_context(_fd_closer(os.dup(old_fd[1] or 2))),
-                        mode="w",
+                        mode="a",
                         closefd=False,
                     )
                 )
@@ -358,7 +358,7 @@ def _enter_impl(self):
             self._enter_context(LoggingIntercept(log_stream, logger=logger, level=None))
 
             if isinstance(self.output, str):
-                self.output_stream = self._enter_context(open(self.output, 'w'))
+                self.output_stream = self._enter_context(open(self.output, 'a'))
             elif self.output is None:
                 self.output_stream = io.StringIO()
             else:
@@ -415,7 +415,7 @@ def _enter_impl(self):
                                     _fd_closer(os.dup(fd_redirect[fd].original_fd)),
                                     prior_to=self.tee,
                                 ),
-                                mode="w",
+                                mode="a",
                                 closefd=False,
                             ),
                             prior_to=self.tee,
@@ -673,7 +673,7 @@ def STDERR(self):
             self._stderr = self.open(buffering=b)
         return self._stderr
 
-    def open(self, mode='w', buffering=-1, encoding=None, newline=None):
+    def open(self, mode='a', buffering=-1, encoding=None, newline=None):
         if encoding is None:
             encoding = self.encoding
         handle = _StreamHandle(mode, buffering, encoding, newline)
diff --git a/pyomo/common/tests/test_log.py b/pyomo/common/tests/test_log.py
