From f2066ea16e4a92d390f41cd32ee4debc38a1dbd1 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 18 May 2025 09:14:22 -0600 Subject: [PATCH 01/50] working on a model observer --- pyomo/contrib/observer/__init__.py | 0 pyomo/contrib/observer/model_observer.py | 741 +++++++++++++++++++++++ 2 files changed, 741 insertions(+) create mode 100644 pyomo/contrib/observer/__init__.py create mode 100644 pyomo/contrib/observer/model_observer.py diff --git a/pyomo/contrib/observer/__init__.py b/pyomo/contrib/observer/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py new file mode 100644 index 00000000000..b644676f3d2 --- /dev/null +++ b/pyomo/contrib/observer/model_observer.py @@ -0,0 +1,741 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# __________________________________________________________________________ + +import abc +import datetime +from typing import List, Sequence + +from pyomo.common.config import ConfigDict, ConfigValue +from pyomo.core.base.constraint import ConstraintData, Constraint +from pyomo.core.base.sos import SOSConstraintData, SOSConstraint +from pyomo.core.base.var import VarData +from pyomo.core.base.param import ParamData, Param +from pyomo.core.base.objective import ObjectiveData +from pyomo.core.staleflag import StaleFlagManager +from pyomo.common.collections import ComponentMap +from pyomo.common.timing import HierarchicalTimer +from pyomo.contrib.solver.common.results import Results +from pyomo.contrib.solver.common.util import collect_vars_and_named_exprs, get_objective +from pyomo.common.numeric_types import native_numeric_types + + +class AutoUpdateConfig(ConfigDict): + """ + Control which parts of the model are automatically checked and/or updated upon re-solve + """ + + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + if doc is None: + doc = 'Configuration options to detect changes in model between solves' + super().__init__( + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + + self.check_for_new_or_removed_constraints: bool = self.declare( + 'check_for_new_or_removed_constraints', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old constraints will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.add_constraints() + and opt.remove_constraints() or when you are certain constraints are not being + added to/removed from the model.""", + ), + ) + self.check_for_new_or_removed_vars: bool = self.declare( + 'check_for_new_or_removed_vars', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old variables will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.add_variables() and + opt.remove_variables() or when you are certain variables are not being added to / + removed from the model.""", + ), + ) + self.check_for_new_or_removed_params: bool = self.declare( + 'check_for_new_or_removed_params', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old parameters will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.add_parameters() and + opt.remove_parameters() or when you are certain parameters are not being added to / + removed from the model.""", + ), + ) + self.check_for_new_objective: bool = self.declare( + 'check_for_new_objective', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old objectives will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.set_objective() or + when you are certain objectives are not being added to / removed from the model.""", + ), + ) + self.update_constraints: bool = self.declare( + 'update_constraints', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to existing constraints will not be automatically detected on + subsequent solves. This includes changes to the lower, body, and upper attributes of + constraints. Use False only when manually updating the solver with + opt.remove_constraints() and opt.add_constraints() or when you are certain constraints + are not being modified.""", + ), + ) + self.update_vars: bool = self.declare( + 'update_vars', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to existing variables will not be automatically detected on + subsequent solves. This includes changes to the lb, ub, domain, and fixed + attributes of variables. Use False only when manually updating the solver with + opt.update_variables() or when you are certain variables are not being modified.""", + ), + ) + self.update_parameters: bool = self.declare( + 'update_parameters', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to parameter values will not be automatically detected on + subsequent solves. Use False only when manually updating the solver with + opt.update_parameters() or when you are certain parameters are not being modified.""", + ), + ) + self.update_named_expressions: bool = self.declare( + 'update_named_expressions', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to Expressions will not be automatically detected on + subsequent solves. Use False only when manually updating the solver with + opt.remove_constraints() and opt.add_constraints() or when you are certain + Expressions are not being modified.""", + ), + ) + self.update_objective: bool = self.declare( + 'update_objective', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to objectives will not be automatically detected on + subsequent solves. This includes the expr and sense attributes of objectives. Use + False only when manually updating the solver with opt.set_objective() or when you are + certain objectives are not being modified.""", + ), + ) + + +class Observer(abc.ABC): + def __init__(self): + pass + + @abc.abstractmethod + def add_variables(self, variables: List[VarData]): + pass + + @abc.abstractmethod + def add_parameters(self, params: List[ParamData]): + pass + + @abc.abstractmethod + def add_constraints(self, cons: List[ConstraintData]): + pass + + @abc.abstractmethod + def add_sos_constraints(self, cons: List[SOSConstraintData]): + pass + + @abc.abstractmethod + def set_objective(self, obj: ObjectiveData): + pass + + @abc.abstractmethod + def remove_constraints(self, cons: List[ConstraintData]): + pass + + @abc.abstractmethod + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + pass + + @abc.abstractmethod + def remove_variables(self, variables: List[VarData]): + pass + + @abc.abstractmethod + def remove_parameters(self, params: List[ParamData]): + pass + + +class ModelChangeDetector: + def __init__( + self, observers: Sequence[Observer], + treat_fixed_vars_as_params=True, + **kwds, + ): + """ + Parameters + ---------- + observers: Sequence[Observer] + The objects to notify when changes are made to the model + treat_fixed_vars_as_params: bool + This is an advanced option that should only be used in special circumstances. + With the default setting of True, fixed variables will be treated like parameters. + This means that z == x*y will be linear if x or y is fixed and the constraint + can be written to an LP file. If the value of the fixed variable gets changed, we have + to completely reprocess all constraints using that variable. If + treat_fixed_vars_as_params is False, then constraints will be processed as if fixed + variables are not fixed, and the solver will be told the variable is fixed. This means + z == x*y could not be written to an LP file even if x and/or y is fixed. However, + updating the values of fixed variables is much faster this way. + """ + self._observers: List[Observer] = list(observers) + self._model = None + self._active_constraints = {} # maps constraint to (lower, body, upper) + self._vars = {} # maps var id to (var, lb, ub, fixed, domain, value) + self._params = {} # maps param id to param + self._objective = None + self._objective_expr = None + self._objective_sense = None + self._named_expressions = ( + {} + ) # maps constraint to list of tuples (named_expr, named_expr.expr) + self._external_functions = ComponentMap() + self._obj_named_expressions = [] + self._referenced_variables = ( + {} + ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + self._referenced_params = ( + {} + ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + self._vars_referenced_by_con = {} + self._vars_referenced_by_obj = [] + self._params_referenced_by_con = {} + self._params_referenced_by_obj = [] + self._expr_types = None + self._treat_fixed_vars_as_params = treat_fixed_vars_as_params + self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) + + def set_instance(self, model): + saved_config = self.config + self.__init__(observers=self._observers, treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) + self.config = saved_config + self._model = model + self.add_block(model) + if self._objective is None: + self.set_objective(None) + + def _add_variables(self, variables: List[VarData]): + for v in variables: + if id(v) in self._referenced_variables: + raise ValueError(f'Variable {v.name} has already been added') + self._referenced_variables[id(v)] = [{}, {}, None] + self._vars[id(v)] = ( + v, + v._lb, + v._ub, + v.fixed, + v.domain.get_interval(), + v.value, + ) + for obs in self._observers: + obs.add_variables(variables) + + def _add_parameters(self, params: List[ParamData]): + for p in params: + pid = id(p) + if pid in self._referenced_params: + raise ValueError(f'Parameter {p.name} has already been added') + self._referenced_params[pid] = [{}, {}, None] + self._params[id(p)] = (p, p.value) + for obs in self._observers: + obs.add_parameters(params) + + def _check_for_new_vars(self, variables: List[VarData]): + new_vars = {} + for v in variables: + v_id = id(v) + if v_id not in self._referenced_variables: + new_vars[v_id] = v + self._add_variables(list(new_vars.values())) + + def _check_to_remove_vars(self, variables: List[VarData]): + vars_to_remove = {} + for v in variables: + v_id = id(v) + ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] + if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: + vars_to_remove[v_id] = v + self._remove_variables(list(vars_to_remove.values())) + + def _check_for_new_params(self, params: List[ParamData]): + new_params = {} + for p in params: + pid = id(p) + if pid not in self._referenced_params: + new_params[pid] = p + self._add_parameters(list(new_params.values())) + + def _check_to_remove_params(self, params: List[ParamData]): + params_to_remove = {} + for p in params: + p_id = id(p) + ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] + if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: + params_to_remove[p_id] = p + self._remove_parameters(list(params_to_remove.values())) + + def _add_constraints(self, cons: List[ConstraintData]): + all_fixed_vars = {} + for con in cons: + if con in self._named_expressions: + raise ValueError(f'Constraint {con.name} has already been added') + self._active_constraints[con] = con.expr + tmp = collect_vars_and_named_exprs(con.expr) + named_exprs, variables, fixed_vars, parameters, external_functions = tmp + self._check_for_new_vars(variables) + self._check_for_new_params(parameters) + self._named_expressions[con] = [(e, e.expr) for e in named_exprs] + if len(external_functions) > 0: + self._external_functions[con] = external_functions + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = parameters + for v in variables: + self._referenced_variables[id(v)][0][con] = None + for p in parameters: + self._referenced_params[id(p)][0][con] = None + if not self._treat_fixed_vars_as_params: + for v in fixed_vars: + v.unfix() + all_fixed_vars[id(v)] = v + for obs in self._observers: + obs.add_constraints(cons) + for v in all_fixed_vars.values(): + v.fix() + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + all_fixed_vars = {} + for con in cons: + if con in self._vars_referenced_by_con: + raise ValueError(f'Constraint {con.name} has already been added') + self._active_constraints[con] = tuple() + variables = [] + params = [] + for v, p in con.get_items(): + variables.append(v) + if type(p) in native_numeric_types: + continue + if p.is_parameter_type(): + params.append(p) + self._check_for_new_vars(variables) + self._check_for_new_params(params) + self._named_expressions[con] = [] + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = params + for v in variables: + self._referenced_variables[id(v)][1][con] = None + for p in params: + self._referenced_params[id(p)][1][con] = None + if not self._treat_fixed_vars_as_params: + for v in variables: + if v.is_fixed(): + v.unfix() + all_fixed_vars[id(v)] = v + for obs in self._observers: + obs.add_sos_constraints(cons) + for v in all_fixed_vars.values(): + v.fix() + + def _set_objective(self, obj: ObjectiveData): + if self._objective is not None: + for v in self._vars_referenced_by_obj: + self._referenced_variables[id(v)][2] = None + self._check_to_remove_vars(self._vars_referenced_by_obj) + self._external_functions.pop(self._objective, None) + if obj is not None: + self._objective = obj + self._objective_expr = obj.expr + self._objective_sense = obj.sense + tmp = collect_vars_and_named_exprs(obj.expr) + named_exprs, variables, fixed_vars, external_functions = tmp + self._check_for_new_vars(variables) + self._obj_named_expressions = [(i, i.expr) for i in named_exprs] + if len(external_functions) > 0: + self._external_functions[obj] = external_functions + self._vars_referenced_by_obj = variables + for v in variables: + self._referenced_variables[id(v)][2] = obj + if not self._treat_fixed_vars_as_params: + for v in fixed_vars: + v.unfix() + for obs in self._observers: + obs.set_objective(obj) + for v in fixed_vars: + v.fix() + else: + self._vars_referenced_by_obj = [] + self._objective = None + self._objective_expr = None + self._objective_sense = None + self._obj_named_expressions = [] + for obs in self._observers: + obs.set_objective(obj) + + def add_block(self, block): + param_dict = {} + for p in block.component_objects(Param, descend_into=True): + if p.mutable: + for _p in p.values(): + param_dict[id(_p)] = _p + self._add_parameters(list(param_dict.values())) + self._add_constraints( + list( + block.component_data_objects(Constraint, descend_into=True, active=True) + ) + ) + self._add_sos_constraints( + list( + block.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + ) + ) + obj = get_objective(block) + if obj is not None: + self._set_objective(obj) + + def _remove_constraints(self, cons: List[ConstraintData]): + for obs in self._observers: + obs.remove_constraints(cons) + for con in cons: + if con not in self._named_expressions: + raise ValueError( + f'Cannot remove constraint {con.name} - it was not added' + ) + for v in self._vars_referenced_by_con[con]: + self._referenced_variables[id(v)][0].pop(con) + self._check_to_remove_vars(self._vars_referenced_by_con[con]) + del self._active_constraints[con] + del self._named_expressions[con] + self._external_functions.pop(con, None) + del self._vars_referenced_by_con[con] + + def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + for obs in self._observers: + obs.remove_sos_constraints(cons) + for con in cons: + if con not in self._vars_referenced_by_con: + raise ValueError( + f'Cannot remove constraint {con.name} - it was not added' + ) + for v in self._vars_referenced_by_con[con]: + self._referenced_variables[id(v)][1].pop(con) + self._check_to_remove_vars(self._vars_referenced_by_con[con]) + del self._active_constraints[con] + del self._named_expressions[con] + del self._vars_referenced_by_con[con] + + def _remove_variables(self, variables: List[VarData]): + for obs in self._observers: + obs.remove_variables(variables) + for v in variables: + v_id = id(v) + if v_id not in self._referenced_variables: + raise ValueError( + f'Cannot remove variable {v.name} - it has not been added' + ) + cons_using, sos_using, obj_using = self._referenced_variables[v_id] + if cons_using or sos_using or (obj_using is not None): + raise ValueError( + f'Cannot remove variable {v.name} - it is still being used by constraints or the objective' + ) + del self._referenced_variables[v_id] + del self._vars[v_id] + + def _remove_parameters(self, params: List[ParamData]): + for obs in self._observers: + obs.remove_parameters(params) + for p in params: + del self._params[id(p)] + + def _remove_block(self, block): + self._remove_constraints( + list( + block.component_data_objects( + ctype=Constraint, descend_into=True, active=True + ) + ) + ) + self._remove_sos_constraints( + list( + block.component_data_objects( + ctype=SOSConstraint, descend_into=True, active=True + ) + ) + ) + self._remove_parameters( + list( + dict( + (id(p), p) + for p in block.component_data_objects( + ctype=Param, descend_into=True + ) + ).values() + ) + ) + + @abc.abstractmethod + def _update_variables(self, variables: List[VarData]): + pass + + def update_variables(self, variables: List[VarData]): + for v in variables: + self._vars[id(v)] = ( + v, + v._lb, + v._ub, + v.fixed, + v.domain.get_interval(), + v.value, + ) + self._update_variables(variables) + + @abc.abstractmethod + def update_parameters(self): + pass + + def update(self, timer: HierarchicalTimer = None): + if timer is None: + timer = HierarchicalTimer() + config = self._active_config.auto_updates + new_vars = [] + old_vars = [] + new_params = [] + old_params = [] + new_cons = [] + old_cons = [] + old_sos = [] + new_sos = [] + current_cons_dict = {} + current_sos_dict = {} + timer.start('vars') + if config.update_vars: + start_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} + timer.stop('vars') + timer.start('params') + if config.check_for_new_or_removed_params: + current_params_dict = {} + for p in self._model.component_objects(Param, descend_into=True): + if p.mutable: + for _p in p.values(): + current_params_dict[id(_p)] = _p + for p_id, p in current_params_dict.items(): + if p_id not in self._params: + new_params.append(p) + for p_id, p in self._params.items(): + if p_id not in current_params_dict: + old_params.append(p) + timer.stop('params') + timer.start('cons') + if config.check_for_new_or_removed_constraints or config.update_constraints: + current_cons_dict = { + c: None + for c in self._model.component_data_objects( + Constraint, descend_into=True, active=True + ) + } + current_sos_dict = { + c: None + for c in self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + } + for c in current_cons_dict.keys(): + if c not in self._vars_referenced_by_con: + new_cons.append(c) + for c in current_sos_dict.keys(): + if c not in self._vars_referenced_by_con: + new_sos.append(c) + for c in self._vars_referenced_by_con: + if c not in current_cons_dict and c not in current_sos_dict: + if (c.ctype is Constraint) or ( + c.ctype is None and isinstance(c, ConstraintData) + ): + old_cons.append(c) + else: + assert (c.ctype is SOSConstraint) or ( + c.ctype is None and isinstance(c, SOSConstraintData) + ) + old_sos.append(c) + self.remove_constraints(old_cons) + self.remove_sos_constraints(old_sos) + timer.stop('cons') + timer.start('params') + self.remove_parameters(old_params) + + # sticking this between removal and addition + # is important so that we don't do unnecessary work + if config.update_parameters: + self.update_parameters() + + self.add_parameters(new_params) + timer.stop('params') + timer.start('vars') + self.add_variables(new_vars) + timer.stop('vars') + timer.start('cons') + self.add_constraints(new_cons) + self.add_sos_constraints(new_sos) + new_cons_set = set(new_cons) + new_sos_set = set(new_sos) + cons_to_remove_and_add = {} + need_to_set_objective = False + if config.update_constraints: + for c in current_cons_dict.keys(): + if c not in new_cons_set and c.expr is not self._active_constraints[c]: + cons_to_remove_and_add[c] = None + sos_to_update = [] + for c in current_sos_dict.keys(): + if c not in new_sos_set: + sos_to_update.append(c) + self.remove_sos_constraints(sos_to_update) + self.add_sos_constraints(sos_to_update) + timer.stop('cons') + timer.start('vars') + if config.update_vars: + end_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} + vars_to_check = [v for v_id, v in end_vars.items() if v_id in start_vars] + if config.update_vars: + vars_to_update = [] + for v in vars_to_check: + _v, lb, ub, fixed, domain_interval, value = self._vars[id(v)] + if (fixed != v.fixed) or (fixed and (value != v.value)): + vars_to_update.append(v) + if self._treat_fixed_vars_as_params: + for c in self._referenced_variables[id(v)][0]: + cons_to_remove_and_add[c] = None + if self._referenced_variables[id(v)][2] is not None: + need_to_set_objective = True + elif lb is not v._lb: + vars_to_update.append(v) + elif ub is not v._ub: + vars_to_update.append(v) + elif domain_interval != v.domain.get_interval(): + vars_to_update.append(v) + self.update_variables(vars_to_update) + timer.stop('vars') + timer.start('cons') + cons_to_remove_and_add = list(cons_to_remove_and_add.keys()) + self.remove_constraints(cons_to_remove_and_add) + self.add_constraints(cons_to_remove_and_add) + timer.stop('cons') + timer.start('named expressions') + if config.update_named_expressions: + cons_to_update = [] + for c, expr_list in self._named_expressions.items(): + if c in new_cons_set: + continue + for named_expr, old_expr in expr_list: + if named_expr.expr is not old_expr: + cons_to_update.append(c) + break + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) + for named_expr, old_expr in self._obj_named_expressions: + if named_expr.expr is not old_expr: + need_to_set_objective = True + break + timer.stop('named expressions') + timer.start('objective') + if self._active_config.auto_updates.check_for_new_objective: + pyomo_obj = get_objective(self._model) + if pyomo_obj is not self._objective: + need_to_set_objective = True + else: + pyomo_obj = self._objective + if self._active_config.auto_updates.update_objective: + if pyomo_obj is not None and pyomo_obj.expr is not self._objective_expr: + need_to_set_objective = True + elif pyomo_obj is not None and pyomo_obj.sense is not self._objective_sense: + # we can definitely do something faster here than resetting the whole objective + need_to_set_objective = True + if need_to_set_objective: + self.set_objective(pyomo_obj) + timer.stop('objective') + + # this has to be done after the objective and constraints in case the + # old objective/constraints use old variables + timer.start('vars') + self.remove_variables(old_vars) + timer.stop('vars') + + +class PersistentSolverMixin: + """ + The `solve` method in Gurobi and Highs is exactly the same, so this Mixin + minimizes the duplicate code + """ + + def solve(self, model, **kwds) -> Results: + start_timestamp = datetime.datetime.now(datetime.timezone.utc) + self._active_config = config = self.config(value=kwds, preserve_implicit=True) + StaleFlagManager.mark_all_as_stale() + + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + if model is not self._model: + timer.start('set_instance') + self.set_instance(model) + timer.stop('set_instance') + else: + timer.start('update') + self.update(timer=timer) + timer.stop('update') + + res = self._solve() + self._last_results_object = res + + end_timestamp = datetime.datetime.now(datetime.timezone.utc) + res.timing_info.start_timestamp = start_timestamp + res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + res.timing_info.timer = timer + self._active_config = self.config + + return res From 97aeb31fd2c6fb2fc30349a19644d719af92b4dc Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 30 Jul 2025 06:37:17 -0600 Subject: [PATCH 02/50] working on a model observer --- pyomo/contrib/observer/model_observer.py | 392 ++++++++++-------- pyomo/contrib/observer/tests/__init__.py | 0 .../observer/tests/test_change_detector.py | 91 ++++ 3 files changed, 306 insertions(+), 177 deletions(-) create mode 100644 pyomo/contrib/observer/tests/__init__.py create mode 100644 pyomo/contrib/observer/tests/test_change_detector.py diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b644676f3d2..b6768f2ac6b 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -11,7 +11,7 @@ import abc import datetime -from typing import List, Sequence +from typing import List, Sequence, Optional from pyomo.common.config import ConfigDict, ConfigValue from pyomo.core.base.constraint import ConstraintData, Constraint @@ -27,6 +27,27 @@ from pyomo.common.numeric_types import native_numeric_types +""" +The ModelChangeDetector is meant to be used to automatically identify changes +in a Pyomo model or block. Here is a list of changes that will be detected. +Note that inactive components (e.g., constraints) are treated as "removed". + - new constraints that have been added to the model + - constraints that have been removed from the model + - new variables that have been detected in new or modified constraints/objectives + - old variables that are no longer used in any constraints/objectives + - new parameters that have been detected in new or modified constraints/objectives + - old parameters that are no longer used in any constraints/objectives + - new objectives that have been added to the model + - objectives that have been removed from the model + - modified constraint expressions (relies on expressions being immutable) + - modified objective expressions (relies on expressions being immutable) + - modified objective sense + - changes to variable bounds, domains, and "fixed" flags + - changes to named expressions (relies on expressions being immutable) + - changes to parameter values and fixed variable values +""" + + class AutoUpdateConfig(ConfigDict): """ Control which parts of the model are automatically checked and/or updated upon re-solve @@ -62,30 +83,6 @@ def __init__( added to/removed from the model.""", ), ) - self.check_for_new_or_removed_vars: bool = self.declare( - 'check_for_new_or_removed_vars', - ConfigValue( - domain=bool, - default=True, - description=""" - If False, new/old variables will not be automatically detected on subsequent - solves. Use False only when manually updating the solver with opt.add_variables() and - opt.remove_variables() or when you are certain variables are not being added to / - removed from the model.""", - ), - ) - self.check_for_new_or_removed_params: bool = self.declare( - 'check_for_new_or_removed_params', - ConfigValue( - domain=bool, - default=True, - description=""" - If False, new/old parameters will not be automatically detected on subsequent - solves. Use False only when manually updating the solver with opt.add_parameters() and - opt.remove_parameters() or when you are certain parameters are not being added to / - removed from the model.""", - ), - ) self.check_for_new_objective: bool = self.declare( 'check_for_new_objective', ConfigValue( @@ -118,19 +115,23 @@ def __init__( description=""" If False, changes to existing variables will not be automatically detected on subsequent solves. This includes changes to the lb, ub, domain, and fixed - attributes of variables. Use False only when manually updating the solver with - opt.update_variables() or when you are certain variables are not being modified.""", + attributes of variables. Use False only when manually updating the observer with + opt.update_variables() or when you are certain variables are not being modified. + Note that changes to values of fixed variables is handled by + update_parameters_and_fixed_vars.""", ), ) - self.update_parameters: bool = self.declare( + self.update_parameters_and_fixed_vars: bool = self.declare( 'update_parameters', ConfigValue( domain=bool, default=True, description=""" - If False, changes to parameter values will not be automatically detected on - subsequent solves. Use False only when manually updating the solver with - opt.update_parameters() or when you are certain parameters are not being modified.""", + If False, changes to parameter values and fixed variable values will + not be automatically detected on subsequent solves. Use False only + when manually updating the observer with + opt.update_parameters_and_fixed_variables() or when you are certain + parameters are not being modified.""", ), ) self.update_named_expressions: bool = self.declare( @@ -199,6 +200,18 @@ def remove_variables(self, variables: List[VarData]): def remove_parameters(self, params: List[ParamData]): pass + @abc.abstractmethod + def update_variables(self, variables: List[VarData]): + pass + + @abc.abstractmethod + def update_parameters_and_fixed_variables( + self, + params: List[ParamData], + variables: List[VarData], + ): + pass + class ModelChangeDetector: def __init__( @@ -224,7 +237,8 @@ def __init__( """ self._observers: List[Observer] = list(observers) self._model = None - self._active_constraints = {} # maps constraint to (lower, body, upper) + self._active_constraints = {} # maps constraint to expression + self._active_sos = {} self._vars = {} # maps var id to (var, lb, ub, fixed, domain, value) self._params = {} # maps param id to param self._objective = None @@ -254,9 +268,7 @@ def set_instance(self, model): self.__init__(observers=self._observers, treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) self.config = saved_config self._model = model - self.add_block(model) - if self._objective is None: - self.set_objective(None) + self._add_block(model) def _add_variables(self, variables: List[VarData]): for v in variables: @@ -321,7 +333,7 @@ def _check_to_remove_params(self, params: List[ParamData]): def _add_constraints(self, cons: List[ConstraintData]): all_fixed_vars = {} for con in cons: - if con in self._named_expressions: + if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') self._active_constraints[con] = con.expr tmp = collect_vars_and_named_exprs(con.expr) @@ -351,10 +363,11 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for con in cons: if con in self._vars_referenced_by_con: raise ValueError(f'Constraint {con.name} has already been added') - self._active_constraints[con] = tuple() + sos_items = list(con.get_items()) + self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) variables = [] params = [] - for v, p in con.get_items(): + for v, p in sos_items: variables.append(v) if type(p) in native_numeric_types: continue @@ -384,20 +397,25 @@ def _set_objective(self, obj: ObjectiveData): for v in self._vars_referenced_by_obj: self._referenced_variables[id(v)][2] = None self._check_to_remove_vars(self._vars_referenced_by_obj) + self._check_to_remove_params(self._params_referenced_by_obj) self._external_functions.pop(self._objective, None) if obj is not None: self._objective = obj self._objective_expr = obj.expr self._objective_sense = obj.sense tmp = collect_vars_and_named_exprs(obj.expr) - named_exprs, variables, fixed_vars, external_functions = tmp + named_exprs, variables, fixed_vars, parameters, external_functions = tmp self._check_for_new_vars(variables) + self._check_for_new_params(parameters) self._obj_named_expressions = [(i, i.expr) for i in named_exprs] if len(external_functions) > 0: self._external_functions[obj] = external_functions self._vars_referenced_by_obj = variables + self._params_referenced_by_obj = parameters for v in variables: self._referenced_variables[id(v)][2] = obj + for p in parameters: + self._referenced_params[id(p)][2] = obj if not self._treat_fixed_vars_as_params: for v in fixed_vars: v.unfix() @@ -407,6 +425,7 @@ def _set_objective(self, obj: ObjectiveData): v.fix() else: self._vars_referenced_by_obj = [] + self._params_referenced_by_obj = [] self._objective = None self._objective_expr = None self._objective_sense = None @@ -414,13 +433,7 @@ def _set_objective(self, obj: ObjectiveData): for obs in self._observers: obs.set_objective(obj) - def add_block(self, block): - param_dict = {} - for p in block.component_objects(Param, descend_into=True): - if p.mutable: - for _p in p.values(): - param_dict[id(_p)] = _p - self._add_parameters(list(param_dict.values())) + def _add_block(self, block): self._add_constraints( list( block.component_data_objects(Constraint, descend_into=True, active=True) @@ -447,11 +460,15 @@ def _remove_constraints(self, cons: List[ConstraintData]): ) for v in self._vars_referenced_by_con[con]: self._referenced_variables[id(v)][0].pop(con) + for p in self._params_referenced_by_con[con]: + self._referenced_params[id(p)][0].pop(con) self._check_to_remove_vars(self._vars_referenced_by_con[con]) + self._check_to_remove_params(self._params_referenced_by_con[con]) del self._active_constraints[con] del self._named_expressions[con] self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] + del self._params_referenced_by_con[con] def _remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: @@ -463,10 +480,14 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): ) for v in self._vars_referenced_by_con[con]: self._referenced_variables[id(v)][1].pop(con) + for p in self._params_referenced_by_con[con]: + self._referenced_params[id(p)][1].pop(con) self._check_to_remove_vars(self._vars_referenced_by_con[con]) - del self._active_constraints[con] + self._check_to_remove_params(self._params_referenced_by_con[con]) + del self._active_sos[con] del self._named_expressions[con] del self._vars_referenced_by_con[con] + del self._params_referenced_by_con[con] def _remove_variables(self, variables: List[VarData]): for obs in self._observers: @@ -489,39 +510,20 @@ def _remove_parameters(self, params: List[ParamData]): for obs in self._observers: obs.remove_parameters(params) for p in params: - del self._params[id(p)] - - def _remove_block(self, block): - self._remove_constraints( - list( - block.component_data_objects( - ctype=Constraint, descend_into=True, active=True + p_id = id(p) + if p_id not in self._referenced_params: + raise ValueError( + f'Cannot remove parameter {p.name} - it has not been added' ) - ) - ) - self._remove_sos_constraints( - list( - block.component_data_objects( - ctype=SOSConstraint, descend_into=True, active=True + cons_using, sos_using, obj_using = self._referenced_params[p_id] + if cons_using or sos_using or (obj_using is not None): + raise ValueError( + f'Cannot remove parameter {p.name} - it is still being used by constraints or the objective' ) - ) - ) - self._remove_parameters( - list( - dict( - (id(p), p) - for p in block.component_data_objects( - ctype=Param, descend_into=True - ) - ).values() - ) - ) + del self._referenced_params[p_id] + del self._params[p_id] - @abc.abstractmethod def _update_variables(self, variables: List[VarData]): - pass - - def update_variables(self, variables: List[VarData]): for v in variables: self._vars[id(v)] = ( v, @@ -531,109 +533,138 @@ def update_variables(self, variables: List[VarData]): v.domain.get_interval(), v.value, ) - self._update_variables(variables) + for obs in self._observers: + obs.update_variables(variables) - @abc.abstractmethod - def update_parameters(self): - pass + def _update_parameters_and_fixed_variables(self, params, variables): + for p in params: + self._params[id(p)] = (p, p.value) + for v in variables: + self._vars[id(v)][5] = v.value + for obs in self._observers: + obs.update_parameters_and_fixed_variables(params, variables) - def update(self, timer: HierarchicalTimer = None): - if timer is None: - timer = HierarchicalTimer() - config = self._active_config.auto_updates - new_vars = [] - old_vars = [] - new_params = [] - old_params = [] + def _check_for_new_or_removed_sos(self): + new_sos = [] + old_sos = [] + current_sos_dict = { + c: None + for c in self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + } + for c in current_sos_dict.keys(): + if c not in self._active_sos: + new_sos.append(c) + for c in self._active_sos: + if c not in current_sos_dict: + old_sos.append(c) + return new_sos, old_sos + + def _check_for_new_or_removed_constraints(self): new_cons = [] old_cons = [] - old_sos = [] - new_sos = [] - current_cons_dict = {} - current_sos_dict = {} - timer.start('vars') - if config.update_vars: - start_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} - timer.stop('vars') - timer.start('params') - if config.check_for_new_or_removed_params: - current_params_dict = {} - for p in self._model.component_objects(Param, descend_into=True): - if p.mutable: - for _p in p.values(): - current_params_dict[id(_p)] = _p - for p_id, p in current_params_dict.items(): - if p_id not in self._params: - new_params.append(p) - for p_id, p in self._params.items(): - if p_id not in current_params_dict: - old_params.append(p) - timer.stop('params') - timer.start('cons') - if config.check_for_new_or_removed_constraints or config.update_constraints: - current_cons_dict = { - c: None - for c in self._model.component_data_objects( - Constraint, descend_into=True, active=True - ) - } - current_sos_dict = { - c: None - for c in self._model.component_data_objects( - SOSConstraint, descend_into=True, active=True - ) - } - for c in current_cons_dict.keys(): - if c not in self._vars_referenced_by_con: - new_cons.append(c) - for c in current_sos_dict.keys(): - if c not in self._vars_referenced_by_con: - new_sos.append(c) - for c in self._vars_referenced_by_con: - if c not in current_cons_dict and c not in current_sos_dict: - if (c.ctype is Constraint) or ( - c.ctype is None and isinstance(c, ConstraintData) - ): - old_cons.append(c) - else: - assert (c.ctype is SOSConstraint) or ( - c.ctype is None and isinstance(c, SOSConstraintData) - ) - old_sos.append(c) - self.remove_constraints(old_cons) - self.remove_sos_constraints(old_sos) - timer.stop('cons') - timer.start('params') - self.remove_parameters(old_params) + current_cons_dict = { + c: None + for c in self._model.component_data_objects( + Constraint, descend_into=True, active=True + ) + } + for c in current_cons_dict.keys(): + if c not in self._active_constraints: + new_cons.append(c) + for c in self._active_constraints: + if c not in current_cons_dict: + old_cons.append(c) + return new_cons, old_cons + + def _check_for_modified_sos(self): + sos_to_update = [] + for c, (old_vlist, old_plist) in self._active_sos.items(): + sos_items = list(c.get_items()) + new_vlist = [i[0] for i in sos_items] + new_plist = [i[1] for i in sos_items] + if len(old_vlist) != len(new_vlist): + sos_to_update.append(c) + elif len(old_plist) != len(new_plist): + sos_to_update.append(c) + else: + needs_update = False + for v1, v2 in zip(old_vlist, new_vlist): + if v1 is not v2: + needs_update = True + break + for p1, p2 in zip(old_plist, new_plist): + if p1 is not p2: + needs_update = True + if needs_update: + break + if needs_update: + sos_to_update.append(c) + return sos_to_update + + def _check_for_modified_constraints(self): + cons_to_update = [] + for c, expr in self._active_constraints.items(): + if c.expr is not expr: + cons_to_update.append(c) + return cons_to_update + + def _check_for_var_changes(self): + vars_to_update = [] + cons_to_update = {} + update_obj = False + for vid, (v, _lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): + if v.fixed != _fixed: + vars_to_update.append(v) + if self._treat_fixed_vars_as_params: + for c in self._referenced_variables[vid][0]: + cons_to_update[c] = None + + elif v._lb is not _lb: + vars_to_update.append(v) + elif v._ub is not _ub: + vars_to_update.append(v) + + + def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): + if timer is None: + timer = HierarchicalTimer() + config: AutoUpdateConfig = self.config(value=kwds, preserve_implicit=True) + + added_cons = set() + added_sos = set() + + if config.check_for_new_or_removed_constraints: + timer.start('sos') + new_sos, old_sos = self._check_for_new_or_removed_sos() + self._add_sos_constraints(new_sos) + self._remove_sos_constraints(old_sos) + added_sos.update(new_sos) + timer.stop('cons') + timer.start('cons') + new_cons, old_cons = self._check_for_new_or_removed_constraints() + self._add_constraints(new_cons) + self._remove_constraints(old_cons) + added_cons.update(new_cons) + timer.stop('cons') - # sticking this between removal and addition - # is important so that we don't do unnecessary work - if config.update_parameters: - self.update_parameters() + if config.update_constraints: + timer.start('cons') + cons_to_update = self._check_for_modified_constraints() + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + timer.stop('cons') + timer.start('sos') + sos_to_update = self._check_for_modified_sos() + self._remove_sos_constraints(sos_to_update) + self._add_sos_constraints(sos_to_update) + added_sos.update(sos_to_update) + timer.stop('sos') - self.add_parameters(new_params) - timer.stop('params') - timer.start('vars') - self.add_variables(new_vars) - timer.stop('vars') - timer.start('cons') - self.add_constraints(new_cons) - self.add_sos_constraints(new_sos) - new_cons_set = set(new_cons) - new_sos_set = set(new_sos) - cons_to_remove_and_add = {} need_to_set_objective = False - if config.update_constraints: - for c in current_cons_dict.keys(): - if c not in new_cons_set and c.expr is not self._active_constraints[c]: - cons_to_remove_and_add[c] = None - sos_to_update = [] - for c in current_sos_dict.keys(): - if c not in new_sos_set: - sos_to_update.append(c) - self.remove_sos_constraints(sos_to_update) - self.add_sos_constraints(sos_to_update) - timer.stop('cons') + timer.start('vars') if config.update_vars: end_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} @@ -696,11 +727,18 @@ def update(self, timer: HierarchicalTimer = None): self.set_objective(pyomo_obj) timer.stop('objective') - # this has to be done after the objective and constraints in case the - # old objective/constraints use old variables - timer.start('vars') - self.remove_variables(old_vars) - timer.stop('vars') + if config.update_parameters: + timer.start('params') + modified_params = [] + for pid, (p, old_val) in self._params.items(): + if p.value != old_val: + modified_params.append(p) + modified_vars = [] + for vid, (v, _lb, _ub, _fixed, _domain_interval, _val) in self._vars.items(): + if _fixed and _val != v.value: + modified_vars.append(v) + self._update_parameters_and_fixed_variables(modified_params, modified_vars) + timer.stop('params') class PersistentSolverMixin: diff --git a/pyomo/contrib/observer/tests/__init__.py b/pyomo/contrib/observer/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py new file mode 100644 index 00000000000..46faacae9bb --- /dev/null +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -0,0 +1,91 @@ +from pyomo.core.base.constraint import ConstraintData +from pyomo.core.base.objective import ObjectiveData +from pyomo.core.base.param import ParamData +from pyomo.core.base.sos import SOSConstraintData +from pyomo.core.base.var import VarData +import pyomo.environ as pe +from pyomo.common import unittest +from typing import List +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.common.collections import ComponentMap +import logging + + +logger = logging.getLogger(__name__) + + +class ObserverChecker(Observer): + def __init__(self): + super().__init__() + self.counts = ComponentMap() + """ + counts is be a mapping from component (e.g., variable) to another + mapping from string ('add', 'remove', 'update', or 'value') to an int that + indicates the number of times the corresponding method has been called + """ + + def check(self, expected): + unittest.assertStructuredAlmostEqual( + first=expected, + second=self.counts, + places=7, + ) + + def _process(self, comps, key): + for c in comps: + if c not in self.counts: + self.counts[c] = {'add': 0, 'remove': 0, 'update': 0, 'value': 0} + self.counts[c][key] += 1 + + def add_variables(self, variables: List[VarData]): + self._process(variables, 'add') + + def add_parameters(self, params: List[ParamData]): + self._process(params, 'add') + + def add_constraints(self, cons: List[ConstraintData]): + self._process(cons, 'add') + + def add_sos_constraints(self, cons: List[SOSConstraintData]): + self._process(cons, 'add') + + def set_objective(self, obj: ObjectiveData): + self._process([obj], 'add') + + def remove_constraints(self, cons: List[ConstraintData]): + self._process(cons, 'remove') + + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + self._process(cons, 'remove') + + def remove_variables(self, variables: List[VarData]): + self._process(variables, 'remove') + + def remove_parameters(self, params: List[ParamData]): + self._process(params, 'remove') + + def update_variables(self, variables: List[VarData]): + self._process(variables, 'update') + + def update_parameters_and_fixed_variables(self, params: List[ParamData], variables: List[VarData]): + self._process(params, 'value') + self._process(variables, 'value') + + +class TestChangeDetector(unittest.TestCase): + def test_basics(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + + detector.set_instance(m) + + expected = ComponentMap() + + obs.check(expected) + + def test_vars_and_params_elsewhere(self): + pass \ No newline at end of file From a69d5e69ea9729a4c1ae0396719fd173065d55a9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 6 Aug 2025 06:20:19 -0600 Subject: [PATCH 03/50] working on model change detector --- pyomo/contrib/observer/component_collector.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 pyomo/contrib/observer/component_collector.py diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py new file mode 100644 index 00000000000..149c4038bc6 --- /dev/null +++ b/pyomo/contrib/observer/component_collector.py @@ -0,0 +1,77 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.core.expr.visitor import StreamBasedExpressionVisitor +from pyomo.core.expr.numeric_expr import ExternalFunctionExpression, NPV_ExternalFunctionExpression +from pyomo.core.base.var import VarData, ScalarVar +from pyomo.core.base.param import ParamData, ScalarParam +from pyomo.core.base.expression import ExpressionData, ScalarExpression + + +def handle_var(node, collector): + collector.variables[id(node)] = node + return None + + +def handle_param(node, collector): + collector.params[id(node)] = node + return None + + +def handle_named_expression(node, collector): + collector.named_expressions[id(node)] = node + return None + + +def handle_external_function(node, collector): + collector.external_functions[id(node)] = node + return None + + +collector_handlers = { + VarData: handle_var, + ScalarVar: handle_var, + ParamData: handle_param, + ScalarParam: handle_param, + ExpressionData: handle_named_expression, + ScalarExpression: handle_named_expression, + ExternalFunctionExpression: handle_external_function, + NPV_ExternalFunctionExpression: handle_external_function, +} + + +class _ComponentFromExprCollector(StreamBasedExpressionVisitor): + def __init__(self): + self.named_expressions = {} + self.variables = {} + self.params = {} + self.external_functions = {} + + def exitNode(self, node, data): + nt = type(node) + if nt in collector_handlers: + return collector_handlers[nt](node, self) + else: + return None + + +_visitor = _ComponentFromExprCollector() + + +def collect_components_from_expr(expr): + _visitor.__init__() + _visitor.walk_expression(expr) + return ( + list(_visitor.named_expressions.values()), + list(_visitor.variables.values()), + list(_visitor.params.values()), + list(_visitor.external_functions.values()), + ) From 9763e9a2f594e74a26bd5003682b963812d0acb3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 7 Aug 2025 08:26:14 -0600 Subject: [PATCH 04/50] observer --- pyomo/contrib/observer/component_collector.py | 3 +- pyomo/contrib/observer/model_observer.py | 313 ++++++++---------- .../observer/tests/test_change_detector.py | 118 ++++++- 3 files changed, 250 insertions(+), 184 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 149c4038bc6..5cbbdaf31bd 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -49,11 +49,12 @@ def handle_external_function(node, collector): class _ComponentFromExprCollector(StreamBasedExpressionVisitor): - def __init__(self): + def __init__(self, **kwds): self.named_expressions = {} self.variables = {} self.params = {} self.external_functions = {} + super().__init__(**kwds) def exitNode(self, node, data): nt = type(node) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b6768f2ac6b..422eb1da574 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -23,7 +23,8 @@ from pyomo.common.collections import ComponentMap from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.results import Results -from pyomo.contrib.solver.common.util import collect_vars_and_named_exprs, get_objective +from pyomo.contrib.solver.common.util import get_objective +from .component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types @@ -121,7 +122,7 @@ def __init__( update_parameters_and_fixed_vars.""", ), ) - self.update_parameters_and_fixed_vars: bool = self.declare( + self.update_parameters: bool = self.declare( 'update_parameters', ConfigValue( domain=bool, @@ -181,7 +182,7 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): pass @abc.abstractmethod - def set_objective(self, obj: ObjectiveData): + def set_objective(self, obj: Optional[ObjectiveData]): pass @abc.abstractmethod @@ -205,18 +206,13 @@ def update_variables(self, variables: List[VarData]): pass @abc.abstractmethod - def update_parameters_and_fixed_variables( - self, - params: List[ParamData], - variables: List[VarData], - ): + def update_parameters(self, params: List[ParamData]): pass class ModelChangeDetector: def __init__( self, observers: Sequence[Observer], - treat_fixed_vars_as_params=True, **kwds, ): """ @@ -224,16 +220,6 @@ def __init__( ---------- observers: Sequence[Observer] The objects to notify when changes are made to the model - treat_fixed_vars_as_params: bool - This is an advanced option that should only be used in special circumstances. - With the default setting of True, fixed variables will be treated like parameters. - This means that z == x*y will be linear if x or y is fixed and the constraint - can be written to an LP file. If the value of the fixed variable gets changed, we have - to completely reprocess all constraints using that variable. If - treat_fixed_vars_as_params is False, then constraints will be processed as if fixed - variables are not fixed, and the solver will be told the variable is fixed. This means - z == x*y could not be written to an LP file even if x and/or y is fixed. However, - updating the values of fixed variables is much faster this way. """ self._observers: List[Observer] = list(observers) self._model = None @@ -260,12 +246,11 @@ def __init__( self._params_referenced_by_con = {} self._params_referenced_by_obj = [] self._expr_types = None - self._treat_fixed_vars_as_params = treat_fixed_vars_as_params self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) def set_instance(self, model): saved_config = self.config - self.__init__(observers=self._observers, treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) + self.__init__(observers=self._observers) self.config = saved_config self._model = model self._add_block(model) @@ -331,37 +316,38 @@ def _check_to_remove_params(self, params: List[ParamData]): self._remove_parameters(list(params_to_remove.values())) def _add_constraints(self, cons: List[ConstraintData]): - all_fixed_vars = {} + vars_to_check = [] + params_to_check = [] for con in cons: if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') self._active_constraints[con] = con.expr - tmp = collect_vars_and_named_exprs(con.expr) - named_exprs, variables, fixed_vars, parameters, external_functions = tmp - self._check_for_new_vars(variables) - self._check_for_new_params(parameters) + tmp = collect_components_from_expr(con.expr) + named_exprs, variables, parameters, external_functions = tmp + vars_to_check.extend(variables) + params_to_check.extend(parameters) self._named_expressions[con] = [(e, e.expr) for e in named_exprs] if len(external_functions) > 0: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = parameters + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for con in cons: + variables = self._vars_referenced_by_con[con] + parameters = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[id(v)][0][con] = None for p in parameters: self._referenced_params[id(p)][0][con] = None - if not self._treat_fixed_vars_as_params: - for v in fixed_vars: - v.unfix() - all_fixed_vars[id(v)] = v for obs in self._observers: obs.add_constraints(cons) - for v in all_fixed_vars.values(): - v.fix() def _add_sos_constraints(self, cons: List[SOSConstraintData]): - all_fixed_vars = {} + vars_to_check = [] + params_to_check = [] for con in cons: - if con in self._vars_referenced_by_con: + if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') sos_items = list(con.get_items()) self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) @@ -373,8 +359,8 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): continue if p.is_parameter_type(): params.append(p) - self._check_for_new_vars(variables) - self._check_for_new_params(params) + vars_to_check.extend(variables) + params_to_check.extend(params) self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params @@ -382,29 +368,28 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._referenced_variables[id(v)][1][con] = None for p in params: self._referenced_params[id(p)][1][con] = None - if not self._treat_fixed_vars_as_params: - for v in variables: - if v.is_fixed(): - v.unfix() - all_fixed_vars[id(v)] = v + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) for obs in self._observers: obs.add_sos_constraints(cons) - for v in all_fixed_vars.values(): - v.fix() - def _set_objective(self, obj: ObjectiveData): + def _set_objective(self, obj: Optional[ObjectiveData]): + vars_to_remove_check = [] + params_to_remove_check = [] if self._objective is not None: for v in self._vars_referenced_by_obj: self._referenced_variables[id(v)][2] = None - self._check_to_remove_vars(self._vars_referenced_by_obj) - self._check_to_remove_params(self._params_referenced_by_obj) + for p in self._params_referenced_by_obj: + self._referenced_params[id(p)][2] = None + vars_to_remove_check.extend(self._vars_referenced_by_obj) + params_to_remove_check.extend(self._params_referenced_by_obj) self._external_functions.pop(self._objective, None) if obj is not None: self._objective = obj self._objective_expr = obj.expr self._objective_sense = obj.sense - tmp = collect_vars_and_named_exprs(obj.expr) - named_exprs, variables, fixed_vars, parameters, external_functions = tmp + tmp = collect_components_from_expr(obj.expr) + named_exprs, variables, parameters, external_functions = tmp self._check_for_new_vars(variables) self._check_for_new_params(parameters) self._obj_named_expressions = [(i, i.expr) for i in named_exprs] @@ -416,13 +401,6 @@ def _set_objective(self, obj: ObjectiveData): self._referenced_variables[id(v)][2] = obj for p in parameters: self._referenced_params[id(p)][2] = obj - if not self._treat_fixed_vars_as_params: - for v in fixed_vars: - v.unfix() - for obs in self._observers: - obs.set_objective(obj) - for v in fixed_vars: - v.fix() else: self._vars_referenced_by_obj = [] self._params_referenced_by_obj = [] @@ -430,8 +408,10 @@ def _set_objective(self, obj: ObjectiveData): self._objective_expr = None self._objective_sense = None self._obj_named_expressions = [] - for obs in self._observers: - obs.set_objective(obj) + for obs in self._observers: + obs.set_objective(obj) + self._check_to_remove_vars(vars_to_remove_check) + self._check_to_remove_params(params_to_remove_check) def _add_block(self, block): self._add_constraints( @@ -447,14 +427,15 @@ def _add_block(self, block): ) ) obj = get_objective(block) - if obj is not None: - self._set_objective(obj) + self._set_objective(obj) def _remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.remove_constraints(cons) + vars_to_check = [] + params_to_check = [] for con in cons: - if con not in self._named_expressions: + if con not in self._active_constraints: raise ValueError( f'Cannot remove constraint {con.name} - it was not added' ) @@ -462,19 +443,23 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._referenced_variables[id(v)][0].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][0].pop(con) - self._check_to_remove_vars(self._vars_referenced_by_con[con]) - self._check_to_remove_params(self._params_referenced_by_con[con]) + vars_to_check.extend(self._vars_referenced_by_con[con]) + params_to_check.extend(self._params_referenced_by_con[con]) del self._active_constraints[con] del self._named_expressions[con] self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] + self._check_to_remove_vars(vars_to_check) + self._check_to_remove_params(params_to_check) def _remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.remove_sos_constraints(cons) + vars_to_check = [] + params_to_check = [] for con in cons: - if con not in self._vars_referenced_by_con: + if con not in self._active_sos: raise ValueError( f'Cannot remove constraint {con.name} - it was not added' ) @@ -482,12 +467,14 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): self._referenced_variables[id(v)][1].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][1].pop(con) - self._check_to_remove_vars(self._vars_referenced_by_con[con]) - self._check_to_remove_params(self._params_referenced_by_con[con]) + vars_to_check.extend(self._vars_referenced_by_con[con]) + params_to_check.extend(self._params_referenced_by_con[con]) del self._active_sos[con] del self._named_expressions[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] + self._check_to_remove_vars(vars_to_check) + self._check_to_remove_params(params_to_check) def _remove_variables(self, variables: List[VarData]): for obs in self._observers: @@ -536,13 +523,11 @@ def _update_variables(self, variables: List[VarData]): for obs in self._observers: obs.update_variables(variables) - def _update_parameters_and_fixed_variables(self, params, variables): + def _update_parameters(self, params): for p in params: self._params[id(p)] = (p, p.value) - for v in variables: - self._vars[id(v)][5] = v.value for obs in self._observers: - obs.update_parameters_and_fixed_variables(params, variables) + obs.update_parameters(params) def _check_for_new_or_removed_sos(self): new_sos = [] @@ -617,15 +602,60 @@ def _check_for_var_changes(self): for vid, (v, _lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): if v.fixed != _fixed: vars_to_update.append(v) - if self._treat_fixed_vars_as_params: - for c in self._referenced_variables[vid][0]: - cons_to_update[c] = None - + for c in self._referenced_variables[vid][0]: + cons_to_update[c] = None + if self._referenced_variables[vid][2] is not None: + update_obj = True elif v._lb is not _lb: vars_to_update.append(v) elif v._ub is not _ub: vars_to_update.append(v) - + elif _domain_interval != v.domain.get_interval(): + vars_to_update.append(v) + elif v.value != _value: + vars_to_update.append(v) + cons_to_update = list(cons_to_update.keys()) + return vars_to_update, cons_to_update, update_obj + + def _check_for_param_changes(self): + params_to_update = [] + for pid, (p, val) in self._params.items(): + if p.value != val: + params_to_update.append(p) + return params_to_update + + def _check_for_named_expression_changes(self): + cons_to_update = [] + for con, ne_list in self._named_expressions.items(): + for named_expr, old_expr in ne_list: + if named_expr.expr is not old_expr: + cons_to_update.append(con) + break + update_obj = False + ne_list = self._obj_named_expressions + for named_expr, old_expr in ne_list: + if named_expr.expr is not old_expr: + update_obj = True + break + return cons_to_update, update_obj + + def _check_for_new_objective(self): + update_obj = False + new_obj = get_objective(self._model) + if new_obj is not self._objective: + update_obj = True + return new_obj, update_obj + + def _check_for_objective_changes(self): + update_obj = False + if self._objective is None: + return update_obj + if self._objective.expr is not self._objective_expr: + update_obj = True + elif self._objective.sense != self._objective_sense: + # we can definitely do something faster here than resetting the whole objective + update_obj = True + return update_obj def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if timer is None: @@ -641,7 +671,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): self._add_sos_constraints(new_sos) self._remove_sos_constraints(old_sos) added_sos.update(new_sos) - timer.stop('cons') + timer.stop('sos') timer.start('cons') new_cons, old_cons = self._check_for_new_or_removed_constraints() self._add_constraints(new_cons) @@ -665,115 +695,46 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): need_to_set_objective = False - timer.start('vars') if config.update_vars: - end_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} - vars_to_check = [v for v_id, v in end_vars.items() if v_id in start_vars] - if config.update_vars: - vars_to_update = [] - for v in vars_to_check: - _v, lb, ub, fixed, domain_interval, value = self._vars[id(v)] - if (fixed != v.fixed) or (fixed and (value != v.value)): - vars_to_update.append(v) - if self._treat_fixed_vars_as_params: - for c in self._referenced_variables[id(v)][0]: - cons_to_remove_and_add[c] = None - if self._referenced_variables[id(v)][2] is not None: - need_to_set_objective = True - elif lb is not v._lb: - vars_to_update.append(v) - elif ub is not v._ub: - vars_to_update.append(v) - elif domain_interval != v.domain.get_interval(): - vars_to_update.append(v) - self.update_variables(vars_to_update) - timer.stop('vars') - timer.start('cons') - cons_to_remove_and_add = list(cons_to_remove_and_add.keys()) - self.remove_constraints(cons_to_remove_and_add) - self.add_constraints(cons_to_remove_and_add) - timer.stop('cons') - timer.start('named expressions') + timer.start('vars') + vars_to_update, cons_to_update, update_obj = self._check_for_var_changes() + self._update_variables(vars_to_update) + cons_to_update = [i for i in cons_to_update if i not in added_cons] + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + if update_obj: + need_to_set_objective = True + timer.stop('vars') + if config.update_named_expressions: - cons_to_update = [] - for c, expr_list in self._named_expressions.items(): - if c in new_cons_set: - continue - for named_expr, old_expr in expr_list: - if named_expr.expr is not old_expr: - cons_to_update.append(c) - break - self.remove_constraints(cons_to_update) - self.add_constraints(cons_to_update) - for named_expr, old_expr in self._obj_named_expressions: - if named_expr.expr is not old_expr: - need_to_set_objective = True - break - timer.stop('named expressions') - timer.start('objective') - if self._active_config.auto_updates.check_for_new_objective: - pyomo_obj = get_objective(self._model) - if pyomo_obj is not self._objective: + timer.start('named expressions') + cons_to_update, update_obj = self._check_for_named_expression_changes() + cons_to_update = [i for i in cons_to_update if i not in added_cons] + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + if update_obj: need_to_set_objective = True - else: - pyomo_obj = self._objective - if self._active_config.auto_updates.update_objective: - if pyomo_obj is not None and pyomo_obj.expr is not self._objective_expr: + timer.stop('named expressions') + + timer.start('objective') + new_obj = self._objective + if config.check_for_new_objective: + new_obj, update_obj = self._check_for_new_objective() + if update_obj: need_to_set_objective = True - elif pyomo_obj is not None and pyomo_obj.sense is not self._objective_sense: - # we can definitely do something faster here than resetting the whole objective + if config.update_objective: + update_obj = self._check_for_objective_changes() + if update_obj: need_to_set_objective = True + if need_to_set_objective: - self.set_objective(pyomo_obj) + self._set_objective(new_obj) timer.stop('objective') if config.update_parameters: timer.start('params') - modified_params = [] - for pid, (p, old_val) in self._params.items(): - if p.value != old_val: - modified_params.append(p) - modified_vars = [] - for vid, (v, _lb, _ub, _fixed, _domain_interval, _val) in self._vars.items(): - if _fixed and _val != v.value: - modified_vars.append(v) - self._update_parameters_and_fixed_variables(modified_params, modified_vars) + params_to_update = self._check_for_param_changes() + self._update_parameters(params_to_update) timer.stop('params') - - -class PersistentSolverMixin: - """ - The `solve` method in Gurobi and Highs is exactly the same, so this Mixin - minimizes the duplicate code - """ - - def solve(self, model, **kwds) -> Results: - start_timestamp = datetime.datetime.now(datetime.timezone.utc) - self._active_config = config = self.config(value=kwds, preserve_implicit=True) - StaleFlagManager.mark_all_as_stale() - - if self._last_results_object is not None: - self._last_results_object.solution_loader.invalidate() - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - if model is not self._model: - timer.start('set_instance') - self.set_instance(model) - timer.stop('set_instance') - else: - timer.start('update') - self.update(timer=timer) - timer.stop('update') - - res = self._solve() - self._last_results_object = res - - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - res.timing_info.start_timestamp = start_timestamp - res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() - res.timing_info.timer = timer - self._active_config = self.config - - return res diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 46faacae9bb..4ed0fff8e45 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -14,13 +14,18 @@ logger = logging.getLogger(__name__) +def make_count_dict(): + d = {'add': 0, 'remove': 0, 'update': 0, 'set': 0} + return d + + class ObserverChecker(Observer): def __init__(self): super().__init__() self.counts = ComponentMap() """ counts is be a mapping from component (e.g., variable) to another - mapping from string ('add', 'remove', 'update', or 'value') to an int that + mapping from string ('add', 'remove', 'update', or 'set') to an int that indicates the number of times the corresponding method has been called """ @@ -34,57 +39,156 @@ def check(self, expected): def _process(self, comps, key): for c in comps: if c not in self.counts: - self.counts[c] = {'add': 0, 'remove': 0, 'update': 0, 'value': 0} + self.counts[c] = make_count_dict() self.counts[c][key] += 1 def add_variables(self, variables: List[VarData]): + for v in variables: + assert v.is_variable_type() self._process(variables, 'add') def add_parameters(self, params: List[ParamData]): + for p in params: + assert p.is_parameter_type() self._process(params, 'add') def add_constraints(self, cons: List[ConstraintData]): + for c in cons: + assert isinstance(c, ConstraintData) self._process(cons, 'add') def add_sos_constraints(self, cons: List[SOSConstraintData]): + for c in cons: + assert isinstance(c, SOSConstraintData) self._process(cons, 'add') def set_objective(self, obj: ObjectiveData): - self._process([obj], 'add') + assert obj is None or isinstance(obj, ObjectiveData) + self._process([obj], 'set') def remove_constraints(self, cons: List[ConstraintData]): + for c in cons: + assert isinstance(c, ConstraintData) self._process(cons, 'remove') def remove_sos_constraints(self, cons: List[SOSConstraintData]): + for c in cons: + assert isinstance(c, SOSConstraintData) self._process(cons, 'remove') def remove_variables(self, variables: List[VarData]): + for v in variables: + assert v.is_variable_type() self._process(variables, 'remove') def remove_parameters(self, params: List[ParamData]): + for p in params: + assert p.is_parameter_type() self._process(params, 'remove') def update_variables(self, variables: List[VarData]): + for v in variables: + assert v.is_variable_type() self._process(variables, 'update') - def update_parameters_and_fixed_variables(self, params: List[ParamData], variables: List[VarData]): - self._process(params, 'value') - self._process(variables, 'value') + def update_parameters(self, params: List[ParamData]): + for p in params: + assert p.is_parameter_type() + self._process(params, 'update') class TestChangeDetector(unittest.TestCase): - def test_basics(self): + def test_objective(self): m = pe.ConcreteModel() m.x = pe.Var() m.y = pe.Var() + m.p = pe.Param(mutable=True, initialize=1) obs = ObserverChecker() detector = ModelChangeDetector([obs]) + expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + detector.set_instance(m) + obs.check(expected) + + m.obj = pe.Objective(expr=m.x**2 + m.p*m.y**2) + detector.update() + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + expected[m.x] = make_count_dict() + expected[m.x]['add'] += 1 + expected[m.y] = make_count_dict() + expected[m.y]['add'] += 1 + expected[m.p] = make_count_dict() + expected[m.p]['add'] += 1 + obs.check(expected) + + m.y.setlb(0) + detector.update() + expected[m.y]['update'] += 1 + obs.check(expected) + + m.x.fix(2) + detector.update() + expected[m.x]['update'] += 1 + expected[m.obj]['set'] += 1 + obs.check(expected) + + m.x.unfix() + detector.update() + expected[m.x]['update'] += 1 + expected[m.obj]['set'] += 1 + obs.check(expected) + + m.p.value = 2 + detector.update() + expected[m.p]['update'] += 1 + obs.check(expected) + + m.obj.expr = m.x**2 + m.y**2 + detector.update() + expected[m.p]['remove'] += 1 + expected[m.obj]['set'] += 1 + obs.check(expected) + + del m.obj + m.obj = pe.Objective(expr=m.p*m.x) + detector.update() + expected[m.p]['add'] += 1 + expected[m.y]['remove'] += 1 + # remember, m.obj is a different object now + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + + def test_constraints(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + m.p = pe.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + + detector.set_instance(m) + obs.check(expected) + m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) + detector.update() + expected[m.x] = make_count_dict() + expected[m.y] = make_count_dict() + expected[m.p] = make_count_dict() + expected[m.x]['add'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['add'] += 1 + expected[m.c1] = make_count_dict() + expected[m.c1]['add'] += 1 obs.check(expected) def test_vars_and_params_elsewhere(self): From ff635b88c03e6a8ba22148cf32dfe7061c066954 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 8 Aug 2025 07:31:55 -0600 Subject: [PATCH 05/50] working on a model observer --- .../observer/tests/test_change_detector.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 4ed0fff8e45..efda8a181d9 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -42,6 +42,12 @@ def _process(self, comps, key): self.counts[c] = make_count_dict() self.counts[c][key] += 1 + def pprint(self): + for k, d in self.counts.items(): + print(f'{k}:') + for a, v in d.items(): + print(f' {a}: {v}') + def add_variables(self, variables: List[VarData]): for v in variables: assert v.is_variable_type() @@ -179,6 +185,7 @@ def test_constraints(self): detector.set_instance(m) obs.check(expected) + m.obj = pe.Objective(expr=m.y) m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) detector.update() expected[m.x] = make_count_dict() @@ -189,6 +196,27 @@ def test_constraints(self): expected[m.p]['add'] += 1 expected[m.c1] = make_count_dict() expected[m.c1]['add'] += 1 + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + obs.check(expected) + + # now fix a variable and make sure the + # constraint gets removed and added + m.x.fix(1) + obs.pprint() + detector.update() + obs.pprint() + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then + # added again when the constraint is added + expected[m.x]['update'] += 1 + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.p]['remove'] += 1 + expected[m.p]['add'] += 1 obs.check(expected) def test_vars_and_params_elsewhere(self): From db0fda4310d498ff7d225df06a035a377356b7bb Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Tue, 12 Aug 2025 07:31:39 -0600 Subject: [PATCH 06/50] Apply black --- pyomo/contrib/observer/component_collector.py | 5 +++- pyomo/contrib/observer/model_observer.py | 24 ++++++++-------- .../observer/tests/test_change_detector.py | 28 ++++++++++--------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 5cbbdaf31bd..d52ec46086c 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -10,7 +10,10 @@ # ___________________________________________________________________________ from pyomo.core.expr.visitor import StreamBasedExpressionVisitor -from pyomo.core.expr.numeric_expr import ExternalFunctionExpression, NPV_ExternalFunctionExpression +from pyomo.core.expr.numeric_expr import ( + ExternalFunctionExpression, + NPV_ExternalFunctionExpression, +) from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 422eb1da574..39b832cc266 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -211,10 +211,7 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: - def __init__( - self, observers: Sequence[Observer], - **kwds, - ): + def __init__(self, observers: Sequence[Observer], **kwds): """ Parameters ---------- @@ -240,13 +237,15 @@ def __init__( ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._referenced_params = ( {} - ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._vars_referenced_by_con = {} self._vars_referenced_by_obj = [] self._params_referenced_by_con = {} self._params_referenced_by_obj = [] self._expr_types = None - self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) + self.config: AutoUpdateConfig = AutoUpdateConfig()( + value=kwds, preserve_implicit=True + ) def set_instance(self, model): saved_config = self.config @@ -350,7 +349,10 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') sos_items = list(con.get_items()) - self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) + self._active_sos[con] = ( + [i[0] for i in sos_items], + [i[1] for i in sos_items], + ) variables = [] params = [] for v, p in sos_items: @@ -616,14 +618,14 @@ def _check_for_var_changes(self): vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) return vars_to_update, cons_to_update, update_obj - + def _check_for_param_changes(self): params_to_update = [] for pid, (p, val) in self._params.items(): if p.value != val: params_to_update.append(p) return params_to_update - + def _check_for_named_expression_changes(self): cons_to_update = [] for con, ne_list in self._named_expressions.items(): @@ -644,7 +646,7 @@ def _check_for_new_objective(self): new_obj = get_objective(self._model) if new_obj is not self._objective: update_obj = True - return new_obj, update_obj + return new_obj, update_obj def _check_for_objective_changes(self): update_obj = False @@ -717,7 +719,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if update_obj: need_to_set_objective = True timer.stop('named expressions') - + timer.start('objective') new_obj = self._objective if config.check_for_new_objective: diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index efda8a181d9..29e0de01eb9 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -6,7 +6,11 @@ import pyomo.environ as pe from pyomo.common import unittest from typing import List -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) from pyomo.common.collections import ComponentMap import logging @@ -31,11 +35,9 @@ def __init__(self): def check(self, expected): unittest.assertStructuredAlmostEqual( - first=expected, - second=self.counts, - places=7, + first=expected, second=self.counts, places=7 ) - + def _process(self, comps, key): for c in comps: if c not in self.counts: @@ -120,7 +122,7 @@ def test_objective(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.x**2 + m.p*m.y**2) + m.obj = pe.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() expected[m.obj]['set'] += 1 @@ -131,7 +133,7 @@ def test_objective(self): expected[m.p] = make_count_dict() expected[m.p]['add'] += 1 obs.check(expected) - + m.y.setlb(0) detector.update() expected[m.y]['update'] += 1 @@ -161,7 +163,7 @@ def test_objective(self): obs.check(expected) del m.obj - m.obj = pe.Objective(expr=m.p*m.x) + m.obj = pe.Objective(expr=m.p * m.x) detector.update() expected[m.p]['add'] += 1 expected[m.y]['remove'] += 1 @@ -186,7 +188,7 @@ def test_constraints(self): obs.check(expected) m.obj = pe.Objective(expr=m.y) - m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) + m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p) ** 2) detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() @@ -208,9 +210,9 @@ def test_constraints(self): obs.pprint() expected[m.c1]['remove'] += 1 expected[m.c1]['add'] += 1 - # because x and p are only used in the - # one constraint, they get removed when - # the constraint is removed and then + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then # added again when the constraint is added expected[m.x]['update'] += 1 expected[m.x]['remove'] += 1 @@ -220,4 +222,4 @@ def test_constraints(self): obs.check(expected) def test_vars_and_params_elsewhere(self): - pass \ No newline at end of file + pass From 42c8cc8423ea87b2e839c8b820b6e3bb643934b2 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 09:54:25 -0600 Subject: [PATCH 07/50] adding copyright statements --- pyomo/contrib/observer/__init__.py | 10 ++++++++++ pyomo/contrib/observer/tests/__init__.py | 10 ++++++++++ pyomo/contrib/observer/tests/test_change_detector.py | 11 +++++++++++ 3 files changed, 31 insertions(+) diff --git a/pyomo/contrib/observer/__init__.py b/pyomo/contrib/observer/__init__.py index e69de29bb2d..6eb9ea8b81d 100644 --- a/pyomo/contrib/observer/__init__.py +++ b/pyomo/contrib/observer/__init__.py @@ -0,0 +1,10 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ diff --git a/pyomo/contrib/observer/tests/__init__.py b/pyomo/contrib/observer/tests/__init__.py index e69de29bb2d..6eb9ea8b81d 100644 --- a/pyomo/contrib/observer/tests/__init__.py +++ b/pyomo/contrib/observer/tests/__init__.py @@ -0,0 +1,10 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 29e0de01eb9..dd7951342da 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -1,3 +1,14 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.objective import ObjectiveData from pyomo.core.base.param import ParamData From 23ba4d96173c363953f574498ca1827c62c3ddd7 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 09:35:47 -0600 Subject: [PATCH 08/50] typo --- pyomo/contrib/observer/tests/test_change_detector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index efda8a181d9..b88877a9f63 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -24,7 +24,7 @@ def __init__(self): super().__init__() self.counts = ComponentMap() """ - counts is be a mapping from component (e.g., variable) to another + counts is a mapping from component (e.g., variable) to another mapping from string ('add', 'remove', 'update', or 'set') to an int that indicates the number of times the corresponding method has been called """ @@ -220,4 +220,4 @@ def test_constraints(self): obs.check(expected) def test_vars_and_params_elsewhere(self): - pass \ No newline at end of file + pass From d7b991825afbbc3eb2010bb5d4823a97f1378bea Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 10:00:38 -0600 Subject: [PATCH 09/50] update observer tests --- pyomo/contrib/observer/model_observer.py | 7 +- .../observer/tests/test_change_detector.py | 158 ++++++++++++++++-- 2 files changed, 149 insertions(+), 16 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 39b832cc266..bd89d4f600d 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -366,12 +366,15 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for con in cons: + variables = self._vars_referenced_by_con[con] + params = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[id(v)][1][con] = None for p in params: self._referenced_params[id(p)][1][con] = None - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) for obs in self._observers: obs.add_sos_constraints(cons) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 92b7bbff390..46527618ab8 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -14,7 +14,7 @@ from pyomo.core.base.param import ParamData from pyomo.core.base.sos import SOSConstraintData from pyomo.core.base.var import VarData -import pyomo.environ as pe +import pyomo.environ as pyo from pyomo.common import unittest from typing import List from pyomo.contrib.observer.model_observer import ( @@ -118,10 +118,10 @@ def update_parameters(self, params: List[ParamData]): class TestChangeDetector(unittest.TestCase): def test_objective(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.p = pe.Param(mutable=True, initialize=1) + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() detector = ModelChangeDetector([obs]) @@ -133,7 +133,7 @@ def test_objective(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.x**2 + m.p * m.y**2) + m.obj = pyo.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() expected[m.obj]['set'] += 1 @@ -174,7 +174,7 @@ def test_objective(self): obs.check(expected) del m.obj - m.obj = pe.Objective(expr=m.p * m.x) + m.obj = pyo.Objective(expr=m.p * m.x) detector.update() expected[m.p]['add'] += 1 expected[m.y]['remove'] += 1 @@ -183,10 +183,10 @@ def test_objective(self): expected[m.obj]['set'] += 1 def test_constraints(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.p = pe.Param(mutable=True, initialize=1) + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() detector = ModelChangeDetector([obs]) @@ -198,8 +198,8 @@ def test_constraints(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.y) - m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p) ** 2) + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= (m.x - m.p) ** 2) detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() @@ -232,5 +232,135 @@ def test_constraints(self): expected[m.p]['add'] += 1 obs.check(expected) + def test_sos(self): + m = pyo.ConcreteModel() + m.a = pyo.Set(initialize=[1, 2, 3], ordered=True) + m.x = pyo.Var(m.a, within=pyo.Binary) + m.y = pyo.Var(within=pyo.Binary) + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.SOSConstraint(var=m.x, sos=1) + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + detector.set_instance(m) + + expected = ComponentMap() + expected[m.obj] = make_count_dict() + for i in m.a: + expected[m.x[i]] = make_count_dict() + expected[m.y] = make_count_dict() + expected[m.c1] = make_count_dict() + expected[m.obj]['set'] += 1 + for i in m.a: + expected[m.x[i]]['add'] += 1 + expected[m.y]['add'] += 1 + expected[m.c1]['add'] += 1 + obs.check(expected) + + for i in m.a: + expected[m.x[i]]['remove'] += 1 + expected[m.c1]['remove'] += 1 + del m.c1 + detector.update() + obs.check(expected) + def test_vars_and_params_elsewhere(self): - pass + m1 = pyo.ConcreteModel() + m1.x = pyo.Var() + m1.y = pyo.Var() + m1.p = pyo.Param(mutable=True, initialize=1) + + m2 = pyo.ConcreteModel() + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + + expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + + detector.set_instance(m2) + obs.check(expected) + + m2.obj = pyo.Objective(expr=m1.y) + m2.c1 = pyo.Constraint(expr=m1.y >= (m1.x - m1.p) ** 2) + detector.update() + expected[m1.x] = make_count_dict() + expected[m1.y] = make_count_dict() + expected[m1.p] = make_count_dict() + expected[m1.x]['add'] += 1 + expected[m1.y]['add'] += 1 + expected[m1.p]['add'] += 1 + expected[m2.c1] = make_count_dict() + expected[m2.c1]['add'] += 1 + expected[m2.obj] = make_count_dict() + expected[m2.obj]['set'] += 1 + obs.check(expected) + + # now fix a variable and make sure the + # constraint gets removed and added + m1.x.fix(1) + obs.pprint() + detector.update() + obs.pprint() + expected[m2.c1]['remove'] += 1 + expected[m2.c1]['add'] += 1 + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then + # added again when the constraint is added + expected[m1.x]['update'] += 1 + expected[m1.x]['remove'] += 1 + expected[m1.x]['add'] += 1 + expected[m1.p]['remove'] += 1 + expected[m1.p]['add'] += 1 + obs.check(expected) + + def test_named_expression(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + + expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + + detector.set_instance(m) + obs.check(expected) + + m.obj = pyo.Objective(expr=m.y) + m.e = pyo.Expression(expr=m.x - m.p) + m.c1 = pyo.Constraint(expr=m.y >= m.e) + detector.update() + expected[m.x] = make_count_dict() + expected[m.y] = make_count_dict() + expected[m.p] = make_count_dict() + expected[m.x]['add'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['add'] += 1 + expected[m.c1] = make_count_dict() + expected[m.c1]['add'] += 1 + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + obs.check(expected) + + # now modify the named expression and make sure the + # constraint gets removed and added + m.e.expr = (m.x - m.p) ** 2 + detector.update() + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then + # added again when the constraint is added + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.p]['remove'] += 1 + expected[m.p]['add'] += 1 + obs.check(expected) + From c313fe53f890bdfc000dfd2d68a76fcef8f9de8b Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 15:48:39 -0600 Subject: [PATCH 10/50] run black --- pyomo/contrib/observer/tests/test_change_detector.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 46527618ab8..f3ce6e28b5c 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -363,4 +363,3 @@ def test_named_expression(self): expected[m.p]['remove'] += 1 expected[m.p]['add'] += 1 obs.check(expected) - From a424cfb0ae4f81f9b90b8e7e229efccb7eeb93f2 Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Mon, 25 Aug 2025 08:14:07 -0600 Subject: [PATCH 11/50] Minor changes - removing unused imports --- pyomo/contrib/observer/component_collector.py | 3 +-- pyomo/contrib/observer/model_observer.py | 9 +++------ pyomo/contrib/observer/tests/test_change_detector.py | 7 ++++--- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index d52ec46086c..e09fe666bbf 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -63,8 +63,7 @@ def exitNode(self, node, data): nt = type(node) if nt in collector_handlers: return collector_handlers[nt](node, self) - else: - return None + return None _visitor = _ComponentFromExprCollector() diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index bd89d4f600d..1452a3d673d 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -10,21 +10,18 @@ # __________________________________________________________________________ import abc -import datetime from typing import List, Sequence, Optional from pyomo.common.config import ConfigDict, ConfigValue from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.var import VarData -from pyomo.core.base.param import ParamData, Param +from pyomo.core.base.param import ParamData from pyomo.core.base.objective import ObjectiveData -from pyomo.core.staleflag import StaleFlagManager from pyomo.common.collections import ComponentMap from pyomo.common.timing import HierarchicalTimer -from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import get_objective -from .component_collector import collect_components_from_expr +from pyomo.contrib.observer.component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types @@ -624,7 +621,7 @@ def _check_for_var_changes(self): def _check_for_param_changes(self): params_to_update = [] - for pid, (p, val) in self._params.items(): + for _, (p, val) in self._params.items(): if p.value != val: params_to_update.append(p) return params_to_update diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index f3ce6e28b5c..802d4482acc 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -9,21 +9,22 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import logging +from typing import List + +import pyomo.environ as pyo from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.objective import ObjectiveData from pyomo.core.base.param import ParamData from pyomo.core.base.sos import SOSConstraintData from pyomo.core.base.var import VarData -import pyomo.environ as pyo from pyomo.common import unittest -from typing import List from pyomo.contrib.observer.model_observer import ( Observer, ModelChangeDetector, AutoUpdateConfig, ) from pyomo.common.collections import ComponentMap -import logging logger = logging.getLogger(__name__) From 33f831ac27389da9664cc519208a8288c20fe191 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 11:02:04 -0600 Subject: [PATCH 12/50] observer updates --- pyomo/contrib/observer/component_collector.py | 56 ++- pyomo/contrib/observer/model_observer.py | 410 +++++++++++------- .../observer/tests/test_change_detector.py | 46 +- 3 files changed, 315 insertions(+), 197 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index e09fe666bbf..2fe2170aa43 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -12,11 +12,27 @@ from pyomo.core.expr.visitor import StreamBasedExpressionVisitor from pyomo.core.expr.numeric_expr import ( ExternalFunctionExpression, - NPV_ExternalFunctionExpression, + NegationExpression, + PowExpression, + MaxExpression, + MinExpression, + ProductExpression, + MonomialTermExpression, + DivisionExpression, + SumExpression, + Expr_ifExpression, + UnaryFunctionExpression, + AbsExpression, +) +from pyomo.core.expr.relational_expr import ( + RangedExpression, + InequalityExpression, + EqualityExpression, ) from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression +from pyomo.repn.util import ExitNodeDispatcher def handle_var(node, collector): @@ -39,16 +55,29 @@ def handle_external_function(node, collector): return None -collector_handlers = { - VarData: handle_var, - ScalarVar: handle_var, - ParamData: handle_param, - ScalarParam: handle_param, - ExpressionData: handle_named_expression, - ScalarExpression: handle_named_expression, - ExternalFunctionExpression: handle_external_function, - NPV_ExternalFunctionExpression: handle_external_function, -} +def handle_skip(node, collector): + return None + + +collector_handlers = ExitNodeDispatcher() +collector_handlers[VarData] = handle_var +collector_handlers[ParamData] = handle_param +collector_handlers[ExpressionData] = handle_named_expression +collector_handlers[ExternalFunctionExpression] = handle_external_function +collector_handlers[NegationExpression] = handle_skip +collector_handlers[PowExpression] = handle_skip +collector_handlers[MaxExpression] = handle_skip +collector_handlers[MinExpression] = handle_skip +collector_handlers[ProductExpression] = handle_skip +collector_handlers[MonomialTermExpression] = handle_skip +collector_handlers[DivisionExpression] = handle_skip +collector_handlers[SumExpression] = handle_skip +collector_handlers[Expr_ifExpression] = handle_skip +collector_handlers[UnaryFunctionExpression] = handle_skip +collector_handlers[AbsExpression] = handle_skip +collector_handlers[RangedExpression] = handle_skip +collector_handlers[InequalityExpression] = handle_skip +collector_handlers[EqualityExpression] = handle_skip class _ComponentFromExprCollector(StreamBasedExpressionVisitor): @@ -60,10 +89,7 @@ def __init__(self, **kwds): super().__init__(**kwds) def exitNode(self, node, data): - nt = type(node) - if nt in collector_handlers: - return collector_handlers[nt](node, self) - return None + return collector_handlers[node.__class__](node, self) _visitor = _ComponentFromExprCollector() diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 1452a3d673d..f429a8ad907 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -17,7 +17,9 @@ from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.var import VarData from pyomo.core.base.param import ParamData -from pyomo.core.base.objective import ObjectiveData +from pyomo.core.base.objective import ObjectiveData, Objective +from pyomo.core.base.block import BlockData +from pyomo.core.base.component import ActiveComponent from pyomo.common.collections import ComponentMap from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.util import get_objective @@ -75,21 +77,24 @@ def __init__( domain=bool, default=True, description=""" - If False, new/old constraints will not be automatically detected on subsequent - solves. Use False only when manually updating the solver with opt.add_constraints() - and opt.remove_constraints() or when you are certain constraints are not being - added to/removed from the model.""", + If False, new/old constraints will not be automatically detected on + subsequent solves. Use False only when manually updating the solver + with opt.add_constraints() and opt.remove_constraints() or when you + are certain constraints are not being added to/removed from the + model.""", ), ) - self.check_for_new_objective: bool = self.declare( - 'check_for_new_objective', + self.check_for_new_or_removed_objectives: bool = self.declare( + 'check_for_new_or_removed_objectives', ConfigValue( domain=bool, default=True, description=""" - If False, new/old objectives will not be automatically detected on subsequent - solves. Use False only when manually updating the solver with opt.set_objective() or - when you are certain objectives are not being added to / removed from the model.""", + If False, new/old objectives will not be automatically detected on + subsequent solves. Use False only when manually updating the solver + with opt.add_objectives() and opt.remove_objectives() or when you + are certain objectives are not being added to/removed from the + model.""", ), ) self.update_constraints: bool = self.declare( @@ -98,11 +103,12 @@ def __init__( domain=bool, default=True, description=""" - If False, changes to existing constraints will not be automatically detected on - subsequent solves. This includes changes to the lower, body, and upper attributes of - constraints. Use False only when manually updating the solver with - opt.remove_constraints() and opt.add_constraints() or when you are certain constraints - are not being modified.""", + If False, changes to existing constraints will not be automatically + detected on subsequent solves. This includes changes to the lower, + body, and upper attributes of constraints. Use False only when + manually updating the solver with opt.remove_constraints() and + opt.add_constraints() or when you are certain constraints are not + being modified.""", ), ) self.update_vars: bool = self.declare( @@ -111,11 +117,12 @@ def __init__( domain=bool, default=True, description=""" - If False, changes to existing variables will not be automatically detected on - subsequent solves. This includes changes to the lb, ub, domain, and fixed - attributes of variables. Use False only when manually updating the observer with - opt.update_variables() or when you are certain variables are not being modified. - Note that changes to values of fixed variables is handled by + If False, changes to existing variables will not be automatically + detected on subsequent solves. This includes changes to the lb, ub, + domain, and fixed attributes of variables. Use False only when + manually updating the observer with opt.update_variables() or when + you are certain variables are not being modified. Note that changes + to values of fixed variables is handled by update_parameters_and_fixed_vars.""", ), ) @@ -139,21 +146,22 @@ def __init__( default=True, description=""" If False, changes to Expressions will not be automatically detected on - subsequent solves. Use False only when manually updating the solver with - opt.remove_constraints() and opt.add_constraints() or when you are certain - Expressions are not being modified.""", + subsequent solves. Use False only when manually updating the solver + with opt.remove_constraints() and opt.add_constraints() or when you + are certain Expressions are not being modified.""", ), ) - self.update_objective: bool = self.declare( - 'update_objective', + self.update_objectives: bool = self.declare( + 'update_objectives', ConfigValue( domain=bool, default=True, description=""" If False, changes to objectives will not be automatically detected on - subsequent solves. This includes the expr and sense attributes of objectives. Use - False only when manually updating the solver with opt.set_objective() or when you are - certain objectives are not being modified.""", + subsequent solves. This includes the expr and sense attributes of + objectives. Use False only when manually updating the solver with + opt.set_objective() or when you are certain objectives are not being + modified.""", ), ) @@ -179,7 +187,11 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): pass @abc.abstractmethod - def set_objective(self, obj: Optional[ObjectiveData]): + def add_objectives(self, objs: List[ObjectiveData]): + pass + + @abc.abstractmethod + def remove_objectives(self, objs: List[ObjectiveData]): pass @abc.abstractmethod @@ -208,54 +220,61 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: - def __init__(self, observers: Sequence[Observer], **kwds): + def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): """ Parameters ---------- observers: Sequence[Observer] The objects to notify when changes are made to the model """ + self._known_active_ctypes = {Constraint, SOSConstraint, Objective} self._observers: List[Observer] = list(observers) - self._model = None self._active_constraints = {} # maps constraint to expression self._active_sos = {} self._vars = {} # maps var id to (var, lb, ub, fixed, domain, value) self._params = {} # maps param id to param - self._objective = None - self._objective_expr = None - self._objective_sense = None - self._named_expressions = ( - {} - ) # maps constraint to list of tuples (named_expr, named_expr.expr) + self._objectives = {} # maps objective id to (objective, expression, sense) + + # maps constraints/objectives to list of tuples (named_expr, named_expr.expr) + self._named_expressions = {} + self._obj_named_expressions = {} + self._external_functions = ComponentMap() - self._obj_named_expressions = [] - self._referenced_variables = ( - {} - ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] - self._referenced_params = ( - {} - ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + + # the dictionaries below are really just ordered sets, but we need to + # stick with built-in types for performance + + # var_id: ( + # dict[constraints, None], + # dict[sos constraints, None], + # dict[objectives, None], + # ) + self._referenced_variables = {} + + # param_id: ( + # dict[constraints, None], + # dict[sos constraints, None], + # dict[objectives, None], + # ) + self._referenced_params = {} + self._vars_referenced_by_con = {} - self._vars_referenced_by_obj = [] + self._vars_referenced_by_obj = {} self._params_referenced_by_con = {} - self._params_referenced_by_obj = [] - self._expr_types = None + self._params_referenced_by_obj = {} + self.config: AutoUpdateConfig = AutoUpdateConfig()( value=kwds, preserve_implicit=True ) - def set_instance(self, model): - saved_config = self.config - self.__init__(observers=self._observers) - self.config = saved_config self._model = model - self._add_block(model) + self._set_instance() def _add_variables(self, variables: List[VarData]): for v in variables: if id(v) in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') - self._referenced_variables[id(v)] = [{}, {}, None] + self._referenced_variables[id(v)] = ({}, {}, {}) self._vars[id(v)] = ( v, v._lb, @@ -272,44 +291,42 @@ def _add_parameters(self, params: List[ParamData]): pid = id(p) if pid in self._referenced_params: raise ValueError(f'Parameter {p.name} has already been added') - self._referenced_params[pid] = [{}, {}, None] + self._referenced_params[pid] = ({}, {}, {}) self._params[id(p)] = (p, p.value) for obs in self._observers: obs.add_parameters(params) def _check_for_new_vars(self, variables: List[VarData]): - new_vars = {} + new_vars = [] for v in variables: - v_id = id(v) - if v_id not in self._referenced_variables: - new_vars[v_id] = v - self._add_variables(list(new_vars.values())) + if id(v) not in self._referenced_variables: + new_vars.append(v) + self._add_variables(new_vars) def _check_to_remove_vars(self, variables: List[VarData]): - vars_to_remove = {} + vars_to_remove = [] for v in variables: v_id = id(v) ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] - if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: - vars_to_remove[v_id] = v - self._remove_variables(list(vars_to_remove.values())) + if not ref_cons and not ref_sos and not ref_obj: + vars_to_remove.append(v) + self._remove_variables(vars_to_remove) def _check_for_new_params(self, params: List[ParamData]): - new_params = {} + new_params = [] for p in params: - pid = id(p) - if pid not in self._referenced_params: - new_params[pid] = p - self._add_parameters(list(new_params.values())) + if id(p) not in self._referenced_params: + new_params.append(p) + self._add_parameters(new_params) def _check_to_remove_params(self, params: List[ParamData]): - params_to_remove = {} + params_to_remove = [] for p in params: p_id = id(p) ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] - if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: - params_to_remove[p_id] = p - self._remove_parameters(list(params_to_remove.values())) + if not ref_cons and not ref_sos and not ref_obj: + params_to_remove.append(p) + self._remove_parameters(params_to_remove) def _add_constraints(self, cons: List[ConstraintData]): vars_to_check = [] @@ -322,7 +339,8 @@ def _add_constraints(self, cons: List[ConstraintData]): named_exprs, variables, parameters, external_functions = tmp vars_to_check.extend(variables) params_to_check.extend(parameters) - self._named_expressions[con] = [(e, e.expr) for e in named_exprs] + if len(named_exprs) > 0: + self._named_expressions[con] = [(e, e.expr) for e in named_exprs] if len(external_functions) > 0: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables @@ -375,61 +393,103 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.add_sos_constraints(cons) - def _set_objective(self, obj: Optional[ObjectiveData]): - vars_to_remove_check = [] - params_to_remove_check = [] - if self._objective is not None: - for v in self._vars_referenced_by_obj: - self._referenced_variables[id(v)][2] = None - for p in self._params_referenced_by_obj: - self._referenced_params[id(p)][2] = None - vars_to_remove_check.extend(self._vars_referenced_by_obj) - params_to_remove_check.extend(self._params_referenced_by_obj) - self._external_functions.pop(self._objective, None) - if obj is not None: - self._objective = obj - self._objective_expr = obj.expr - self._objective_sense = obj.sense - tmp = collect_components_from_expr(obj.expr) - named_exprs, variables, parameters, external_functions = tmp - self._check_for_new_vars(variables) - self._check_for_new_params(parameters) - self._obj_named_expressions = [(i, i.expr) for i in named_exprs] + def _add_objectives(self, objs: List[ObjectiveData]): + vars_to_check = [] + params_to_check = [] + for obj in objs: + obj_id = id(obj) + self._objectives[obj_id] = (obj, obj.expr, obj.sense) + ( + named_exprs, + variables, + parameters, + external_functions, + ) = collect_components_from_expr(obj.expr) + vars_to_check.extend(variables) + params_to_check.extend(parameters) + if len(named_exprs) > 0: + self._obj_named_expressions[obj_id] = [(e, e.expr) for e in named_exprs] if len(external_functions) > 0: self._external_functions[obj] = external_functions - self._vars_referenced_by_obj = variables - self._params_referenced_by_obj = parameters + self._vars_referenced_by_obj[obj_id] = variables + self._params_referenced_by_obj[obj_id] = parameters + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for obj in objs: + obj_id = id(obj) + variables = self._vars_referenced_by_obj[obj_id] + parameters = self._params_referenced_by_obj[obj_id] for v in variables: - self._referenced_variables[id(v)][2] = obj + self._referenced_variables[id(v)][2][obj_id] = None for p in parameters: - self._referenced_params[id(p)][2] = obj - else: - self._vars_referenced_by_obj = [] - self._params_referenced_by_obj = [] - self._objective = None - self._objective_expr = None - self._objective_sense = None - self._obj_named_expressions = [] + self._referenced_params[id(p)][2][obj_id] = None + for obs in self._observers: + obs.add_objectives(objs) + + def _remove_objectives(self, objs: List[ObjectiveData]): for obs in self._observers: - obs.set_objective(obj) - self._check_to_remove_vars(vars_to_remove_check) - self._check_to_remove_params(params_to_remove_check) + obs.remove_objectives(objs) + + vars_to_check = [] + params_to_check = [] + for obj in objs: + obj_id = id(obj) + if obj_id not in self._objectives: + raise ValueError( + f'cannot remove objective {obj.name} - it was not added' + ) + for v in self._vars_referenced_by_obj[obj_id]: + self._referenced_variables[id(v)][2].pop(obj_id) + for p in self._params_referenced_by_obj[obj_id]: + self._referenced_params[id(p)][2].pop(obj_id) + vars_to_check.extend(self._vars_referenced_by_obj[obj_id]) + params_to_check.extend(self._params_referenced_by_obj[obj_id]) + del self._objectives[obj_id] + del self._obj_named_expressions[obj_id] + self._external_functions.pop(obj, None) + del self._vars_referenced_by_obj[obj_id] + del self._params_referenced_by_obj[obj_id] + self._check_to_remove_vars(vars_to_check) + self._check_to_remove_params(params_to_check) + + def _check_for_unknown_active_components(self): + for ctype in self._model.collect_ctypes(): + if not issubclass(ctype, ActiveComponent): + continue + if ctype in self._known_active_ctypes: + continue + for comp in self._model.component_data_objects( + ctype, + active=True, + descend_into=True + ): + raise NotImplementedError( + f'ModelChangeDetector does not know how to ' + 'handle compents with ctype {ctype}' + ) + + def _set_instance(self): + self._check_for_unknown_active_components() - def _add_block(self, block): self._add_constraints( list( - block.component_data_objects(Constraint, descend_into=True, active=True) + self._model.component_data_objects(Constraint, descend_into=True, active=True) ) ) self._add_sos_constraints( list( - block.component_data_objects( - SOSConstraint, descend_into=True, active=True + self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True, + ) + ) + ) + self._add_objectives( + list( + self._model.component_data_objects( + Objective, descend_into=True, active=True, ) ) ) - obj = get_objective(block) - self._set_objective(obj) def _remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: @@ -488,9 +548,9 @@ def _remove_variables(self, variables: List[VarData]): f'Cannot remove variable {v.name} - it has not been added' ) cons_using, sos_using, obj_using = self._referenced_variables[v_id] - if cons_using or sos_using or (obj_using is not None): + if cons_using or sos_using or obj_using: raise ValueError( - f'Cannot remove variable {v.name} - it is still being used by constraints or the objective' + f'Cannot remove variable {v.name} - it is still being used by constraints/objectives' ) del self._referenced_variables[v_id] del self._vars[v_id] @@ -505,9 +565,9 @@ def _remove_parameters(self, params: List[ParamData]): f'Cannot remove parameter {p.name} - it has not been added' ) cons_using, sos_using, obj_using = self._referenced_params[p_id] - if cons_using or sos_using or (obj_using is not None): + if cons_using or sos_using or obj_using: raise ValueError( - f'Cannot remove parameter {p.name} - it is still being used by constraints or the objective' + f'Cannot remove parameter {p.name} - it is still being used by constraints/objectives' ) del self._referenced_params[p_id] del self._params[p_id] @@ -600,14 +660,14 @@ def _check_for_modified_constraints(self): def _check_for_var_changes(self): vars_to_update = [] cons_to_update = {} - update_obj = False + objs_to_update = {} for vid, (v, _lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): if v.fixed != _fixed: vars_to_update.append(v) for c in self._referenced_variables[vid][0]: cons_to_update[c] = None - if self._referenced_variables[vid][2] is not None: - update_obj = True + for obj_id in self._referenced_variables[vid][2]: + objs_to_update[obj_id] = None elif v._lb is not _lb: vars_to_update.append(v) elif v._ub is not _ub: @@ -617,7 +677,8 @@ def _check_for_var_changes(self): elif v.value != _value: vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) - return vars_to_update, cons_to_update, update_obj + objs_to_update = [self._objectives[obj_id][0] for obj_id in objs_to_update.keys()] + return vars_to_update, cons_to_update, objs_to_update def _check_for_param_changes(self): params_to_update = [] @@ -633,39 +694,47 @@ def _check_for_named_expression_changes(self): if named_expr.expr is not old_expr: cons_to_update.append(con) break - update_obj = False - ne_list = self._obj_named_expressions - for named_expr, old_expr in ne_list: - if named_expr.expr is not old_expr: - update_obj = True - break - return cons_to_update, update_obj - - def _check_for_new_objective(self): - update_obj = False - new_obj = get_objective(self._model) - if new_obj is not self._objective: - update_obj = True - return new_obj, update_obj - - def _check_for_objective_changes(self): - update_obj = False - if self._objective is None: - return update_obj - if self._objective.expr is not self._objective_expr: - update_obj = True - elif self._objective.sense != self._objective_sense: - # we can definitely do something faster here than resetting the whole objective - update_obj = True - return update_obj + objs_to_update = [] + for obj_id, ne_list in self._obj_named_expressions.items(): + for named_expr, old_expr in ne_list: + if named_expr.expr is not old_expr: + objs_to_update.append(self._objectives[obj_id][0]) + break + return cons_to_update, objs_to_update + + def _check_for_new_or_removed_objectives(self): + new_objs = [] + old_objs = [] + current_objs_dict = { + id(obj): obj for obj in self._model.component_data_objects( + Objective, descend_into=True, active=True + ) + } + for obj_id, obj in current_objs_dict.items(): + if obj_id not in self._objectives: + new_objs.append(obj) + for obj_id, (obj, obj_expr, obj_sense) in self._objectives.items(): + if obj_id not in current_objs_dict: + old_objs.append(obj) + return new_objs, old_objs + + def _check_for_modified_objectives(self): + objs_to_update = [] + for obj_id, (obj, obj_expr, obj_sense) in self._objectives.items(): + if obj.expr is not obj_expr or obj.sense != obj_sense: + objs_to_update.append(obj) + return objs_to_update def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if timer is None: timer = HierarchicalTimer() config: AutoUpdateConfig = self.config(value=kwds, preserve_implicit=True) + self._check_for_unknown_active_components() + added_cons = set() added_sos = set() + added_objs = {} if config.check_for_new_or_removed_constraints: timer.start('sos') @@ -695,46 +764,51 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): added_sos.update(sos_to_update) timer.stop('sos') - need_to_set_objective = False + if config.check_for_new_or_removed_objectives: + timer.start('objective') + new_objs, old_objs = self._check_for_new_or_removed_objectives() + # many solvers require one objective, so we have to remove the + # old objective first + self._remove_objectives(old_objs) + self._add_objectives(new_objs) + added_objs.update((id(i), i) for i in new_objs) + timer.stop('objective') + + if config.update_objectives: + timer.start('objective') + objs_to_update = self._check_for_modified_objectives() + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) + timer.stop('objective') if config.update_vars: timer.start('vars') - vars_to_update, cons_to_update, update_obj = self._check_for_var_changes() + vars_to_update, cons_to_update, objs_to_update = self._check_for_var_changes() self._update_variables(vars_to_update) cons_to_update = [i for i in cons_to_update if i not in added_cons] + objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] self._remove_constraints(cons_to_update) self._add_constraints(cons_to_update) added_cons.update(cons_to_update) - if update_obj: - need_to_set_objective = True + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) timer.stop('vars') if config.update_named_expressions: timer.start('named expressions') - cons_to_update, update_obj = self._check_for_named_expression_changes() + cons_to_update, objs_to_update = self._check_for_named_expression_changes() cons_to_update = [i for i in cons_to_update if i not in added_cons] + objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] self._remove_constraints(cons_to_update) self._add_constraints(cons_to_update) added_cons.update(cons_to_update) - if update_obj: - need_to_set_objective = True + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) timer.stop('named expressions') - timer.start('objective') - new_obj = self._objective - if config.check_for_new_objective: - new_obj, update_obj = self._check_for_new_objective() - if update_obj: - need_to_set_objective = True - if config.update_objective: - update_obj = self._check_for_objective_changes() - if update_obj: - need_to_set_objective = True - - if need_to_set_objective: - self._set_objective(new_obj) - timer.stop('objective') - if config.update_parameters: timer.start('params') params_to_update = self._check_for_param_changes() diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 802d4482acc..a9d367fad01 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -82,15 +82,21 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): assert isinstance(c, SOSConstraintData) self._process(cons, 'add') - def set_objective(self, obj: ObjectiveData): - assert obj is None or isinstance(obj, ObjectiveData) - self._process([obj], 'set') + def add_objectives(self, objs: List[ObjectiveData]): + for obj in objs: + assert isinstance(obj, ObjectiveData) + self._process(objs, 'add') def remove_constraints(self, cons: List[ConstraintData]): for c in cons: assert isinstance(c, ConstraintData) self._process(cons, 'remove') + def remove_objectives(self, objs: List[ObjectiveData]): + for obj in objs: + assert isinstance(obj, ObjectiveData) + self._process(objs, 'remove') + def remove_sos_constraints(self, cons: List[SOSConstraintData]): for c in cons: assert isinstance(c, SOSConstraintData) @@ -125,19 +131,15 @@ def test_objective(self): m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() - detector = ModelChangeDetector([obs]) + detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() - expected[None] = make_count_dict() - expected[None]['set'] += 1 - - detector.set_instance(m) obs.check(expected) m.obj = pyo.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() - expected[m.obj]['set'] += 1 + expected[m.obj]['add'] += 1 expected[m.x] = make_count_dict() expected[m.x]['add'] += 1 expected[m.y] = make_count_dict() @@ -154,13 +156,15 @@ def test_objective(self): m.x.fix(2) detector.update() expected[m.x]['update'] += 1 - expected[m.obj]['set'] += 1 + expected[m.obj]['remove'] += 1 + expected[m.obj]['add'] += 1 obs.check(expected) m.x.unfix() detector.update() expected[m.x]['update'] += 1 - expected[m.obj]['set'] += 1 + expected[m.obj]['remove'] += 1 + expected[m.obj]['add'] += 1 obs.check(expected) m.p.value = 2 @@ -171,9 +175,11 @@ def test_objective(self): m.obj.expr = m.x**2 + m.y**2 detector.update() expected[m.p]['remove'] += 1 - expected[m.obj]['set'] += 1 + expected[m.obj]['remove'] += 1 + expected[m.obj]['add'] += 1 obs.check(expected) + expected[m.obj]['remove'] += 1 del m.obj m.obj = pyo.Objective(expr=m.p * m.x) detector.update() @@ -181,7 +187,7 @@ def test_objective(self): expected[m.y]['remove'] += 1 # remember, m.obj is a different object now expected[m.obj] = make_count_dict() - expected[m.obj]['set'] += 1 + expected[m.obj]['add'] += 1 def test_constraints(self): m = pyo.ConcreteModel() @@ -190,7 +196,7 @@ def test_constraints(self): m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() - detector = ModelChangeDetector([obs]) + detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() expected[None] = make_count_dict() @@ -364,3 +370,15 @@ def test_named_expression(self): expected[m.p]['remove'] += 1 expected[m.p]['add'] += 1 obs.check(expected) + + def test_update_config(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.obj = pyo.Objective(expr=m.x**2 + m.y**2) + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + + expected = ComponentMap() + expected[None] = make_count_dict() From 5f3f403bfabac655d04530794f1f5b5af5e161c2 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 12:38:07 -0600 Subject: [PATCH 13/50] observer fixes --- pyomo/contrib/observer/component_collector.py | 3 + pyomo/contrib/observer/model_observer.py | 6 +- .../observer/tests/test_change_detector.py | 95 ++++++++++++------- 3 files changed, 68 insertions(+), 36 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 2fe2170aa43..638f85327b4 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -63,6 +63,7 @@ def handle_skip(node, collector): collector_handlers[VarData] = handle_var collector_handlers[ParamData] = handle_param collector_handlers[ExpressionData] = handle_named_expression +collector_handlers[ScalarExpression] = handle_named_expression collector_handlers[ExternalFunctionExpression] = handle_external_function collector_handlers[NegationExpression] = handle_skip collector_handlers[PowExpression] = handle_skip @@ -78,6 +79,8 @@ def handle_skip(node, collector): collector_handlers[RangedExpression] = handle_skip collector_handlers[InequalityExpression] = handle_skip collector_handlers[EqualityExpression] = handle_skip +collector_handlers[int] = handle_skip +collector_handlers[float] = handle_skip class _ComponentFromExprCollector(StreamBasedExpressionVisitor): diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index f429a8ad907..4b873dd858e 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -378,7 +378,6 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): params.append(p) vars_to_check.extend(variables) params_to_check.extend(params) - self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params self._check_for_new_vars(vars_to_check) @@ -445,7 +444,7 @@ def _remove_objectives(self, objs: List[ObjectiveData]): vars_to_check.extend(self._vars_referenced_by_obj[obj_id]) params_to_check.extend(self._params_referenced_by_obj[obj_id]) del self._objectives[obj_id] - del self._obj_named_expressions[obj_id] + self._obj_named_expressions.pop(obj_id, None) self._external_functions.pop(obj, None) del self._vars_referenced_by_obj[obj_id] del self._params_referenced_by_obj[obj_id] @@ -508,7 +507,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): vars_to_check.extend(self._vars_referenced_by_con[con]) params_to_check.extend(self._params_referenced_by_con[con]) del self._active_constraints[con] - del self._named_expressions[con] + self._named_expressions.pop(con, None) self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] @@ -532,7 +531,6 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): vars_to_check.extend(self._vars_referenced_by_con[con]) params_to_check.extend(self._params_referenced_by_con[con]) del self._active_sos[con] - del self._named_expressions[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] self._check_to_remove_vars(vars_to_check) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index a9d367fad01..d6b5fcbcc62 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -156,8 +156,18 @@ def test_objective(self): m.x.fix(2) detector.update() expected[m.x]['update'] += 1 + # the variable gets updated + # the objective must get removed and added + # that causes x,y, and p to all get removed + # and added expected[m.obj]['remove'] += 1 expected[m.obj]['add'] += 1 + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.y]['remove'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['remove'] += 1 + expected[m.p]['add'] += 1 obs.check(expected) m.x.unfix() @@ -165,6 +175,12 @@ def test_objective(self): expected[m.x]['update'] += 1 expected[m.obj]['remove'] += 1 expected[m.obj]['add'] += 1 + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.y]['remove'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['remove'] += 1 + expected[m.p]['add'] += 1 obs.check(expected) m.p.value = 2 @@ -174,20 +190,27 @@ def test_objective(self): m.obj.expr = m.x**2 + m.y**2 detector.update() - expected[m.p]['remove'] += 1 expected[m.obj]['remove'] += 1 expected[m.obj]['add'] += 1 + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.y]['remove'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['remove'] += 1 obs.check(expected) expected[m.obj]['remove'] += 1 del m.obj - m.obj = pyo.Objective(expr=m.p * m.x) + m.obj2 = pyo.Objective(expr=m.p * m.x) detector.update() - expected[m.p]['add'] += 1 - expected[m.y]['remove'] += 1 # remember, m.obj is a different object now - expected[m.obj] = make_count_dict() - expected[m.obj]['add'] += 1 + expected[m.obj2] = make_count_dict() + expected[m.obj2]['add'] += 1 + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.y]['remove'] += 1 + expected[m.p]['add'] += 1 + obs.check(expected) def test_constraints(self): m = pyo.ConcreteModel() @@ -199,10 +222,6 @@ def test_constraints(self): detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() - expected[None] = make_count_dict() - expected[None]['set'] += 1 - - detector.set_instance(m) obs.check(expected) m.obj = pyo.Objective(expr=m.y) @@ -217,15 +236,13 @@ def test_constraints(self): expected[m.c1] = make_count_dict() expected[m.c1]['add'] += 1 expected[m.obj] = make_count_dict() - expected[m.obj]['set'] += 1 + expected[m.obj]['add'] += 1 obs.check(expected) # now fix a variable and make sure the # constraint gets removed and added m.x.fix(1) - obs.pprint() detector.update() - obs.pprint() expected[m.c1]['remove'] += 1 expected[m.c1]['add'] += 1 # because x and p are only used in the @@ -248,8 +265,7 @@ def test_sos(self): m.c1 = pyo.SOSConstraint(var=m.x, sos=1) obs = ObserverChecker() - detector = ModelChangeDetector([obs]) - detector.set_instance(m) + detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() expected[m.obj] = make_count_dict() @@ -257,7 +273,7 @@ def test_sos(self): expected[m.x[i]] = make_count_dict() expected[m.y] = make_count_dict() expected[m.c1] = make_count_dict() - expected[m.obj]['set'] += 1 + expected[m.obj]['add'] += 1 for i in m.a: expected[m.x[i]]['add'] += 1 expected[m.y]['add'] += 1 @@ -280,13 +296,9 @@ def test_vars_and_params_elsewhere(self): m2 = pyo.ConcreteModel() obs = ObserverChecker() - detector = ModelChangeDetector([obs]) + detector = ModelChangeDetector(m2, [obs]) expected = ComponentMap() - expected[None] = make_count_dict() - expected[None]['set'] += 1 - - detector.set_instance(m2) obs.check(expected) m2.obj = pyo.Objective(expr=m1.y) @@ -301,15 +313,13 @@ def test_vars_and_params_elsewhere(self): expected[m2.c1] = make_count_dict() expected[m2.c1]['add'] += 1 expected[m2.obj] = make_count_dict() - expected[m2.obj]['set'] += 1 + expected[m2.obj]['add'] += 1 obs.check(expected) # now fix a variable and make sure the # constraint gets removed and added m1.x.fix(1) - obs.pprint() detector.update() - obs.pprint() expected[m2.c1]['remove'] += 1 expected[m2.c1]['add'] += 1 # because x and p are only used in the @@ -330,13 +340,9 @@ def test_named_expression(self): m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() - detector = ModelChangeDetector([obs]) + detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() - expected[None] = make_count_dict() - expected[None]['set'] += 1 - - detector.set_instance(m) obs.check(expected) m.obj = pyo.Objective(expr=m.y) @@ -352,7 +358,7 @@ def test_named_expression(self): expected[m.c1] = make_count_dict() expected[m.c1]['add'] += 1 expected[m.obj] = make_count_dict() - expected[m.obj]['set'] += 1 + expected[m.obj]['add'] += 1 obs.check(expected) # now modify the named expression and make sure the @@ -376,9 +382,34 @@ def test_update_config(self): m.x = pyo.Var() m.y = pyo.Var() m.obj = pyo.Objective(expr=m.x**2 + m.y**2) + m.c1 = pyo.Constraint(expr=m.y >= pyo.exp(m.x)) obs = ObserverChecker() - detector = ModelChangeDetector([obs]) + detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() - expected[None] = make_count_dict() + expected[m.x] = make_count_dict() + expected[m.y] = make_count_dict() + expected[m.obj] = make_count_dict() + expected[m.c1] = make_count_dict() + expected[m.x]['add'] += 1 + expected[m.y]['add'] += 1 + expected[m.obj]['add'] += 1 + expected[m.c1]['add'] += 1 + obs.check(expected) + + detector.config.check_for_new_or_removed_constraints = False + detector.config.update_constraints = False + m.c2 = pyo.Constraint(expr=m.y >= (m.x - 1)**2) + detector.update() + obs.check(expected) + + m.x.setlb(0) + detector.update() + expected[m.x]['update'] += 1 + obs.check(expected) + + detector.config.check_for_new_or_removed_constraints = True + detector.update() + expected[m.c2] = make_count_dict() + expected[m.c2]['add'] += 1 From 79e1b472c549c30918068ececab9aa21ca9601c8 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 12:46:02 -0600 Subject: [PATCH 14/50] observer config updates --- pyomo/contrib/observer/model_observer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 4b873dd858e..335619a9f2e 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -12,7 +12,7 @@ import abc from typing import List, Sequence, Optional -from pyomo.common.config import ConfigDict, ConfigValue +from pyomo.common.config import ConfigDict, ConfigValue, document_configdict from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.var import VarData @@ -48,6 +48,7 @@ """ +@document_configdict() class AutoUpdateConfig(ConfigDict): """ Control which parts of the model are automatically checked and/or updated upon re-solve @@ -71,6 +72,7 @@ def __init__( visibility=visibility, ) + # automatically detect new/removed constraints on subsequent solves self.check_for_new_or_removed_constraints: bool = self.declare( 'check_for_new_or_removed_constraints', ConfigValue( @@ -84,6 +86,7 @@ def __init__( model.""", ), ) + # automatically detect new/removed objectives on subsequent solves self.check_for_new_or_removed_objectives: bool = self.declare( 'check_for_new_or_removed_objectives', ConfigValue( @@ -97,6 +100,7 @@ def __init__( model.""", ), ) + # automatically detect changes to constraints on subsequent solves self.update_constraints: bool = self.declare( 'update_constraints', ConfigValue( @@ -111,6 +115,7 @@ def __init__( being modified.""", ), ) + # automatically detect changes to variables on subsequent solves self.update_vars: bool = self.declare( 'update_vars', ConfigValue( @@ -126,6 +131,7 @@ def __init__( update_parameters_and_fixed_vars.""", ), ) + # automatically detect changes to parameters on subsequent solves self.update_parameters: bool = self.declare( 'update_parameters', ConfigValue( @@ -139,6 +145,7 @@ def __init__( parameters are not being modified.""", ), ) + # automatically detect changes to named expressions on subsequent solves self.update_named_expressions: bool = self.declare( 'update_named_expressions', ConfigValue( @@ -151,6 +158,7 @@ def __init__( are certain Expressions are not being modified.""", ), ) + # automatically detect changes to objectives on subsequent solves self.update_objectives: bool = self.declare( 'update_objectives', ConfigValue( From 7275176961665ea610ee35c676f37b9bca2525a9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 12:50:32 -0600 Subject: [PATCH 15/50] minor observer updates --- pyomo/contrib/observer/model_observer.py | 59 +++++++++++++++--------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 335619a9f2e..fcf18dcb310 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -745,28 +745,34 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.check_for_new_or_removed_constraints: timer.start('sos') new_sos, old_sos = self._check_for_new_or_removed_sos() - self._add_sos_constraints(new_sos) - self._remove_sos_constraints(old_sos) + if new_sos: + self._add_sos_constraints(new_sos) + if old_sos: + self._remove_sos_constraints(old_sos) added_sos.update(new_sos) timer.stop('sos') timer.start('cons') new_cons, old_cons = self._check_for_new_or_removed_constraints() - self._add_constraints(new_cons) - self._remove_constraints(old_cons) + if new_cons: + self._add_constraints(new_cons) + if old_cons: + self._remove_constraints(old_cons) added_cons.update(new_cons) timer.stop('cons') if config.update_constraints: timer.start('cons') cons_to_update = self._check_for_modified_constraints() - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) added_cons.update(cons_to_update) timer.stop('cons') timer.start('sos') sos_to_update = self._check_for_modified_sos() - self._remove_sos_constraints(sos_to_update) - self._add_sos_constraints(sos_to_update) + if sos_to_update: + self._remove_sos_constraints(sos_to_update) + self._add_sos_constraints(sos_to_update) added_sos.update(sos_to_update) timer.stop('sos') @@ -775,30 +781,36 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): new_objs, old_objs = self._check_for_new_or_removed_objectives() # many solvers require one objective, so we have to remove the # old objective first - self._remove_objectives(old_objs) - self._add_objectives(new_objs) + if old_objs: + self._remove_objectives(old_objs) + if new_objs: + self._add_objectives(new_objs) added_objs.update((id(i), i) for i in new_objs) timer.stop('objective') if config.update_objectives: timer.start('objective') objs_to_update = self._check_for_modified_objectives() - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) added_objs.update((id(i), i) for i in objs_to_update) timer.stop('objective') if config.update_vars: timer.start('vars') vars_to_update, cons_to_update, objs_to_update = self._check_for_var_changes() - self._update_variables(vars_to_update) + if vars_to_update: + self._update_variables(vars_to_update) cons_to_update = [i for i in cons_to_update if i not in added_cons] objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) added_cons.update(cons_to_update) - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) added_objs.update((id(i), i) for i in objs_to_update) timer.stop('vars') @@ -807,16 +819,19 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): cons_to_update, objs_to_update = self._check_for_named_expression_changes() cons_to_update = [i for i in cons_to_update if i not in added_cons] objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) added_cons.update(cons_to_update) - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) added_objs.update((id(i), i) for i in objs_to_update) timer.stop('named expressions') if config.update_parameters: timer.start('params') params_to_update = self._check_for_param_changes() - self._update_parameters(params_to_update) + if params_to_update: + self._update_parameters(params_to_update) timer.stop('params') From 823e15a410bbac6e6950ea7ae6008c017ca750bf Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 12:58:02 -0600 Subject: [PATCH 16/50] disable gc for expensive parts of observer --- pyomo/contrib/observer/model_observer.py | 244 ++++++++++++----------- 1 file changed, 130 insertions(+), 114 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index fcf18dcb310..48dac5ccb62 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -25,6 +25,7 @@ from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.observer.component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types +import gc """ @@ -476,27 +477,35 @@ def _check_for_unknown_active_components(self): ) def _set_instance(self): - self._check_for_unknown_active_components() - self._add_constraints( - list( - self._model.component_data_objects(Constraint, descend_into=True, active=True) + is_gc_enabled = gc.isenabled() + gc.disable() + + try: + self._check_for_unknown_active_components() + + self._add_constraints( + list( + self._model.component_data_objects(Constraint, descend_into=True, active=True) + ) ) - ) - self._add_sos_constraints( - list( - self._model.component_data_objects( - SOSConstraint, descend_into=True, active=True, + self._add_sos_constraints( + list( + self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True, + ) ) ) - ) - self._add_objectives( - list( - self._model.component_data_objects( - Objective, descend_into=True, active=True, + self._add_objectives( + list( + self._model.component_data_objects( + Objective, descend_into=True, active=True, + ) ) ) - ) + finally: + if is_gc_enabled: + gc.enable() def _remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: @@ -736,102 +745,109 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): timer = HierarchicalTimer() config: AutoUpdateConfig = self.config(value=kwds, preserve_implicit=True) - self._check_for_unknown_active_components() - - added_cons = set() - added_sos = set() - added_objs = {} - - if config.check_for_new_or_removed_constraints: - timer.start('sos') - new_sos, old_sos = self._check_for_new_or_removed_sos() - if new_sos: - self._add_sos_constraints(new_sos) - if old_sos: - self._remove_sos_constraints(old_sos) - added_sos.update(new_sos) - timer.stop('sos') - timer.start('cons') - new_cons, old_cons = self._check_for_new_or_removed_constraints() - if new_cons: - self._add_constraints(new_cons) - if old_cons: - self._remove_constraints(old_cons) - added_cons.update(new_cons) - timer.stop('cons') - - if config.update_constraints: - timer.start('cons') - cons_to_update = self._check_for_modified_constraints() - if cons_to_update: - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) - added_cons.update(cons_to_update) - timer.stop('cons') - timer.start('sos') - sos_to_update = self._check_for_modified_sos() - if sos_to_update: - self._remove_sos_constraints(sos_to_update) - self._add_sos_constraints(sos_to_update) - added_sos.update(sos_to_update) - timer.stop('sos') - - if config.check_for_new_or_removed_objectives: - timer.start('objective') - new_objs, old_objs = self._check_for_new_or_removed_objectives() - # many solvers require one objective, so we have to remove the - # old objective first - if old_objs: - self._remove_objectives(old_objs) - if new_objs: - self._add_objectives(new_objs) - added_objs.update((id(i), i) for i in new_objs) - timer.stop('objective') - - if config.update_objectives: - timer.start('objective') - objs_to_update = self._check_for_modified_objectives() - if objs_to_update: - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) - added_objs.update((id(i), i) for i in objs_to_update) - timer.stop('objective') - - if config.update_vars: - timer.start('vars') - vars_to_update, cons_to_update, objs_to_update = self._check_for_var_changes() - if vars_to_update: - self._update_variables(vars_to_update) - cons_to_update = [i for i in cons_to_update if i not in added_cons] - objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] - if cons_to_update: - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) - added_cons.update(cons_to_update) - if objs_to_update: - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) - added_objs.update((id(i), i) for i in objs_to_update) - timer.stop('vars') - - if config.update_named_expressions: - timer.start('named expressions') - cons_to_update, objs_to_update = self._check_for_named_expression_changes() - cons_to_update = [i for i in cons_to_update if i not in added_cons] - objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] - if cons_to_update: - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) - added_cons.update(cons_to_update) - if objs_to_update: - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) - added_objs.update((id(i), i) for i in objs_to_update) - timer.stop('named expressions') - - if config.update_parameters: - timer.start('params') - params_to_update = self._check_for_param_changes() - if params_to_update: - self._update_parameters(params_to_update) - timer.stop('params') + is_gc_enabled = gc.isenabled() + gc.disable() + + try: + self._check_for_unknown_active_components() + + added_cons = set() + added_sos = set() + added_objs = {} + + if config.check_for_new_or_removed_constraints: + timer.start('sos') + new_sos, old_sos = self._check_for_new_or_removed_sos() + if new_sos: + self._add_sos_constraints(new_sos) + if old_sos: + self._remove_sos_constraints(old_sos) + added_sos.update(new_sos) + timer.stop('sos') + timer.start('cons') + new_cons, old_cons = self._check_for_new_or_removed_constraints() + if new_cons: + self._add_constraints(new_cons) + if old_cons: + self._remove_constraints(old_cons) + added_cons.update(new_cons) + timer.stop('cons') + + if config.update_constraints: + timer.start('cons') + cons_to_update = self._check_for_modified_constraints() + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + timer.stop('cons') + timer.start('sos') + sos_to_update = self._check_for_modified_sos() + if sos_to_update: + self._remove_sos_constraints(sos_to_update) + self._add_sos_constraints(sos_to_update) + added_sos.update(sos_to_update) + timer.stop('sos') + + if config.check_for_new_or_removed_objectives: + timer.start('objective') + new_objs, old_objs = self._check_for_new_or_removed_objectives() + # many solvers require one objective, so we have to remove the + # old objective first + if old_objs: + self._remove_objectives(old_objs) + if new_objs: + self._add_objectives(new_objs) + added_objs.update((id(i), i) for i in new_objs) + timer.stop('objective') + + if config.update_objectives: + timer.start('objective') + objs_to_update = self._check_for_modified_objectives() + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) + timer.stop('objective') + + if config.update_vars: + timer.start('vars') + vars_to_update, cons_to_update, objs_to_update = self._check_for_var_changes() + if vars_to_update: + self._update_variables(vars_to_update) + cons_to_update = [i for i in cons_to_update if i not in added_cons] + objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) + timer.stop('vars') + + if config.update_named_expressions: + timer.start('named expressions') + cons_to_update, objs_to_update = self._check_for_named_expression_changes() + cons_to_update = [i for i in cons_to_update if i not in added_cons] + objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) + timer.stop('named expressions') + + if config.update_parameters: + timer.start('params') + params_to_update = self._check_for_param_changes() + if params_to_update: + self._update_parameters(params_to_update) + timer.stop('params') + finally: + if is_gc_enabled: + gc.enable() From 73258f50838414992f2778a12b58183e2c186231 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 12:59:57 -0600 Subject: [PATCH 17/50] minor observer updates --- pyomo/contrib/observer/model_observer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 48dac5ccb62..0c4254d25f9 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -344,8 +344,12 @@ def _add_constraints(self, cons: List[ConstraintData]): if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') self._active_constraints[con] = con.expr - tmp = collect_components_from_expr(con.expr) - named_exprs, variables, parameters, external_functions = tmp + ( + named_exprs, + variables, + parameters, + external_functions, + ) = collect_components_from_expr(con.expr) vars_to_check.extend(variables) params_to_check.extend(parameters) if len(named_exprs) > 0: From 5a8a5a683496c74e216becb256c2b6e8e42d5a88 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 13:01:11 -0600 Subject: [PATCH 18/50] minor observer updates --- pyomo/contrib/observer/model_observer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 0c4254d25f9..dd6cd28892f 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -352,9 +352,9 @@ def _add_constraints(self, cons: List[ConstraintData]): ) = collect_components_from_expr(con.expr) vars_to_check.extend(variables) params_to_check.extend(parameters) - if len(named_exprs) > 0: + if named_exprs: self._named_expressions[con] = [(e, e.expr) for e in named_exprs] - if len(external_functions) > 0: + if external_functions: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = parameters @@ -419,9 +419,9 @@ def _add_objectives(self, objs: List[ObjectiveData]): ) = collect_components_from_expr(obj.expr) vars_to_check.extend(variables) params_to_check.extend(parameters) - if len(named_exprs) > 0: + if named_exprs: self._obj_named_expressions[obj_id] = [(e, e.expr) for e in named_exprs] - if len(external_functions) > 0: + if external_functions: self._external_functions[obj] = external_functions self._vars_referenced_by_obj[obj_id] = variables self._params_referenced_by_obj[obj_id] = parameters From 321755a3460eac135be15e6cbf02b0b1ee7fae1c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 16:01:51 -0600 Subject: [PATCH 19/50] minor observer updates --- pyomo/contrib/observer/model_observer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index dd6cd28892f..e00331b47ed 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -687,14 +687,14 @@ def _check_for_var_changes(self): cons_to_update[c] = None for obj_id in self._referenced_variables[vid][2]: objs_to_update[obj_id] = None + elif _fixed and v.value != _value: + vars_to_update.append(v) elif v._lb is not _lb: vars_to_update.append(v) elif v._ub is not _ub: vars_to_update.append(v) elif _domain_interval != v.domain.get_interval(): vars_to_update.append(v) - elif v.value != _value: - vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) objs_to_update = [self._objectives[obj_id][0] for obj_id in objs_to_update.keys()] return vars_to_update, cons_to_update, objs_to_update From 2adefbc724afbbe63a19f1a03966a55579930e0c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 17:09:12 -0600 Subject: [PATCH 20/50] docstring for the model change detector --- pyomo/contrib/observer/model_observer.py | 121 +++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index e00331b47ed..647c8aef5d8 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -229,10 +229,131 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: + """ + This class "watches" a pyomo model and notifies the observers when any + changes to the model are made (but only when ModelChangeDetector.update + is called). An example use case is for the persistent solver interfaces. + + The ModelChangeDetector considers the model to be defined by its set of + active components and any components used by those active components. For + example, the observers will not be notified of the addition of a variable + if that variable is not used in any constraints. + + The Observer/ModelChangeDetector are most useful when a small number + of changes are being relative to the size of the model. For example, + the persistent solver interfaces can be very efficient when repeatedly + solving the same model but with different values for mutable parameters. + + If you know that certain changes will not be made to the model, the + config can be modified to improve performance. For example, if you + know that no constraints will be added to or removed from the model, + then ``check_for_new_or_removed_constraints`` can be set to ``False``, + which will save some time when ``update`` is called. + + Here are some examples: + + >>> import pyomo.environ as pyo + >>> from pyomo.contrib.observer.model_observer import ( + ... AutoUpdateConfig, + ... Observer, + ... ModelChangeDetector, + ... ) + >>> class PrintObserver(Observer): + ... def add_variables(self, variables): + ... for i in variables: + ... print(f'{i} was added to the model') + ... def add_parameters(self, params): + ... for i in params: + ... print(f'{i} was added to the model') + ... def add_constraints(self, cons): + ... for i in cons: + ... print(f'{i} was added to the model') + ... def add_sos_constraints(self, cons): + ... for i in cons: + ... print(f'{i} was added to the model') + ... def add_objectives(self, objs): + ... for i in objs: + ... print(f'{i} was added to the model') + ... def remove_objectives(self, objs): + ... for i in objs: + ... print(f'{i} was removed from the model') + ... def remove_constraints(self, cons): + ... for i in cons: + ... print(f'{i} was removed from the model') + ... def remove_sos_constraints(self, cons): + ... for i in cons: + ... print(f'{i} was removed from the model') + ... def remove_variables(self, variables): + ... for i in variables: + ... print(f'{i} was removed from the model') + ... def remove_parameters(self, params): + ... for i in params: + ... print(f'{i} was removed from the model') + ... def update_variables(self, variables): + ... for i in variables: + ... print(f'{i} was modified') + ... def update_parameters(self, params): + ... for i in params: + ... print(f'{i} was modified') + >>> m = pyo.ConcreteModel() + >>> obs = PrintObserver() + >>> detector = ModelChangeDetector(m, [obs]) + >>> m.x = pyo.Var(bounds=()) + >>> m.y = pyo.Var() + >>> detector.update() # no output because the variables are not used + >>> m.obj = pyo.Objective(expr=m.x**2 + m.y**2) + >>> detector.update() + x was added to the model + y was added to the model + obj was added to the model + >>> del m.obj + >>> detector.update() + obj was removed from the model + x was removed from the model + y was removed from the model + >>> m.px = pyo.Param(mutable=True, initialize=1) + >>> m.py = pyo.Param(mutable=True, initialize=1) + >>> m.obj = pyo.Objective(expr=m.px*m.x + m.py*m.y) + >>> detector.update() + x was added to the model + y was added to the model + px was added to the model + py was added to the model + obj was added to the model + >>> detector.config.check_for_new_or_removed_constraints = False + >>> detector.config.check_for_new_or_removed_objectives = False + >>> detector.config.update_constraints = False + >>> detector.config.update_vars = False + >>> detector.config.update_parameters = True + >>> detector.config.update_named_expressions = False + >>> detector.config.update_objectives = False + >>> for i in range(10): + ... m.py.value = i + ... detector.update() # this will be faster because it is only checking for changes to parameters + py was modified + py was modified + py was modified + py was modified + py was modified + py was modified + py was modified + py was modified + py was modified + py was modified + >>> m.c = pyo.Constraint(expr=m.y >= pyo.exp(m.x)) + >>> detector.update() # no output because we did not check for new constraints + >>> detector.config.check_for_new_or_removed_constraints = True + >>> detector.update() + c was added to the model + + """ + def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): """ Parameters ---------- + model: BlockData + The model for which changes should be detected observers: Sequence[Observer] The objects to notify when changes are made to the model """ From df26c4e96f6bcf2e1c23aa5fd691fd9a86135c57 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 17:10:05 -0600 Subject: [PATCH 21/50] run black --- pyomo/contrib/observer/model_observer.py | 73 ++++++++++--------- .../observer/tests/test_change_detector.py | 4 +- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 647c8aef5d8..b1442a43ef7 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -230,31 +230,31 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: """ - This class "watches" a pyomo model and notifies the observers when any + This class "watches" a pyomo model and notifies the observers when any changes to the model are made (but only when ModelChangeDetector.update is called). An example use case is for the persistent solver interfaces. - - The ModelChangeDetector considers the model to be defined by its set of + + The ModelChangeDetector considers the model to be defined by its set of active components and any components used by those active components. For example, the observers will not be notified of the addition of a variable - if that variable is not used in any constraints. + if that variable is not used in any constraints. - The Observer/ModelChangeDetector are most useful when a small number - of changes are being relative to the size of the model. For example, + The Observer/ModelChangeDetector are most useful when a small number + of changes are being relative to the size of the model. For example, the persistent solver interfaces can be very efficient when repeatedly solving the same model but with different values for mutable parameters. - If you know that certain changes will not be made to the model, the - config can be modified to improve performance. For example, if you + If you know that certain changes will not be made to the model, the + config can be modified to improve performance. For example, if you know that no constraints will be added to or removed from the model, then ``check_for_new_or_removed_constraints`` can be set to ``False``, which will save some time when ``update`` is called. - + Here are some examples: >>> import pyomo.environ as pyo >>> from pyomo.contrib.observer.model_observer import ( - ... AutoUpdateConfig, + ... AutoUpdateConfig, ... Observer, ... ModelChangeDetector, ... ) @@ -371,18 +371,18 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): self._external_functions = ComponentMap() - # the dictionaries below are really just ordered sets, but we need to + # the dictionaries below are really just ordered sets, but we need to # stick with built-in types for performance # var_id: ( # dict[constraints, None], - # dict[sos constraints, None], + # dict[sos constraints, None], # dict[objectives, None], # ) self._referenced_variables = {} # param_id: ( - # dict[constraints, None], + # dict[constraints, None], # dict[sos constraints, None], # dict[objectives, None], # ) @@ -465,12 +465,9 @@ def _add_constraints(self, cons: List[ConstraintData]): if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') self._active_constraints[con] = con.expr - ( - named_exprs, - variables, - parameters, - external_functions, - ) = collect_components_from_expr(con.expr) + (named_exprs, variables, parameters, external_functions) = ( + collect_components_from_expr(con.expr) + ) vars_to_check.extend(variables) params_to_check.extend(parameters) if named_exprs: @@ -532,12 +529,9 @@ def _add_objectives(self, objs: List[ObjectiveData]): for obj in objs: obj_id = id(obj) self._objectives[obj_id] = (obj, obj.expr, obj.sense) - ( - named_exprs, - variables, - parameters, - external_functions, - ) = collect_components_from_expr(obj.expr) + (named_exprs, variables, parameters, external_functions) = ( + collect_components_from_expr(obj.expr) + ) vars_to_check.extend(variables) params_to_check.extend(parameters) if named_exprs: @@ -592,9 +586,7 @@ def _check_for_unknown_active_components(self): if ctype in self._known_active_ctypes: continue for comp in self._model.component_data_objects( - ctype, - active=True, - descend_into=True + ctype, active=True, descend_into=True ): raise NotImplementedError( f'ModelChangeDetector does not know how to ' @@ -611,20 +603,22 @@ def _set_instance(self): self._add_constraints( list( - self._model.component_data_objects(Constraint, descend_into=True, active=True) + self._model.component_data_objects( + Constraint, descend_into=True, active=True + ) ) ) self._add_sos_constraints( list( self._model.component_data_objects( - SOSConstraint, descend_into=True, active=True, + SOSConstraint, descend_into=True, active=True ) ) ) self._add_objectives( list( self._model.component_data_objects( - Objective, descend_into=True, active=True, + Objective, descend_into=True, active=True ) ) ) @@ -817,7 +811,9 @@ def _check_for_var_changes(self): elif _domain_interval != v.domain.get_interval(): vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) - objs_to_update = [self._objectives[obj_id][0] for obj_id in objs_to_update.keys()] + objs_to_update = [ + self._objectives[obj_id][0] for obj_id in objs_to_update.keys() + ] return vars_to_update, cons_to_update, objs_to_update def _check_for_param_changes(self): @@ -846,7 +842,8 @@ def _check_for_new_or_removed_objectives(self): new_objs = [] old_objs = [] current_objs_dict = { - id(obj): obj for obj in self._model.component_data_objects( + id(obj): obj + for obj in self._model.component_data_objects( Objective, descend_into=True, active=True ) } @@ -917,7 +914,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.check_for_new_or_removed_objectives: timer.start('objective') new_objs, old_objs = self._check_for_new_or_removed_objectives() - # many solvers require one objective, so we have to remove the + # many solvers require one objective, so we have to remove the # old objective first if old_objs: self._remove_objectives(old_objs) @@ -937,7 +934,9 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.update_vars: timer.start('vars') - vars_to_update, cons_to_update, objs_to_update = self._check_for_var_changes() + vars_to_update, cons_to_update, objs_to_update = ( + self._check_for_var_changes() + ) if vars_to_update: self._update_variables(vars_to_update) cons_to_update = [i for i in cons_to_update if i not in added_cons] @@ -954,7 +953,9 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.update_named_expressions: timer.start('named expressions') - cons_to_update, objs_to_update = self._check_for_named_expression_changes() + cons_to_update, objs_to_update = ( + self._check_for_named_expression_changes() + ) cons_to_update = [i for i in cons_to_update if i not in added_cons] objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] if cons_to_update: diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index d6b5fcbcc62..19609182daf 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -158,7 +158,7 @@ def test_objective(self): expected[m.x]['update'] += 1 # the variable gets updated # the objective must get removed and added - # that causes x,y, and p to all get removed + # that causes x,y, and p to all get removed # and added expected[m.obj]['remove'] += 1 expected[m.obj]['add'] += 1 @@ -400,7 +400,7 @@ def test_update_config(self): detector.config.check_for_new_or_removed_constraints = False detector.config.update_constraints = False - m.c2 = pyo.Constraint(expr=m.y >= (m.x - 1)**2) + m.c2 = pyo.Constraint(expr=m.y >= (m.x - 1) ** 2) detector.update() obs.check(expected) From 2dc566ec3d3f93150194642282470c75b57ac624 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 17:34:26 -0600 Subject: [PATCH 22/50] some comments for the observer --- pyomo/contrib/observer/model_observer.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b1442a43ef7..0fd5a489b43 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -863,6 +863,30 @@ def _check_for_modified_objectives(self): return objs_to_update def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): + """ + Check for changes to the model and notify the observers. + + Parameters + ---------- + timer: Optional[HierarchicalTimer] + The timer to use for tracking how much time is spent detecting + different kinds of changes + """ + + """ + When possible, it is better to add new constraints before removing old + constraints. This prevents unnecessarily removing and adding variables. + If a constraint is removed, any variables that are used only by that + constraint will be removed. If there is a new constraint that uses + the same variable, then we don't actually need to remove the variable. + This is hard to avoid when we are modifying a constraint or changing + the objective. When the objective changes, we remove the old one + first just because most things don't handle multiple objectives. + + We check for changes to constraints/objectives before variables/parameters + so that we don't waste time updating a variable/parameter that is going to + get removed. + """ if timer is None: timer = HierarchicalTimer() config: AutoUpdateConfig = self.config(value=kwds, preserve_implicit=True) From 6b2ffbd84153a069880c39b964ed00e4a4e0f8b0 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 17:51:22 -0600 Subject: [PATCH 23/50] only need to descend into named expressions once --- pyomo/contrib/observer/component_collector.py | 5 ++++ .../tests/test_component_collector.py | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 pyomo/contrib/observer/tests/test_component_collector.py diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 638f85327b4..c79c1182869 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -93,6 +93,11 @@ def __init__(self, **kwds): def exitNode(self, node, data): return collector_handlers[node.__class__](node, self) + + def beforeChild(self, node, child, child_idx): + if id(child) in self.named_expressions: + return False, None + return True, None _visitor = _ComponentFromExprCollector() diff --git a/pyomo/contrib/observer/tests/test_component_collector.py b/pyomo/contrib/observer/tests/test_component_collector.py new file mode 100644 index 00000000000..99d00074418 --- /dev/null +++ b/pyomo/contrib/observer/tests/test_component_collector.py @@ -0,0 +1,25 @@ +import pyomo.environ as pyo +from pyomo.common import unittest +from pyomo.contrib.observer.component_collector import collect_components_from_expr +from pyomo.common.collections import ComponentSet + + +class TestComponentCollector(unittest.TestCase): + def test_nested_named_expressions(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.z = pyo.Var() + m.e1 = pyo.Expression(expr=m.x + m.y) + m.e2 = pyo.Expression(expr=m.e1 + m.z) + e = m.e2 * pyo.exp(m.e2) + ( + named_exprs, + vars, + params, + external_funcs, + ) = collect_components_from_expr(e) + self.assertEqual(len(named_exprs), 2) + named_exprs = ComponentSet(named_exprs) + self.assertIn(m.e1, named_exprs) + self.assertIn(m.e2, named_exprs) From e68ee73049a9fa538b37c30b433333b2459478f1 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 17:51:43 -0600 Subject: [PATCH 24/50] only need to descend into named expressions once --- pyomo/contrib/observer/component_collector.py | 2 +- pyomo/contrib/observer/tests/test_component_collector.py | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index c79c1182869..d30bb128758 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -93,7 +93,7 @@ def __init__(self, **kwds): def exitNode(self, node, data): return collector_handlers[node.__class__](node, self) - + def beforeChild(self, node, child, child_idx): if id(child) in self.named_expressions: return False, None diff --git a/pyomo/contrib/observer/tests/test_component_collector.py b/pyomo/contrib/observer/tests/test_component_collector.py index 99d00074418..ca272b06cfa 100644 --- a/pyomo/contrib/observer/tests/test_component_collector.py +++ b/pyomo/contrib/observer/tests/test_component_collector.py @@ -13,12 +13,7 @@ def test_nested_named_expressions(self): m.e1 = pyo.Expression(expr=m.x + m.y) m.e2 = pyo.Expression(expr=m.e1 + m.z) e = m.e2 * pyo.exp(m.e2) - ( - named_exprs, - vars, - params, - external_funcs, - ) = collect_components_from_expr(e) + (named_exprs, vars, params, external_funcs) = collect_components_from_expr(e) self.assertEqual(len(named_exprs), 2) named_exprs = ComponentSet(named_exprs) self.assertIn(m.e1, named_exprs) From 82e40b213a43ce2f2c64adbdece6e8d796af7bdd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 19:47:56 -0600 Subject: [PATCH 25/50] update observer tests --- .../observer/tests/test_change_detector.py | 89 ++++++++++++++++--- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 19609182daf..c12398746e0 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -280,6 +280,18 @@ def test_sos(self): expected[m.c1]['add'] += 1 obs.check(expected) + detector.update() + obs.check(expected) + + m.c1.set_items([m.x[2], m.x[1], m.x[3]], [1, 2, 3]) + detector.update() + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + for i in m.a: + expected[m.x[i]]['remove'] += 1 + expected[m.x[i]]['add'] += 1 + obs.check(expected) + for i in m.a: expected[m.x[i]]['remove'] += 1 expected[m.c1]['remove'] += 1 @@ -381,35 +393,90 @@ def test_update_config(self): m = pyo.ConcreteModel() m.x = pyo.Var() m.y = pyo.Var() - m.obj = pyo.Objective(expr=m.x**2 + m.y**2) - m.c1 = pyo.Constraint(expr=m.y >= pyo.exp(m.x)) + m.p = pyo.Param(initialize=1, mutable=True) obs = ObserverChecker() detector = ModelChangeDetector(m, [obs]) - expected = ComponentMap() + obs.check(expected) + + detector.config.check_for_new_or_removed_constraints = False + detector.config.check_for_new_or_removed_objectives = False + detector.config.update_constraints = False + detector.config.update_objectives = False + detector.config.update_vars = False + detector.config.update_parameters = False + detector.config.update_named_expressions = False + + m.e = pyo.Expression(expr=pyo.exp(m.x)) + m.obj = pyo.Objective(expr=m.x**2 + m.p*m.y**2) + m.c1 = pyo.Constraint(expr=m.y >= m.e + m.p) + + detector.update() + obs.check(expected) + + detector.config.check_for_new_or_removed_constraints = True + detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() - expected[m.obj] = make_count_dict() + expected[m.p] = make_count_dict() expected[m.c1] = make_count_dict() expected[m.x]['add'] += 1 expected[m.y]['add'] += 1 - expected[m.obj]['add'] += 1 + expected[m.p]['add'] += 1 expected[m.c1]['add'] += 1 obs.check(expected) - detector.config.check_for_new_or_removed_constraints = False - detector.config.update_constraints = False - m.c2 = pyo.Constraint(expr=m.y >= (m.x - 1) ** 2) + detector.config.check_for_new_or_removed_objectives = True detector.update() + expected[m.obj] = make_count_dict() + expected[m.obj]['add'] += 1 obs.check(expected) m.x.setlb(0) detector.update() + obs.check(expected) + + detector.config.update_vars = True + detector.update() expected[m.x]['update'] += 1 obs.check(expected) - detector.config.check_for_new_or_removed_constraints = True + m.p.value = 2 + detector.update() + obs.check(expected) + + detector.config.update_parameters = True + detector.update() + expected[m.p]['update'] += 1 + obs.check(expected) + + m.e.expr += 1 + detector.update() + obs.check(expected) + + detector.config.update_named_expressions = True + detector.update() + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + obs.check(expected) + + m.obj.expr += 1 + detector.update() + obs.check(expected) + + detector.config.update_objectives = True + detector.update() + expected[m.obj]['remove'] += 1 + expected[m.obj]['add'] += 1 + obs.check(expected) + + m.c1 = m.y >= m.e + detector.update() + obs.check(expected) + + detector.config.update_constraints = True detector.update() - expected[m.c2] = make_count_dict() - expected[m.c2]['add'] += 1 + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + obs.check(expected) From ae7e0311302b10974286d021dc4226d54b1da1e0 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 19:49:07 -0600 Subject: [PATCH 26/50] run black --- pyomo/contrib/observer/tests/test_change_detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index c12398746e0..5aea43bdd58 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -409,7 +409,7 @@ def test_update_config(self): detector.config.update_named_expressions = False m.e = pyo.Expression(expr=pyo.exp(m.x)) - m.obj = pyo.Objective(expr=m.x**2 + m.p*m.y**2) + m.obj = pyo.Objective(expr=m.x**2 + m.p * m.y**2) m.c1 = pyo.Constraint(expr=m.y >= m.e + m.p) detector.update() From 5778688d5d209bbce0062d74727e2fc060c47aa3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 17 Sep 2025 06:20:09 -0600 Subject: [PATCH 27/50] fix docs --- pyomo/contrib/observer/model_observer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 0fd5a489b43..dddc96010c7 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -298,7 +298,7 @@ class ModelChangeDetector: >>> m = pyo.ConcreteModel() >>> obs = PrintObserver() >>> detector = ModelChangeDetector(m, [obs]) - >>> m.x = pyo.Var(bounds=()) + >>> m.x = pyo.Var() >>> m.y = pyo.Var() >>> detector.update() # no output because the variables are not used >>> m.obj = pyo.Objective(expr=m.x**2 + m.y**2) From cd1c4eff1cb8e79c7c7e9aead6f53d0fcf321772 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 18 Sep 2025 07:43:21 -0600 Subject: [PATCH 28/50] observer updates --- pyomo/contrib/observer/model_observer.py | 144 +++++++++++++++++- .../tests/test_component_collector.py | 11 ++ 2 files changed, 147 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index dddc96010c7..32822e34104 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -73,7 +73,7 @@ def __init__( visibility=visibility, ) - # automatically detect new/removed constraints on subsequent solves + #: automatically detect new/removed constraints on subsequent solves self.check_for_new_or_removed_constraints: bool = self.declare( 'check_for_new_or_removed_constraints', ConfigValue( @@ -87,7 +87,7 @@ def __init__( model.""", ), ) - # automatically detect new/removed objectives on subsequent solves + #: automatically detect new/removed objectives on subsequent solves self.check_for_new_or_removed_objectives: bool = self.declare( 'check_for_new_or_removed_objectives', ConfigValue( @@ -101,7 +101,7 @@ def __init__( model.""", ), ) - # automatically detect changes to constraints on subsequent solves + #: automatically detect changes to constraints on subsequent solves self.update_constraints: bool = self.declare( 'update_constraints', ConfigValue( @@ -116,7 +116,7 @@ def __init__( being modified.""", ), ) - # automatically detect changes to variables on subsequent solves + #: automatically detect changes to variables on subsequent solves self.update_vars: bool = self.declare( 'update_vars', ConfigValue( @@ -132,7 +132,7 @@ def __init__( update_parameters_and_fixed_vars.""", ), ) - # automatically detect changes to parameters on subsequent solves + #: automatically detect changes to parameters on subsequent solves self.update_parameters: bool = self.declare( 'update_parameters', ConfigValue( @@ -146,7 +146,7 @@ def __init__( parameters are not being modified.""", ), ) - # automatically detect changes to named expressions on subsequent solves + #: automatically detect changes to named expressions on subsequent solves self.update_named_expressions: bool = self.declare( 'update_named_expressions', ConfigValue( @@ -159,7 +159,7 @@ def __init__( are certain Expressions are not being modified.""", ), ) - # automatically detect changes to objectives on subsequent solves + #: automatically detect changes to objectives on subsequent solves self.update_objectives: bool = self.declare( 'update_objectives', ConfigValue( @@ -181,50 +181,178 @@ def __init__(self): @abc.abstractmethod def add_variables(self, variables: List[VarData]): + """ + This method gets called by the ModelChangeDetector when new + "active" variables are detected in the model. This means variables + that are used within an active component such as a constraint or + an objective. + + Parameters + ---------- + variables: List[VarData] + The list of variables added to the model + """ pass @abc.abstractmethod def add_parameters(self, params: List[ParamData]): + """ + This method gets called by the ModelChangeDetector when new + "active" parameters are detected in the model. This means parameters + that are used within an active component such as a constraint or + an objective. + + Parameters + ---------- + params: List[ParamData] + The list of parameters added to the model + """ pass @abc.abstractmethod def add_constraints(self, cons: List[ConstraintData]): + """ + This method gets called by the ModelChangeDetector when new + active constraints are detected in the model. + + Parameters + ---------- + cons: List[ConstraintData] + The list of constraints added to the model + """ pass @abc.abstractmethod def add_sos_constraints(self, cons: List[SOSConstraintData]): + """ + This method gets called by the ModelChangeDetector when new + active SOS constraints are detected in the model. + + Parameters + ---------- + cons: List[SOSConstraintData] + The list of SOS constraints added to the model + """ pass @abc.abstractmethod def add_objectives(self, objs: List[ObjectiveData]): + """ + This method gets called by the ModelChangeDetector when new + active objectives are detected in the model. + + Parameters + ---------- + objs: List[ObjectiveData] + The list of objectives added to the model + """ pass @abc.abstractmethod def remove_objectives(self, objs: List[ObjectiveData]): + """ + This method gets called by the ModelChangeDetector when it detects + objectives that have been deactivated or removed from the model. + If the ModelChangeDetector detects changes in the underlying + expression for the objective, then ``remove_objectives`` will be + called followed by ``add_objectives``. + + Parameters + ---------- + objs: List[ObjectiveData] + The list of objectives that are no longer part of the model + """ pass @abc.abstractmethod def remove_constraints(self, cons: List[ConstraintData]): + """ + This method gets called by the ModelChangeDetector when it detects + constraints that have been deactivated or removed from the model. + If the ModelChangeDetector detects changes in the underlying + expression for the constraint, then ``remove_constraints`` will be + called followed by ``add_constraints``. + + Parameters + ---------- + cons: List[ConstraintData] + The list of constraints that are no longer part of the model + """ pass @abc.abstractmethod def remove_sos_constraints(self, cons: List[SOSConstraintData]): + """ + This method gets called by the ModelChangeDetector when it detects + SOS constraints that have been deactivated or removed from the model. + If the ModelChangeDetector detects changes in the underlying + data for the constraint, then ``remove_sos_constraints`` will be + called followed by ``add_sos_constraints``. + + Parameters + ---------- + cons: List[SOSConstraintData] + The list of SOS constraints that are no longer part of the model + """ pass @abc.abstractmethod def remove_variables(self, variables: List[VarData]): + """ + This method gets called by the ModelChangeDetector when it detects + variables that are no longer used in any active components ( + objectives or constraints). + + Parameters + ---------- + variables: List[VarData] + The list of variables that are no longer part of the model + """ pass @abc.abstractmethod def remove_parameters(self, params: List[ParamData]): + """ + This method gets called by the ModelChangeDetector when it detects + parameters that are no longer used in any active components ( + objectives or constraints). + + Parameters + ---------- + params: List[ParamData] + The list of parameters that are no longer part of the model + """ pass @abc.abstractmethod def update_variables(self, variables: List[VarData]): + """ + This method gets called by the ModelChangeDetector when it detects + variables that have been modified in some way (e.g., the bounds + change). This is only true for changes that are considered + "inputs" to the model. For example, the value of the variable is + considered an "output" (unless the variable is fixed), so changing + the value of an unfixed variable will not cause this method to be + called. + + Parameters + ---------- + variables: List[VarData] + The list of variables that have been modified + """ pass @abc.abstractmethod def update_parameters(self, params: List[ParamData]): + """ + This method gets called by the ModelChangeDetector when it detects + parameters that have been modified (i.e., the value changed). + + Parameters + ---------- + params: List[ParamData] + The list of parameters that have been modified + """ pass @@ -240,7 +368,7 @@ class ModelChangeDetector: if that variable is not used in any constraints. The Observer/ModelChangeDetector are most useful when a small number - of changes are being relative to the size of the model. For example, + of changes are being made relative to the size of the model. For example, the persistent solver interfaces can be very efficient when repeatedly solving the same model but with different values for mutable parameters. diff --git a/pyomo/contrib/observer/tests/test_component_collector.py b/pyomo/contrib/observer/tests/test_component_collector.py index ca272b06cfa..70d01a08ccf 100644 --- a/pyomo/contrib/observer/tests/test_component_collector.py +++ b/pyomo/contrib/observer/tests/test_component_collector.py @@ -1,3 +1,14 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# __________________________________________________________________________ + import pyomo.environ as pyo from pyomo.common import unittest from pyomo.contrib.observer.component_collector import collect_components_from_expr From 3f308935dc9e9dea431d9f287e96a753e8fb436e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 18 Sep 2025 07:43:47 -0600 Subject: [PATCH 29/50] observer updates --- pyomo/contrib/observer/model_observer.py | 42 ++++++++++++------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 32822e34104..c5135504c48 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -182,10 +182,10 @@ def __init__(self): @abc.abstractmethod def add_variables(self, variables: List[VarData]): """ - This method gets called by the ModelChangeDetector when new + This method gets called by the ModelChangeDetector when new "active" variables are detected in the model. This means variables - that are used within an active component such as a constraint or - an objective. + that are used within an active component such as a constraint or + an objective. Parameters ---------- @@ -197,10 +197,10 @@ def add_variables(self, variables: List[VarData]): @abc.abstractmethod def add_parameters(self, params: List[ParamData]): """ - This method gets called by the ModelChangeDetector when new + This method gets called by the ModelChangeDetector when new "active" parameters are detected in the model. This means parameters - that are used within an active component such as a constraint or - an objective. + that are used within an active component such as a constraint or + an objective. Parameters ---------- @@ -212,8 +212,8 @@ def add_parameters(self, params: List[ParamData]): @abc.abstractmethod def add_constraints(self, cons: List[ConstraintData]): """ - This method gets called by the ModelChangeDetector when new - active constraints are detected in the model. + This method gets called by the ModelChangeDetector when new + active constraints are detected in the model. Parameters ---------- @@ -225,8 +225,8 @@ def add_constraints(self, cons: List[ConstraintData]): @abc.abstractmethod def add_sos_constraints(self, cons: List[SOSConstraintData]): """ - This method gets called by the ModelChangeDetector when new - active SOS constraints are detected in the model. + This method gets called by the ModelChangeDetector when new + active SOS constraints are detected in the model. Parameters ---------- @@ -238,8 +238,8 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): @abc.abstractmethod def add_objectives(self, objs: List[ObjectiveData]): """ - This method gets called by the ModelChangeDetector when new - active objectives are detected in the model. + This method gets called by the ModelChangeDetector when new + active objectives are detected in the model. Parameters ---------- @@ -253,8 +253,8 @@ def remove_objectives(self, objs: List[ObjectiveData]): """ This method gets called by the ModelChangeDetector when it detects objectives that have been deactivated or removed from the model. - If the ModelChangeDetector detects changes in the underlying - expression for the objective, then ``remove_objectives`` will be + If the ModelChangeDetector detects changes in the underlying + expression for the objective, then ``remove_objectives`` will be called followed by ``add_objectives``. Parameters @@ -269,8 +269,8 @@ def remove_constraints(self, cons: List[ConstraintData]): """ This method gets called by the ModelChangeDetector when it detects constraints that have been deactivated or removed from the model. - If the ModelChangeDetector detects changes in the underlying - expression for the constraint, then ``remove_constraints`` will be + If the ModelChangeDetector detects changes in the underlying + expression for the constraint, then ``remove_constraints`` will be called followed by ``add_constraints``. Parameters @@ -285,8 +285,8 @@ def remove_sos_constraints(self, cons: List[SOSConstraintData]): """ This method gets called by the ModelChangeDetector when it detects SOS constraints that have been deactivated or removed from the model. - If the ModelChangeDetector detects changes in the underlying - data for the constraint, then ``remove_sos_constraints`` will be + If the ModelChangeDetector detects changes in the underlying + data for the constraint, then ``remove_sos_constraints`` will be called followed by ``add_sos_constraints``. Parameters @@ -328,11 +328,11 @@ def remove_parameters(self, params: List[ParamData]): def update_variables(self, variables: List[VarData]): """ This method gets called by the ModelChangeDetector when it detects - variables that have been modified in some way (e.g., the bounds - change). This is only true for changes that are considered + variables that have been modified in some way (e.g., the bounds + change). This is only true for changes that are considered "inputs" to the model. For example, the value of the variable is considered an "output" (unless the variable is fixed), so changing - the value of an unfixed variable will not cause this method to be + the value of an unfixed variable will not cause this method to be called. Parameters From dc19b170db098ee7d515847149aacdda88a9ee0d Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 2 Oct 2025 03:55:23 -0600 Subject: [PATCH 30/50] observer: docstring updates --- pyomo/contrib/observer/model_observer.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index c5135504c48..6ee5302ddb8 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -378,7 +378,17 @@ class ModelChangeDetector: then ``check_for_new_or_removed_constraints`` can be set to ``False``, which will save some time when ``update`` is called. - Here are some examples: + We have discussed expanding the interface of the ``ModelChangeDetector`` + with methods to request extra information. For example, if the value + of a fixed variable changes, an observer may want to know all of the + constraints that use the variables. This class alredy has that + information, so the observer should not have to waste time recomputing + that. We have not yet added methods like this because we do not have + an immediate use case or need, and it's not yet clear waht those + methods should look like. If a need arises, please create an issue or + pull request. + + Here are some usage examples: >>> import pyomo.environ as pyo >>> from pyomo.contrib.observer.model_observer import ( @@ -718,7 +728,7 @@ def _check_for_unknown_active_components(self): ): raise NotImplementedError( f'ModelChangeDetector does not know how to ' - 'handle compents with ctype {ctype}' + 'handle components with ctype {ctype}' ) def _set_instance(self): From bb609590b75a63e8befaa1da91dca8f5b2d9df21 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 2 Oct 2025 04:05:11 -0600 Subject: [PATCH 31/50] observer: typos --- pyomo/contrib/observer/model_observer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 6ee5302ddb8..d72b3e06ca3 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -381,10 +381,10 @@ class ModelChangeDetector: We have discussed expanding the interface of the ``ModelChangeDetector`` with methods to request extra information. For example, if the value of a fixed variable changes, an observer may want to know all of the - constraints that use the variables. This class alredy has that + constraints that use the variables. This class already has that information, so the observer should not have to waste time recomputing that. We have not yet added methods like this because we do not have - an immediate use case or need, and it's not yet clear waht those + an immediate use case or need, and it's not yet clear what those methods should look like. If a need arises, please create an issue or pull request. From 99ac089ed3e99154e2de04d18484f68dca8d7c0a Mon Sep 17 00:00:00 2001 From: Michael Bynum <20401710+michaelbynum@users.noreply.github.com> Date: Mon, 6 Oct 2025 14:44:44 -0600 Subject: [PATCH 32/50] Update pyomo/contrib/observer/model_observer.py Co-authored-by: John Siirola --- pyomo/contrib/observer/model_observer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index d72b3e06ca3..ff22afb55e1 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -718,7 +718,7 @@ def _remove_objectives(self, objs: List[ObjectiveData]): self._check_to_remove_params(params_to_check) def _check_for_unknown_active_components(self): - for ctype in self._model.collect_ctypes(): + for ctype in self._model.collect_ctypes(active=True, descend_into=True): if not issubclass(ctype, ActiveComponent): continue if ctype in self._known_active_ctypes: From 213d353406bee59de2cd495d3155476b0f384dab Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 14:50:45 -0600 Subject: [PATCH 33/50] update check for unknown ctypes --- pyomo/contrib/observer/model_observer.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index d72b3e06ca3..c4b754e1dc9 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -723,13 +723,10 @@ def _check_for_unknown_active_components(self): continue if ctype in self._known_active_ctypes: continue - for comp in self._model.component_data_objects( - ctype, active=True, descend_into=True - ): - raise NotImplementedError( - f'ModelChangeDetector does not know how to ' - 'handle components with ctype {ctype}' - ) + raise NotImplementedError( + f'ModelChangeDetector does not know how to ' + 'handle components with ctype {ctype}' + ) def _set_instance(self): From ddf0a39fd0f965763a6ec17c0f33fc84691759ce Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 14:54:11 -0600 Subject: [PATCH 34/50] observer: use PauseGC() --- pyomo/contrib/observer/model_observer.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 380292852bc..db9e69d7977 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -21,11 +21,11 @@ from pyomo.core.base.block import BlockData from pyomo.core.base.component import ActiveComponent from pyomo.common.collections import ComponentMap +from pyomo.common.gc_manager import PauseGC from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.observer.component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types -import gc """ @@ -730,10 +730,7 @@ def _check_for_unknown_active_components(self): def _set_instance(self): - is_gc_enabled = gc.isenabled() - gc.disable() - - try: + with PauseGC() as pgc: self._check_for_unknown_active_components() self._add_constraints( @@ -757,9 +754,6 @@ def _set_instance(self): ) ) ) - finally: - if is_gc_enabled: - gc.enable() def _remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: @@ -1026,10 +1020,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): timer = HierarchicalTimer() config: AutoUpdateConfig = self.config(value=kwds, preserve_implicit=True) - is_gc_enabled = gc.isenabled() - gc.disable() - - try: + with PauseGC() as pgc: self._check_for_unknown_active_components() added_cons = set() @@ -1133,6 +1124,3 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if params_to_update: self._update_parameters(params_to_update) timer.stop('params') - finally: - if is_gc_enabled: - gc.enable() From f569d430be48f4baa35906ebcd1732d2f13aef39 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 8 Oct 2025 22:08:42 -0600 Subject: [PATCH 35/50] observer: give reason for variable changes --- pyomo/contrib/observer/model_observer.py | 185 ++++++++---------- .../observer/tests/test_change_detector.py | 47 +---- 2 files changed, 85 insertions(+), 147 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index db9e69d7977..40754751599 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -26,6 +26,7 @@ from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.observer.component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types +import enum """ @@ -127,9 +128,7 @@ def __init__( detected on subsequent solves. This includes changes to the lb, ub, domain, and fixed attributes of variables. Use False only when manually updating the observer with opt.update_variables() or when - you are certain variables are not being modified. Note that changes - to values of fixed variables is handled by - update_parameters_and_fixed_vars.""", + you are certain variables are not being modified.""", ), ) #: automatically detect changes to parameters on subsequent solves @@ -175,6 +174,14 @@ def __init__( ) +class Reason(enum.Flag): + bounds = enum.auto() + fixed = enum.auto() + unfixed = enum.auto() + domain = enum.auto() + value = enum.auto() + + class Observer(abc.ABC): def __init__(self): pass @@ -325,7 +332,7 @@ def remove_parameters(self, params: List[ParamData]): pass @abc.abstractmethod - def update_variables(self, variables: List[VarData]): + def update_variables(self, variables: List[VarData], reasons: List[Reason]): """ This method gets called by the ModelChangeDetector when it detects variables that have been modified in some way (e.g., the bounds @@ -333,12 +340,17 @@ def update_variables(self, variables: List[VarData]): "inputs" to the model. For example, the value of the variable is considered an "output" (unless the variable is fixed), so changing the value of an unfixed variable will not cause this method to be - called. + called. Parameters ---------- variables: List[VarData] The list of variables that have been modified + reasons: List[Reason] + A list of reasons the variables changed. This list will be the + same length as `variables`. The Reason enum is a Flag enum so + that if multiple changes are made, this can be easily indicated + with the bitwise | operator. """ pass @@ -427,7 +439,7 @@ class ModelChangeDetector: ... def remove_parameters(self, params): ... for i in params: ... print(f'{i} was removed from the model') - ... def update_variables(self, variables): + ... def update_variables(self, variables, reasons): ... for i in variables: ... print(f'{i} was modified') ... def update_parameters(self, params): @@ -835,7 +847,7 @@ def _remove_parameters(self, params: List[ParamData]): del self._referenced_params[p_id] del self._params[p_id] - def _update_variables(self, variables: List[VarData]): + def _update_variables(self, variables: List[VarData], reasons: List[Reason]): for v in variables: self._vars[id(v)] = ( v, @@ -846,7 +858,7 @@ def _update_variables(self, variables: List[VarData]): v.value, ) for obs in self._observers: - obs.update_variables(variables) + obs.update_variables(variables, reasons) def _update_parameters(self, params): for p in params: @@ -922,28 +934,23 @@ def _check_for_modified_constraints(self): def _check_for_var_changes(self): vars_to_update = [] - cons_to_update = {} - objs_to_update = {} + reasons = [] for vid, (v, _lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): - if v.fixed != _fixed: - vars_to_update.append(v) - for c in self._referenced_variables[vid][0]: - cons_to_update[c] = None - for obj_id in self._referenced_variables[vid][2]: - objs_to_update[obj_id] = None - elif _fixed and v.value != _value: - vars_to_update.append(v) - elif v._lb is not _lb: + reason = Reason(0) + if _fixed and not v.fixed: + reason = reason | Reason.unfixed + elif not _fixed and v.fixed: + reason = reason | Reason.fixed + elif _fixed and (v.value != _value): + reason = reason | Reason.value + if v._lb is not _lb or v._ub is not _ub: + reason = reason | Reason.bounds + if _domain_interval != v.domain.get_interval(): + reason = reason | Reason.domain + if reason: vars_to_update.append(v) - elif v._ub is not _ub: - vars_to_update.append(v) - elif _domain_interval != v.domain.get_interval(): - vars_to_update.append(v) - cons_to_update = list(cons_to_update.keys()) - objs_to_update = [ - self._objectives[obj_id][0] for obj_id in objs_to_update.keys() - ] - return vars_to_update, cons_to_update, objs_to_update + reasons.append(reason) + return vars_to_update, reasons def _check_for_param_changes(self): params_to_update = [] @@ -1023,9 +1030,54 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): with PauseGC() as pgc: self._check_for_unknown_active_components() - added_cons = set() - added_sos = set() - added_objs = {} + if config.update_vars: + timer.start('vars') + vars_to_update, reasons = self._check_for_var_changes() + if vars_to_update: + self._update_variables(vars_to_update, reasons) + timer.stop('vars') + + if config.update_parameters: + timer.start('params') + params_to_update = self._check_for_param_changes() + if params_to_update: + self._update_parameters(params_to_update) + timer.stop('params') + + if config.update_named_expressions: + timer.start('named expressions') + cons_to_update, objs_to_update = ( + self._check_for_named_expression_changes() + ) + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + timer.stop('named expressions') + + if config.update_constraints: + timer.start('cons') + cons_to_update = self._check_for_modified_constraints() + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + timer.stop('cons') + timer.start('sos') + sos_to_update = self._check_for_modified_sos() + if sos_to_update: + self._remove_sos_constraints(sos_to_update) + self._add_sos_constraints(sos_to_update) + timer.stop('sos') + + if config.update_objectives: + timer.start('objective') + objs_to_update = self._check_for_modified_objectives() + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + timer.stop('objective') if config.check_for_new_or_removed_constraints: timer.start('sos') @@ -1034,7 +1086,6 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): self._add_sos_constraints(new_sos) if old_sos: self._remove_sos_constraints(old_sos) - added_sos.update(new_sos) timer.stop('sos') timer.start('cons') new_cons, old_cons = self._check_for_new_or_removed_constraints() @@ -1042,24 +1093,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): self._add_constraints(new_cons) if old_cons: self._remove_constraints(old_cons) - added_cons.update(new_cons) - timer.stop('cons') - - if config.update_constraints: - timer.start('cons') - cons_to_update = self._check_for_modified_constraints() - if cons_to_update: - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) - added_cons.update(cons_to_update) timer.stop('cons') - timer.start('sos') - sos_to_update = self._check_for_modified_sos() - if sos_to_update: - self._remove_sos_constraints(sos_to_update) - self._add_sos_constraints(sos_to_update) - added_sos.update(sos_to_update) - timer.stop('sos') if config.check_for_new_or_removed_objectives: timer.start('objective') @@ -1070,57 +1104,4 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): self._remove_objectives(old_objs) if new_objs: self._add_objectives(new_objs) - added_objs.update((id(i), i) for i in new_objs) timer.stop('objective') - - if config.update_objectives: - timer.start('objective') - objs_to_update = self._check_for_modified_objectives() - if objs_to_update: - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) - added_objs.update((id(i), i) for i in objs_to_update) - timer.stop('objective') - - if config.update_vars: - timer.start('vars') - vars_to_update, cons_to_update, objs_to_update = ( - self._check_for_var_changes() - ) - if vars_to_update: - self._update_variables(vars_to_update) - cons_to_update = [i for i in cons_to_update if i not in added_cons] - objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] - if cons_to_update: - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) - added_cons.update(cons_to_update) - if objs_to_update: - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) - added_objs.update((id(i), i) for i in objs_to_update) - timer.stop('vars') - - if config.update_named_expressions: - timer.start('named expressions') - cons_to_update, objs_to_update = ( - self._check_for_named_expression_changes() - ) - cons_to_update = [i for i in cons_to_update if i not in added_cons] - objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] - if cons_to_update: - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) - added_cons.update(cons_to_update) - if objs_to_update: - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) - added_objs.update((id(i), i) for i in objs_to_update) - timer.stop('named expressions') - - if config.update_parameters: - timer.start('params') - params_to_update = self._check_for_param_changes() - if params_to_update: - self._update_parameters(params_to_update) - timer.stop('params') diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 5aea43bdd58..1a22fe7f2a0 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -23,6 +23,7 @@ Observer, ModelChangeDetector, AutoUpdateConfig, + Reason, ) from pyomo.common.collections import ComponentMap @@ -112,7 +113,7 @@ def remove_parameters(self, params: List[ParamData]): assert p.is_parameter_type() self._process(params, 'remove') - def update_variables(self, variables: List[VarData]): + def update_variables(self, variables: List[VarData], reasons: List[Reason]): for v in variables: assert v.is_variable_type() self._process(variables, 'update') @@ -156,31 +157,11 @@ def test_objective(self): m.x.fix(2) detector.update() expected[m.x]['update'] += 1 - # the variable gets updated - # the objective must get removed and added - # that causes x,y, and p to all get removed - # and added - expected[m.obj]['remove'] += 1 - expected[m.obj]['add'] += 1 - expected[m.x]['remove'] += 1 - expected[m.x]['add'] += 1 - expected[m.y]['remove'] += 1 - expected[m.y]['add'] += 1 - expected[m.p]['remove'] += 1 - expected[m.p]['add'] += 1 obs.check(expected) m.x.unfix() detector.update() expected[m.x]['update'] += 1 - expected[m.obj]['remove'] += 1 - expected[m.obj]['add'] += 1 - expected[m.x]['remove'] += 1 - expected[m.x]['add'] += 1 - expected[m.y]['remove'] += 1 - expected[m.y]['add'] += 1 - expected[m.p]['remove'] += 1 - expected[m.p]['add'] += 1 obs.check(expected) m.p.value = 2 @@ -239,21 +220,9 @@ def test_constraints(self): expected[m.obj]['add'] += 1 obs.check(expected) - # now fix a variable and make sure the - # constraint gets removed and added m.x.fix(1) detector.update() - expected[m.c1]['remove'] += 1 - expected[m.c1]['add'] += 1 - # because x and p are only used in the - # one constraint, they get removed when - # the constraint is removed and then - # added again when the constraint is added expected[m.x]['update'] += 1 - expected[m.x]['remove'] += 1 - expected[m.x]['add'] += 1 - expected[m.p]['remove'] += 1 - expected[m.p]['add'] += 1 obs.check(expected) def test_sos(self): @@ -328,21 +297,9 @@ def test_vars_and_params_elsewhere(self): expected[m2.obj]['add'] += 1 obs.check(expected) - # now fix a variable and make sure the - # constraint gets removed and added m1.x.fix(1) detector.update() - expected[m2.c1]['remove'] += 1 - expected[m2.c1]['add'] += 1 - # because x and p are only used in the - # one constraint, they get removed when - # the constraint is removed and then - # added again when the constraint is added expected[m1.x]['update'] += 1 - expected[m1.x]['remove'] += 1 - expected[m1.x]['add'] += 1 - expected[m1.p]['remove'] += 1 - expected[m1.p]['add'] += 1 obs.check(expected) def test_named_expression(self): From be0e04380c21a982b758fdb68bcaacd9139c854d Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 16 Oct 2025 11:36:40 -0600 Subject: [PATCH 36/50] observer: variable bounds might depend on parameters --- pyomo/contrib/observer/model_observer.py | 265 ++++++++++++++--------- 1 file changed, 167 insertions(+), 98 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 40754751599..862bd878f9c 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -16,16 +16,17 @@ from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.var import VarData -from pyomo.core.base.param import ParamData +from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.objective import ObjectiveData, Objective -from pyomo.core.base.block import BlockData -from pyomo.core.base.component import ActiveComponent +from pyomo.core.base.block import BlockData, Block +from pyomo.core.base.suffix import Suffix from pyomo.common.collections import ComponentMap from pyomo.common.gc_manager import PauseGC from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.observer.component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types +import warnings import enum @@ -50,6 +51,9 @@ """ +_param_types = {ParamData, ScalarParam} + + @document_configdict() class AutoUpdateConfig(ConfigDict): """ @@ -183,9 +187,6 @@ class Reason(enum.Flag): class Observer(abc.ABC): - def __init__(self): - pass - @abc.abstractmethod def add_variables(self, variables: List[VarData]): """ @@ -507,7 +508,7 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): observers: Sequence[Observer] The objects to notify when changes are made to the model """ - self._known_active_ctypes = {Constraint, SOSConstraint, Objective} + self._known_active_ctypes = {Constraint, SOSConstraint, Objective, Block} self._observers: List[Observer] = list(observers) self._active_constraints = {} # maps constraint to expression self._active_sos = {} @@ -535,12 +536,16 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): # dict[constraints, None], # dict[sos constraints, None], # dict[objectives, None], + # dict[var_id, None], # ) self._referenced_params = {} self._vars_referenced_by_con = {} self._vars_referenced_by_obj = {} self._params_referenced_by_con = {} + self._params_referenced_by_var = ( + {} + ) # for when parameters show up in variable bounds self._params_referenced_by_obj = {} self.config: AutoUpdateConfig = AutoUpdateConfig()( @@ -550,12 +555,14 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): self._model = model self._set_instance() - def _add_variables(self, variables: List[VarData]): + def add_variables(self, variables: List[VarData]): + params_to_check = {} for v in variables: - if id(v) in self._referenced_variables: + vid = id(v) + if vid in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') - self._referenced_variables[id(v)] = ({}, {}, {}) - self._vars[id(v)] = ( + self._referenced_variables[vid] = ({}, {}, {}) + self._vars[vid] = ( v, v._lb, v._ub, @@ -563,54 +570,87 @@ def _add_variables(self, variables: List[VarData]): v.domain.get_interval(), v.value, ) + ref_params = set() + for bnd in (v._lb, v._ub): + if bnd is None or type(bnd) in native_numeric_types: + continue + (named_exprs, _vars, parameters, external_functions) = ( + collect_components_from_expr(bnd) + ) + if _vars: + raise NotImplementedError( + 'ModelChangeDetector does not support variables in the bounds of other variables' + ) + if named_exprs: + raise NotImplementedError( + 'ModelChangeDetector does not support Expressions in the bounds of other variables' + ) + if external_functions: + raise NotImplementedError( + 'ModelChangeDetector does not support external functions in the bounds of other variables' + ) + params_to_check.update((id(p), p) for p in parameters) + if vid not in self._params_referenced_by_var: + self._params_referenced_by_var[vid] = [] + self._params_referenced_by_var[vid].extend( + p for p in parameters if id(p) not in ref_params + ) + ref_params.update(id(p) for p in parameters) + self._check_for_new_params(list(params_to_check.values())) + for v in variables: + if id(v) not in self._params_referenced_by_var: + continue + parameters = self._params_referenced_by_var[id(v)] + for p in parameters: + self._referenced_params[id(p)][3][id(v)] = None for obs in self._observers: obs.add_variables(variables) - def _add_parameters(self, params: List[ParamData]): + def add_parameters(self, params: List[ParamData]): for p in params: pid = id(p) if pid in self._referenced_params: raise ValueError(f'Parameter {p.name} has already been added') - self._referenced_params[pid] = ({}, {}, {}) + self._referenced_params[pid] = ({}, {}, {}, {}) self._params[id(p)] = (p, p.value) for obs in self._observers: obs.add_parameters(params) def _check_for_new_vars(self, variables: List[VarData]): - new_vars = [] + new_vars = {} for v in variables: if id(v) not in self._referenced_variables: - new_vars.append(v) - self._add_variables(new_vars) + new_vars[id(v)] = v + self.add_variables(list(new_vars.values())) def _check_to_remove_vars(self, variables: List[VarData]): - vars_to_remove = [] + vars_to_remove = {} for v in variables: v_id = id(v) ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] if not ref_cons and not ref_sos and not ref_obj: - vars_to_remove.append(v) - self._remove_variables(vars_to_remove) + vars_to_remove[v_id] = v + self.remove_variables(list(vars_to_remove.values())) def _check_for_new_params(self, params: List[ParamData]): - new_params = [] + new_params = {} for p in params: if id(p) not in self._referenced_params: - new_params.append(p) - self._add_parameters(new_params) + new_params[id(p)] = p + self.add_parameters(list(new_params.values())) def _check_to_remove_params(self, params: List[ParamData]): - params_to_remove = [] + params_to_remove = {} for p in params: p_id = id(p) - ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] - if not ref_cons and not ref_sos and not ref_obj: - params_to_remove.append(p) - self._remove_parameters(params_to_remove) + ref_cons, ref_sos, ref_obj, ref_vars = self._referenced_params[p_id] + if not ref_cons and not ref_sos and not ref_obj and not ref_vars: + params_to_remove[p_id] = p + self.remove_parameters(list(params_to_remove.values())) - def _add_constraints(self, cons: List[ConstraintData]): - vars_to_check = [] - params_to_check = [] + def add_constraints(self, cons: List[ConstraintData]): + vars_to_check = {} + params_to_check = {} for con in cons: if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') @@ -618,16 +658,16 @@ def _add_constraints(self, cons: List[ConstraintData]): (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(con.expr) ) - vars_to_check.extend(variables) - params_to_check.extend(parameters) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in parameters) if named_exprs: self._named_expressions[con] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = parameters - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for con in cons: variables = self._vars_referenced_by_con[con] parameters = self._params_referenced_by_con[con] @@ -638,9 +678,9 @@ def _add_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.add_constraints(cons) - def _add_sos_constraints(self, cons: List[SOSConstraintData]): - vars_to_check = [] - params_to_check = [] + def add_sos_constraints(self, cons: List[SOSConstraintData]): + vars_to_check = {} + params_to_check = {} for con in cons: if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') @@ -657,12 +697,12 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): continue if p.is_parameter_type(): params.append(p) - vars_to_check.extend(variables) - params_to_check.extend(params) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in params) self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for con in cons: variables = self._vars_referenced_by_con[con] params = self._params_referenced_by_con[con] @@ -673,25 +713,25 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.add_sos_constraints(cons) - def _add_objectives(self, objs: List[ObjectiveData]): - vars_to_check = [] - params_to_check = [] + def add_objectives(self, objs: List[ObjectiveData]): + vars_to_check = {} + params_to_check = {} for obj in objs: obj_id = id(obj) self._objectives[obj_id] = (obj, obj.expr, obj.sense) (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(obj.expr) ) - vars_to_check.extend(variables) - params_to_check.extend(parameters) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in parameters) if named_exprs: self._obj_named_expressions[obj_id] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[obj] = external_functions self._vars_referenced_by_obj[obj_id] = variables self._params_referenced_by_obj[obj_id] = parameters - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for obj in objs: obj_id = id(obj) variables = self._vars_referenced_by_obj[obj_id] @@ -703,12 +743,12 @@ def _add_objectives(self, objs: List[ObjectiveData]): for obs in self._observers: obs.add_objectives(objs) - def _remove_objectives(self, objs: List[ObjectiveData]): + def remove_objectives(self, objs: List[ObjectiveData]): for obs in self._observers: obs.remove_objectives(objs) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for obj in objs: obj_id = id(obj) if obj_id not in self._objectives: @@ -719,22 +759,29 @@ def _remove_objectives(self, objs: List[ObjectiveData]): self._referenced_variables[id(v)][2].pop(obj_id) for p in self._params_referenced_by_obj[obj_id]: self._referenced_params[id(p)][2].pop(obj_id) - vars_to_check.extend(self._vars_referenced_by_obj[obj_id]) - params_to_check.extend(self._params_referenced_by_obj[obj_id]) + vars_to_check.update( + (id(v), v) for v in self._vars_referenced_by_obj[obj_id] + ) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_obj[obj_id] + ) del self._objectives[obj_id] self._obj_named_expressions.pop(obj_id, None) self._external_functions.pop(obj, None) del self._vars_referenced_by_obj[obj_id] del self._params_referenced_by_obj[obj_id] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) def _check_for_unknown_active_components(self): for ctype in self._model.collect_ctypes(active=True, descend_into=True): - if not issubclass(ctype, ActiveComponent): - continue if ctype in self._known_active_ctypes: continue + if ctype is Suffix: + warnings.warn( + 'ModelChangeDetector does not detect changes to suffixes' + ) + continue raise NotImplementedError( f'ModelChangeDetector does not know how to ' 'handle components with ctype {ctype}' @@ -745,21 +792,21 @@ def _set_instance(self): with PauseGC() as pgc: self._check_for_unknown_active_components() - self._add_constraints( + self.add_constraints( list( self._model.component_data_objects( Constraint, descend_into=True, active=True ) ) ) - self._add_sos_constraints( + self.add_sos_constraints( list( self._model.component_data_objects( SOSConstraint, descend_into=True, active=True ) ) ) - self._add_objectives( + self.add_objectives( list( self._model.component_data_objects( Objective, descend_into=True, active=True @@ -767,11 +814,11 @@ def _set_instance(self): ) ) - def _remove_constraints(self, cons: List[ConstraintData]): + def remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.remove_constraints(cons) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con not in self._active_constraints: raise ValueError( @@ -781,21 +828,23 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._referenced_variables[id(v)][0].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][0].pop(con) - vars_to_check.extend(self._vars_referenced_by_con[con]) - params_to_check.extend(self._params_referenced_by_con[con]) + vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_con[con] + ) del self._active_constraints[con] self._named_expressions.pop(con, None) self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) - def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + def remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.remove_sos_constraints(cons) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con not in self._active_sos: raise ValueError( @@ -805,23 +854,33 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): self._referenced_variables[id(v)][1].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][1].pop(con) - vars_to_check.extend(self._vars_referenced_by_con[con]) - params_to_check.extend(self._params_referenced_by_con[con]) + vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_con[con] + ) del self._active_sos[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) - def _remove_variables(self, variables: List[VarData]): + def remove_variables(self, variables: List[VarData]): for obs in self._observers: obs.remove_variables(variables) + params_to_check = {} for v in variables: v_id = id(v) if v_id not in self._referenced_variables: raise ValueError( f'Cannot remove variable {v.name} - it has not been added' ) + if v_id in self._params_referenced_by_var: + for p in self._params_referenced_by_var[v_id]: + self._referenced_params[id(p)][3].pop(v_id) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_var[v_id] + ) + self._params_referenced_by_var.pop(v_id) cons_using, sos_using, obj_using = self._referenced_variables[v_id] if cons_using or sos_using or obj_using: raise ValueError( @@ -829,8 +888,9 @@ def _remove_variables(self, variables: List[VarData]): ) del self._referenced_variables[v_id] del self._vars[v_id] + self._check_to_remove_params(list(params_to_check.values())) - def _remove_parameters(self, params: List[ParamData]): + def remove_parameters(self, params: List[ParamData]): for obs in self._observers: obs.remove_parameters(params) for p in params: @@ -839,15 +899,15 @@ def _remove_parameters(self, params: List[ParamData]): raise ValueError( f'Cannot remove parameter {p.name} - it has not been added' ) - cons_using, sos_using, obj_using = self._referenced_params[p_id] - if cons_using or sos_using or obj_using: + cons_using, sos_using, obj_using, vars_using = self._referenced_params[p_id] + if cons_using or sos_using or obj_using or vars_using: raise ValueError( f'Cannot remove parameter {p.name} - it is still being used by constraints/objectives' ) del self._referenced_params[p_id] del self._params[p_id] - def _update_variables(self, variables: List[VarData], reasons: List[Reason]): + def update_variables(self, variables: List[VarData], reasons: List[Reason]): for v in variables: self._vars[id(v)] = ( v, @@ -860,7 +920,7 @@ def _update_variables(self, variables: List[VarData], reasons: List[Reason]): for obs in self._observers: obs.update_variables(variables, reasons) - def _update_parameters(self, params): + def update_parameters(self, params): for p in params: self._params[id(p)] = (p, p.value) for obs in self._observers: @@ -1034,14 +1094,14 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): timer.start('vars') vars_to_update, reasons = self._check_for_var_changes() if vars_to_update: - self._update_variables(vars_to_update, reasons) + self.update_variables(vars_to_update, reasons) timer.stop('vars') if config.update_parameters: timer.start('params') params_to_update = self._check_for_param_changes() if params_to_update: - self._update_parameters(params_to_update) + self.update_parameters(params_to_update) timer.stop('params') if config.update_named_expressions: @@ -1050,49 +1110,49 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): self._check_for_named_expression_changes() ) if cons_to_update: - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) if objs_to_update: - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) + self.remove_objectives(objs_to_update) + self.add_objectives(objs_to_update) timer.stop('named expressions') if config.update_constraints: timer.start('cons') cons_to_update = self._check_for_modified_constraints() if cons_to_update: - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) timer.stop('cons') timer.start('sos') sos_to_update = self._check_for_modified_sos() if sos_to_update: - self._remove_sos_constraints(sos_to_update) - self._add_sos_constraints(sos_to_update) + self.remove_sos_constraints(sos_to_update) + self.add_sos_constraints(sos_to_update) timer.stop('sos') if config.update_objectives: timer.start('objective') objs_to_update = self._check_for_modified_objectives() if objs_to_update: - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) + self.remove_objectives(objs_to_update) + self.add_objectives(objs_to_update) timer.stop('objective') if config.check_for_new_or_removed_constraints: timer.start('sos') new_sos, old_sos = self._check_for_new_or_removed_sos() if new_sos: - self._add_sos_constraints(new_sos) + self.add_sos_constraints(new_sos) if old_sos: - self._remove_sos_constraints(old_sos) + self.remove_sos_constraints(old_sos) timer.stop('sos') timer.start('cons') new_cons, old_cons = self._check_for_new_or_removed_constraints() if new_cons: - self._add_constraints(new_cons) + self.add_constraints(new_cons) if old_cons: - self._remove_constraints(old_cons) + self.remove_constraints(old_cons) timer.stop('cons') if config.check_for_new_or_removed_objectives: @@ -1101,7 +1161,16 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): # many solvers require one objective, so we have to remove the # old objective first if old_objs: - self._remove_objectives(old_objs) + self.remove_objectives(old_objs) if new_objs: - self._add_objectives(new_objs) + self.add_objectives(new_objs) timer.stop('objective') + + def get_variables_impacted_by_param(self, p: ParamData): + return [self._vars[vid][0] for vid in self._referenced_params[id(p)][3]] + + def get_constraints_impacted_by_param(self, p: ParamData): + return list(self._referenced_params[id(p)][0]) + + def get_constraints_impacted_by_var(self, v: VarData): + return list(self._referenced_variables[id(v)][0]) From 0739cf6f498f8652c37ea452336bbc3b8d112b0c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 16 Oct 2025 11:37:28 -0600 Subject: [PATCH 37/50] observer: imports --- pyomo/contrib/observer/model_observer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 862bd878f9c..0b4396ba570 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -20,7 +20,7 @@ from pyomo.core.base.objective import ObjectiveData, Objective from pyomo.core.base.block import BlockData, Block from pyomo.core.base.suffix import Suffix -from pyomo.common.collections import ComponentMap +from pyomo.common.collections import ComponentMap, ComponentSet, DefaultComponentMap from pyomo.common.gc_manager import PauseGC from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.util import get_objective From dfbdce03b0177408fde2710ce1d3f573381cfa3f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 17 Oct 2025 16:31:47 -0600 Subject: [PATCH 38/50] refactoring observer --- pyomo/contrib/observer/model_observer.py | 828 +++++++++++------------ 1 file changed, 377 insertions(+), 451 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 0b4396ba570..356f55612be 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -10,17 +10,21 @@ # __________________________________________________________________________ import abc -from typing import List, Sequence, Optional +from typing import List, Sequence, Optional, Mapping, MutableMapping, MutableSet, Tuple, Collection, Union +from pyomo.common.enums import ObjectiveSense from pyomo.common.config import ConfigDict, ConfigValue, document_configdict from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.var import VarData from pyomo.core.base.param import ParamData, ScalarParam +from pyomo.core.base.expression import ExpressionData from pyomo.core.base.objective import ObjectiveData, Objective from pyomo.core.base.block import BlockData, Block from pyomo.core.base.suffix import Suffix -from pyomo.common.collections import ComponentMap, ComponentSet, DefaultComponentMap +from pyomo.core.expr.numeric_expr import NumericValue +from pyomo.core.expr.relational_expr import RelationalExpression +from pyomo.common.collections import ComponentMap, ComponentSet, OrderedSet, DefaultComponentMap from pyomo.common.gc_manager import PauseGC from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.util import get_objective @@ -28,27 +32,26 @@ from pyomo.common.numeric_types import native_numeric_types import warnings import enum - - -""" -The ModelChangeDetector is meant to be used to automatically identify changes -in a Pyomo model or block. Here is a list of changes that will be detected. -Note that inactive components (e.g., constraints) are treated as "removed". - - new constraints that have been added to the model - - constraints that have been removed from the model - - new variables that have been detected in new or modified constraints/objectives - - old variables that are no longer used in any constraints/objectives - - new parameters that have been detected in new or modified constraints/objectives - - old parameters that are no longer used in any constraints/objectives - - new objectives that have been added to the model - - objectives that have been removed from the model - - modified constraint expressions (relies on expressions being immutable) - - modified objective expressions (relies on expressions being immutable) - - modified objective sense - - changes to variable bounds, domains, and "fixed" flags - - changes to named expressions (relies on expressions being immutable) - - changes to parameter values and fixed variable values -""" +from collections import defaultdict + + +# The ModelChangeDetector is meant to be used to automatically identify changes +# in a Pyomo model or block. Here is a list of changes that will be detected. +# Note that inactive components (e.g., constraints) are treated as "removed". +# - new constraints that have been added to the model +# - constraints that have been removed from the model +# - new variables that have been detected in new or modified constraints/objectives +# - old variables that are no longer used in any constraints/objectives +# - new parameters that have been detected in new or modified constraints/objectives +# - old parameters that are no longer used in any constraints/objectives +# - new objectives that have been added to the model +# - objectives that have been removed from the model +# - modified constraint expressions (relies on expressions being immutable) +# - modified objective expressions (relies on expressions being immutable) +# - modified objective sense +# - changes to variable bounds, domains, and "fixed" flags +# - changes to named expressions (relies on expressions being immutable) +# - changes to parameter values and fixed variable values _param_types = {ParamData, ScalarParam} @@ -86,9 +89,9 @@ def __init__( default=True, description=""" If False, new/old constraints will not be automatically detected on - subsequent solves. Use False only when manually updating the solver - with opt.add_constraints() and opt.remove_constraints() or when you - are certain constraints are not being added to/removed from the + subsequent solves. Use False only when manually updating the change + detector with cd.add_constraints() and cd.remove_constraints() or + when you are certain constraints are not being added to/removed from the model.""", ), ) @@ -179,194 +182,118 @@ def __init__( class Reason(enum.Flag): - bounds = enum.auto() - fixed = enum.auto() - unfixed = enum.auto() - domain = enum.auto() - value = enum.auto() + no_change = 0 + bounds = 1 + fixed = 2 + domain = 4 + value = 8 + added = 16 + removed = 32 + expr = 64 + sense = 128 + sos_items = 256 class Observer(abc.ABC): @abc.abstractmethod - def add_variables(self, variables: List[VarData]): + def _update_variables(self, variables: Mapping[VarData, Reason]): """ - This method gets called by the ModelChangeDetector when new - "active" variables are detected in the model. This means variables + This method gets called by the ModelChangeDetector when there are + any modifications to the set of "active" variables in the model being + observed. By "active" variables, we mean variables that are used within an active component such as a constraint or - an objective. + an objective. Changes include new variables being added to the model, + variables being removed from the model, or changes to variables + already in the model Parameters ---------- - variables: List[VarData] - The list of variables added to the model + variables: Mapping[VarData, Reason] + The variables and what changed about them """ pass @abc.abstractmethod - def add_parameters(self, params: List[ParamData]): + def _update_parameters(self, params: Mapping[ParamData, Reason]): """ - This method gets called by the ModelChangeDetector when new - "active" parameters are detected in the model. This means parameters - that are used within an active component such as a constraint or - an objective. + This method gets called by the ModelChangeDetector when there are any + modifications to the set of "active" parameters in the model being + observed. By "active" parameters, we mean parameters that are used within + an active component such as a constraint or an objective. Changes include + parameters being added to the model, parameters being removed from the model, + or changes to parameters already in the model Parameters ---------- - params: List[ParamData] - The list of parameters added to the model + params: Mapping[ParamData, Reason] + The parameters and what changed about them """ pass @abc.abstractmethod - def add_constraints(self, cons: List[ConstraintData]): + def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): """ - This method gets called by the ModelChangeDetector when new - active constraints are detected in the model. + This method gets called by the ModelChangeDetector when there are any + modifications to the set of active constraints in the model being observed. + Changes include constraints being added to the model, constraints being + removed from the model, or changes to constraints already in the model. Parameters ---------- - cons: List[ConstraintData] - The list of constraints added to the model + cons: Mapping[ConstraintData, Reason] + The constraints and what changed about them """ pass @abc.abstractmethod - def add_sos_constraints(self, cons: List[SOSConstraintData]): + def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): """ - This method gets called by the ModelChangeDetector when new - active SOS constraints are detected in the model. + This method gets called by the ModelChangeDetector when there are any + modifications to the set of active SOS constraints in the model being + observed. Changes include constraints being added to the model, constraints + being removed from the model, or changes to constraints already in the model. Parameters ---------- - cons: List[SOSConstraintData] - The list of SOS constraints added to the model + cons: Mapping[SOSConstraintData, Reason] + The SOS constraints and what changed about them """ pass @abc.abstractmethod - def add_objectives(self, objs: List[ObjectiveData]): + def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): """ - This method gets called by the ModelChangeDetector when new - active objectives are detected in the model. + This method gets called by the ModelChangeDetector when there are any + modifications to the set of active objectives in the model being observed. + Changes include objectives being added to the model, objectives being + removed from the model, or changes to objectives already in the model. Parameters ---------- - objs: List[ObjectiveData] - The list of objectives added to the model + objs: Mapping[ObjectiveData, Reason] + The objectives and what changed about them """ pass - @abc.abstractmethod - def remove_objectives(self, objs: List[ObjectiveData]): - """ - This method gets called by the ModelChangeDetector when it detects - objectives that have been deactivated or removed from the model. - If the ModelChangeDetector detects changes in the underlying - expression for the objective, then ``remove_objectives`` will be - called followed by ``add_objectives``. - - Parameters - ---------- - objs: List[ObjectiveData] - The list of objectives that are no longer part of the model - """ - pass - - @abc.abstractmethod - def remove_constraints(self, cons: List[ConstraintData]): - """ - This method gets called by the ModelChangeDetector when it detects - constraints that have been deactivated or removed from the model. - If the ModelChangeDetector detects changes in the underlying - expression for the constraint, then ``remove_constraints`` will be - called followed by ``add_constraints``. - - Parameters - ---------- - cons: List[ConstraintData] - The list of constraints that are no longer part of the model - """ - pass - - @abc.abstractmethod - def remove_sos_constraints(self, cons: List[SOSConstraintData]): - """ - This method gets called by the ModelChangeDetector when it detects - SOS constraints that have been deactivated or removed from the model. - If the ModelChangeDetector detects changes in the underlying - data for the constraint, then ``remove_sos_constraints`` will be - called followed by ``add_sos_constraints``. - - Parameters - ---------- - cons: List[SOSConstraintData] - The list of SOS constraints that are no longer part of the model - """ - pass - @abc.abstractmethod - def remove_variables(self, variables: List[VarData]): - """ - This method gets called by the ModelChangeDetector when it detects - variables that are no longer used in any active components ( - objectives or constraints). +def _default_reason(): + return Reason.no_change - Parameters - ---------- - variables: List[VarData] - The list of variables that are no longer part of the model - """ - pass - @abc.abstractmethod - def remove_parameters(self, params: List[ParamData]): - """ - This method gets called by the ModelChangeDetector when it detects - parameters that are no longer used in any active components ( - objectives or constraints). +class _Updates: + def __init__(self) -> None: + self.vars_to_update = DefaultComponentMap(_default_reason) + self.params_to_update = DefaultComponentMap(_default_reason) + self.cons_to_update = defaultdict(_default_reason) + self.sos_to_update = defaultdict(_default_reason) + self.objs_to_update = DefaultComponentMap(_default_reason) - Parameters - ---------- - params: List[ParamData] - The list of parameters that are no longer part of the model - """ - pass - - @abc.abstractmethod - def update_variables(self, variables: List[VarData], reasons: List[Reason]): - """ - This method gets called by the ModelChangeDetector when it detects - variables that have been modified in some way (e.g., the bounds - change). This is only true for changes that are considered - "inputs" to the model. For example, the value of the variable is - considered an "output" (unless the variable is fixed), so changing - the value of an unfixed variable will not cause this method to be - called. - - Parameters - ---------- - variables: List[VarData] - The list of variables that have been modified - reasons: List[Reason] - A list of reasons the variables changed. This list will be the - same length as `variables`. The Reason enum is a Flag enum so - that if multiple changes are made, this can be easily indicated - with the bitwise | operator. - """ - pass - - @abc.abstractmethod - def update_parameters(self, params: List[ParamData]): - """ - This method gets called by the ModelChangeDetector when it detects - parameters that have been modified (i.e., the value changed). - - Parameters - ---------- - params: List[ParamData] - The list of parameters that have been modified - """ - pass + def clear(self): + self.vars_to_update.clear() + self.params_to_update.clear() + self.cons_to_update.clear() + self.sos_to_update.clear() + self.objs_to_update.clear() class ModelChangeDetector: @@ -510,67 +437,92 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): """ self._known_active_ctypes = {Constraint, SOSConstraint, Objective, Block} self._observers: List[Observer] = list(observers) - self._active_constraints = {} # maps constraint to expression + + self._active_constraints: MutableMapping[ + ConstraintData, Union[RelationalExpression, None] + ] = {} + self._active_sos = {} - self._vars = {} # maps var id to (var, lb, ub, fixed, domain, value) - self._params = {} # maps param id to param - self._objectives = {} # maps objective id to (objective, expression, sense) + + # maps var to (lb, ub, fixed, domain, value) + self._vars: MutableMapping[VarData, Tuple] = ComponentMap() + + # maps param to value + self._params: MutableMapping[ParamData, float] = ComponentMap() + + self._objectives: MutableMapping[ + ObjectiveData, Tuple[Union[NumericValue, float, int, None], ObjectiveSense] + ] = ComponentMap() # maps objective to (expression, sense) # maps constraints/objectives to list of tuples (named_expr, named_expr.expr) - self._named_expressions = {} - self._obj_named_expressions = {} + self._named_expressions: MutableMapping[ + ConstraintData, List[Tuple[ExpressionData, Union[NumericValue, float, int, None]]] + ] = {} + self._obj_named_expressions: MutableMapping[ + ObjectiveData, List[Tuple[ExpressionData, Union[NumericValue, float, int, None]]] + ] = ComponentMap() self._external_functions = ComponentMap() - # the dictionaries below are really just ordered sets, but we need to - # stick with built-in types for performance - - # var_id: ( - # dict[constraints, None], - # dict[sos constraints, None], - # dict[objectives, None], - # ) - self._referenced_variables = {} - - # param_id: ( - # dict[constraints, None], - # dict[sos constraints, None], - # dict[objectives, None], - # dict[var_id, None], - # ) - self._referenced_params = {} - - self._vars_referenced_by_con = {} - self._vars_referenced_by_obj = {} - self._params_referenced_by_con = {} - self._params_referenced_by_var = ( - {} - ) # for when parameters show up in variable bounds - self._params_referenced_by_obj = {} + self._referenced_variables: MutableMapping[ + VarData, + Tuple[ + MutableSet[ConstraintData], + MutableSet[SOSConstraintData], + MutableSet[ObjectiveData] + ] + ] = ComponentMap() + + self._referenced_params: MutableMapping[ + ParamData, + Tuple[ + MutableSet[ConstraintData], + MutableSet[SOSConstraintData], + MutableSet[ObjectiveData], + MutableSet[VarData], + ] + ] = ComponentMap() + + self._vars_referenced_by_con: MutableMapping[ + Union[ConstraintData, SOSConstraintData], List[VarData] + ] = {} + self._vars_referenced_by_obj: MutableMapping[ + ObjectiveData, List[VarData] + ] = ComponentMap() + self._params_referenced_by_con: MutableMapping[ + Union[ConstraintData, SOSConstraintData], List[ParamData] + ] = {} + # for when parameters show up in variable bounds + self._params_referenced_by_var: MutableMapping[ + VarData, List[ParamData] + ] = ComponentMap() + self._params_referenced_by_obj: MutableMapping[ + ObjectiveData, List[ParamData] + ] = ComponentMap() self.config: AutoUpdateConfig = AutoUpdateConfig()( value=kwds, preserve_implicit=True ) - self._model = model + self._updates = _Updates() + + self._model: BlockData = model self._set_instance() - def add_variables(self, variables: List[VarData]): - params_to_check = {} + def add_variables(self, variables: Collection[VarData]): + params_to_check = ComponentSet() for v in variables: - vid = id(v) - if vid in self._referenced_variables: + if v in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') - self._referenced_variables[vid] = ({}, {}, {}) - self._vars[vid] = ( - v, + self._referenced_variables[v] = (OrderedSet(), OrderedSet(), ComponentSet()) + self._vars[v] = ( v._lb, v._ub, v.fixed, v.domain.get_interval(), v.value, ) - ref_params = set() + ref_params = ComponentSet() for bnd in (v._lb, v._ub): if bnd is None or type(bnd) in native_numeric_types: continue @@ -589,68 +541,58 @@ def add_variables(self, variables: List[VarData]): raise NotImplementedError( 'ModelChangeDetector does not support external functions in the bounds of other variables' ) - params_to_check.update((id(p), p) for p in parameters) - if vid not in self._params_referenced_by_var: - self._params_referenced_by_var[vid] = [] - self._params_referenced_by_var[vid].extend( - p for p in parameters if id(p) not in ref_params - ) - ref_params.update(id(p) for p in parameters) - self._check_for_new_params(list(params_to_check.values())) + params_to_check.update(parameters) + ref_params.update(parameters) + if ref_params: + self._params_referenced_by_var[v] = list(ref_params) + self._check_for_new_params(params_to_check) for v in variables: - if id(v) not in self._params_referenced_by_var: + if v not in self._params_referenced_by_var: continue - parameters = self._params_referenced_by_var[id(v)] + parameters = self._params_referenced_by_var[v] for p in parameters: - self._referenced_params[id(p)][3][id(v)] = None + self._referenced_params[p][3].add(v) for obs in self._observers: - obs.add_variables(variables) + obs._update_variables(ComponentMap((v, Reason.added) for v in variables)) - def add_parameters(self, params: List[ParamData]): + def add_parameters(self, params: Collection[ParamData]): for p in params: - pid = id(p) - if pid in self._referenced_params: + if p in self._referenced_params: raise ValueError(f'Parameter {p.name} has already been added') - self._referenced_params[pid] = ({}, {}, {}, {}) - self._params[id(p)] = (p, p.value) + self._referenced_params[p] = (OrderedSet(), OrderedSet(), ComponentSet(), ComponentSet()) + self._params[p] = p.value for obs in self._observers: - obs.add_parameters(params) + obs._update_parameters(ComponentMap((p, Reason.added) for p in params)) - def _check_for_new_vars(self, variables: List[VarData]): - new_vars = {} - for v in variables: - if id(v) not in self._referenced_variables: - new_vars[id(v)] = v - self.add_variables(list(new_vars.values())) + def _check_for_new_vars(self, variables: Collection[VarData]): + new_vars = ComponentSet( + v for v in variables if v not in self._referenced_variables + ) + self.add_variables(new_vars) - def _check_to_remove_vars(self, variables: List[VarData]): - vars_to_remove = {} + def _check_to_remove_vars(self, variables: Collection[VarData]): + vars_to_remove = ComponentSet() for v in variables: - v_id = id(v) - ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] - if not ref_cons and not ref_sos and not ref_obj: - vars_to_remove[v_id] = v - self.remove_variables(list(vars_to_remove.values())) - - def _check_for_new_params(self, params: List[ParamData]): - new_params = {} - for p in params: - if id(p) not in self._referenced_params: - new_params[id(p)] = p - self.add_parameters(list(new_params.values())) + if not any(self._referenced_variables[v]): + vars_to_remove.add(v) + self.remove_variables(vars_to_remove) - def _check_to_remove_params(self, params: List[ParamData]): - params_to_remove = {} + def _check_for_new_params(self, params: Collection[ParamData]): + new_params = ComponentSet( + p for p in params if p not in self._referenced_params + ) + self.add_parameters(new_params) + + def _check_to_remove_params(self, params: Collection[ParamData]): + params_to_remove = ComponentSet() for p in params: - p_id = id(p) - ref_cons, ref_sos, ref_obj, ref_vars = self._referenced_params[p_id] - if not ref_cons and not ref_sos and not ref_obj and not ref_vars: - params_to_remove[p_id] = p - self.remove_parameters(list(params_to_remove.values())) - - def add_constraints(self, cons: List[ConstraintData]): - vars_to_check = {} - params_to_check = {} + if not any(self._referenced_params[p]): + params_to_remove.add(p) + self.remove_parameters(params_to_remove) + + def add_constraints(self, cons: Collection[ConstraintData]): + vars_to_check = ComponentSet() + params_to_check = ComponentSet() for con in cons: if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') @@ -658,29 +600,29 @@ def add_constraints(self, cons: List[ConstraintData]): (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(con.expr) ) - vars_to_check.update((id(v), v) for v in variables) - params_to_check.update((id(p), p) for p in parameters) + vars_to_check.update(variables) + params_to_check.update(parameters) if named_exprs: self._named_expressions[con] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = parameters - self._check_for_new_vars(list(vars_to_check.values())) - self._check_for_new_params(list(params_to_check.values())) + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) for con in cons: variables = self._vars_referenced_by_con[con] parameters = self._params_referenced_by_con[con] for v in variables: - self._referenced_variables[id(v)][0][con] = None + self._referenced_variables[v][0].add(con) for p in parameters: - self._referenced_params[id(p)][0][con] = None + self._referenced_params[p][0].add(con) for obs in self._observers: - obs.add_constraints(cons) + obs._update_constraints({c: Reason.added for c in cons}) - def add_sos_constraints(self, cons: List[SOSConstraintData]): - vars_to_check = {} - params_to_check = {} + def add_sos_constraints(self, cons: Collection[SOSConstraintData]): + vars_to_check = ComponentSet() + params_to_check = ComponentSet() for con in cons: if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') @@ -697,81 +639,74 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): continue if p.is_parameter_type(): params.append(p) - vars_to_check.update((id(v), v) for v in variables) - params_to_check.update((id(p), p) for p in params) + vars_to_check.update(variables) + params_to_check.update(params) self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params - self._check_for_new_vars(list(vars_to_check.values())) - self._check_for_new_params(list(params_to_check.values())) + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) for con in cons: variables = self._vars_referenced_by_con[con] params = self._params_referenced_by_con[con] for v in variables: - self._referenced_variables[id(v)][1][con] = None + self._referenced_variables[v][1].add(con) for p in params: - self._referenced_params[id(p)][1][con] = None + self._referenced_params[p][1].add(con) for obs in self._observers: - obs.add_sos_constraints(cons) + obs._update_sos_constraints(ComponentMap((c, Reason.added) for c in cons)) - def add_objectives(self, objs: List[ObjectiveData]): - vars_to_check = {} - params_to_check = {} + def add_objectives(self, objs: Collection[ObjectiveData]): + vars_to_check = ComponentSet() + params_to_check = ComponentSet() for obj in objs: - obj_id = id(obj) - self._objectives[obj_id] = (obj, obj.expr, obj.sense) + self._objectives[obj] = (obj.expr, obj.sense) (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(obj.expr) ) - vars_to_check.update((id(v), v) for v in variables) - params_to_check.update((id(p), p) for p in parameters) + vars_to_check.update(variables) + params_to_check.update(parameters) if named_exprs: - self._obj_named_expressions[obj_id] = [(e, e.expr) for e in named_exprs] + self._obj_named_expressions[obj] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[obj] = external_functions - self._vars_referenced_by_obj[obj_id] = variables - self._params_referenced_by_obj[obj_id] = parameters - self._check_for_new_vars(list(vars_to_check.values())) - self._check_for_new_params(list(params_to_check.values())) + self._vars_referenced_by_obj[obj] = variables + self._params_referenced_by_obj[obj] = parameters + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) for obj in objs: - obj_id = id(obj) - variables = self._vars_referenced_by_obj[obj_id] - parameters = self._params_referenced_by_obj[obj_id] + variables = self._vars_referenced_by_obj[obj] + parameters = self._params_referenced_by_obj[obj] for v in variables: - self._referenced_variables[id(v)][2][obj_id] = None + self._referenced_variables[v][2].add(obj) for p in parameters: - self._referenced_params[id(p)][2][obj_id] = None + self._referenced_params[p][2].add(obj) for obs in self._observers: - obs.add_objectives(objs) + obs._update_objectives(ComponentMap((obj, Reason.added) for obj in objs)) - def remove_objectives(self, objs: List[ObjectiveData]): + def remove_objectives(self, objs: Collection[ObjectiveData]): for obs in self._observers: - obs.remove_objectives(objs) + obs._update_objectives(ComponentMap((obj, Reason.removed) for obj in objs)) - vars_to_check = {} - params_to_check = {} + vars_to_check = ComponentSet() + params_to_check = ComponentSet() for obj in objs: - obj_id = id(obj) - if obj_id not in self._objectives: + if obj not in self._objectives: raise ValueError( f'cannot remove objective {obj.name} - it was not added' ) - for v in self._vars_referenced_by_obj[obj_id]: - self._referenced_variables[id(v)][2].pop(obj_id) - for p in self._params_referenced_by_obj[obj_id]: - self._referenced_params[id(p)][2].pop(obj_id) - vars_to_check.update( - (id(v), v) for v in self._vars_referenced_by_obj[obj_id] - ) - params_to_check.update( - (id(p), p) for p in self._params_referenced_by_obj[obj_id] - ) - del self._objectives[obj_id] - self._obj_named_expressions.pop(obj_id, None) + for v in self._vars_referenced_by_obj[obj]: + self._referenced_variables[v][2].remove(obj) + for p in self._params_referenced_by_obj[obj]: + self._referenced_params[p][2].remove(obj) + vars_to_check.update(self._vars_referenced_by_obj[obj]) + params_to_check.update(self._params_referenced_by_obj[obj]) + del self._objectives[obj] + self._obj_named_expressions.pop(obj, None) self._external_functions.pop(obj, None) - del self._vars_referenced_by_obj[obj_id] - del self._params_referenced_by_obj[obj_id] - self._check_to_remove_vars(list(vars_to_check.values())) - self._check_to_remove_params(list(params_to_check.values())) + self._vars_referenced_by_obj.pop(obj) + self._params_referenced_by_obj.pop(obj) + self._check_to_remove_vars(vars_to_check) + self._check_to_remove_params(params_to_check) def _check_for_unknown_active_components(self): for ctype in self._model.collect_ctypes(active=True, descend_into=True): @@ -814,103 +749,92 @@ def _set_instance(self): ) ) - def remove_constraints(self, cons: List[ConstraintData]): + def remove_constraints(self, cons: Collection[ConstraintData]): for obs in self._observers: - obs.remove_constraints(cons) - vars_to_check = {} - params_to_check = {} + obs._update_constraints({c: Reason.removed for c in cons}) + vars_to_check = ComponentSet() + params_to_check = ComponentSet() for con in cons: if con not in self._active_constraints: raise ValueError( f'Cannot remove constraint {con.name} - it was not added' ) for v in self._vars_referenced_by_con[con]: - self._referenced_variables[id(v)][0].pop(con) + self._referenced_variables[v][0].remove(con) for p in self._params_referenced_by_con[con]: - self._referenced_params[id(p)][0].pop(con) - vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) - params_to_check.update( - (id(p), p) for p in self._params_referenced_by_con[con] - ) - del self._active_constraints[con] + self._referenced_params[p][0].remove(con) + vars_to_check.update(self._vars_referenced_by_con[con]) + params_to_check.update(self._params_referenced_by_con[con]) + self._active_constraints.pop(con) self._named_expressions.pop(con, None) self._external_functions.pop(con, None) - del self._vars_referenced_by_con[con] - del self._params_referenced_by_con[con] - self._check_to_remove_vars(list(vars_to_check.values())) - self._check_to_remove_params(list(params_to_check.values())) + self._vars_referenced_by_con.pop(con) + self._params_referenced_by_con.pop(con) + self._check_to_remove_vars(vars_to_check) + self._check_to_remove_params(params_to_check) - def remove_sos_constraints(self, cons: List[SOSConstraintData]): + def remove_sos_constraints(self, cons: Collection[SOSConstraintData]): for obs in self._observers: - obs.remove_sos_constraints(cons) - vars_to_check = {} - params_to_check = {} + obs._update_sos_constraints({c: Reason.removed for c in cons}) + vars_to_check = ComponentSet() + params_to_check = ComponentSet() for con in cons: if con not in self._active_sos: raise ValueError( f'Cannot remove constraint {con.name} - it was not added' ) for v in self._vars_referenced_by_con[con]: - self._referenced_variables[id(v)][1].pop(con) + self._referenced_variables[v][1].remove(con) for p in self._params_referenced_by_con[con]: - self._referenced_params[id(p)][1].pop(con) - vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) - params_to_check.update( - (id(p), p) for p in self._params_referenced_by_con[con] - ) - del self._active_sos[con] - del self._vars_referenced_by_con[con] - del self._params_referenced_by_con[con] - self._check_to_remove_vars(list(vars_to_check.values())) - self._check_to_remove_params(list(params_to_check.values())) - - def remove_variables(self, variables: List[VarData]): + self._referenced_params[p][1].remove(con) + vars_to_check.update(self._vars_referenced_by_con[con]) + params_to_check.update(self._params_referenced_by_con[con]) + self._active_sos.pop(con) + self._vars_referenced_by_con.pop(con) + self._params_referenced_by_con.pop(con) + self._check_to_remove_vars(vars_to_check) + self._check_to_remove_params(params_to_check) + + def remove_variables(self, variables: Collection[VarData]): for obs in self._observers: - obs.remove_variables(variables) - params_to_check = {} + obs._update_variables(ComponentMap((v, Reason.removed) for v in variables)) + params_to_check = ComponentSet() for v in variables: - v_id = id(v) - if v_id not in self._referenced_variables: + if v not in self._referenced_variables: raise ValueError( f'Cannot remove variable {v.name} - it has not been added' ) - if v_id in self._params_referenced_by_var: - for p in self._params_referenced_by_var[v_id]: - self._referenced_params[id(p)][3].pop(v_id) - params_to_check.update( - (id(p), p) for p in self._params_referenced_by_var[v_id] - ) - self._params_referenced_by_var.pop(v_id) - cons_using, sos_using, obj_using = self._referenced_variables[v_id] - if cons_using or sos_using or obj_using: + if v in self._params_referenced_by_var: + for p in self._params_referenced_by_var[v]: + self._referenced_params[p][3].remove(v) + params_to_check.update(self._params_referenced_by_var[v]) + self._params_referenced_by_var.pop(v) + if any(self._referenced_variables[v]): raise ValueError( f'Cannot remove variable {v.name} - it is still being used by constraints/objectives' ) - del self._referenced_variables[v_id] - del self._vars[v_id] - self._check_to_remove_params(list(params_to_check.values())) + self._referenced_variables.pop(v) + self._vars.pop(v) + self._check_to_remove_params(params_to_check) - def remove_parameters(self, params: List[ParamData]): + def remove_parameters(self, params: Collection[ParamData]): for obs in self._observers: - obs.remove_parameters(params) + obs._update_parameters(ComponentMap((p, Reason.removed) for p in params)) for p in params: - p_id = id(p) - if p_id not in self._referenced_params: + if p not in self._referenced_params: raise ValueError( f'Cannot remove parameter {p.name} - it has not been added' ) - cons_using, sos_using, obj_using, vars_using = self._referenced_params[p_id] - if cons_using or sos_using or obj_using or vars_using: + if any(self._referenced_params[p]): raise ValueError( f'Cannot remove parameter {p.name} - it is still being used by constraints/objectives' ) - del self._referenced_params[p_id] - del self._params[p_id] + self._referenced_params.pop(p) + self._params.pop(p) - def update_variables(self, variables: List[VarData], reasons: List[Reason]): + def update_variables(self, variables: Mapping[VarData, Reason]): for v in variables: - self._vars[id(v)] = ( - v, + self._vars[v] = ( v._lb, v._ub, v.fixed, @@ -918,58 +842,57 @@ def update_variables(self, variables: List[VarData], reasons: List[Reason]): v.value, ) for obs in self._observers: - obs.update_variables(variables, reasons) + obs._update_variables(variables) - def update_parameters(self, params): + def update_parameters(self, params: Mapping[ParamData, Reason]): for p in params: - self._params[id(p)] = (p, p.value) + self._params[p] = p.value for obs in self._observers: - obs.update_parameters(params) - - def _check_for_new_or_removed_sos(self): - new_sos = [] - old_sos = [] - current_sos_dict = { - c: None - for c in self._model.component_data_objects( + obs._update_parameters(params) + + def _check_for_new_or_removed_sos(self, sos_to_update=None): + if sos_to_update is None: + sos_to_update = defaultdict(_default_reason) + current_sos_set = OrderedSet( + self._model.component_data_objects( SOSConstraint, descend_into=True, active=True ) - } - for c in current_sos_dict.keys(): + ) + for c in current_sos_set: if c not in self._active_sos: - new_sos.append(c) + sos_to_update[c] |= Reason.added for c in self._active_sos: - if c not in current_sos_dict: - old_sos.append(c) - return new_sos, old_sos - - def _check_for_new_or_removed_constraints(self): - new_cons = [] - old_cons = [] - current_cons_dict = { - c: None - for c in self._model.component_data_objects( + if c not in current_sos_set: + sos_to_update[c] |= Reason.removed + return sos_to_update + + def _check_for_new_or_removed_constraints(self, cons_to_update=None): + if cons_to_update is None: + cons_to_update = defaultdict(_default_reason) + current_cons_set = OrderedSet( + self._model.component_data_objects( Constraint, descend_into=True, active=True ) - } - for c in current_cons_dict.keys(): + ) + for c in current_cons_set: if c not in self._active_constraints: - new_cons.append(c) + cons_to_update[c] |= Reason.added for c in self._active_constraints: - if c not in current_cons_dict: - old_cons.append(c) - return new_cons, old_cons + if c not in current_cons_set: + cons_to_update[c] |= Reason.removed + return cons_to_update - def _check_for_modified_sos(self): - sos_to_update = [] + def _check_for_modified_sos(self, sos_to_update=None): + if sos_to_update is None: + sos_to_update = defaultdict(_default_reason) for c, (old_vlist, old_plist) in self._active_sos.items(): sos_items = list(c.get_items()) new_vlist = [i[0] for i in sos_items] new_plist = [i[1] for i in sos_items] if len(old_vlist) != len(new_vlist): - sos_to_update.append(c) + sos_to_update[c] |= Reason.sos_items elif len(old_plist) != len(new_plist): - sos_to_update.append(c) + sos_to_update[c] |= Reason.sos_items else: needs_update = False for v1, v2 in zip(old_vlist, new_vlist): @@ -982,24 +905,23 @@ def _check_for_modified_sos(self): if needs_update: break if needs_update: - sos_to_update.append(c) + sos_to_update[c] |= Reason.sos_items return sos_to_update - def _check_for_modified_constraints(self): - cons_to_update = [] + def _check_for_modified_constraints(self, cons_to_update=None): + if cons_to_update is None: + cons_to_update = defaultdict(_default_reason) for c, expr in self._active_constraints.items(): if c.expr is not expr: - cons_to_update.append(c) + cons_to_update[c] |= Reason.expr return cons_to_update - def _check_for_var_changes(self): - vars_to_update = [] - reasons = [] - for vid, (v, _lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): - reason = Reason(0) - if _fixed and not v.fixed: - reason = reason | Reason.unfixed - elif not _fixed and v.fixed: + def _check_for_var_changes(self, vars_to_update=None): + if vars_to_update is None: + vars_to_update = DefaultComponentMap(_default_reason) + for v, (_lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): + reason = Reason.no_change + if _fixed != v.fixed: reason = reason | Reason.fixed elif _fixed and (v.value != _value): reason = reason | Reason.value @@ -1008,54 +930,58 @@ def _check_for_var_changes(self): if _domain_interval != v.domain.get_interval(): reason = reason | Reason.domain if reason: - vars_to_update.append(v) - reasons.append(reason) - return vars_to_update, reasons + vars_to_update[v] |= reason + return vars_to_update - def _check_for_param_changes(self): - params_to_update = [] - for _, (p, val) in self._params.items(): + def _check_for_param_changes(self, params_to_update=None): + if params_to_update is None: + params_to_update = DefaultComponentMap(_default_reason) + for p, val in self._params.items(): if p.value != val: - params_to_update.append(p) + params_to_update[p] |= Reason.value return params_to_update - def _check_for_named_expression_changes(self): - cons_to_update = [] + def _check_for_named_expression_changes(self, cons_to_update=None, objs_to_update=None): + if cons_to_update is None: + cons_to_update = defaultdict(_default_reason) + if objs_to_update is None: + objs_to_update = DefaultComponentMap(_default_reason) for con, ne_list in self._named_expressions.items(): for named_expr, old_expr in ne_list: if named_expr.expr is not old_expr: - cons_to_update.append(con) + cons_to_update[con] |= Reason.expr break - objs_to_update = [] - for obj_id, ne_list in self._obj_named_expressions.items(): + for obj, ne_list in self._obj_named_expressions.items(): for named_expr, old_expr in ne_list: if named_expr.expr is not old_expr: - objs_to_update.append(self._objectives[obj_id][0]) + objs_to_update[obj] |= Reason.expr break return cons_to_update, objs_to_update - def _check_for_new_or_removed_objectives(self): - new_objs = [] - old_objs = [] - current_objs_dict = { - id(obj): obj - for obj in self._model.component_data_objects( + def _check_for_new_or_removed_objectives(self, objs_to_update=None): + if objs_to_update is None: + objs_to_update = DefaultComponentMap(_default_reason) + current_objs_set = ComponentSet( + self._model.component_data_objects( Objective, descend_into=True, active=True ) - } - for obj_id, obj in current_objs_dict.items(): - if obj_id not in self._objectives: - new_objs.append(obj) - for obj_id, (obj, obj_expr, obj_sense) in self._objectives.items(): - if obj_id not in current_objs_dict: - old_objs.append(obj) - return new_objs, old_objs - - def _check_for_modified_objectives(self): - objs_to_update = [] - for obj_id, (obj, obj_expr, obj_sense) in self._objectives.items(): - if obj.expr is not obj_expr or obj.sense != obj_sense: - objs_to_update.append(obj) + ) + for obj in current_objs_set: + if obj not in self._objectives: + objs_to_update[obj] |= Reason.added + for obj, (obj_expr, obj_sense) in self._objectives.items(): + if obj not in current_objs_set: + objs_to_update[obj] |= Reason.removed + return objs_to_update + + def _check_for_modified_objectives(self, objs_to_update=None): + if objs_to_update is None: + objs_to_update = DefaultComponentMap(_default_reason) + for obj, (obj_expr, obj_sense) in self._objectives.items(): + if obj.expr is not obj_expr: + objs_to_update[obj] |= Reason.expr + if obj.sense != obj_sense: + objs_to_update[obj] |= Reason.sense return objs_to_update def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): @@ -1092,7 +1018,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.update_vars: timer.start('vars') - vars_to_update, reasons = self._check_for_var_changes() + vars_to_update = self._check_for_var_changes() if vars_to_update: self.update_variables(vars_to_update, reasons) timer.stop('vars') From 3408913f029b98669d29f57f829c702ca88d1c70 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 20 Oct 2025 22:55:34 -0600 Subject: [PATCH 39/50] observer: refactor --- pyomo/contrib/observer/component_collector.py | 27 +- pyomo/contrib/observer/model_observer.py | 764 +++++++++++------- 2 files changed, 491 insertions(+), 300 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index d30bb128758..7d756b52d7c 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -33,25 +33,26 @@ from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression from pyomo.repn.util import ExitNodeDispatcher +from pyomo.common.collections import ComponentSet def handle_var(node, collector): - collector.variables[id(node)] = node + collector.variables.add(node) return None def handle_param(node, collector): - collector.params[id(node)] = node + collector.params.add(node) return None def handle_named_expression(node, collector): - collector.named_expressions[id(node)] = node + collector.named_expressions.add(node) return None def handle_external_function(node, collector): - collector.external_functions[id(node)] = node + collector.external_functions.add(node) return None @@ -85,17 +86,17 @@ def handle_skip(node, collector): class _ComponentFromExprCollector(StreamBasedExpressionVisitor): def __init__(self, **kwds): - self.named_expressions = {} - self.variables = {} - self.params = {} - self.external_functions = {} + self.named_expressions = ComponentSet() + self.variables = ComponentSet() + self.params = ComponentSet() + self.external_functions = ComponentSet() super().__init__(**kwds) def exitNode(self, node, data): return collector_handlers[node.__class__](node, self) def beforeChild(self, node, child, child_idx): - if id(child) in self.named_expressions: + if child in self.named_expressions: return False, None return True, None @@ -107,8 +108,8 @@ def collect_components_from_expr(expr): _visitor.__init__() _visitor.walk_expression(expr) return ( - list(_visitor.named_expressions.values()), - list(_visitor.variables.values()), - list(_visitor.params.values()), - list(_visitor.external_functions.values()), + _visitor.named_expressions, + _visitor.variables, + _visitor.params, + _visitor.external_functions, ) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 356f55612be..807f9566f6d 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -9,6 +9,7 @@ # This software is distributed under the 3-clause BSD License. # __________________________________________________________________________ +from __future__ import annotations import abc from typing import List, Sequence, Optional, Mapping, MutableMapping, MutableSet, Tuple, Collection, Union @@ -49,9 +50,9 @@ # - modified constraint expressions (relies on expressions being immutable) # - modified objective expressions (relies on expressions being immutable) # - modified objective sense -# - changes to variable bounds, domains, and "fixed" flags +# - changes to variable bounds, domains, "fixed" flags, and values for fixed variables # - changes to named expressions (relies on expressions being immutable) -# - changes to parameter values and fixed variable values +# - changes to parameter values _param_types = {ParamData, ScalarParam} @@ -281,12 +282,39 @@ def _default_reason(): class _Updates: - def __init__(self) -> None: + def __init__(self, observers: Collection[Observer]) -> None: self.vars_to_update = DefaultComponentMap(_default_reason) self.params_to_update = DefaultComponentMap(_default_reason) self.cons_to_update = defaultdict(_default_reason) self.sos_to_update = defaultdict(_default_reason) self.objs_to_update = DefaultComponentMap(_default_reason) + self.observers = observers + + def run(self): + # split up new, removed, and modified variables + new_vars = ComponentMap((k, v) for k, v in self.vars_to_update.items() if v & Reason.added) + other_vars = ComponentMap((k, v) for k, v in self.vars_to_update.items() if not (v & Reason.added)) + + new_params = ComponentMap((k, v) for k, v in self.params_to_update.items() if v & Reason.added) + other_params = ComponentMap((k, v) for k, v in self.params_to_update.items() if not (v & Reason.added)) + + for obs in observers: + if new_vars: + obs._update_variables(new_vars) + if new_params: + obs._update_parameters(new_params) + if self.cons_to_update: + obs._update_constraints(self.cons_to_update) + if self.sos_to_update: + obs._update_sos_constraints(self.sos_to_update) + if self.objs_to_update: + obs._update_objectives(self.objs_to_update) + if other_vars: + obs._update_variables(other_vars) + if other_params: + obs._update_parameters(other_params) + + self.clear() def clear(self): self.vars_to_update.clear() @@ -296,6 +324,27 @@ def clear(self): self.objs_to_update.clear() +""" +There are three stages: +- identification of differences between the model and the internal data structures of the Change Detector +- synchronization of the model with the internal data structures of the ChangeDetector +- notification of the observers + +The first two really happen at the same time + +Update order when notifying the observers: + - add new variables + - add new constraints + - add new objectives + - remove old constraints + - remove old objectives + - remove old variables + - update modified constraints + - update modified objectives + - update modified variables +""" + + class ModelChangeDetector: """ This class "watches" a pyomo model and notifies the observers when any @@ -484,20 +533,20 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): ] = ComponentMap() self._vars_referenced_by_con: MutableMapping[ - Union[ConstraintData, SOSConstraintData], List[VarData] + Union[ConstraintData, SOSConstraintData], MutableSet[VarData] ] = {} self._vars_referenced_by_obj: MutableMapping[ - ObjectiveData, List[VarData] + ObjectiveData, MutableSet[VarData] ] = ComponentMap() self._params_referenced_by_con: MutableMapping[ - Union[ConstraintData, SOSConstraintData], List[ParamData] + Union[ConstraintData, SOSConstraintData], MutableSet[ParamData] ] = {} # for when parameters show up in variable bounds self._params_referenced_by_var: MutableMapping[ - VarData, List[ParamData] + VarData, MutableSet[ParamData] ] = ComponentMap() self._params_referenced_by_obj: MutableMapping[ - ObjectiveData, List[ParamData] + ObjectiveData, MutableSet[ParamData] ] = ComponentMap() self.config: AutoUpdateConfig = AutoUpdateConfig()( @@ -509,11 +558,11 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): self._model: BlockData = model self._set_instance() - def add_variables(self, variables: Collection[VarData]): - params_to_check = ComponentSet() + def _add_variables(self, variables: Collection[VarData]): for v in variables: if v in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') + self._updates.vars_to_update[v] |= Reason.added self._referenced_variables[v] = (OrderedSet(), OrderedSet(), ComponentSet()) self._vars[v] = ( v._lb, @@ -541,172 +590,158 @@ def add_variables(self, variables: Collection[VarData]): raise NotImplementedError( 'ModelChangeDetector does not support external functions in the bounds of other variables' ) - params_to_check.update(parameters) ref_params.update(parameters) + self._params_referenced_by_var[v] = ref_params if ref_params: - self._params_referenced_by_var[v] = list(ref_params) - self._check_for_new_params(params_to_check) - for v in variables: - if v not in self._params_referenced_by_var: - continue - parameters = self._params_referenced_by_var[v] - for p in parameters: - self._referenced_params[p][3].add(v) - for obs in self._observers: - obs._update_variables(ComponentMap((v, Reason.added) for v in variables)) + self._check_for_new_params(ref_params) + for p in ref_params: + self._referenced_params[p][3].add(v) - def add_parameters(self, params: Collection[ParamData]): + def add_variables(self, variables: Collection[VarData]): + self._add_variables(variables) + self._updates.run() + + def _add_parameters(self, params: Collection[ParamData]): for p in params: if p in self._referenced_params: raise ValueError(f'Parameter {p.name} has already been added') + self._updates.params_to_update[p] |= Reason.added self._referenced_params[p] = (OrderedSet(), OrderedSet(), ComponentSet(), ComponentSet()) self._params[p] = p.value - for obs in self._observers: - obs._update_parameters(ComponentMap((p, Reason.added) for p in params)) + + def add_parameters(self, params: Collection[ParamData]): + self._add_parameters(params) + self._updates.run() def _check_for_new_vars(self, variables: Collection[VarData]): new_vars = ComponentSet( v for v in variables if v not in self._referenced_variables ) - self.add_variables(new_vars) + self._add_variables(new_vars) def _check_to_remove_vars(self, variables: Collection[VarData]): vars_to_remove = ComponentSet() for v in variables: if not any(self._referenced_variables[v]): vars_to_remove.add(v) - self.remove_variables(vars_to_remove) + self._remove_variables(vars_to_remove) def _check_for_new_params(self, params: Collection[ParamData]): new_params = ComponentSet( p for p in params if p not in self._referenced_params ) - self.add_parameters(new_params) + self._add_parameters(new_params) def _check_to_remove_params(self, params: Collection[ParamData]): params_to_remove = ComponentSet() for p in params: if not any(self._referenced_params[p]): params_to_remove.add(p) - self.remove_parameters(params_to_remove) + self._remove_parameters(params_to_remove) - def add_constraints(self, cons: Collection[ConstraintData]): - vars_to_check = ComponentSet() - params_to_check = ComponentSet() + def _add_constraints(self, cons: Collection[ConstraintData]): for con in cons: if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') + self._updates.cons_to_update[con] |= Reason.added self._active_constraints[con] = con.expr (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(con.expr) ) - vars_to_check.update(variables) - params_to_check.update(parameters) + self._check_for_new_vars(variables) + self._check_for_new_params(parameters) if named_exprs: self._named_expressions[con] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = parameters - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) - for con in cons: - variables = self._vars_referenced_by_con[con] - parameters = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[v][0].add(con) for p in parameters: self._referenced_params[p][0].add(con) - for obs in self._observers: - obs._update_constraints({c: Reason.added for c in cons}) - def add_sos_constraints(self, cons: Collection[SOSConstraintData]): - vars_to_check = ComponentSet() - params_to_check = ComponentSet() + def add_constraints(self, cons: Collection[ConstraintData]): + self._add_constraints(cons) + self._updates.run() + + def _add_sos_constraints(self, cons: Collection[SOSConstraintData]): for con in cons: if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') + self._updates.sos_to_update[con] |= Reason.added sos_items = list(con.get_items()) self._active_sos[con] = ( [i[0] for i in sos_items], [i[1] for i in sos_items], ) - variables = [] - params = [] + variables = ComponentSet() + params = ComponentSet() for v, p in sos_items: - variables.append(v) + variables.add(v) if type(p) in native_numeric_types: continue if p.is_parameter_type(): - params.append(p) - vars_to_check.update(variables) - params_to_check.update(params) + params.add(p) + self._check_for_new_vars(variables) + self._check_for_new_params(params) self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) - for con in cons: - variables = self._vars_referenced_by_con[con] - params = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[v][1].add(con) for p in params: self._referenced_params[p][1].add(con) - for obs in self._observers: - obs._update_sos_constraints(ComponentMap((c, Reason.added) for c in cons)) - def add_objectives(self, objs: Collection[ObjectiveData]): - vars_to_check = ComponentSet() - params_to_check = ComponentSet() + def add_sos_constraints(self, cons: Collection[SOSConstraintData]): + self._add_sos_constraints(cons) + self._updates.run() + + def _add_objectives(self, objs: Collection[ObjectiveData]): for obj in objs: + self._updates.objs_to_update[obj] |= Reason.added self._objectives[obj] = (obj.expr, obj.sense) (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(obj.expr) ) - vars_to_check.update(variables) - params_to_check.update(parameters) + self._check_for_new_vars(variables) + self._check_for_new_params(parameters) if named_exprs: self._obj_named_expressions[obj] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[obj] = external_functions self._vars_referenced_by_obj[obj] = variables self._params_referenced_by_obj[obj] = parameters - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) - for obj in objs: - variables = self._vars_referenced_by_obj[obj] - parameters = self._params_referenced_by_obj[obj] for v in variables: self._referenced_variables[v][2].add(obj) for p in parameters: self._referenced_params[p][2].add(obj) - for obs in self._observers: - obs._update_objectives(ComponentMap((obj, Reason.added) for obj in objs)) - def remove_objectives(self, objs: Collection[ObjectiveData]): - for obs in self._observers: - obs._update_objectives(ComponentMap((obj, Reason.removed) for obj in objs)) + def add_objectives(self, objs: Collection[ObjectiveData]): + self._add_objectives(objs) + self._updates.run() - vars_to_check = ComponentSet() - params_to_check = ComponentSet() + def _remove_objectives(self, objs: Collection[ObjectiveData]): for obj in objs: if obj not in self._objectives: raise ValueError( f'cannot remove objective {obj.name} - it was not added' ) + self._updates.objs_to_update[obj] |= Reason.removed for v in self._vars_referenced_by_obj[obj]: self._referenced_variables[v][2].remove(obj) for p in self._params_referenced_by_obj[obj]: self._referenced_params[p][2].remove(obj) - vars_to_check.update(self._vars_referenced_by_obj[obj]) - params_to_check.update(self._params_referenced_by_obj[obj]) + self._check_to_remove_vars(self._vars_referenced_by_obj[obj]) + self._check_to_remove_params(self._params_referenced_by_obj[obj]) del self._objectives[obj] self._obj_named_expressions.pop(obj, None) self._external_functions.pop(obj, None) self._vars_referenced_by_obj.pop(obj) self._params_referenced_by_obj.pop(obj) - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + + def remove_objectives(self, objs: Collection[ObjectiveData]): + self._remove_objectives(objs) + self._updates.run() def _check_for_unknown_active_components(self): for ctype in self._model.collect_ctypes(active=True, descend_into=True): @@ -749,82 +784,79 @@ def _set_instance(self): ) ) - def remove_constraints(self, cons: Collection[ConstraintData]): - for obs in self._observers: - obs._update_constraints({c: Reason.removed for c in cons}) - vars_to_check = ComponentSet() - params_to_check = ComponentSet() + def _remove_constraints(self, cons: Collection[ConstraintData]): for con in cons: if con not in self._active_constraints: raise ValueError( f'Cannot remove constraint {con.name} - it was not added' ) + self._updates.cons_to_update[con] |= Reason.removed for v in self._vars_referenced_by_con[con]: self._referenced_variables[v][0].remove(con) for p in self._params_referenced_by_con[con]: self._referenced_params[p][0].remove(con) - vars_to_check.update(self._vars_referenced_by_con[con]) - params_to_check.update(self._params_referenced_by_con[con]) + self._check_to_remove_vars(self._vars_referenced_by_con[con]) + self._check_to_remove_params(self._params_referenced_by_con[con]) self._active_constraints.pop(con) self._named_expressions.pop(con, None) self._external_functions.pop(con, None) self._vars_referenced_by_con.pop(con) self._params_referenced_by_con.pop(con) - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) - def remove_sos_constraints(self, cons: Collection[SOSConstraintData]): - for obs in self._observers: - obs._update_sos_constraints({c: Reason.removed for c in cons}) - vars_to_check = ComponentSet() - params_to_check = ComponentSet() + def remove_constraints(self, cons: Collection[ConstraintData]): + self._remove_constraints(cons) + self._updates.run() + + def _remove_sos_constraints(self, cons: Collection[SOSConstraintData]): for con in cons: if con not in self._active_sos: raise ValueError( f'Cannot remove constraint {con.name} - it was not added' ) + self._updates.sos_to_update[con] |= Reason.removed for v in self._vars_referenced_by_con[con]: self._referenced_variables[v][1].remove(con) for p in self._params_referenced_by_con[con]: self._referenced_params[p][1].remove(con) - vars_to_check.update(self._vars_referenced_by_con[con]) - params_to_check.update(self._params_referenced_by_con[con]) + self._check_to_remove_vars(self._vars_referenced_by_con[con]) + self._check_to_remove_params(self._params_referenced_by_con[con]) self._active_sos.pop(con) self._vars_referenced_by_con.pop(con) self._params_referenced_by_con.pop(con) - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) - def remove_variables(self, variables: Collection[VarData]): - for obs in self._observers: - obs._update_variables(ComponentMap((v, Reason.removed) for v in variables)) - params_to_check = ComponentSet() + def remove_sos_constraints(self, cons: Collection[SOSConstraintData]): + self._remove_sos_constraints(cons) + self._updates.run() + + def _remove_variables(self, variables: Collection[VarData]): for v in variables: if v not in self._referenced_variables: raise ValueError( f'Cannot remove variable {v.name} - it has not been added' ) - if v in self._params_referenced_by_var: - for p in self._params_referenced_by_var[v]: - self._referenced_params[p][3].remove(v) - params_to_check.update(self._params_referenced_by_var[v]) - self._params_referenced_by_var.pop(v) + self._updates.vars_to_update[v] |= Reason.removed + for p in self._params_referenced_by_var[v]: + self._referenced_params[p][3].remove(v) + self._check_to_remove_params(self._params_referenced_by_var[v]) + self._params_referenced_by_var.pop(v) if any(self._referenced_variables[v]): raise ValueError( f'Cannot remove variable {v.name} - it is still being used by constraints/objectives' ) self._referenced_variables.pop(v) self._vars.pop(v) - self._check_to_remove_params(params_to_check) + + def remove_variables(self, variables: Collection[VarData]): + self._remove_variables(variables) + self._updates.run() - def remove_parameters(self, params: Collection[ParamData]): - for obs in self._observers: - obs._update_parameters(ComponentMap((p, Reason.removed) for p in params)) + def _remove_parameters(self, params: Collection[ParamData]): for p in params: if p not in self._referenced_params: raise ValueError( f'Cannot remove parameter {p.name} - it has not been added' ) + self._updates.params_to_update[p] |= Reason.removed if any(self._referenced_params[p]): raise ValueError( f'Cannot remove parameter {p.name} - it is still being used by constraints/objectives' @@ -832,27 +864,287 @@ def remove_parameters(self, params: Collection[ParamData]): self._referenced_params.pop(p) self._params.pop(p) - def update_variables(self, variables: Mapping[VarData, Reason]): - for v in variables: - self._vars[v] = ( - v._lb, - v._ub, - v.fixed, - v.domain.get_interval(), - v.value, + def remove_parameters(self, params: Collection[ParamData]): + self._remove_parameters(params) + self._updates.run() + + def _update_var_bounds(self, v: VarData): + ref_params = ComponentSet() + for bnd in (v._lb, v._ub): + if bnd is None or type(bnd) in native_numeric_types: + continue + (named_exprs, _vars, parameters, external_functions) = ( + collect_components_from_expr(bnd) ) - for obs in self._observers: - obs._update_variables(variables) + if _vars: + raise NotImplementedError( + 'ModelChangeDetector does not support variables in the bounds of other variables' + ) + if named_exprs: + raise NotImplementedError( + 'ModelChangeDetector does not support Expressions in the bounds of other variables' + ) + if external_functions: + raise NotImplementedError( + 'ModelChangeDetector does not support external functions in the bounds of other variables' + ) + ref_params.update(parameters) + + _ref_params = self._params_referenced_by_var[v] + new_params = ref_params - _ref_params + old_params = _ref_params - ref_params + + self._params_referenced_by_var[v] = ref_params + + if new_params: + self._check_for_new_params(new_params) + if old_params: + self._check_to_remove_params(old_params) + + for p in new_params: + self._referenced_params[p][3].add(v) + for p in old_params: + self._referenced_params[p][3].remove(v) + + def _update_variables(self, variables: Optional[Collection[VarData]] = None): + if variables is None: + variables = self._vars + for v in variables: + _lb, _ub, _fixed, _domain_interval, _value = self._vars[v] + lb, ub, fixed, domain_interval, value = v._lb, v._ub, v.fixed, v.domain.get_interval(), v.value + reason = Reason.no_change + if _fixed != fixed: + reason |= Reason.fixed + elif _fixed and (value != _value): + reason |= Reason.value + if lb is not _lb or ub is not _ub: + reason |= Reason.bounds + if _domain_interval != domain_interval: + reason |= Reason.domain + if reason: + self._updates.vars_to_update[v] |= reason + self._vars[v] = ( + lb, + ub, + fixed, + domain_interval, + value, + ) + if reason & Reason.bounds: + self._update_var_bounds(v) + + def update_variables(self, variables: Optional[Collection[VarData]] = None): + self._update_variables(variables) + self._updates.run() - def update_parameters(self, params: Mapping[ParamData, Reason]): + def _update_parameters(self, params: Optional[Collection[ParamData]] = None): + if params is None: + params = self._params for p in params: - self._params[p] = p.value - for obs in self._observers: - obs._update_parameters(params) + _val = self._params[p] + val = p.value + reason = Reason.no_change + if _val != val: + reason |= Reason.value + if reason: + self._updates.params_to_update[p] |= reason + self._params[p] = val + + def update_parameters(self, params: Optional[Collection[ParamData]]): + self._update_parameters(params) + self._updates.run() - def _check_for_new_or_removed_sos(self, sos_to_update=None): - if sos_to_update is None: - sos_to_update = defaultdict(_default_reason) + def _update_con(self, con: ConstraintData): + self._active_constraints[con] = con.expr + (named_exprs, variables, parameters, external_functions) = ( + collect_components_from_expr(con.expr) + ) + if named_exprs: + self._named_expressions[con] = [(e, e.expr) for e in named_exprs] + else: + self._named_expressions.pop(con, None) + if external_functions: + self._external_functions[con] = external_functions + else: + self._external_functions.pop(con, None) + + _variables = self._vars_referenced_by_con[con] + _parameters = self._params_referenced_by_con[con] + new_vars = variables - _variables + old_vars = _variables - variables + new_params = parameters - _parameters + old_params = _parameters - parameters + + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = parameters + + if new_vars: + self._check_for_new_vars(new_vars) + if old_vars: + self._check_to_remove_vars(old_vars) + if new_params: + self._check_for_new_params(new_params) + if old_params: + self._check_to_remove_params(old_params) + + for v in new_vars: + self._referenced_variables[v][0].add(con) + for v in old_vars: + self._referenced_variables[v][0].remove(con) + for p in new_params: + self._referenced_params[p][0].add(con) + for p in old_params: + self._referenced_params[p][0].remove(con) + + def _update_constraints(self, cons: Optional[Collection[ConstraintData]] = None): + if cons is None: + cons = self._active_constraints + for c in cons: + reason = Reason.no_change + if c.expr is not self._active_constraints[c]: + reason |= Reason.expr + if reason: + self._updates.cons_to_update[c] |= reason + self._update_con(c) + + def update_constraints(self, cons: Optional[Collection[ConstraintData]] = None): + self._update_constraints(cons) + self._updates.run() + + def _update_sos_con(self, con: SOSConstraintData): + sos_items = list(con.get_items()) + self._active_sos[con] = ( + [i[0] for i in sos_items], + [i[1] for i in sos_items], + ) + variables = ComponentSet() + parameters = ComponentSet() + for v, p in sos_items: + variables.add(v) + parameters.add(p) + + _variables = self._vars_referenced_by_con[con] + _parameters = self._params_referenced_by_con[con] + new_vars = variables - _variables + old_vars = _variables - variables + new_params = parameters - _parameters + old_params = _parameters - parameters + + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = parameters + + if new_vars: + self._check_for_new_vars(new_vars) + if old_vars: + self._check_to_remove_vars(old_vars) + if new_params: + self._check_for_new_params(new_params) + if old_params: + self._check_to_remove_params(old_params) + + for v in new_vars: + self._referenced_variables[v][1].add(con) + for v in old_vars: + self._referenced_variables[v][1].remove(con) + for p in new_params: + self._referenced_params[p][1].add(con) + for p in old_params: + self._referenced_params[p][1].remove(con) + + def _update_sos_constraints(self, cons: Optional[Collection[SOSConstraintData]] = None): + if cons is None: + cons = self._active_sos + for c in cons: + reason = Reason.no_change + _vlist, _plist = self._active_sos[c] + sos_items = list(c.get_items()) + vlist = [i[0] for i in sos_items] + plist = [i[1] for i in sos_items] + needs_update = False + if len(_vlist) != len(vlist) or len(_plist) != len(plist): + needs_update = True + else: + for v1, v2 in zip(_vlist, vlist): + if v1 is not v2: + needs_update = True + break + for p1, p2 in zip(_plist, plist): + if p1 is not p2: + needs_update = True + break + if needs_update: + reason |= Reason.sos_items + self._updates.sos_to_update[c] |= reason + self._update_sos_con(c) + + def update_sos_constraints(self, cons: Optional[Collection[SOSConstraintData]] = None): + self._update_sos_constraints(cons) + self._updates.run() + + def _update_obj_expr(self, obj: ObjectiveData): + (named_exprs, variables, parameters, external_functions) = ( + collect_components_from_expr(obj.expr) + ) + if named_exprs: + self._obj_named_expressions[obj] = [(e, e.expr) for e in named_exprs] + else: + self._obj_named_expressions.pop(obj, None) + if external_functions: + self._external_functions[obj] = external_functions + else: + self._external_functions.pop(obj, None) + + _variables = self._vars_referenced_by_obj[obj] + _parameters = self._params_referenced_by_obj[obj] + new_vars = variables - _variables + old_vars = _variables - variables + new_params = parameters - _parameters + old_params = _parameters - parameters + + self._vars_referenced_by_obj[obj] = variables + self._params_referenced_by_obj[obj] = parameters + + if new_vars: + self._check_for_new_vars(new_vars) + if old_vars: + self._check_to_remove_vars(old_vars) + if new_params: + self._check_for_new_params(new_params) + if old_params: + self._check_to_remove_params(old_params) + + for v in new_vars: + self._referenced_variables[v][2].add(obj) + for v in old_vars: + self._referenced_variables[v][2].remove(obj) + for p in new_params: + self._referenced_params[p][2].add(obj) + for p in old_params: + self._referenced_params[p][2].remove(obj) + + def _update_objectives(self, objs: Optional[Collection[ObjectiveData]] = None): + if objs is None: + objs = self._objectives + for obj in objs: + reason = Reason.no_change + _expr, _sense = self._objectives[obj] + if _expr is not obj.expr: + reason |= Reason.expr + if _sense != obj.sense: + reason |= Reason.sense + if reason: + self._updates.objs_to_update[obj] |= reason + self._objectives[obj] = (obj.expr, obj.sense) + if reason & Reason.expr: + self._update_obj_expr(obj) + + def update_objectives(self, objs: Optional[Collection[ObjectiveData]] = None): + self._update_objectives(objs) + self._updates.run() + + def _check_for_new_or_removed_sos(self): + new_sos = [] + old_sos = [] current_sos_set = OrderedSet( self._model.component_data_objects( SOSConstraint, descend_into=True, active=True @@ -860,15 +1152,15 @@ def _check_for_new_or_removed_sos(self, sos_to_update=None): ) for c in current_sos_set: if c not in self._active_sos: - sos_to_update[c] |= Reason.added + new_sos.append(c) for c in self._active_sos: if c not in current_sos_set: - sos_to_update[c] |= Reason.removed - return sos_to_update + old_sos.append(c) + return new_sos, old_sos - def _check_for_new_or_removed_constraints(self, cons_to_update=None): - if cons_to_update is None: - cons_to_update = defaultdict(_default_reason) + def _check_for_new_or_removed_constraints(self): + new_cons = [] + old_cons = [] current_cons_set = OrderedSet( self._model.component_data_objects( Constraint, descend_into=True, active=True @@ -876,91 +1168,29 @@ def _check_for_new_or_removed_constraints(self, cons_to_update=None): ) for c in current_cons_set: if c not in self._active_constraints: - cons_to_update[c] |= Reason.added + new_cons.append(c) for c in self._active_constraints: if c not in current_cons_set: - cons_to_update[c] |= Reason.removed - return cons_to_update + old_cons.append(c) + return new_cons, old_cons - def _check_for_modified_sos(self, sos_to_update=None): - if sos_to_update is None: - sos_to_update = defaultdict(_default_reason) - for c, (old_vlist, old_plist) in self._active_sos.items(): - sos_items = list(c.get_items()) - new_vlist = [i[0] for i in sos_items] - new_plist = [i[1] for i in sos_items] - if len(old_vlist) != len(new_vlist): - sos_to_update[c] |= Reason.sos_items - elif len(old_plist) != len(new_plist): - sos_to_update[c] |= Reason.sos_items - else: - needs_update = False - for v1, v2 in zip(old_vlist, new_vlist): - if v1 is not v2: - needs_update = True - break - for p1, p2 in zip(old_plist, new_plist): - if p1 is not p2: - needs_update = True - if needs_update: - break - if needs_update: - sos_to_update[c] |= Reason.sos_items - return sos_to_update - - def _check_for_modified_constraints(self, cons_to_update=None): - if cons_to_update is None: - cons_to_update = defaultdict(_default_reason) - for c, expr in self._active_constraints.items(): - if c.expr is not expr: - cons_to_update[c] |= Reason.expr - return cons_to_update - - def _check_for_var_changes(self, vars_to_update=None): - if vars_to_update is None: - vars_to_update = DefaultComponentMap(_default_reason) - for v, (_lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): - reason = Reason.no_change - if _fixed != v.fixed: - reason = reason | Reason.fixed - elif _fixed and (v.value != _value): - reason = reason | Reason.value - if v._lb is not _lb or v._ub is not _ub: - reason = reason | Reason.bounds - if _domain_interval != v.domain.get_interval(): - reason = reason | Reason.domain - if reason: - vars_to_update[v] |= reason - return vars_to_update - - def _check_for_param_changes(self, params_to_update=None): - if params_to_update is None: - params_to_update = DefaultComponentMap(_default_reason) - for p, val in self._params.items(): - if p.value != val: - params_to_update[p] |= Reason.value - return params_to_update - - def _check_for_named_expression_changes(self, cons_to_update=None, objs_to_update=None): - if cons_to_update is None: - cons_to_update = defaultdict(_default_reason) - if objs_to_update is None: - objs_to_update = DefaultComponentMap(_default_reason) + def _check_for_named_expression_changes(self): for con, ne_list in self._named_expressions.items(): for named_expr, old_expr in ne_list: if named_expr.expr is not old_expr: - cons_to_update[con] |= Reason.expr + self._updates.cons_to_update[con] |= Reason.expr + self._update_con(con) break for obj, ne_list in self._obj_named_expressions.items(): for named_expr, old_expr in ne_list: if named_expr.expr is not old_expr: - objs_to_update[obj] |= Reason.expr + self._updates.objs_to_update[obj] |= Reason.expr + self._update_obj_expr(obj) break - return cons_to_update, objs_to_update - def _check_for_new_or_removed_objectives(self, objs_to_update=None): - if objs_to_update is None: - objs_to_update = DefaultComponentMap(_default_reason) + def _check_for_new_or_removed_objectives(self): + new_objs = [] + old_objs = [] current_objs_set = ComponentSet( self._model.component_data_objects( Objective, descend_into=True, active=True @@ -968,21 +1198,11 @@ def _check_for_new_or_removed_objectives(self, objs_to_update=None): ) for obj in current_objs_set: if obj not in self._objectives: - objs_to_update[obj] |= Reason.added - for obj, (obj_expr, obj_sense) in self._objectives.items(): + new_objs.append(obj) + for obj in self._objectives.keys(): if obj not in current_objs_set: - objs_to_update[obj] |= Reason.removed - return objs_to_update - - def _check_for_modified_objectives(self, objs_to_update=None): - if objs_to_update is None: - objs_to_update = DefaultComponentMap(_default_reason) - for obj, (obj_expr, obj_sense) in self._objectives.items(): - if obj.expr is not obj_expr: - objs_to_update[obj] |= Reason.expr - if obj.sense != obj_sense: - objs_to_update[obj] |= Reason.sense - return objs_to_update + old_objs.append(obj) + return new_objs, old_objs def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): """ @@ -1016,81 +1236,51 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): with PauseGC() as pgc: self._check_for_unknown_active_components() - if config.update_vars: - timer.start('vars') - vars_to_update = self._check_for_var_changes() - if vars_to_update: - self.update_variables(vars_to_update, reasons) - timer.stop('vars') - - if config.update_parameters: - timer.start('params') - params_to_update = self._check_for_param_changes() - if params_to_update: - self.update_parameters(params_to_update) - timer.stop('params') + if config.check_for_new_or_removed_constraints: + new_cons, old_cons = self._check_for_new_or_removed_constraints() + new_sos, old_sos = self._check_for_new_or_removed_sos() + else: + new_cons = [] + old_cons = [] + new_sos = [] + old_sos = [] - if config.update_named_expressions: - timer.start('named expressions') - cons_to_update, objs_to_update = ( - self._check_for_named_expression_changes() - ) - if cons_to_update: - self.remove_constraints(cons_to_update) - self.add_constraints(cons_to_update) - if objs_to_update: - self.remove_objectives(objs_to_update) - self.add_objectives(objs_to_update) - timer.stop('named expressions') + if config.check_for_new_or_removed_objectives: + new_objs, old_objs = self._check_for_new_or_removed_objectives() + else: + new_objs = [] + old_objs = [] + + if new_cons: + self._add_constraints(new_cons) + if new_sos: + self._add_sos_constraints(new_sos) + if new_objs: + self._add_objectives(new_objs) + + if old_cons: + self._remove_constraints(old_cons) + if old_sos: + self._remove_sos_constraints(old_sos) + if old_objs: + self._remove_objectives(old_objs) if config.update_constraints: - timer.start('cons') - cons_to_update = self._check_for_modified_constraints() - if cons_to_update: - self.remove_constraints(cons_to_update) - self.add_constraints(cons_to_update) - timer.stop('cons') - timer.start('sos') - sos_to_update = self._check_for_modified_sos() - if sos_to_update: - self.remove_sos_constraints(sos_to_update) - self.add_sos_constraints(sos_to_update) - timer.stop('sos') - + self._update_constraints() + self._update_sos_constraints() if config.update_objectives: - timer.start('objective') - objs_to_update = self._check_for_modified_objectives() - if objs_to_update: - self.remove_objectives(objs_to_update) - self.add_objectives(objs_to_update) - timer.stop('objective') + self._update_objectives() - if config.check_for_new_or_removed_constraints: - timer.start('sos') - new_sos, old_sos = self._check_for_new_or_removed_sos() - if new_sos: - self.add_sos_constraints(new_sos) - if old_sos: - self.remove_sos_constraints(old_sos) - timer.stop('sos') - timer.start('cons') - new_cons, old_cons = self._check_for_new_or_removed_constraints() - if new_cons: - self.add_constraints(new_cons) - if old_cons: - self.remove_constraints(old_cons) - timer.stop('cons') + if config.update_named_expressions: + self._check_for_named_expression_changes() - if config.check_for_new_or_removed_objectives: - timer.start('objective') - new_objs, old_objs = self._check_for_new_or_removed_objectives() - # many solvers require one objective, so we have to remove the - # old objective first - if old_objs: - self.remove_objectives(old_objs) - if new_objs: - self.add_objectives(new_objs) - timer.stop('objective') + if config.update_vars: + self._update_variables() + + if config.update_parameters: + self._update_parameters() + + self._updates.run() def get_variables_impacted_by_param(self, p: ParamData): return [self._vars[vid][0] for vid in self._referenced_params[id(p)][3]] From 9bee2bd740222eb116c79643769fb9d3f2ae4b74 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 21 Oct 2025 00:00:23 -0600 Subject: [PATCH 40/50] observer: typos --- pyomo/contrib/observer/model_observer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 807f9566f6d..678463b8f9b 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -23,6 +23,7 @@ from pyomo.core.base.objective import ObjectiveData, Objective from pyomo.core.base.block import BlockData, Block from pyomo.core.base.suffix import Suffix +from pyomo.core.base.component import ActiveComponent from pyomo.core.expr.numeric_expr import NumericValue from pyomo.core.expr.relational_expr import RelationalExpression from pyomo.common.collections import ComponentMap, ComponentSet, OrderedSet, DefaultComponentMap @@ -298,7 +299,7 @@ def run(self): new_params = ComponentMap((k, v) for k, v in self.params_to_update.items() if v & Reason.added) other_params = ComponentMap((k, v) for k, v in self.params_to_update.items() if not (v & Reason.added)) - for obs in observers: + for obs in self.observers: if new_vars: obs._update_variables(new_vars) if new_params: @@ -553,7 +554,7 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): value=kwds, preserve_implicit=True ) - self._updates = _Updates() + self._updates = _Updates(self._observers) self._model: BlockData = model self._set_instance() @@ -745,6 +746,8 @@ def remove_objectives(self, objs: Collection[ObjectiveData]): def _check_for_unknown_active_components(self): for ctype in self._model.collect_ctypes(active=True, descend_into=True): + if not issubclass(ctype, ActiveComponent): + continue if ctype in self._known_active_ctypes: continue if ctype is Suffix: @@ -754,7 +757,7 @@ def _check_for_unknown_active_components(self): continue raise NotImplementedError( f'ModelChangeDetector does not know how to ' - 'handle components with ctype {ctype}' + f'handle components with ctype {ctype}' ) def _set_instance(self): From 1568e1f95481248a97862d5f70cfe3acb51c7041 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 21 Oct 2025 00:11:18 -0600 Subject: [PATCH 41/50] observer: bug --- pyomo/contrib/observer/model_observer.py | 32 +++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 678463b8f9b..45ac1f861d5 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -901,14 +901,15 @@ def _update_var_bounds(self, v: VarData): if new_params: self._check_for_new_params(new_params) - if old_params: - self._check_to_remove_params(old_params) for p in new_params: self._referenced_params[p][3].add(v) for p in old_params: self._referenced_params[p][3].remove(v) + if old_params: + self._check_to_remove_params(old_params) + def _update_variables(self, variables: Optional[Collection[VarData]] = None): if variables is None: variables = self._vars @@ -983,12 +984,8 @@ def _update_con(self, con: ConstraintData): if new_vars: self._check_for_new_vars(new_vars) - if old_vars: - self._check_to_remove_vars(old_vars) if new_params: self._check_for_new_params(new_params) - if old_params: - self._check_to_remove_params(old_params) for v in new_vars: self._referenced_variables[v][0].add(con) @@ -999,6 +996,11 @@ def _update_con(self, con: ConstraintData): for p in old_params: self._referenced_params[p][0].remove(con) + if old_vars: + self._check_to_remove_vars(old_vars) + if old_params: + self._check_to_remove_params(old_params) + def _update_constraints(self, cons: Optional[Collection[ConstraintData]] = None): if cons is None: cons = self._active_constraints @@ -1038,12 +1040,8 @@ def _update_sos_con(self, con: SOSConstraintData): if new_vars: self._check_for_new_vars(new_vars) - if old_vars: - self._check_to_remove_vars(old_vars) if new_params: self._check_for_new_params(new_params) - if old_params: - self._check_to_remove_params(old_params) for v in new_vars: self._referenced_variables[v][1].add(con) @@ -1054,6 +1052,11 @@ def _update_sos_con(self, con: SOSConstraintData): for p in old_params: self._referenced_params[p][1].remove(con) + if old_vars: + self._check_to_remove_vars(old_vars) + if old_params: + self._check_to_remove_params(old_params) + def _update_sos_constraints(self, cons: Optional[Collection[SOSConstraintData]] = None): if cons is None: cons = self._active_sos @@ -1109,12 +1112,8 @@ def _update_obj_expr(self, obj: ObjectiveData): if new_vars: self._check_for_new_vars(new_vars) - if old_vars: - self._check_to_remove_vars(old_vars) if new_params: self._check_for_new_params(new_params) - if old_params: - self._check_to_remove_params(old_params) for v in new_vars: self._referenced_variables[v][2].add(obj) @@ -1125,6 +1124,11 @@ def _update_obj_expr(self, obj: ObjectiveData): for p in old_params: self._referenced_params[p][2].remove(obj) + if old_vars: + self._check_to_remove_vars(old_vars) + if old_params: + self._check_to_remove_params(old_params) + def _update_objectives(self, objs: Optional[Collection[ObjectiveData]] = None): if objs is None: objs = self._objectives From 817f274bbacab76c6078fd014e93b1501cfb8835 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 21 Oct 2025 00:30:55 -0600 Subject: [PATCH 42/50] observer: update tests --- pyomo/contrib/observer/model_observer.py | 5 +- .../observer/tests/test_change_detector.py | 259 ++++++------------ 2 files changed, 84 insertions(+), 180 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 45ac1f861d5..29101877811 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -1026,7 +1026,10 @@ def _update_sos_con(self, con: SOSConstraintData): parameters = ComponentSet() for v, p in sos_items: variables.add(v) - parameters.add(p) + if type(p) in native_numeric_types: + continue + if p.is_parameter_type(): + parameters.add(p) _variables = self._vars_referenced_by_con[con] _parameters = self._params_referenced_by_con[con] diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 1a22fe7f2a0..fbc42e0e591 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -10,7 +10,7 @@ # ___________________________________________________________________________ import logging -from typing import List +from typing import List, Mapping import pyomo.environ as pyo from pyomo.core.base.constraint import ConstraintData @@ -25,25 +25,25 @@ AutoUpdateConfig, Reason, ) -from pyomo.common.collections import ComponentMap +from pyomo.common.collections import DefaultComponentMap, ComponentMap logger = logging.getLogger(__name__) def make_count_dict(): - d = {'add': 0, 'remove': 0, 'update': 0, 'set': 0} + d = {i: 0 for i in Reason} return d class ObserverChecker(Observer): def __init__(self): super().__init__() - self.counts = ComponentMap() + self.counts = DefaultComponentMap(make_count_dict) """ counts is a mapping from component (e.g., variable) to another - mapping from string ('add', 'remove', 'update', or 'set') to an int that - indicates the number of times the corresponding method has been called + mapping from Reason to an int that indicates the number of times + the corresponding method has been called """ def check(self, expected): @@ -51,77 +51,31 @@ def check(self, expected): first=expected, second=self.counts, places=7 ) - def _process(self, comps, key): - for c in comps: - if c not in self.counts: - self.counts[c] = make_count_dict() - self.counts[c][key] += 1 - def pprint(self): for k, d in self.counts.items(): print(f'{k}:') for a, v in d.items(): print(f' {a}: {v}') - def add_variables(self, variables: List[VarData]): - for v in variables: - assert v.is_variable_type() - self._process(variables, 'add') - - def add_parameters(self, params: List[ParamData]): - for p in params: - assert p.is_parameter_type() - self._process(params, 'add') - - def add_constraints(self, cons: List[ConstraintData]): - for c in cons: - assert isinstance(c, ConstraintData) - self._process(cons, 'add') - - def add_sos_constraints(self, cons: List[SOSConstraintData]): - for c in cons: - assert isinstance(c, SOSConstraintData) - self._process(cons, 'add') - - def add_objectives(self, objs: List[ObjectiveData]): - for obj in objs: - assert isinstance(obj, ObjectiveData) - self._process(objs, 'add') - - def remove_constraints(self, cons: List[ConstraintData]): - for c in cons: - assert isinstance(c, ConstraintData) - self._process(cons, 'remove') - - def remove_objectives(self, objs: List[ObjectiveData]): - for obj in objs: - assert isinstance(obj, ObjectiveData) - self._process(objs, 'remove') - - def remove_sos_constraints(self, cons: List[SOSConstraintData]): - for c in cons: - assert isinstance(c, SOSConstraintData) - self._process(cons, 'remove') - - def remove_variables(self, variables: List[VarData]): - for v in variables: - assert v.is_variable_type() - self._process(variables, 'remove') - - def remove_parameters(self, params: List[ParamData]): - for p in params: - assert p.is_parameter_type() - self._process(params, 'remove') - - def update_variables(self, variables: List[VarData], reasons: List[Reason]): - for v in variables: - assert v.is_variable_type() - self._process(variables, 'update') - - def update_parameters(self, params: List[ParamData]): - for p in params: - assert p.is_parameter_type() - self._process(params, 'update') + def _update_variables(self, variables: Mapping[VarData, Reason]): + for v, reason in variables.items(): + self.counts[v][reason] += 1 + + def _update_parameters(self, params: Mapping[ParamData, Reason]): + for p, reason in params.items(): + self.counts[p][reason] += 1 + + def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): + for c, reason in cons.items(): + self.counts[c][reason] += 1 + + def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): + for c, reason in cons.items(): + self.counts[c][reason] += 1 + + def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): + for obj, reason in objs.items(): + self.counts[obj][reason] += 1 class TestChangeDetector(unittest.TestCase): @@ -134,63 +88,51 @@ def test_objective(self): obs = ObserverChecker() detector = ModelChangeDetector(m, [obs]) - expected = ComponentMap() + expected = DefaultComponentMap(make_count_dict) obs.check(expected) m.obj = pyo.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() - expected[m.obj] = make_count_dict() - expected[m.obj]['add'] += 1 - expected[m.x] = make_count_dict() - expected[m.x]['add'] += 1 - expected[m.y] = make_count_dict() - expected[m.y]['add'] += 1 - expected[m.p] = make_count_dict() - expected[m.p]['add'] += 1 + expected[m.obj][Reason.added] += 1 + expected[m.x][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.p][Reason.added] += 1 obs.check(expected) m.y.setlb(0) detector.update() - expected[m.y]['update'] += 1 + expected[m.y][Reason.bounds] += 1 obs.check(expected) m.x.fix(2) detector.update() - expected[m.x]['update'] += 1 + expected[m.x][Reason.fixed] += 1 obs.check(expected) m.x.unfix() detector.update() - expected[m.x]['update'] += 1 + expected[m.x][Reason.fixed] += 1 obs.check(expected) m.p.value = 2 detector.update() - expected[m.p]['update'] += 1 + expected[m.p][Reason.value] += 1 obs.check(expected) m.obj.expr = m.x**2 + m.y**2 detector.update() - expected[m.obj]['remove'] += 1 - expected[m.obj]['add'] += 1 - expected[m.x]['remove'] += 1 - expected[m.x]['add'] += 1 - expected[m.y]['remove'] += 1 - expected[m.y]['add'] += 1 - expected[m.p]['remove'] += 1 + expected[m.obj][Reason.expr] += 1 + expected[m.p][Reason.removed] += 1 obs.check(expected) - expected[m.obj]['remove'] += 1 + expected[m.obj][Reason.removed] += 1 del m.obj m.obj2 = pyo.Objective(expr=m.p * m.x) detector.update() # remember, m.obj is a different object now - expected[m.obj2] = make_count_dict() - expected[m.obj2]['add'] += 1 - expected[m.x]['remove'] += 1 - expected[m.x]['add'] += 1 - expected[m.y]['remove'] += 1 - expected[m.p]['add'] += 1 + expected[m.obj2][Reason.added] += 1 + expected[m.y][Reason.removed] += 1 + expected[m.p][Reason.added] += 1 obs.check(expected) def test_constraints(self): @@ -202,27 +144,22 @@ def test_constraints(self): obs = ObserverChecker() detector = ModelChangeDetector(m, [obs]) - expected = ComponentMap() + expected = DefaultComponentMap(make_count_dict) obs.check(expected) m.obj = pyo.Objective(expr=m.y) m.c1 = pyo.Constraint(expr=m.y >= (m.x - m.p) ** 2) detector.update() - expected[m.x] = make_count_dict() - expected[m.y] = make_count_dict() - expected[m.p] = make_count_dict() - expected[m.x]['add'] += 1 - expected[m.y]['add'] += 1 - expected[m.p]['add'] += 1 - expected[m.c1] = make_count_dict() - expected[m.c1]['add'] += 1 - expected[m.obj] = make_count_dict() - expected[m.obj]['add'] += 1 + expected[m.x][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 + expected[m.obj][Reason.added] += 1 obs.check(expected) m.x.fix(1) detector.update() - expected[m.x]['update'] += 1 + expected[m.x][Reason.fixed] += 1 obs.check(expected) def test_sos(self): @@ -236,17 +173,12 @@ def test_sos(self): obs = ObserverChecker() detector = ModelChangeDetector(m, [obs]) - expected = ComponentMap() - expected[m.obj] = make_count_dict() - for i in m.a: - expected[m.x[i]] = make_count_dict() - expected[m.y] = make_count_dict() - expected[m.c1] = make_count_dict() - expected[m.obj]['add'] += 1 + expected = DefaultComponentMap(make_count_dict) + expected[m.obj][Reason.added] += 1 for i in m.a: - expected[m.x[i]]['add'] += 1 - expected[m.y]['add'] += 1 - expected[m.c1]['add'] += 1 + expected[m.x[i]][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 obs.check(expected) detector.update() @@ -254,16 +186,12 @@ def test_sos(self): m.c1.set_items([m.x[2], m.x[1], m.x[3]], [1, 2, 3]) detector.update() - expected[m.c1]['remove'] += 1 - expected[m.c1]['add'] += 1 - for i in m.a: - expected[m.x[i]]['remove'] += 1 - expected[m.x[i]]['add'] += 1 + expected[m.c1][Reason.sos_items] += 1 obs.check(expected) for i in m.a: - expected[m.x[i]]['remove'] += 1 - expected[m.c1]['remove'] += 1 + expected[m.x[i]][Reason.removed] += 1 + expected[m.c1][Reason.removed] += 1 del m.c1 detector.update() obs.check(expected) @@ -279,27 +207,22 @@ def test_vars_and_params_elsewhere(self): obs = ObserverChecker() detector = ModelChangeDetector(m2, [obs]) - expected = ComponentMap() + expected = DefaultComponentMap(make_count_dict) obs.check(expected) m2.obj = pyo.Objective(expr=m1.y) m2.c1 = pyo.Constraint(expr=m1.y >= (m1.x - m1.p) ** 2) detector.update() - expected[m1.x] = make_count_dict() - expected[m1.y] = make_count_dict() - expected[m1.p] = make_count_dict() - expected[m1.x]['add'] += 1 - expected[m1.y]['add'] += 1 - expected[m1.p]['add'] += 1 - expected[m2.c1] = make_count_dict() - expected[m2.c1]['add'] += 1 - expected[m2.obj] = make_count_dict() - expected[m2.obj]['add'] += 1 + expected[m1.x][Reason.added] += 1 + expected[m1.y][Reason.added] += 1 + expected[m1.p][Reason.added] += 1 + expected[m2.c1][Reason.added] += 1 + expected[m2.obj][Reason.added] += 1 obs.check(expected) m1.x.fix(1) detector.update() - expected[m1.x]['update'] += 1 + expected[m1.x][Reason.fixed] += 1 obs.check(expected) def test_named_expression(self): @@ -311,39 +234,25 @@ def test_named_expression(self): obs = ObserverChecker() detector = ModelChangeDetector(m, [obs]) - expected = ComponentMap() + expected = DefaultComponentMap(make_count_dict) obs.check(expected) m.obj = pyo.Objective(expr=m.y) m.e = pyo.Expression(expr=m.x - m.p) m.c1 = pyo.Constraint(expr=m.y >= m.e) detector.update() - expected[m.x] = make_count_dict() - expected[m.y] = make_count_dict() - expected[m.p] = make_count_dict() - expected[m.x]['add'] += 1 - expected[m.y]['add'] += 1 - expected[m.p]['add'] += 1 - expected[m.c1] = make_count_dict() - expected[m.c1]['add'] += 1 - expected[m.obj] = make_count_dict() - expected[m.obj]['add'] += 1 + expected[m.x][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 + expected[m.obj][Reason.added] += 1 obs.check(expected) # now modify the named expression and make sure the # constraint gets removed and added m.e.expr = (m.x - m.p) ** 2 detector.update() - expected[m.c1]['remove'] += 1 - expected[m.c1]['add'] += 1 - # because x and p are only used in the - # one constraint, they get removed when - # the constraint is removed and then - # added again when the constraint is added - expected[m.x]['remove'] += 1 - expected[m.x]['add'] += 1 - expected[m.p]['remove'] += 1 - expected[m.p]['add'] += 1 + expected[m.c1][Reason.expr] += 1 obs.check(expected) def test_update_config(self): @@ -354,7 +263,7 @@ def test_update_config(self): obs = ObserverChecker() detector = ModelChangeDetector(m, [obs]) - expected = ComponentMap() + expected = DefaultComponentMap(make_count_dict) obs.check(expected) detector.config.check_for_new_or_removed_constraints = False @@ -374,20 +283,15 @@ def test_update_config(self): detector.config.check_for_new_or_removed_constraints = True detector.update() - expected[m.x] = make_count_dict() - expected[m.y] = make_count_dict() - expected[m.p] = make_count_dict() - expected[m.c1] = make_count_dict() - expected[m.x]['add'] += 1 - expected[m.y]['add'] += 1 - expected[m.p]['add'] += 1 - expected[m.c1]['add'] += 1 + expected[m.x][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 obs.check(expected) detector.config.check_for_new_or_removed_objectives = True detector.update() - expected[m.obj] = make_count_dict() - expected[m.obj]['add'] += 1 + expected[m.obj][Reason.added] += 1 obs.check(expected) m.x.setlb(0) @@ -396,7 +300,7 @@ def test_update_config(self): detector.config.update_vars = True detector.update() - expected[m.x]['update'] += 1 + expected[m.x][Reason.bounds] += 1 obs.check(expected) m.p.value = 2 @@ -405,7 +309,7 @@ def test_update_config(self): detector.config.update_parameters = True detector.update() - expected[m.p]['update'] += 1 + expected[m.p][Reason.value] += 1 obs.check(expected) m.e.expr += 1 @@ -414,8 +318,7 @@ def test_update_config(self): detector.config.update_named_expressions = True detector.update() - expected[m.c1]['remove'] += 1 - expected[m.c1]['add'] += 1 + expected[m.c1][Reason.expr] += 1 obs.check(expected) m.obj.expr += 1 @@ -424,8 +327,7 @@ def test_update_config(self): detector.config.update_objectives = True detector.update() - expected[m.obj]['remove'] += 1 - expected[m.obj]['add'] += 1 + expected[m.obj][Reason.expr] += 1 obs.check(expected) m.c1 = m.y >= m.e @@ -434,6 +336,5 @@ def test_update_config(self): detector.config.update_constraints = True detector.update() - expected[m.c1]['remove'] += 1 - expected[m.c1]['add'] += 1 + expected[m.c1][Reason.expr] += 1 obs.check(expected) From 2d54b54c69da6b380c5984d1017ba3fe03ee8981 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 21 Oct 2025 00:37:43 -0600 Subject: [PATCH 43/50] run black --- pyomo/contrib/observer/model_observer.py | 147 +++++++++++++---------- 1 file changed, 84 insertions(+), 63 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 29101877811..5e8b41b7051 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -11,7 +11,17 @@ from __future__ import annotations import abc -from typing import List, Sequence, Optional, Mapping, MutableMapping, MutableSet, Tuple, Collection, Union +from typing import ( + List, + Sequence, + Optional, + Mapping, + MutableMapping, + MutableSet, + Tuple, + Collection, + Union, +) from pyomo.common.enums import ObjectiveSense from pyomo.common.config import ConfigDict, ConfigValue, document_configdict @@ -26,7 +36,12 @@ from pyomo.core.base.component import ActiveComponent from pyomo.core.expr.numeric_expr import NumericValue from pyomo.core.expr.relational_expr import RelationalExpression -from pyomo.common.collections import ComponentMap, ComponentSet, OrderedSet, DefaultComponentMap +from pyomo.common.collections import ( + ComponentMap, + ComponentSet, + OrderedSet, + DefaultComponentMap, +) from pyomo.common.gc_manager import PauseGC from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.util import get_objective @@ -38,7 +53,7 @@ # The ModelChangeDetector is meant to be used to automatically identify changes -# in a Pyomo model or block. Here is a list of changes that will be detected. +# in a Pyomo model or block. Here is a list of changes that will be detected. # Note that inactive components (e.g., constraints) are treated as "removed". # - new constraints that have been added to the model # - constraints that have been removed from the model @@ -200,12 +215,12 @@ class Observer(abc.ABC): @abc.abstractmethod def _update_variables(self, variables: Mapping[VarData, Reason]): """ - This method gets called by the ModelChangeDetector when there are + This method gets called by the ModelChangeDetector when there are any modifications to the set of "active" variables in the model being observed. By "active" variables, we mean variables that are used within an active component such as a constraint or an objective. Changes include new variables being added to the model, - variables being removed from the model, or changes to variables + variables being removed from the model, or changes to variables already in the model Parameters @@ -218,12 +233,12 @@ def _update_variables(self, variables: Mapping[VarData, Reason]): @abc.abstractmethod def _update_parameters(self, params: Mapping[ParamData, Reason]): """ - This method gets called by the ModelChangeDetector when there are any + This method gets called by the ModelChangeDetector when there are any modifications to the set of "active" parameters in the model being - observed. By "active" parameters, we mean parameters that are used within - an active component such as a constraint or an objective. Changes include + observed. By "active" parameters, we mean parameters that are used within + an active component such as a constraint or an objective. Changes include parameters being added to the model, parameters being removed from the model, - or changes to parameters already in the model + or changes to parameters already in the model Parameters ---------- @@ -235,9 +250,9 @@ def _update_parameters(self, params: Mapping[ParamData, Reason]): @abc.abstractmethod def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): """ - This method gets called by the ModelChangeDetector when there are any + This method gets called by the ModelChangeDetector when there are any modifications to the set of active constraints in the model being observed. - Changes include constraints being added to the model, constraints being + Changes include constraints being added to the model, constraints being removed from the model, or changes to constraints already in the model. Parameters @@ -250,8 +265,8 @@ def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): @abc.abstractmethod def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): """ - This method gets called by the ModelChangeDetector when there are any - modifications to the set of active SOS constraints in the model being + This method gets called by the ModelChangeDetector when there are any + modifications to the set of active SOS constraints in the model being observed. Changes include constraints being added to the model, constraints being removed from the model, or changes to constraints already in the model. @@ -265,9 +280,9 @@ def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): @abc.abstractmethod def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): """ - This method gets called by the ModelChangeDetector when there are any + This method gets called by the ModelChangeDetector when there are any modifications to the set of active objectives in the model being observed. - Changes include objectives being added to the model, objectives being + Changes include objectives being added to the model, objectives being removed from the model, or changes to objectives already in the model. Parameters @@ -293,11 +308,19 @@ def __init__(self, observers: Collection[Observer]) -> None: def run(self): # split up new, removed, and modified variables - new_vars = ComponentMap((k, v) for k, v in self.vars_to_update.items() if v & Reason.added) - other_vars = ComponentMap((k, v) for k, v in self.vars_to_update.items() if not (v & Reason.added)) + new_vars = ComponentMap( + (k, v) for k, v in self.vars_to_update.items() if v & Reason.added + ) + other_vars = ComponentMap( + (k, v) for k, v in self.vars_to_update.items() if not (v & Reason.added) + ) - new_params = ComponentMap((k, v) for k, v in self.params_to_update.items() if v & Reason.added) - other_params = ComponentMap((k, v) for k, v in self.params_to_update.items() if not (v & Reason.added)) + new_params = ComponentMap( + (k, v) for k, v in self.params_to_update.items() if v & Reason.added + ) + other_params = ComponentMap( + (k, v) for k, v in self.params_to_update.items() if not (v & Reason.added) + ) for obs in self.observers: if new_vars: @@ -506,21 +529,23 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): # maps constraints/objectives to list of tuples (named_expr, named_expr.expr) self._named_expressions: MutableMapping[ - ConstraintData, List[Tuple[ExpressionData, Union[NumericValue, float, int, None]]] + ConstraintData, + List[Tuple[ExpressionData, Union[NumericValue, float, int, None]]], ] = {} self._obj_named_expressions: MutableMapping[ - ObjectiveData, List[Tuple[ExpressionData, Union[NumericValue, float, int, None]]] + ObjectiveData, + List[Tuple[ExpressionData, Union[NumericValue, float, int, None]]], ] = ComponentMap() self._external_functions = ComponentMap() self._referenced_variables: MutableMapping[ - VarData, + VarData, Tuple[ - MutableSet[ConstraintData], - MutableSet[SOSConstraintData], - MutableSet[ObjectiveData] - ] + MutableSet[ConstraintData], + MutableSet[SOSConstraintData], + MutableSet[ObjectiveData], + ], ] = ComponentMap() self._referenced_params: MutableMapping[ @@ -530,7 +555,7 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): MutableSet[SOSConstraintData], MutableSet[ObjectiveData], MutableSet[VarData], - ] + ], ] = ComponentMap() self._vars_referenced_by_con: MutableMapping[ @@ -565,13 +590,7 @@ def _add_variables(self, variables: Collection[VarData]): raise ValueError(f'Variable {v.name} has already been added') self._updates.vars_to_update[v] |= Reason.added self._referenced_variables[v] = (OrderedSet(), OrderedSet(), ComponentSet()) - self._vars[v] = ( - v._lb, - v._ub, - v.fixed, - v.domain.get_interval(), - v.value, - ) + self._vars[v] = (v._lb, v._ub, v.fixed, v.domain.get_interval(), v.value) ref_params = ComponentSet() for bnd in (v._lb, v._ub): if bnd is None or type(bnd) in native_numeric_types: @@ -607,7 +626,12 @@ def _add_parameters(self, params: Collection[ParamData]): if p in self._referenced_params: raise ValueError(f'Parameter {p.name} has already been added') self._updates.params_to_update[p] |= Reason.added - self._referenced_params[p] = (OrderedSet(), OrderedSet(), ComponentSet(), ComponentSet()) + self._referenced_params[p] = ( + OrderedSet(), + OrderedSet(), + ComponentSet(), + ComponentSet(), + ) self._params[p] = p.value def add_parameters(self, params: Collection[ParamData]): @@ -628,9 +652,7 @@ def _check_to_remove_vars(self, variables: Collection[VarData]): self._remove_variables(vars_to_remove) def _check_for_new_params(self, params: Collection[ParamData]): - new_params = ComponentSet( - p for p in params if p not in self._referenced_params - ) + new_params = ComponentSet(p for p in params if p not in self._referenced_params) self._add_parameters(new_params) def _check_to_remove_params(self, params: Collection[ParamData]): @@ -751,10 +773,8 @@ def _check_for_unknown_active_components(self): if ctype in self._known_active_ctypes: continue if ctype is Suffix: - warnings.warn( - 'ModelChangeDetector does not detect changes to suffixes' - ) - continue + warnings.warn('ModelChangeDetector does not detect changes to suffixes') + continue raise NotImplementedError( f'ModelChangeDetector does not know how to ' f'handle components with ctype {ctype}' @@ -848,7 +868,7 @@ def _remove_variables(self, variables: Collection[VarData]): ) self._referenced_variables.pop(v) self._vars.pop(v) - + def remove_variables(self, variables: Collection[VarData]): self._remove_variables(variables) self._updates.run() @@ -915,7 +935,13 @@ def _update_variables(self, variables: Optional[Collection[VarData]] = None): variables = self._vars for v in variables: _lb, _ub, _fixed, _domain_interval, _value = self._vars[v] - lb, ub, fixed, domain_interval, value = v._lb, v._ub, v.fixed, v.domain.get_interval(), v.value + lb, ub, fixed, domain_interval, value = ( + v._lb, + v._ub, + v.fixed, + v.domain.get_interval(), + v.value, + ) reason = Reason.no_change if _fixed != fixed: reason |= Reason.fixed @@ -927,13 +953,7 @@ def _update_variables(self, variables: Optional[Collection[VarData]] = None): reason |= Reason.domain if reason: self._updates.vars_to_update[v] |= reason - self._vars[v] = ( - lb, - ub, - fixed, - domain_interval, - value, - ) + self._vars[v] = (lb, ub, fixed, domain_interval, value) if reason & Reason.bounds: self._update_var_bounds(v) @@ -971,7 +991,7 @@ def _update_con(self, con: ConstraintData): self._external_functions[con] = external_functions else: self._external_functions.pop(con, None) - + _variables = self._vars_referenced_by_con[con] _parameters = self._params_referenced_by_con[con] new_vars = variables - _variables @@ -986,7 +1006,7 @@ def _update_con(self, con: ConstraintData): self._check_for_new_vars(new_vars) if new_params: self._check_for_new_params(new_params) - + for v in new_vars: self._referenced_variables[v][0].add(con) for v in old_vars: @@ -1018,10 +1038,7 @@ def update_constraints(self, cons: Optional[Collection[ConstraintData]] = None): def _update_sos_con(self, con: SOSConstraintData): sos_items = list(con.get_items()) - self._active_sos[con] = ( - [i[0] for i in sos_items], - [i[1] for i in sos_items], - ) + self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) variables = ComponentSet() parameters = ComponentSet() for v, p in sos_items: @@ -1030,7 +1047,7 @@ def _update_sos_con(self, con: SOSConstraintData): continue if p.is_parameter_type(): parameters.add(p) - + _variables = self._vars_referenced_by_con[con] _parameters = self._params_referenced_by_con[con] new_vars = variables - _variables @@ -1045,7 +1062,7 @@ def _update_sos_con(self, con: SOSConstraintData): self._check_for_new_vars(new_vars) if new_params: self._check_for_new_params(new_params) - + for v in new_vars: self._referenced_variables[v][1].add(con) for v in old_vars: @@ -1060,7 +1077,9 @@ def _update_sos_con(self, con: SOSConstraintData): if old_params: self._check_to_remove_params(old_params) - def _update_sos_constraints(self, cons: Optional[Collection[SOSConstraintData]] = None): + def _update_sos_constraints( + self, cons: Optional[Collection[SOSConstraintData]] = None + ): if cons is None: cons = self._active_sos for c in cons: @@ -1086,7 +1105,9 @@ def _update_sos_constraints(self, cons: Optional[Collection[SOSConstraintData]] self._updates.sos_to_update[c] |= reason self._update_sos_con(c) - def update_sos_constraints(self, cons: Optional[Collection[SOSConstraintData]] = None): + def update_sos_constraints( + self, cons: Optional[Collection[SOSConstraintData]] = None + ): self._update_sos_constraints(cons) self._updates.run() @@ -1102,7 +1123,7 @@ def _update_obj_expr(self, obj: ObjectiveData): self._external_functions[obj] = external_functions else: self._external_functions.pop(obj, None) - + _variables = self._vars_referenced_by_obj[obj] _parameters = self._params_referenced_by_obj[obj] new_vars = variables - _variables @@ -1117,7 +1138,7 @@ def _update_obj_expr(self, obj: ObjectiveData): self._check_for_new_vars(new_vars) if new_params: self._check_for_new_params(new_params) - + for v in new_vars: self._referenced_variables[v][2].add(obj) for v in old_vars: From 036a320bb559e827060bc955c766df0726598d95 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 21 Oct 2025 00:48:22 -0600 Subject: [PATCH 44/50] observer typos --- pyomo/contrib/observer/model_observer.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 5e8b41b7051..b02fc5899d8 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -1314,10 +1314,16 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): self._updates.run() def get_variables_impacted_by_param(self, p: ParamData): - return [self._vars[vid][0] for vid in self._referenced_params[id(p)][3]] + return list(self._referenced_params[p][3]) def get_constraints_impacted_by_param(self, p: ParamData): - return list(self._referenced_params[id(p)][0]) + return list(self._referenced_params[p][0]) def get_constraints_impacted_by_var(self, v: VarData): - return list(self._referenced_variables[id(v)][0]) + return list(self._referenced_variables[v][0]) + + def get_objectives_impacted_by_param(self, p: ParamData): + return list(self._referenced_params[p][2]) + + def get_objectives_impacted_by_var(self, v: VarData): + return list(self._referenced_variables[v][2]) From 9b236b6c5510813b8de69681107eb2e2b94b3c6b Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 21 Oct 2025 01:01:28 -0600 Subject: [PATCH 45/50] observer: more tests --- .../observer/tests/test_change_detector.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index fbc42e0e591..618adb4ae29 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -162,6 +162,21 @@ def test_constraints(self): expected[m.x][Reason.fixed] += 1 obs.check(expected) + m.z = pyo.Var() + m.c1.set_value(m.y == 2*m.z) + detector.update() + expected[m.z][Reason.added] += 1 + expected[m.c1][Reason.expr] += 1 + expected[m.p][Reason.removed] += 1 + expected[m.x][Reason.removed] += 1 + obs.check(expected) + + expected[m.c1][Reason.removed] += 1 + del m.c1 + detector.update() + expected[m.z][Reason.removed] += 1 + obs.check(expected) + def test_sos(self): m = pyo.ConcreteModel() m.a = pyo.Set(initialize=[1, 2, 3], ordered=True) @@ -338,3 +353,34 @@ def test_update_config(self): detector.update() expected[m.c1][Reason.expr] += 1 obs.check(expected) + + def test_param_in_bounds(self): + m = pyo.ConcreteModel() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector(m, [obs]) + + expected = DefaultComponentMap(make_count_dict) + obs.check(expected) + + m.obj = pyo.Objective(expr=m.y) + m.y.setlb(m.p - 1) + detector.update() + expected[m.y][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + expected[m.obj][Reason.added] += 1 + obs.check(expected) + + m.p.value = 2 + detector.update() + expected[m.p][Reason.value] += 1 + obs.check(expected) + + m.p2 = pyo.Param(mutable=True, initialize=1) + m.y.setub(m.p2 + 1) + detector.update() + expected[m.p2][Reason.added] += 1 + expected[m.y][Reason.bounds] += 1 + obs.check(expected) From ab3400ffa84ff763564114dd617f614d669a998e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 21 Oct 2025 09:27:36 -0600 Subject: [PATCH 46/50] run black --- pyomo/contrib/observer/tests/test_change_detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 618adb4ae29..4c31a85a629 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -163,7 +163,7 @@ def test_constraints(self): obs.check(expected) m.z = pyo.Var() - m.c1.set_value(m.y == 2*m.z) + m.c1.set_value(m.y == 2 * m.z) detector.update() expected[m.z][Reason.added] += 1 expected[m.c1][Reason.expr] += 1 From af519718cd13b187fd05fe6a96ccf105b88c92c3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 21 Oct 2025 12:11:21 -0600 Subject: [PATCH 47/50] observer: update docstring --- pyomo/contrib/observer/model_observer.py | 117 ++++++++++------------- 1 file changed, 49 insertions(+), 68 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b02fc5899d8..5468a9a205c 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -391,61 +391,39 @@ class ModelChangeDetector: then ``check_for_new_or_removed_constraints`` can be set to ``False``, which will save some time when ``update`` is called. - We have discussed expanding the interface of the ``ModelChangeDetector`` - with methods to request extra information. For example, if the value - of a fixed variable changes, an observer may want to know all of the - constraints that use the variables. This class already has that - information, so the observer should not have to waste time recomputing - that. We have not yet added methods like this because we do not have - an immediate use case or need, and it's not yet clear what those - methods should look like. If a need arises, please create an issue or - pull request. - Here are some usage examples: >>> import pyomo.environ as pyo + >>> from typing import Mapping >>> from pyomo.contrib.observer.model_observer import ( ... AutoUpdateConfig, ... Observer, ... ModelChangeDetector, + ... Reason, + ... ) + >>> from pyomo.core.base import ( + ... VarData, + ... ParamData, + ... ConstraintData, + ... SOSConstraintData, + ... ObjectiveData, ... ) >>> class PrintObserver(Observer): - ... def add_variables(self, variables): - ... for i in variables: - ... print(f'{i} was added to the model') - ... def add_parameters(self, params): - ... for i in params: - ... print(f'{i} was added to the model') - ... def add_constraints(self, cons): - ... for i in cons: - ... print(f'{i} was added to the model') - ... def add_sos_constraints(self, cons): - ... for i in cons: - ... print(f'{i} was added to the model') - ... def add_objectives(self, objs): - ... for i in objs: - ... print(f'{i} was added to the model') - ... def remove_objectives(self, objs): - ... for i in objs: - ... print(f'{i} was removed from the model') - ... def remove_constraints(self, cons): - ... for i in cons: - ... print(f'{i} was removed from the model') - ... def remove_sos_constraints(self, cons): - ... for i in cons: - ... print(f'{i} was removed from the model') - ... def remove_variables(self, variables): - ... for i in variables: - ... print(f'{i} was removed from the model') - ... def remove_parameters(self, params): - ... for i in params: - ... print(f'{i} was removed from the model') - ... def update_variables(self, variables, reasons): - ... for i in variables: - ... print(f'{i} was modified') - ... def update_parameters(self, params): - ... for i in params: - ... print(f'{i} was modified') + ... def _update_variables(self, vars: Mapping[VarData, Reason]): + ... for v, r in vars.items(): + ... print(f'{v}: {r.name}') + ... def _update_parameters(self, params: Mapping[ParamData, Reason]): + ... for p, r in params.items(): + ... print(f'{p}: {r.name}') + ... def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): + ... for c, r in cons.items(): + ... print(f'{c}: {r.name}') + ... def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): + ... for c, r in cons.items(): + ... print(f'{c}: {r.name}') + ... def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): + ... for o, r in objs.items(): + ... print(f'{o}: {r.name}') >>> m = pyo.ConcreteModel() >>> obs = PrintObserver() >>> detector = ModelChangeDetector(m, [obs]) @@ -454,23 +432,26 @@ class ModelChangeDetector: >>> detector.update() # no output because the variables are not used >>> m.obj = pyo.Objective(expr=m.x**2 + m.y**2) >>> detector.update() - x was added to the model - y was added to the model - obj was added to the model + x: added + y: added + obj: added >>> del m.obj >>> detector.update() - obj was removed from the model - x was removed from the model - y was removed from the model + obj: removed + x: removed + y: removed >>> m.px = pyo.Param(mutable=True, initialize=1) >>> m.py = pyo.Param(mutable=True, initialize=1) >>> m.obj = pyo.Objective(expr=m.px*m.x + m.py*m.y) >>> detector.update() - x was added to the model - y was added to the model - px was added to the model - py was added to the model - obj was added to the model + x: added + y: added + px: added + py: added + obj: added + >>> m.px.value = 2 + >>> detector.update() + px: value >>> detector.config.check_for_new_or_removed_constraints = False >>> detector.config.check_for_new_or_removed_objectives = False >>> detector.config.update_constraints = False @@ -481,21 +462,21 @@ class ModelChangeDetector: >>> for i in range(10): ... m.py.value = i ... detector.update() # this will be faster because it is only checking for changes to parameters - py was modified - py was modified - py was modified - py was modified - py was modified - py was modified - py was modified - py was modified - py was modified - py was modified + py: value + py: value + py: value + py: value + py: value + py: value + py: value + py: value + py: value + py: value >>> m.c = pyo.Constraint(expr=m.y >= pyo.exp(m.x)) >>> detector.update() # no output because we did not check for new constraints >>> detector.config.check_for_new_or_removed_constraints = True >>> detector.update() - c was added to the model + c: added """ From 7d09c273c7e4458836b5415ad227671d5c3e253d Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 21 Oct 2025 12:12:00 -0600 Subject: [PATCH 48/50] run black --- pyomo/contrib/observer/model_observer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 5468a9a205c..3c3a83ce26f 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -402,10 +402,10 @@ class ModelChangeDetector: ... Reason, ... ) >>> from pyomo.core.base import ( - ... VarData, - ... ParamData, - ... ConstraintData, - ... SOSConstraintData, + ... VarData, + ... ParamData, + ... ConstraintData, + ... SOSConstraintData, ... ObjectiveData, ... ) >>> class PrintObserver(Observer): From 880dfe8e65753ea3b1e8eb25b82f3592b6583309 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 23 Oct 2025 10:54:34 -0600 Subject: [PATCH 49/50] observer: more tests --- pyomo/contrib/observer/model_observer.py | 16 -- .../observer/tests/test_change_detector.py | 157 ++++++++++++++++++ 2 files changed, 157 insertions(+), 16 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 3c3a83ce26f..97910eb2bf1 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -598,10 +598,6 @@ def _add_variables(self, variables: Collection[VarData]): for p in ref_params: self._referenced_params[p][3].add(v) - def add_variables(self, variables: Collection[VarData]): - self._add_variables(variables) - self._updates.run() - def _add_parameters(self, params: Collection[ParamData]): for p in params: if p in self._referenced_params: @@ -615,10 +611,6 @@ def _add_parameters(self, params: Collection[ParamData]): ) self._params[p] = p.value - def add_parameters(self, params: Collection[ParamData]): - self._add_parameters(params) - self._updates.run() - def _check_for_new_vars(self, variables: Collection[VarData]): new_vars = ComponentSet( v for v in variables if v not in self._referenced_variables @@ -850,10 +842,6 @@ def _remove_variables(self, variables: Collection[VarData]): self._referenced_variables.pop(v) self._vars.pop(v) - def remove_variables(self, variables: Collection[VarData]): - self._remove_variables(variables) - self._updates.run() - def _remove_parameters(self, params: Collection[ParamData]): for p in params: if p not in self._referenced_params: @@ -868,10 +856,6 @@ def _remove_parameters(self, params: Collection[ParamData]): self._referenced_params.pop(p) self._params.pop(p) - def remove_parameters(self, params: Collection[ParamData]): - self._remove_parameters(params) - self._updates.run() - def _update_var_bounds(self, v: VarData): ref_params = ComponentSet() for bnd in (v._lb, v._ub): diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 4c31a85a629..aa756d7332e 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -26,6 +26,7 @@ Reason, ) from pyomo.common.collections import DefaultComponentMap, ComponentMap +from pyomo.common.errors import PyomoException logger = logging.getLogger(__name__) @@ -204,6 +205,18 @@ def test_sos(self): expected[m.c1][Reason.sos_items] += 1 obs.check(expected) + m.c1.set_items([m.x[2], m.x[1]], [1, 2]) + detector.update() + expected[m.c1][Reason.sos_items] += 1 + expected[m.x[3]][Reason.removed] += 1 + obs.check(expected) + + m.c1.set_items([m.x[2], m.x[1], m.x[3]], [1, 2, 3]) + detector.update() + expected[m.c1][Reason.sos_items] += 1 + expected[m.x[3]][Reason.added] += 1 + obs.check(expected) + for i in m.a: expected[m.x[i]][Reason.removed] += 1 expected[m.c1][Reason.removed] += 1 @@ -384,3 +397,147 @@ def test_param_in_bounds(self): expected[m.p2][Reason.added] += 1 expected[m.y][Reason.bounds] += 1 obs.check(expected) + + def test_incidence(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.z = pyo.Var() + m.p1 = pyo.Param(mutable=True, initialize=1) + m.p2 = pyo.Param(mutable=True, initialize=1) + m.x.setlb(m.p1) + + m.e1 = pyo.Expression(expr=m.x + m.p1) + m.e2 = pyo.Expression(expr=(m.e1**2)) + m.obj = pyo.Objective(expr=m.e2 + m.y**2) + m.c1 = pyo.Constraint(expr=m.z + m.p2 == 0) + m.c2 = pyo.Constraint(expr=m.x + m.p2 == 0) + + obs = ObserverChecker() + detector = ModelChangeDetector(m, [obs]) + + expected = DefaultComponentMap(make_count_dict) + expected[m.x][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.z][Reason.added] += 1 + expected[m.p1][Reason.added] += 1 + expected[m.p2][Reason.added] += 1 + expected[m.obj][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 + expected[m.c2][Reason.added] += 1 + obs.check(expected) + + self.assertEqual(detector.get_variables_impacted_by_param(m.p1), [m.x]) + self.assertEqual(detector.get_variables_impacted_by_param(m.p2), []) + self.assertEqual(detector.get_constraints_impacted_by_param(m.p1), []) + self.assertEqual(detector.get_constraints_impacted_by_param(m.p2), [m.c1, m.c2]) + self.assertEqual(detector.get_constraints_impacted_by_var(m.x), [m.c2]) + self.assertEqual(detector.get_constraints_impacted_by_var(m.y), []) + self.assertEqual(detector.get_constraints_impacted_by_var(m.z), [m.c1]) + self.assertEqual(detector.get_objectives_impacted_by_param(m.p1), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_param(m.p2), []) + self.assertEqual(detector.get_objectives_impacted_by_var(m.x), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_var(m.y), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_var(m.z), []) + + m.e1.expr += m.z + detector.update() + expected[m.obj][Reason.expr] += 1 + obs.check(expected) + + self.assertEqual(detector.get_objectives_impacted_by_param(m.p1), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_param(m.p2), []) + self.assertEqual(detector.get_objectives_impacted_by_var(m.x), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_var(m.y), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_var(m.z), [m.obj]) + + def test_manual_updates(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector(m, [obs]) + + expected = DefaultComponentMap(make_count_dict) + obs.check(expected) + + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= (m.x - m.p) ** 2) + m.c2 = pyo.Constraint(expr=m.x + m.y == 0) + + detector.add_objectives([m.obj]) + expected[m.obj][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + obs.check(expected) + + detector.add_constraints([m.c1]) + expected[m.x][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 + obs.check(expected) + + detector.add_constraints([m.c2]) + expected[m.c2][Reason.added] += 1 + obs.check(expected) + + detector.remove_constraints([m.c1]) + expected[m.c1][Reason.removed] += 1 + expected[m.p][Reason.removed] += 1 + obs.check(expected) + + detector.add_constraints([m.c1]) + expected[m.c1][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + obs.check(expected) + + detector.remove_objectives([m.obj]) + expected[m.obj][Reason.removed] += 1 + obs.check(expected) + + detector.add_objectives([m.obj]) + expected[m.obj][Reason.added] += 1 + obs.check(expected) + + m.x.setlb(0) + detector.update_variables([m.x, m.y]) + expected[m.x][Reason.bounds] += 1 + obs.check(expected) + + m.p.value = 2 + detector.update_parameters([m.p]) + expected[m.p][Reason.value] += 1 + obs.check(expected) + + m.c1.set_value(m.y >= m.x**2) + detector.update_constraints([m.c1, m.c2]) + expected[m.p][Reason.removed] += 1 + expected[m.c1][Reason.expr] += 1 + obs.check(expected) + + m.obj.expr += m.x + detector.update_objectives([m.obj]) + expected[m.obj][Reason.expr] += 1 + obs.check(expected) + + def test_mutable_parameters_in_sos(self): + """ + There is logic in the ModelChangeDetector to handle + mutable parameters in SOS constraints. However, we cannot + currently test it because of #3769. For now, we will + just make sure that an error is raised when attempting to + use a mutable parameter in an SOS constraint. If #3769 is + resolved, we will just need to update this test to make + sure the ModelChangeDetector does the right thing. + """ + m = pyo.ConcreteModel() + m.a = pyo.Set(initialize=[1,2,3]) + m.x = pyo.Var(m.a) + m.p = pyo.Param(m.a, mutable=True) + m.p[1].value = 1 + m.p[2].value = 2 + m.p[3].value = 3 + + with self.assertRaisesRegex(PyomoException, 'Cannot convert non-constant Pyomo expression .* to bool.*'): + m.c = pyo.SOSConstraint(var=m.x, sos=1, weights=m.p) From 4cea68528cf8692a3330705d7dd2bf5047c67a28 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 23 Oct 2025 10:55:26 -0600 Subject: [PATCH 50/50] run black --- pyomo/contrib/observer/tests/test_change_detector.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index aa756d7332e..b342bed9e4f 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -523,21 +523,23 @@ def test_manual_updates(self): def test_mutable_parameters_in_sos(self): """ - There is logic in the ModelChangeDetector to handle + There is logic in the ModelChangeDetector to handle mutable parameters in SOS constraints. However, we cannot currently test it because of #3769. For now, we will just make sure that an error is raised when attempting to use a mutable parameter in an SOS constraint. If #3769 is - resolved, we will just need to update this test to make + resolved, we will just need to update this test to make sure the ModelChangeDetector does the right thing. """ m = pyo.ConcreteModel() - m.a = pyo.Set(initialize=[1,2,3]) + m.a = pyo.Set(initialize=[1, 2, 3]) m.x = pyo.Var(m.a) m.p = pyo.Param(m.a, mutable=True) m.p[1].value = 1 m.p[2].value = 2 m.p[3].value = 3 - with self.assertRaisesRegex(PyomoException, 'Cannot convert non-constant Pyomo expression .* to bool.*'): + with self.assertRaisesRegex( + PyomoException, 'Cannot convert non-constant Pyomo expression .* to bool.*' + ): m.c = pyo.SOSConstraint(var=m.x, sos=1, weights=m.p)