Canonicalizers

Canonicalizers translate a verification problem into a solver-ready formulation (MIP or SDP), including optimization-based bound tightening (OBBT).

Canonicalizer

A base class for mathematical-optimization canonicalizers. This class defines a common interface for mixed-integer programming (MIP) and semidefinite programming (SDP) encodings.

Subclasses declare a class-level _CANONICALIZERS dict mapping the lowercase step class name (e.g. "relu") to the function that encodes it. canonicalize looks up the function in this dict and raises NotImplementedError if absent.

Attributes:

Name	Type	Description
solver	Solver	A ready-to-use instance of a subclass of Solver (e.g., gurobi.Gurobi()).

Source code in fomtools/canonicalizers/canonicalizer.py

class Canonicalizer:
    """
    A base class for mathematical-optimization canonicalizers. This class
    defines a common interface for mixed-integer programming (MIP)
    and semidefinite programming (SDP) encodings.

    Subclasses declare a class-level ``_CANONICALIZERS`` dict mapping the
    lowercase step class name (e.g. ``"relu"``) to the function that
    encodes it. ``canonicalize`` looks up the function in this dict and
    raises ``NotImplementedError`` if absent.

    Attributes
    ----------
    solver: Solver
        A ready-to-use instance of a subclass of `Solver`
        (e.g., ``gurobi.Gurobi()``).

    """

    _CANONICALIZERS: ClassVar[dict] = {}

    def __init__(self, solver_params: dict = None, solver: str = "highs"):
        """
        Initialize an instance of `Canonicalizer`.

        Parameters
        ----------
        solver_params: dict or None, optional
            Key-value pairs with solver parameters and their corresponding
            configuration values.

        solver: str, default "gurobi"
            Case-insensitive identifier for the solver to use. Supported
            values and common aliases include:
            - ``"gurobi"``, ``"grb"``
            - ``"highs"``
        """
        self.solver_params = solver_params
        self.obbt_handler = None
        if solver is not None:
            self.solver_name = solver.lower()
            self.solver = self._get_solver(solver, solver_params)
        else:
            self.solver_name = None
            self.solver = None

    @staticmethod
    def _get_solver(solver: str, solver_params: dict) -> Solver:
        """
        Return an initialized solver instance for the requested backend.

        Parameters
        ----------
        solver: str
            Case-insensitive identifier for the solver to use. Supported values
            (and common aliases) include:
            - ``"gurobi"``, ``"grb"``
            - ``"highs"``

        solver_params: dict or None, optional
            Key-value pairs with solver parameters and their corresponding
            configuration values.

        Returns
        -------
        Solver
            A ready-to-use instance of a subclass of `Solver`
            (e.g., ``gurobi.Gurobi()``).

        Raises
        ------
        ValueError
            If `solver` is not one of the supported options.
        """
        return make_solver(solver, solver_params)

    def canonicalize(self, step: Step):
        """
        Dispatch the provided step into a backend-specific encoding.

        Looks up the lowercase step class name (e.g. ``"relu"``) in this
        canonicalizer's ``_CANONICALIZERS`` registry and invokes the
        corresponding encoder. Raises ``NotImplementedError`` if the step
        type is not registered.

        Parameters
        ----------
        step: Step
            The concrete step class to canonicalize. This class encapsulates
            the thresholding or activation operator.

        Returns
        -------
        None

        Raises
        ------
        NotImplementedError
            If this canonicalizer's ``_CANONICALIZERS`` registry has no
            entry for the concrete type of ``step``.

        """
        canonicalizer = self._CANONICALIZERS.get(type(step).__name__.lower())
        if canonicalizer is None:
            raise NotImplementedError(
                f"{self.__class__.__name__} does not support {type(step).__name__}"
            )
        canonicalizer(self, step)

    def add_var(self, iterate, **kwargs) -> None:
        """
        Add a variable to the canonicalizer instance.

        Parameters
        ----------
        iterate : LinearExpression
            Linear expression object to add to the canonicalizer.

        Raises
        ------
        NotImplementedError
            Must be implemented by subclasses.
        """
        raise NotImplementedError

    def add_constraint(self, constraint: Constraint) -> None:
        """
        Add a constraint to the canonicalizer instance.

        Parameters
        ----------
        constraint : Constraint
            Constraint object to add to the canonicalizer.

        Raises
        ------
        NotImplementedError
            Must be implemented by subclasses.
        """
        raise NotImplementedError

    def update_bounds(
        self, expression: LinearExpression, lb: float | np.ndarray, ub: float | np.ndarray
    ) -> None:
        """
        Update the lower and upper bounds of the solver variables corresponding
        to the given ``LinearExpression``.

        Parameters
        ----------
        expression : LinearExpression
            A ``LinearExpression`` object whose underlying variable
            bounds are to be updated.

        lb : float or np.ndarray
            New lower bound(s).

        ub : float or np.ndarray
            New upper bound(s).

        Raises
        ------
        NotImplementedError
            Must be implemented by subclasses.
        """
        raise NotImplementedError

    def set_two_norm_objective(
        self, expr_list: list[LinearExpression], lb_list=None, ub_list=None
    ) -> None:
        """
        Set the objective to maximize the sum of squared two-norms.

        Configures the optimization objective to compute
        sum_i ||expr_i||_2^2 over the given list of expressions.

        Parameters
        ----------
        expr_list : list[LinearExpression]
            List of expressions whose squared norms are summed in the
            objective.
        lb_list : list[np.ndarray], optional
            Lower bounds for each expression (may be used by some backends).
        ub_list : list[np.ndarray], optional
            Upper bounds for each expression (may be used by some backends).

        Raises
        ------
        NotImplementedError
            Must be implemented by subclasses that support this objective type.
        """
        raise NotImplementedError(
            f"{self.__class__.__name__} does not support set_two_norm_objective"
        )

    def set_one_norm_objective(self, expr_list: list[LinearExpression]) -> None:
        """
        Set the objective to maximize the one-norm (sum of absolute values).

        Parameters
        ----------
        expr_list : list[LinearExpression]
            List of expressions whose one-norms are summed in the objective.

        Raises
        ------
        NotImplementedError
            Must be implemented by subclasses that support this objective type.
        """
        raise NotImplementedError(
            f"{self.__class__.__name__} does not support set_one_norm_objective"
        )

    def set_infinity_norm_objective(
        self, expr_list: list[LinearExpression], lb=None, ub=None
    ) -> None:
        """
        Set the objective to maximize the infinity-norm (maximum absolute value).

        Parameters
        ----------
        expr_list : list[LinearExpression]
            List of expressions whose maximum absolute value is the objective.
        lb : np.ndarray, optional
            Lower bounds for the expressions (may be used by some backends).
        ub : np.ndarray, optional
            Upper bounds for the expressions (may be used by some backends).

        Raises
        ------
        NotImplementedError
            Must be implemented by subclasses that support this objective type.
        """
        raise NotImplementedError(
            f"{self.__class__.__name__} does not support set_infinity_norm_objective"
        )

    def get_expression_bounds(self, expr: LinearExpression) -> tuple[np.ndarray, np.ndarray]:
        """
        Compute bounds for a linear expression.

        Uses OBBT if available for tighter bounds, otherwise falls back
        to interval bound propagation.

        Parameters
        ----------
        expr : LinearExpression
            The expression whose bounds are to be computed.

        Returns
        -------
        tuple[np.ndarray, np.ndarray]
            A tuple (lb, ub) of lower and upper bounds.
        """
        lb, ub = expr.bound_propagation()

        if self.obbt_handler is not None:
            result = self.obbt_handler.obbt(expr, lb, ub)
            if result is not None:
                lb, ub = result

        return lb, ub

    @abstractmethod
    def copy(self) -> "Canonicalizer":
        """
        Create an independent copy of this canonicalizer.

        The copy shares no mutable state with the original and can be
        modified (e.g., adding objectives) without affecting the original.

        Returns
        -------
        Canonicalizer
            A new canonicalizer instance with copied state.

        Raises
        ------
        NotImplementedError
            Must be implemented by subclasses.
        """
        raise NotImplementedError

__init__

__init__(solver_params: dict = None, solver: str = 'highs')

Parameters:

Name	Type	Description	Default
solver_params	dict	Key-value pairs with solver parameters and their corresponding configuration values.	None
solver	str	Case-insensitive identifier for the solver to use. Supported values and common aliases include: - "gurobi", "grb" - "highs"	'highs'

Source code in fomtools/canonicalizers/canonicalizer.py

def __init__(self, solver_params: dict = None, solver: str = "highs"):
    """
    Initialize an instance of `Canonicalizer`.

    Parameters
    ----------
    solver_params: dict or None, optional
        Key-value pairs with solver parameters and their corresponding
        configuration values.

    solver: str, default "gurobi"
        Case-insensitive identifier for the solver to use. Supported
        values and common aliases include:
        - ``"gurobi"``, ``"grb"``
        - ``"highs"``
    """
    self.solver_params = solver_params
    self.obbt_handler = None
    if solver is not None:
        self.solver_name = solver.lower()
        self.solver = self._get_solver(solver, solver_params)
    else:
        self.solver_name = None
        self.solver = None

add_constraint

add_constraint(constraint: Constraint) -> None

Add a constraint to the canonicalizer instance.

Parameters:

Name	Type	Description	Default
constraint	Constraint	Constraint object to add to the canonicalizer.	required

Raises:

Type	Description
NotImplementedError	Must be implemented by subclasses.

Source code in fomtools/canonicalizers/canonicalizer.py

def add_constraint(self, constraint: Constraint) -> None:
    """
    Add a constraint to the canonicalizer instance.

    Parameters
    ----------
    constraint : Constraint
        Constraint object to add to the canonicalizer.

    Raises
    ------
    NotImplementedError
        Must be implemented by subclasses.
    """
    raise NotImplementedError

add_var

add_var(iterate, **kwargs) -> None

Add a variable to the canonicalizer instance.

Parameters:

Name	Type	Description	Default
iterate	LinearExpression	Linear expression object to add to the canonicalizer.	required

Raises:

Type	Description
NotImplementedError	Must be implemented by subclasses.

Source code in fomtools/canonicalizers/canonicalizer.py

def add_var(self, iterate, **kwargs) -> None:
    """
    Add a variable to the canonicalizer instance.

    Parameters
    ----------
    iterate : LinearExpression
        Linear expression object to add to the canonicalizer.

    Raises
    ------
    NotImplementedError
        Must be implemented by subclasses.
    """
    raise NotImplementedError

canonicalize

canonicalize(step: Step)

Dispatch the provided step into a backend-specific encoding.

Looks up the lowercase step class name (e.g. "relu") in this canonicalizer's _CANONICALIZERS registry and invokes the corresponding encoder. Raises NotImplementedError if the step type is not registered.

Parameters:

Name	Type	Description	Default
step	Step	The concrete step class to canonicalize. This class encapsulates the thresholding or activation operator.	required

Returns:

Type	Description
None

Raises:

Type	Description
NotImplementedError	If this canonicalizer's _CANONICALIZERS registry has no entry for the concrete type of step.

Source code in fomtools/canonicalizers/canonicalizer.py

def canonicalize(self, step: Step):
    """
    Dispatch the provided step into a backend-specific encoding.

    Looks up the lowercase step class name (e.g. ``"relu"``) in this
    canonicalizer's ``_CANONICALIZERS`` registry and invokes the
    corresponding encoder. Raises ``NotImplementedError`` if the step
    type is not registered.

    Parameters
    ----------
    step: Step
        The concrete step class to canonicalize. This class encapsulates
        the thresholding or activation operator.

    Returns
    -------
    None

    Raises
    ------
    NotImplementedError
        If this canonicalizer's ``_CANONICALIZERS`` registry has no
        entry for the concrete type of ``step``.

    """
    canonicalizer = self._CANONICALIZERS.get(type(step).__name__.lower())
    if canonicalizer is None:
        raise NotImplementedError(
            f"{self.__class__.__name__} does not support {type(step).__name__}"
        )
    canonicalizer(self, step)

copy abstractmethod

copy() -> Canonicalizer

Create an independent copy of this canonicalizer.

The copy shares no mutable state with the original and can be modified (e.g., adding objectives) without affecting the original.

Returns:

Type	Description
Canonicalizer	A new canonicalizer instance with copied state.

Raises:

Type	Description
NotImplementedError	Must be implemented by subclasses.

Source code in fomtools/canonicalizers/canonicalizer.py

@abstractmethod
def copy(self) -> "Canonicalizer":
    """
    Create an independent copy of this canonicalizer.

    The copy shares no mutable state with the original and can be
    modified (e.g., adding objectives) without affecting the original.

    Returns
    -------
    Canonicalizer
        A new canonicalizer instance with copied state.

    Raises
    ------
    NotImplementedError
        Must be implemented by subclasses.
    """
    raise NotImplementedError

get_expression_bounds

get_expression_bounds(expr: LinearExpression) -> tuple[ndarray, ndarray]

Compute bounds for a linear expression.

Uses OBBT if available for tighter bounds, otherwise falls back to interval bound propagation.

Parameters:

Name	Type	Description	Default
expr	LinearExpression	The expression whose bounds are to be computed.	required

Returns:

Type	Description
tuple[ndarray, ndarray]	A tuple (lb, ub) of lower and upper bounds.

Source code in fomtools/canonicalizers/canonicalizer.py

def get_expression_bounds(self, expr: LinearExpression) -> tuple[np.ndarray, np.ndarray]:
    """
    Compute bounds for a linear expression.

    Uses OBBT if available for tighter bounds, otherwise falls back
    to interval bound propagation.

    Parameters
    ----------
    expr : LinearExpression
        The expression whose bounds are to be computed.

    Returns
    -------
    tuple[np.ndarray, np.ndarray]
        A tuple (lb, ub) of lower and upper bounds.
    """
    lb, ub = expr.bound_propagation()

    if self.obbt_handler is not None:
        result = self.obbt_handler.obbt(expr, lb, ub)
        if result is not None:
            lb, ub = result

    return lb, ub

set_infinity_norm_objective

set_infinity_norm_objective(expr_list: list[LinearExpression], lb=None, ub=None) -> None

Set the objective to maximize the infinity-norm (maximum absolute value).

Parameters:

Name	Type	Description	Default
expr_list	list[LinearExpression]	List of expressions whose maximum absolute value is the objective.	required
lb	ndarray	Lower bounds for the expressions (may be used by some backends).	None
ub	ndarray	Upper bounds for the expressions (may be used by some backends).	None

Raises:

Type	Description
NotImplementedError	Must be implemented by subclasses that support this objective type.

Source code in fomtools/canonicalizers/canonicalizer.py

def set_infinity_norm_objective(
    self, expr_list: list[LinearExpression], lb=None, ub=None
) -> None:
    """
    Set the objective to maximize the infinity-norm (maximum absolute value).

    Parameters
    ----------
    expr_list : list[LinearExpression]
        List of expressions whose maximum absolute value is the objective.
    lb : np.ndarray, optional
        Lower bounds for the expressions (may be used by some backends).
    ub : np.ndarray, optional
        Upper bounds for the expressions (may be used by some backends).

    Raises
    ------
    NotImplementedError
        Must be implemented by subclasses that support this objective type.
    """
    raise NotImplementedError(
        f"{self.__class__.__name__} does not support set_infinity_norm_objective"
    )

set_one_norm_objective

set_one_norm_objective(expr_list: list[LinearExpression]) -> None

Set the objective to maximize the one-norm (sum of absolute values).

Parameters:

Name	Type	Description	Default
expr_list	list[LinearExpression]	List of expressions whose one-norms are summed in the objective.	required

Raises:

Type	Description
NotImplementedError	Must be implemented by subclasses that support this objective type.

Source code in fomtools/canonicalizers/canonicalizer.py

def set_one_norm_objective(self, expr_list: list[LinearExpression]) -> None:
    """
    Set the objective to maximize the one-norm (sum of absolute values).

    Parameters
    ----------
    expr_list : list[LinearExpression]
        List of expressions whose one-norms are summed in the objective.

    Raises
    ------
    NotImplementedError
        Must be implemented by subclasses that support this objective type.
    """
    raise NotImplementedError(
        f"{self.__class__.__name__} does not support set_one_norm_objective"
    )

set_two_norm_objective

set_two_norm_objective(expr_list: list[LinearExpression], lb_list=None, ub_list=None) -> None

Set the objective to maximize the sum of squared two-norms.

Configures the optimization objective to compute sum_i ||expr_i||_2^2 over the given list of expressions.

Parameters:

Name	Type	Description	Default
expr_list	list[LinearExpression]	List of expressions whose squared norms are summed in the objective.	required
lb_list	list[ndarray]	Lower bounds for each expression (may be used by some backends).	None
ub_list	list[ndarray]	Upper bounds for each expression (may be used by some backends).	None

Raises:

Type	Description
NotImplementedError	Must be implemented by subclasses that support this objective type.

Source code in fomtools/canonicalizers/canonicalizer.py

def set_two_norm_objective(
    self, expr_list: list[LinearExpression], lb_list=None, ub_list=None
) -> None:
    """
    Set the objective to maximize the sum of squared two-norms.

    Configures the optimization objective to compute
    sum_i ||expr_i||_2^2 over the given list of expressions.

    Parameters
    ----------
    expr_list : list[LinearExpression]
        List of expressions whose squared norms are summed in the
        objective.
    lb_list : list[np.ndarray], optional
        Lower bounds for each expression (may be used by some backends).
    ub_list : list[np.ndarray], optional
        Upper bounds for each expression (may be used by some backends).

    Raises
    ------
    NotImplementedError
        Must be implemented by subclasses that support this objective type.
    """
    raise NotImplementedError(
        f"{self.__class__.__name__} does not support set_two_norm_objective"
    )

update_bounds

update_bounds(expression: LinearExpression, lb: float | ndarray, ub: float | ndarray) -> None

Update the lower and upper bounds of the solver variables corresponding to the given LinearExpression.

Parameters:

Name	Type	Description	Default
expression	LinearExpression	A LinearExpression object whose underlying variable bounds are to be updated.	required
lb	float or ndarray	New lower bound(s).	required
ub	float or ndarray	New upper bound(s).	required

Raises:

Type	Description
NotImplementedError	Must be implemented by subclasses.

Source code in fomtools/canonicalizers/canonicalizer.py

def update_bounds(
    self, expression: LinearExpression, lb: float | np.ndarray, ub: float | np.ndarray
) -> None:
    """
    Update the lower and upper bounds of the solver variables corresponding
    to the given ``LinearExpression``.

    Parameters
    ----------
    expression : LinearExpression
        A ``LinearExpression`` object whose underlying variable
        bounds are to be updated.

    lb : float or np.ndarray
        New lower bound(s).

    ub : float or np.ndarray
        New upper bound(s).

    Raises
    ------
    NotImplementedError
        Must be implemented by subclasses.
    """
    raise NotImplementedError

MIP canonicalizer

MIP

Bases: Canonicalizer

An interface for mixed-integer programming (MIP) methods.

Attributes:

Name	Type	Description
solver	Solver	A ready-to-use instance of a subclass of Solver (e.g., gurobi.Gurobi()).
_objective_spec	tuple or None	Stored objective specification for deferred application. Applied to a copy of the solver at solve time.
_solved_copy	Solver or None	Reference to the solved copy of the solver, used for extracting solution values after solve.
objective_branch_priority	int	Branching priority given to the binaries of the infinity-norm objective (component selectors and signs); step binaries keep priority 0. Branching on the objective first splits the worst case by component and sign, which closes the bound much faster once the objective's big-M constants are tight. Set to 0 to disable. Solvers without branching priorities ignore it.

Source code in fomtools/canonicalizers/mip.py

class MIP(Canonicalizer):
    """
    An interface for mixed-integer programming (MIP) methods.

    Attributes
    ----------
    solver: Solver
        A ready-to-use instance of a subclass of `Solver`
        (e.g., ``gurobi.Gurobi()``).
    _objective_spec: tuple or None
        Stored objective specification for deferred application.
        Applied to a copy of the solver at solve time.
    _solved_copy: Solver or None
        Reference to the solved copy of the solver, used for
        extracting solution values after solve.
    objective_branch_priority: int
        Branching priority given to the binaries of the infinity-norm objective
        (component selectors and signs); step binaries keep priority 0. Branching
        on the objective first splits the worst case by component and sign, which
        closes the bound much faster once the objective's big-M constants are
        tight. Set to 0 to disable. Solvers without branching priorities ignore it.

    """

    objective_branch_priority = 10

    def __init__(self, solver_params=None, solver="gurobi"):
        self.steps = []
        super(MIP, self).__init__(solver_params, solver)
        self._objective_spec = None
        self._solved_copy = None
        # Warm-start bookkeeping (see ``_start_values``): the leaves whose values
        # determine every other leaf, the binaries each step created, the leaves
        # the objective created on the last solve copy, the inputs of the last
        # solution, and optional user samples of the inputs.
        self._input_leaves = []
        self._step_binaries = {}
        self._objective_leaves = None
        self._warm_start_inputs = None
        self._start_samples = None
        self.warm_start_stats = None

    def add_var(self, var: LinearExpression, **kwargs) -> None:
        """
        Add a variable to the MIP instance. This method registers
        a new variable with the MIP program and attaches it to the
        internal model so it can be referenced in the constraints.

        Parameters
        ----------
        var: LinearExpression
            Linear expression object to add to the MIP instance. Later to
            be used in an objective function, constraints, or parameters.

        Returns
        -------
        None
        """
        self.solver.add_var(var=var, lb=var.lb, ub=var.ub)
        if kwargs.get("var_type") in ("param", "iterate"):
            self._input_leaves.append(var)

    def add_constraint(self, constraint: Constraint) -> None:
        """
        Add a constraint to the MIP instance. This method registers
        a new constraint with the MIP problem and attaches it to the
        internal model.

        Parameters
        ----------
        constraint: Constraint
             Constraint object to add to the MIP instance.

        Returns
        -------
        None
        """
        if isinstance(constraint, ComplementarityConstraint):
            self.add_constraints(self._build_complementarity_constraints(constraint))
            return
        if isinstance(constraint, QuadraticConstraint):
            # Quadratic (sum-of-squares) constraints require a solver that supports
            # quadratic constraints (MIQCP). Gurobi does; HiGHS does not. Each solver
            # validates this in its own ``add_sum_squares_constr`` (HiGHS raises), so the
            # error still surfaces eagerly at add-time rather than at solve-time.
            self.solver.add_sum_squares_constr(
                constraint.expr, constraint.radius_sq, constraint.sense
            )
            return
        self.solver.add_constr(constraint)

    def _build_complementarity_constraints(
        self, constraint: ComplementarityConstraint
    ) -> List[Constraint]:
        r"""
        Build the big-M MIP formulation of an elementwise complementarity constraint.

        For ``e1 ⊥ e2`` (both ``>= 0``), each coordinate ``i`` gets a fresh binary
        ``z`` and, with finite big-M values ``M1 = e1.ub[i]`` and ``M2 = e2.ub[i]``:

        $$
            e_{1,i} \ge 0,\quad e_{2,i} \ge 0,\quad
            e_{1,i} \le M_{1}\,z,\quad e_{2,i} \le M_{2}\,(1 - z).
        $$

        ``z = 1`` forces ``e2[i] = 0``; ``z = 0`` forces ``e1[i] = 0``. This mirrors the
        ReLU big-M idiom in `_build_relu_constraints`.

        Parameters
        ----------
        constraint : ComplementarityConstraint
            The complementarity constraint to encode.

        Returns
        -------
        List[Constraint]
            The linear (and binary-coupled) constraints encoding the relation.

        Raises
        ------
        ValueError
            If either expression has a non-finite upper bound, so big-M is undefined.
        """
        e1, e2 = constraint.expr1, constraint.expr2
        # Use bound_propagation so the big-M is correct for composite expressions
        # (e.g. ``b - A @ u``), not just leaf variables: a composite's ``.ub`` is the
        # list of its leaves' bounds, whereas bound_propagation returns the elementwise
        # upper bound of the expression itself.
        with np.errstate(invalid="ignore"):  # inf - inf when an input is unbounded
            _, M1 = e1.bound_propagation()
            _, M2 = e2.bound_propagation()
        if not (np.all(np.isfinite(M1)) and np.all(np.isfinite(M2))):
            raise ValueError(
                "Complementarity big-M encoding requires finite upper bounds on both "
                "expressions. Provide explicit variable bounds or enable OBBT."
            )

        constraints = []
        for i in range(constraint.ndim):
            _e1 = e1[i]
            _e2 = e2[i]
            z = LinearExpression.from_int(_e1.ndim, is_binary=True)  # scalar binary
            constraints.extend(
                [
                    _e1 >= 0,
                    _e2 >= 0,
                    _e1 <= M1[i] * z,
                    _e2 <= M2[i] * (1 - z),
                ]
            )
        return constraints

    def add_constraints(self, constraints: List[Constraint]) -> None:
        """
        Add a list of constraints to the MIP instance. This method registers
        multiple constraints with the MIP problem and attaches them to
        the internal model.

        Parameters
        ----------
        constraints: List[Constraint]
            List of constraints to add to the MIP instance.

        Returns
        -------
        None
        """
        self.solver.constraints_to_solver(constraints)

    def add_theory_cut(
        self,
        margin: float,
        target_expr: LinearExpression,
        lb: float | np.ndarray,
        ub: float | np.ndarray,
    ):
        """
        Add relaxed bound constraints on the target expression up to the
        provided relaxation margin.

        Parameters
        ----------
        margin: float
            Relaxation margin for the bounds. If finite, the lower bound is
            shifted to ``lb - margin`` and the upper bound to ``ub + margin``. If
            ``np.inf``, no constraints are added.

        target_expr: LinearExpression
            The expression whose bounds are to be relaxed.

        lb: float | np.ndarray
            Lower bound(s) for ``target_expr`` before relaxation.

        ub: float | np.ndarray
            Upper bound(s) for ``target_expr`` before relaxation.

        Returns
        -------
        None
        """
        if margin != np.inf:
            self.solver.constraints_to_solver(
                [target_expr >= lb - margin, target_expr <= ub + margin]
            )

    def update_bounds(
        self, expression: LinearExpression, lb: float | np.ndarray, ub: float | np.ndarray
    ):
        """
        Update the lower and upper bounds of the solver variables corresponding
        to the given ``LinearExpression``.

        Parameters
        ----------
        expression: LinearExpression
            A ``LinearExpression`` object whose underlying variables
            bounds are to be updated.

        lb: float or np.ndarray
            New lower bound(s).

        ub: float or np.ndarray
            New upper bound(s).


        Returns
        -------
        None
        """
        self.solver.update_bounds(expression, lb, ub)

    def copy(self) -> "MIP":
        """
        Create an independent copy of this MIP canonicalizer.

        Returns a new MIP canonicalizer with a copied solver. The copy
        can be modified without affecting the original.

        Returns
        -------
        MIP
            A new canonicalizer instance with copied state.
        """
        new_canon = MIP.__new__(MIP)
        new_canon.solver_params = dict(self.solver_params) if self.solver_params else None
        new_canon.solver_name = self.solver_name
        new_canon.solver = self.solver.copy()
        new_canon.obbt_handler = self.obbt_handler
        new_canon._objective_spec = self._objective_spec
        new_canon._solved_copy = None
        return new_canon

    def set_zero_objective(self):
        """Store zero objective specification for deferred application."""
        self._objective_spec = ("zero",)

    def set_infinity_norm_objective(self, expr_list, lb_list, ub_list):
        """Store infinity-norm objective specification for deferred application."""
        self._objective_spec = ("infinity_norm", expr_list, lb_list, ub_list)

    def set_two_norm_objective(self, expr_list, lb_list, ub_list):
        """Store two-norm objective specification for deferred application."""
        self._objective_spec = ("two_norm", expr_list, lb_list, ub_list)

    def set_one_norm_objective(self, expr_list):
        """Store one-norm objective specification for deferred application."""
        self._objective_spec = ("one_norm", expr_list)

    def _apply_objective(self, solver):
        """
        Apply stored objective to the given solver.

        Parameters
        ----------
        solver : Solver
            The solver instance to apply the objective to.
        """
        if self._objective_spec is None:
            return

        obj_type = self._objective_spec[0]
        match obj_type:
            case "infinity_norm":
                _, expr_list, lb_list, ub_list = self._objective_spec
                self._apply_infinity_norm_objective(solver, expr_list, lb_list, ub_list)
            case "two_norm":
                _, expr_list, lb_list, ub_list = self._objective_spec
                self._apply_two_norm_objective(solver, expr_list, lb_list, ub_list)
            case "one_norm":
                _, expr_list = self._objective_spec
                self._apply_one_norm_objective(solver, expr_list)
            case "zero":
                solver.set_objective(expression=0)

    def _apply_infinity_norm_objective(self, solver, expr_list, lb_list, ub_list):
        """
        Build and apply infinity-norm objective constraints to solver.

        Parameters
        ----------
        solver : Solver
            The solver instance to apply the objective to.
        expr_list : list[LinearExpression]
            List of expressions for the infinity norm.
        lb_list : list[np.ndarray]
            Lower bounds for each expression.
        ub_list : list[np.ndarray]
            Upper bounds for each expression.
        """
        constraints = []
        M = np.max(np.abs(np.hstack([np.hstack(lb_list), np.hstack(ub_list)])))
        q = LinearExpression.from_int(ndim=1, ub=M)
        all_gammas = []
        self._objective_leaves = {"q": q, "parts": []}

        for expr, lb, ub in zip(expr_list, lb_list, ub_list):
            n = expr.ndim

            up = LinearExpression.from_int(ndim=n, ub=np.abs(ub))
            un = LinearExpression.from_int(ndim=n, ub=np.abs(lb))
            constraints.append(up - un == expr)

            all_gammas.append(
                gamma := LinearExpression.from_int(
                    ndim=n, is_binary=True, branch_priority=self.objective_branch_priority
                )
            )
            signs = {}
            self._objective_leaves["parts"].append(
                {"expr": expr, "up": up, "un": un, "gamma": gamma, "signs": signs}
            )
            for j, (_lb, _ub, _up, _un, _gamma) in enumerate(zip(lb, ub, up, un, gamma)):
                if _lb >= 0:
                    constraints.append(_un == 0)
                elif _ub < 0:
                    constraints.append(_up == 0)
                else:
                    vobji = LinearExpression.from_int(
                        ndim=1, is_binary=True, branch_priority=self.objective_branch_priority
                    )
                    signs[j] = vobji
                    constraints.extend(
                        [_up <= np.abs(_ub) * vobji, _un <= np.abs(_lb) * (1 - vobji)]
                    )

                constraints.extend([q >= _up + _un, q <= _up + _un + M * (1 - _gamma)])

        # Need one gamma across all vectors in the list.
        constraints.append(sum([sum(gamma) for gamma in all_gammas]) == 1)

        solver.constraints_to_solver(constraints)
        solver.set_objective(expression=q, sense="maximize")

    def _apply_two_norm_objective(self, solver, expr_list, lb_list, ub_list):
        """
        Build and apply two-norm objective to solver.

        The two-norm objective requires a quadratic program (MIQP); solvers that
        cannot represent one raise when the objective is set.

        Parameters
        ----------
        solver : Solver
            The solver instance to apply the objective to.
        expr_list : list[LinearExpression]
            List of expressions for the two-norm objective.
        lb_list : list[np.ndarray]
            Lower bounds for each expression.
        ub_list : list[np.ndarray]
            Upper bounds for each expression.

        Raises
        ------
        ValueError
            If the solver does not support quadratic objectives.
        """
        solver.set_two_norm_objective(expr_list, lb_list, ub_list)

    def _apply_one_norm_objective(self, solver, expr_list):
        """
        Build and apply one-norm objective to solver.

        Uses the identity: ||x||_1 = 1^T @ (relu(x) + relu(-x))

        Parameters
        ----------
        solver : Solver
            The solver instance to apply the objective to.
        expr_list : list[LinearExpression]
            List of expressions for the one-norm objective.
        """
        constraints = []
        q = 0

        for expr in expr_list:
            n = expr.ndim

            # Create relu(expr) step
            step_pos = Relu(arg=expr)
            step_pos.update_bounds(*self.get_expression_bounds(expr))
            for leaf in step_pos.output_leaves:
                solver.add_var(leaf)
            constraints.extend(self._build_relu_constraints(step_pos))

            # Create relu(-expr) step
            neg_expr = -expr
            step_neg = Relu(arg=neg_expr)
            step_neg.update_bounds(*self.get_expression_bounds(neg_expr))
            for leaf in step_neg.output_leaves:
                solver.add_var(leaf)
            constraints.extend(self._build_relu_constraints(step_neg))

            # Accumulate: 1^T @ (relu(expr) + relu(-expr))
            ones = np.ones((1, n))
            q = q + ones @ step_pos.output + ones @ step_neg.output

        solver.constraints_to_solver(constraints)
        solver.set_objective(expression=q, sense="maximize")

    def separate_hull_cuts(self, tol: float = 1e-6) -> List[Constraint]:
        """
        Separate the root LP optimum from the convex hulls of the steps.

        Copies the model (which carries every cut added so far), applies the
        objective, solves the continuous relaxation and returns the most violated
        hull inequality of every unstable step component. The model itself is not
        modified; ``Verifier.solve`` adds the returned cuts.

        Parameters
        ----------
        tol: float, default 1e-6
            Minimum violation for a cut to be returned.

        Returns
        -------
        List[Constraint]
            The separated cuts.
        """
        relaxed = self.solver.copy()
        if self.solver_params:
            relaxed.set_solver_params(self.solver_params)
        self._apply_objective(relaxed)
        cuts = relaxed.separate_convex_hull_cuts(self.steps, tol=tol)
        self.last_separation_stats = relaxed.last_separation_stats
        return cuts

    def tighten_objective_bounds(self, pad: float = 1e-6) -> dict | None:
        """
        Replace the interval bounds of an infinity-norm objective with OBBT bounds.

        The infinity-norm encoding takes its big-M constants (the global ``M`` and
        the per-component sign constants) from bounds on each residual component.
        Any bounds that hold on every feasible point keep the optimum exact, so the
        interval-propagation bounds set with the objective are intersected with
        the minimum and maximum of each component over the OBBT LP. The OBBT
        values are padded outward by ``pad * max(1, |bound|)`` to absorb the LP
        solver's feasibility tolerance, then clipped back into the interval
        bounds, which are exactly valid.

        Does nothing unless the objective is an infinity norm and an OBBT handler
        is attached.

        Parameters
        ----------
        pad: float, default 1e-6
            Relative outward padding of the OBBT bounds.

        Returns
        -------
        dict or None
            ``None`` if nothing was done, otherwise ``time`` (seconds),
            ``width_ratio`` (median interval width over tightened width),
            ``M_before``, ``M_after`` and ``sign_fixed`` (components whose sign
            the tightened bounds determine but the interval bounds did not).
        """
        obbt_handler = self.obbt_handler
        if obbt_handler is None or self._objective_spec is None:
            return None
        if self._objective_spec[0] != "infinity_norm":
            return None

        start = time.perf_counter()
        _, expr_list, lb_list, ub_list = self._objective_spec
        new_lb, new_ub = [], []
        for expr, lb, ub in zip(expr_list, lb_list, ub_list):
            lb, ub = np.asarray(lb, dtype=float), np.asarray(ub, dtype=float)
            result = obbt_handler.obbt(expr, lb, ub)
            if result is None:
                new_lb.append(lb)
                new_ub.append(ub)
                continue
            t_lb, t_ub = result
            t_lb = np.maximum(t_lb - pad * np.maximum(1.0, np.abs(t_lb)), lb)
            t_ub = np.minimum(t_ub + pad * np.maximum(1.0, np.abs(t_ub)), ub)
            new_lb.append(t_lb)
            new_ub.append(t_ub)
        self._objective_spec = ("infinity_norm", expr_list, new_lb, new_ub)

        def big_m(lbs, ubs):
            return float(np.max(np.abs(np.hstack([np.hstack(lbs), np.hstack(ubs)]))))

        old_lb, old_ub = np.hstack(lb_list), np.hstack(ub_list)
        t_lb, t_ub = np.hstack(new_lb), np.hstack(new_ub)
        old_fixed = (old_lb >= 0) | (old_ub < 0)
        new_fixed = (t_lb >= 0) | (t_ub < 0)
        return dict(
            time=time.perf_counter() - start,
            width_ratio=float(np.median((old_ub - old_lb) / np.maximum(t_ub - t_lb, 1e-12))),
            M_before=big_m(lb_list, ub_list),
            M_after=big_m(new_lb, new_ub),
            sign_fixed=int(np.sum(new_fixed & ~old_fixed)),
        )

    def set_start_samples(self, samples: dict) -> None:
        """
        Samples of input leaves for the sample-max warm start.

        Parameters
        ----------
        samples: dict[LinearExpression, np.ndarray]
            ``(n, ndim)`` samples per input leaf (parameters or initial
            iterates), sharing the same ``n``. Inputs without samples are held at
            their value if their bounds fix them, and the start is skipped otherwise.
        """
        self._start_samples = {
            leaf: np.atleast_2d(np.asarray(v, dtype=float)) for leaf, v in samples.items()
        }

    def _start_values(self, inputs: dict) -> tuple[dict, dict]:
        """
        Every leaf and binary of the solve copy at the trajectory from ``inputs``.

        ``inputs`` maps input leaves to ``(n, ndim)`` values. The steps run in
        canonicalization order, each binary takes the branch its big-M rows force
        at the computed argument, and the infinity-norm objective's leaves are set
        from the residual. Values are clipped into the leaves' current bounds.
        Leaves that cannot be computed (unsupported steps or missing inputs) are
        left out, so the solver may complete them. Returns the values, with ``n``
        rows each, and the row-wise residual norm under ``"objective"`` in the
        second dict (or ``None`` without an infinity-norm objective).
        """
        values = dict(inputs)
        for step in self.steps:
            arg = _evaluate(step.arg, values)
            if arg is None:
                continue
            binaries = self._step_binaries.get(id(step), {})
            if isinstance(step, Affine):
                values[step.output] = _solve_affine(step, arg)
            elif isinstance(step, Relu):
                out = np.where(
                    np.isin(np.arange(arg.shape[1]), list(step.proj_indices)),
                    np.maximum(arg, 0.0),
                    arg,
                )
                leaf = step.output_leaves[0]
                values[leaf] = out[:, sorted(step.proj_indices)] if leaf is not step.output else out
                for i, b in binaries.items():
                    values[b["z"]] = (arg[:, [i]] > 0).astype(float)
            elif isinstance(step, SaturatedLinear):
                if isinstance(step.l, LinearExpression) or isinstance(step.u, LinearExpression):
                    continue
                l = np.broadcast_to(np.asarray(step.l, dtype=float), (arg.shape[1],))
                u = np.broadcast_to(np.asarray(step.u, dtype=float), (arg.shape[1],))
                values[step.output] = np.clip(arg, l, u)
                for i, b in binaries.items():
                    values[b["up"]] = (arg[:, [i]] > u[i]).astype(float)
                    values[b["low"]] = (arg[:, [i]] < l[i]).astype(float)
            elif isinstance(step, SoftThreshold):
                lambd = np.broadcast_to(np.asarray(step.lambd, dtype=float), (arg.shape[1],))
                values[step.output] = np.sign(arg) * np.maximum(np.abs(arg) - lambd, 0.0)
                for i, b in binaries.items():
                    values[b["right"]] = (arg[:, [i]] > lambd[i]).astype(float)
                    values[b["left"]] = (arg[:, [i]] < -lambd[i]).astype(float)

        info = {"objective": None}
        obj = self._objective_leaves
        if obj is not None and self._objective_spec and self._objective_spec[0] == "infinity_norm":
            residuals = [_evaluate(part["expr"], values) for part in obj["parts"]]
            if all(r is not None for r in residuals):
                absr = np.hstack([np.abs(r) for r in residuals])
                best = np.argmax(absr, axis=1)
                offset = 0
                for part, r in zip(obj["parts"], residuals):
                    values[part["up"]] = np.maximum(r, 0.0)
                    values[part["un"]] = np.maximum(-r, 0.0)
                    cols = np.arange(r.shape[1]) + offset
                    values[part["gamma"]] = (best[:, None] == cols[None, :]).astype(float)
                    for j, v in part["signs"].items():
                        values[v] = (r[:, [j]] > 0).astype(float)
                    offset += r.shape[1]
                values[obj["q"]] = absr.max(axis=1, keepdims=True)
                info["objective"] = absr.max(axis=1)

        for leaf, val in values.items():
            values[leaf] = np.clip(
                val, np.asarray(leaf.lb, dtype=float), np.asarray(leaf.ub, dtype=float)
            )
        return values, info

    def _free_inputs(self) -> list:
        """Input leaves that no step computes (trajectory slots can be registered as iterates)."""
        outputs = {id(leaf) for step in self.steps for leaf in step.output_leaves}
        return [leaf for leaf in self._input_leaves if id(leaf) not in outputs]

    def _warm_starts(self) -> list[dict]:
        """The sequential start and the sample-max start, as ``{leaf: 1-D value}`` maps."""
        starts, stats = [], []
        free = self._free_inputs()
        present = self._solved_copy.vector_var_map

        def seeded(values):
            return sum(1 for leaf in values if leaf in present)

        if self._warm_start_inputs is not None:
            inputs = {
                leaf: np.atleast_2d(self._warm_start_inputs[leaf])
                for leaf in free
                if leaf in self._warm_start_inputs
            }
            values, info = self._start_values(inputs)
            starts.append({leaf: v[0] for leaf, v in values.items()})
            stats.append(
                dict(kind="sequential", objective=_first(info["objective"]), seeded=seeded(values))
            )
        if self._start_samples:
            n = len(next(iter(self._start_samples.values())))
            inputs = {}
            for leaf in free:
                if leaf in self._start_samples:
                    inputs[leaf] = self._start_samples[leaf]
                else:
                    lb, ub = np.asarray(leaf.lb, dtype=float), np.asarray(leaf.ub, dtype=float)
                    if not np.allclose(lb, ub):
                        inputs = None
                        break
                    inputs[leaf] = np.tile(lb, (n, 1))
            if inputs is not None:
                values, info = self._start_values(inputs)
                if info["objective"] is not None:
                    k = int(np.argmax(info["objective"]))
                    starts.append({leaf: v[k] for leaf, v in values.items()})
                    stats.append(
                        dict(
                            kind="sample_max",
                            objective=float(info["objective"][k]),
                            seeded=seeded(values),
                        )
                    )
        for st in stats:
            st["leaves"] = len(present)
        self.warm_start_stats = stats
        return starts

    def solve_model(self, warm_start: bool = True, **solver_kwargs) -> float:
        """
        Optimize the underlying MIP problem.

        Creates a copy of the solver, applies the objective to the copy,
        solves it, and keeps a reference for solution extraction.
        This prevents objective constraints from accumulating across
        multiple solve calls. Convex hull cuts are added to the model
        beforehand by ``Verifier.solve``.

        Parameters
        ----------
        warm_start: bool, default True
            Give the solver up to two starts: the inputs of the previous
            solution's worst case run through the current steps, and the
            worst sample of ``set_start_samples``. Solvers without MIP starts
            ignore them.
        **solver_kwargs
            Native solver options forwarded verbatim to the backend
            (e.g. ``TimeLimit=10``, ``MIPGap=1e-4`` for Gurobi). Applied
            to the solve copy, so they override construction-time
            ``params`` for this call. A ``ValueError`` is raised if the
            backend rejects an option.

        Returns
        -------
        float
            The finite best upper bound on the maximization objective, including
            after a time limit. Raises ``RuntimeError`` when none is available.
            Incumbent values and status remain on the solved solver copy.

        """
        # Create copy to apply objective without modifying original
        self._solved_copy = self.solver.copy()

        # Re-apply construction-time options: HiGHS's copy() uses passModel,
        # which copies the model data but not solver options, so the copy
        # would otherwise reset them to defaults. (Gurobi's copy() preserves
        # them, making this a harmless no-op there.) solver_kwargs are applied
        # last so solve-time options override construction-time params.
        if self.solver_params:
            self._solved_copy.set_solver_params(self.solver_params)

        if solver_kwargs:
            self._solved_copy.set_solver_params(solver_kwargs)

        self._apply_objective(self._solved_copy)
        self.warm_start_stats = None
        if warm_start:
            starts = self._warm_starts()
            if starts:
                self._solved_copy.set_starts(starts)
        self._solved_copy.optimize()
        if self._solved_copy.has_solution():
            self._warm_start_inputs = {
                leaf: self._solved_copy.get_solution_value(leaf) for leaf in self._input_leaves
            }
        return self._solved_copy.get_best_bound()

    def get_solution_value(self, expression: LinearExpression) -> np.ndarray:
        """
        Return the solution value for the given ``LinearExpression``.

        Parameters
        ----------
        expression: LinearExpression
            The expression whose solution value is to be computed. Each element in
            its decomposition must have been registered with the solver.

        Returns
        -------
        np.ndarray[float]
            A 1-D array of shape ``(expression.ndim,)`` containing the
            evaluated values at the current model solution.

        Raises
        ------
        RuntimeError
            If no solution is available (solve_model() not called).
        """
        if self._solved_copy is None or not self._solved_copy.has_solution():
            raise RuntimeError("No feasible incumbent available. Call solve() first.")
        for leaf in expression.decomposition_dict:
            if leaf not in self._solved_copy.vector_var_map:
                if leaf in self.solver.vector_var_map:
                    raise RuntimeError(
                        "The expression uses variables added after the last solve; "
                        "call solve() again."
                    )
                raise RuntimeError("The expression has variables not registered with this model.")
        return self._solved_copy.get_solution_value(expression)

    def canonicalize_affine(self, step: Affine) -> None:
        """
        Construct the linear equality constraint corresponding to the
        affine iteration step. The constraint is used to express the affine
        update step in the verification problem.

        Parameters
        ----------
        step: Affine
            An instance of the ``Affine`` class to canonicalize to a MIP problem.

        Returns
        -------
        None
        """
        self.steps.append(step)
        self.add_constraint(step.output_mat @ step.output == step.arg)

    def canonicalize_relu(self, step: Relu):
        r"""
        Construct the convex-hull linear constraints corresponding
        to the `Rectified Linear Unit (ReLU)` operator.

        Given a ReLU step: $(\phi(v,x))_i = \mathbf{ReLU}(a^T y) = \max\{a^T y, 0\}.$

        This method derives the convex hull of the region defined by:

        $$
            \phi = \left\{ (y, w) \in \left [\underline{y}, \overline{y} \right ] \times
            \mathbb{R} | w = \textbf{ReLU}(a^{T}y) \right\},
        $$

        It then expresses it as a set of Constraint objects. The convex hull
        of the region is:


        $$
            \operatorname{conv}(\Phi) =
            \left\{
            (y, w) \in [\underline{y}, \overline{y}] \times \mathbb{R}
            \;\middle|\;
            \begin{cases}
            w = a^{\mathsf{T}} y, & \ell_J \ge 0, \\[6pt]
            w = 0, & \ell_J < 0, \\[6pt]
            (y, w) \in Q, & \text{otherwise}
            \end{cases}
            \right\},
        $$

        with

        $$
            Q =
            \left\{
            (y, w) \in \mathbb{R}^{d + p + 1}
            \;\middle|\;
            \begin{aligned}
            & w \ge a^{\mathsf{T}} y, \quad y \ge 0, \\[4pt]
            & w \le \sum_{i \in I} a_i (y_i - \ell_i^{0})
               + \frac{\ell_I}{u_o^{0} - \ell_o^{0}} (y_o - \ell_o^{0}),
               \quad \forall (I, o) \in \mathcal{I}
            \end{aligned}
            \right\},
        $$

        where:

        - vector $l^{0} \in \mathbb{R}^{d}$ with entries $l_i^{0} = \underline{y}_i \text{ if }
        a_i \geq 0 \text{ and } \overline{y}_i$ otherwise,
        - vector $u^{0} \in \mathbb{R}^{d}$ with entries $u_i^{0} = \overline{y}_i \text{ if }
        a_i \geq 0 \text{ and } \underline{y}_i$ otherwise,
        - Given $\textbf{I} \subseteq \textbf{J} = \left\{1, ..., n\right\}$, the lower bound is
        defined as $l_I = \sum_{i \in I} a_i l_i^{0} + \sum_{i \notin I} a_i u_i^{0}$

        Parameters
        ----------
        step: Relu
            An instance of the `Relu` step to canonicalize.

        Returns
        -------
        None
        """
        constraints = self._build_relu_constraints(step)
        self.add_constraints(constraints)
        self.steps.append(step)

    def _build_relu_constraints(self, step: Relu) -> List[Constraint]:
        """
        Build MIP formulation constraints for a ReLU step.

        Parameters
        ----------
        step : Relu
            An instance of the `Relu` step.

        Returns
        -------
        List[Constraint]
            MIP constraints for the ReLU activation.
        """
        constraints = []
        # Iterate proj indices only. For partial projection, ``step.output`` is a
        # composite whose non-proj rows share decomposition with ``step.arg``;
        # those rows already satisfy ``output[i] == arg[i]`` structurally, so no
        # explicit constraint is needed. For full projection, ``proj_indices``
        # covers every row and behavior matches the previous implementation.
        for i in sorted(step.proj_indices):
            _output = step.output[i]
            _arg = step.arg[i]
            if step.ub[i] <= 0:
                constraints.append(_output == 0)
            elif step.lb[i] > 0:
                constraints.append(_output == _arg)
            else:
                _require_finite_bounds("ReLU", i, step.lb[i], step.ub[i])
                new_var = LinearExpression.from_int(_arg.ndim, is_binary=True)
                self._step_binaries.setdefault(id(step), {})[i] = {"z": new_var}
                step.idx_with_binary_vars.add(i)
                constraints.append(_output <= step.ub[i] * new_var)
                constraints.extend(
                    [
                        _output <= step.ub[i] / (step.ub[i] - step.lb[i]) * (_arg - step.lb[i]),
                        _output >= _arg,
                        _output <= _arg - step.lb[i] * (1 - new_var),
                    ]
                )
        return constraints

    def canonicalize_saturatedlinear(self, step: SaturatedLinear):
        r"""
        Construct the convex-hull linear constraints corresponding to
        the saturated linear unit operator.

        Given a saturated linear unit operator, defined as:

        $$
            (\phi(v, b))_i = S_{[b, c]}(a^T y) =
            \begin{cases}
            b & a^T y < b, \\\\
            a^T y & b \leq a^T y \leq c, \\\\
            c & a^T y > c,
            \end{cases}
            \quad \text{with } y = (v, b), \text{and }
             b, c \in \mathbb{R},
        $$

        This method derives the convex hull of the region defined by:

        $$
            \Phi = \{ (y, w) \in [\underline{y}, \bar{y}] \times [b, c] \mid w =
            \mathcal{S}_{[b,c]}(a^\top y) \}.
        $$

        It then expresses it as a set of Constraint objects. The convex hull of the
        region is:

        $$
            \operatorname{conv}(\Phi) =
            \left\{
            (y, w) \in [\underline{y}, \bar{y}] \times [b, c]
            \;\middle|\;
            \begin{cases}
            w = b & u_J < b, \\[4pt]
            w = c & \ell_J > c, \\[4pt]
            w = a^\top y & b \le \ell_J \le u_J \le c, \\[4pt]
            (y, w) \in Q & \text{otherwise}
            \end{cases}
            \right\},
        $$

        with

        $$
            Q =
            \left\{
            (y, w) \in \mathbb{R}^{p + d + 1}
            \;\middle|\;
            \begin{aligned}
            & b \le w \le c, \\[4pt]
            & \frac{b - c}{b - u_J}(a^\top y - u_J) + c
            \;\le\;
            w
            \;\le\;
            \frac{c - b}{c - \ell_J}(a^\top y - \ell_J) + b, \\[6pt]
            & w \le \sum_{i \in I} a_i (y_i - \ell^0_i)
            + \frac{\ell_I - b}{u^0_o - \ell^0_o}(y_o - \ell^0_o) + b,
            \qquad \forall (I, o) \in \mathcal{I}^+, \\[6pt]
            & w \ge \sum_{i \in I} a_i (y_i - u^0_i)
            + \frac{u_I - c}{\ell^0_o - u^0_o}(y_o - u^0_o) + c,
            \qquad \forall (I, o) \in \mathcal{I}^-
            \end{aligned}
            \right\},
        $$

        where:

        - vector $l^{0} \in \mathbb{R}^{d}$ with entries $l_i^{0} = \underline{y}_i \text{ if }
        a_i \geq 0 \text{ and } \overline{y}_i$ otherwise,
        - vector $u^{0} \in \mathbb{R}^{d}$ with entries $u_i^{0} = \overline{y}_i \text{ if }
        a_i \geq 0 \text{ and } \underline{y}_i$ otherwise,
        - Given $\textbf{I} \subseteq \textbf{J} = \left\{1, ..., n\right\}$, the lower bound is
        defined as $l_I = \sum_{i \in I} a_i l_i^{0} + \sum_{i \notin I} a_i u_i^{0}$ and the
        upper bound is $u_I = \sum_{i \in I} a_i u_i^{0} + \sum_{i \notin I} a_i l_i^{0}$,
        - $\mathcal{I}^+ = \{ (I, o) \in 2^J \times J \mid \ell_I \ge b,\; \ell_{I \cup \{o\}}
        < b,\; w_I \neq 0 \},$
        - $\mathcal{I}^- = \{ (I, o) \in 2^J \times J \mid u_I \le c,\; u_{I \cup \{o\}} >
        c,\; w_I \neq 0 \}.$

        Parameters
        ----------
        step: SaturatedLinear
            An instance of the `SaturatedLinear` step to canonicalize.

        Returns
        -------
        None
        """
        constraints = []

        for row, (l, u, lb, ub, output, arg) in enumerate(
            zip(step.l, step.u, step.lb, step.ub, step.output, step.arg)
        ):
            if lb >= u:
                constraints.append(output == u)
            elif ub <= l:
                constraints.append(output == l)
            elif lb >= l and ub <= u:
                constraints.append(output == arg)
            else:
                if not step.relax_binary_vars:
                    _require_finite_bounds("Saturated-linear", row, lb, ub)
                    w1 = LinearExpression.from_int(arg.ndim, is_binary=True)
                    w2 = LinearExpression.from_int(arg.ndim, is_binary=True)
                    # w1: upper saturation (arg >= u), w2: lower saturation (arg <= l).
                    self._step_binaries.setdefault(id(step), {})[row] = {"up": w1, "low": w2}

                    if lb < l:
                        constraints.extend(
                            [
                                output <= arg + (ub - lb) * w2,
                                arg >= l + (lb - l) * w2,
                                arg <= l + (ub - l) * (1 - w2),
                            ]
                        )

                        if ub > u:
                            constraints.extend(
                                [
                                    output >= u - (u - l) * (1 - w1),
                                    output <= l + (u - l) * (1 - w2),
                                    output >= l + (lb - l) * w2,
                                    w1 + w2 <= 1,
                                ]
                            )
                        else:
                            # Lower-or-middle (lb < l, ub <= u): the upper face is
                            # never active, so only w2 (lower-saturation vs interior)
                            # matters. Pin the saturated branch: w2=1 => output <= l
                            # (with the output-variable bound output >= l this forces
                            # output = l); w2=0 => output <= ub (interior, slack).
                            # Without this the lower-sat output was unpinned and the
                            # MIP admitted off-graph points (mirror of the
                            # soft-threshold left-or-middle bug).
                            constraints.append(output <= l + (ub - l) * (1 - w2))

                    if ub > u:
                        constraints.extend(
                            [
                                output >= arg - (ub - lb) * w1,
                                arg >= u + (lb - u) * (1 - w1),
                                arg <= u + (ub - u) * w1,
                            ]
                        )
                        if lb >= l:
                            constraints.append(output >= u - (u - lb) * (1 - w1))

                if lb < l and ub > u:
                    constraints.extend(
                        [
                            output <= (u - l) / (u - lb) * (arg - u) + u,
                            output >= (u - l) / (ub - l) * (arg - l) + l,
                        ]
                    )

                # Non-strict, so that the boundary cases lb == l and ub == u still pin
                # the output to arg on the unsaturated side.
                elif lb < l and ub <= u:
                    constraints.extend(
                        [output >= arg, output <= ((ub - l) / (ub - lb)) * (arg - ub) + ub]
                    )

                elif lb >= l and ub > u:
                    constraints.extend(
                        [output <= arg, output >= ((u - lb) / (ub - lb) * (arg - lb) + lb)]
                    )

        self.add_constraints(constraints)
        self.steps.append(step)

    def canonicalize_softthreshold(self, step: SoftThreshold):
        r"""
        Construct the convex-hull linear constraints corresponding to the
        soft-thresholding operator.

        Given a soft-thresholding operator for shrinkage, defined as:

        $$
            (\phi(v,x))_i=\mathcal{T}_{\lambda}(a^\top y)=
            \begin{cases}
            a^\top y-\lambda,& a^\top y>\lambda,\\[6pt]
            0,& |a^\top y|\le \lambda,\\[6pt]
            a^\top y+\lambda,& a^\top y<-\lambda,
            \end{cases}
            \quad \text{with } y=(v,x),
        $$

        where $\mathcal{T}_{\lambda}$ is the soft-thresholding operator with constant $\lambda>0$,
        and $a\in\mathbf{R}^d$.

        This method derives the convex hull of the region defined by:

        $$
            \Phi = \{ (y, w) \in [\underline{y}, \bar{y}] \times [b, c]
            \mid w = \mathcal{S}_{[b,c]}(a^\top y) \}.
        $$

        It then expresses it as a set of Constraint objects. The convex hull of the
        region is:

        $$
            \operatorname{conv}(\Phi) =
            \left\{
            (y, w) \in [\underline{y}, \bar{y}] \times \mathbf{R}
            \;\middle|\;
            \begin{cases}
            w = a^\top y - \lambda & \ell_J > \lambda, \\[4pt]
            w = a^\top y  + \lambda & u_J < - \lambda, \\[4pt]
            w = 0 & - \lambda \le \ell_J \le u_J \le \lambda, \\[4pt]
            (y, w) \in Q & \text{otherwise}
            \end{cases}
            \right\},
        $$

        with

        $$
            Q = \left\{
            (y, w) \in \mathbb{R}^{p + d + 1}
            \;\middle|\;
            \begin{aligned}
            & a^\top y - \lambda \le w \le a^\top y + \lambda, \\[4pt]
            & \frac{l_J + \lambda}{l_J - \lambda}(a^\top y - \lambda)
            \;\le\;
            w
            \;\le\;
            \frac{u_J - \lambda}{u_J + \lambda}(a^\top y + \lambda), \\[6pt]
            & w \le \sum_{i \in I} a_i (y_i - \ell^0_i)
            + \frac{\ell_I - \lambda}{u^0_o - \ell^0_o}(y_o - \ell^0_o),
            \qquad \forall (I, o) \in \mathcal{I}^+, \\[6pt]
            & w \ge \sum_{i \in I} a_i (y_i - u^0_i)
            + \frac{u_I + \lambda}{u^0_o - u^0_o}(y_o - u^0_o),
            \qquad \forall (I, o) \in \mathcal{I}^- ,
            \end{aligned}
            \right\}.
        $$

        where:

        - vector $l^{0} \in \mathbb{R}^{d}$ with entries $l_i^{0} = \underline{y}_i \text{ if }
        a_i \geq 0 \text{ and } \overline{y}_i$ otherwise,
        - vector $u^{0} \in \mathbb{R}^{d}$ with entries $u_i^{0} = \overline{y}_i \text{ if }
        a_i \geq 0 \text{ and } \underline{y}_i$ otherwise,
        - Given $\textbf{I} \subseteq \textbf{J} = \left\{1, ..., n\right\}$, the lower bound is
        defined as $l_I = \sum_{i \in I} a_i l_i^{0} + \sum_{i \notin I} a_i u_i^{0}$ and the
        upper bound is $u_I = \sum_{i \in I} a_i u_i^{0} + \sum_{i \notin I} a_i l_i^{0}$,
        - $\mathcal{I}^+ = \{ (I, o) \in 2^J \times J \mid \ell_I \ge \lambda,\; \ell_{I \cup \{o\}}
        < \lambda,\; w_I \neq 0 \},$
        -  $\mathcal{I}^- = \{ (I, o) \in 2^J \times J \mid u_I \le - \lambda,\; u_{I \cup \{o\}} >
        -\lambda,\; w_I \neq 0 \}.$
        - $w_I$ is the slice of vector $w$ over $I$.

        Parameters
        ----------
        step: SoftThreshold
            An instance of the `SoftThreshold` step to canonicalize.

        Returns
        -------
        None
        """
        constraints = []

        i = 0
        for lb, ub, output_lb, output_ub, output, arg in zip(
            step.lb, step.ub, step.output.lb, step.output.ub, step.output, step.arg
        ):
            if lb >= step.lambd:
                constraints.append(output == arg - step.lambd)
            elif ub <= -step.lambd:
                constraints.append(output == arg + step.lambd)
            elif lb >= -step.lambd and ub <= step.lambd:
                constraints.append(output == 0)
            else:
                if not step.relax_binary_vars:
                    _require_finite_bounds("Soft-threshold", i, lb, ub)
                    w1 = LinearExpression.from_int(arg.ndim, is_binary=True)
                    w2 = LinearExpression.from_int(arg.ndim, is_binary=True)
                    # w1: right branch (arg >= lambd), w2: left branch (arg <= -lambd).
                    self._step_binaries.setdefault(id(step), {})[i] = {"right": w1, "left": w2}

                    if ub > step.lambd:
                        step.idx_with_right_binary_vars.add(i)
                        constraints.extend(
                            [
                                arg >= step.lambd + (lb - step.lambd) * (1 - w1),
                                arg <= step.lambd + (ub - step.lambd) * w1,
                                output <= arg - step.lambd + (2 * step.lambd) * (1 - w1),
                            ]
                        )

                        if lb < -step.lambd:
                            constraints.extend(
                                [
                                    output <= output_ub * w1,
                                    output >= output_lb * w2,
                                    w1 + w2 <= 1,
                                ]
                            )

                        # Right-or-middle: the middle (zero) sub-region sits on the
                        # LOWER side, so it is reachable iff lb is inside [-lambd,
                        # lambd]. There w1=0 selects the middle and this pin forces
                        # output <= 0; combined with output >= 0 (line 918) => 0.
                        if -step.lambd <= lb <= step.lambd:
                            constraints.append(output <= output_ub * w1)

                    if lb < -step.lambd:
                        step.idx_with_left_binary_vars.add(i)
                        constraints.extend(
                            [
                                arg <= -step.lambd + (ub + step.lambd) * (1 - w2),
                                arg >= -step.lambd + (lb + step.lambd) * w2,
                                output >= arg + step.lambd + (-2 * step.lambd) * (1 - w2),
                            ]
                        )

                        # Left-or-middle: the middle (zero) sub-region sits on the
                        # UPPER side, so it is reachable iff ub is inside [-lambd,
                        # lambd] (guard on ub, mirroring the lb guard above for the
                        # right side). There w2=0 selects the middle and this pin
                        # forces output >= 0; combined with output <= 0 (line 927)
                        # => 0. Guarding on lb here was a bug: this block already
                        # requires lb < -lambd, so an ``lb`` guard is never true and
                        # the middle output was left unpinned, admitting off-graph
                        # points (output in [arg-lambd, 0] when |arg| < lambd).
                        if -step.lambd <= ub <= step.lambd:
                            constraints.append(output >= output_lb * w2)

                constraints.extend(
                    softthreshold_envelope(output, arg, lb, ub, output_lb, output_ub, step.lambd)
                )

            i += 1

        self.add_constraints(constraints)
        self.steps.append(step)

    _CANONICALIZERS = {
        "affine": canonicalize_affine,
        "relu": canonicalize_relu,
        "saturatedlinear": canonicalize_saturatedlinear,
        "softthreshold": canonicalize_softthreshold,
    }

add_constraint

add_constraint(constraint: Constraint) -> None

Add a constraint to the MIP instance. This method registers a new constraint with the MIP problem and attaches it to the internal model.

Parameters:

Name	Type	Description	Default
constraint	Constraint	Constraint object to add to the MIP instance.	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/mip.py

def add_constraint(self, constraint: Constraint) -> None:
    """
    Add a constraint to the MIP instance. This method registers
    a new constraint with the MIP problem and attaches it to the
    internal model.

    Parameters
    ----------
    constraint: Constraint
         Constraint object to add to the MIP instance.

    Returns
    -------
    None
    """
    if isinstance(constraint, ComplementarityConstraint):
        self.add_constraints(self._build_complementarity_constraints(constraint))
        return
    if isinstance(constraint, QuadraticConstraint):
        # Quadratic (sum-of-squares) constraints require a solver that supports
        # quadratic constraints (MIQCP). Gurobi does; HiGHS does not. Each solver
        # validates this in its own ``add_sum_squares_constr`` (HiGHS raises), so the
        # error still surfaces eagerly at add-time rather than at solve-time.
        self.solver.add_sum_squares_constr(
            constraint.expr, constraint.radius_sq, constraint.sense
        )
        return
    self.solver.add_constr(constraint)

add_constraints

add_constraints(constraints: List[Constraint]) -> None

Add a list of constraints to the MIP instance. This method registers multiple constraints with the MIP problem and attaches them to the internal model.

Parameters:

Name	Type	Description	Default
constraints	List[Constraint]	List of constraints to add to the MIP instance.	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/mip.py

def add_constraints(self, constraints: List[Constraint]) -> None:
    """
    Add a list of constraints to the MIP instance. This method registers
    multiple constraints with the MIP problem and attaches them to
    the internal model.

    Parameters
    ----------
    constraints: List[Constraint]
        List of constraints to add to the MIP instance.

    Returns
    -------
    None
    """
    self.solver.constraints_to_solver(constraints)

add_theory_cut

add_theory_cut(margin: float, target_expr: LinearExpression, lb: float | ndarray, ub: float | ndarray)

Add relaxed bound constraints on the target expression up to the provided relaxation margin.

Parameters:

Name	Type	Description	Default
margin	float	Relaxation margin for the bounds. If finite, the lower bound is shifted to lb - margin and the upper bound to ub + margin. If np.inf, no constraints are added.	required
target_expr	LinearExpression	The expression whose bounds are to be relaxed.	required
lb	float | ndarray	Lower bound(s) for target_expr before relaxation.	required
ub	float | ndarray	Upper bound(s) for target_expr before relaxation.	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/mip.py

def add_theory_cut(
    self,
    margin: float,
    target_expr: LinearExpression,
    lb: float | np.ndarray,
    ub: float | np.ndarray,
):
    """
    Add relaxed bound constraints on the target expression up to the
    provided relaxation margin.

    Parameters
    ----------
    margin: float
        Relaxation margin for the bounds. If finite, the lower bound is
        shifted to ``lb - margin`` and the upper bound to ``ub + margin``. If
        ``np.inf``, no constraints are added.

    target_expr: LinearExpression
        The expression whose bounds are to be relaxed.

    lb: float | np.ndarray
        Lower bound(s) for ``target_expr`` before relaxation.

    ub: float | np.ndarray
        Upper bound(s) for ``target_expr`` before relaxation.

    Returns
    -------
    None
    """
    if margin != np.inf:
        self.solver.constraints_to_solver(
            [target_expr >= lb - margin, target_expr <= ub + margin]
        )

add_var

add_var(var: LinearExpression, **kwargs) -> None

Add a variable to the MIP instance. This method registers a new variable with the MIP program and attaches it to the internal model so it can be referenced in the constraints.

Parameters:

Name	Type	Description	Default
var	LinearExpression	Linear expression object to add to the MIP instance. Later to be used in an objective function, constraints, or parameters.	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/mip.py

def add_var(self, var: LinearExpression, **kwargs) -> None:
    """
    Add a variable to the MIP instance. This method registers
    a new variable with the MIP program and attaches it to the
    internal model so it can be referenced in the constraints.

    Parameters
    ----------
    var: LinearExpression
        Linear expression object to add to the MIP instance. Later to
        be used in an objective function, constraints, or parameters.

    Returns
    -------
    None
    """
    self.solver.add_var(var=var, lb=var.lb, ub=var.ub)
    if kwargs.get("var_type") in ("param", "iterate"):
        self._input_leaves.append(var)

canonicalize_affine

canonicalize_affine(step: Affine) -> None

Construct the linear equality constraint corresponding to the affine iteration step. The constraint is used to express the affine update step in the verification problem.

Parameters:

Name	Type	Description	Default
step	Affine	An instance of the Affine class to canonicalize to a MIP problem.	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/mip.py

def canonicalize_affine(self, step: Affine) -> None:
    """
    Construct the linear equality constraint corresponding to the
    affine iteration step. The constraint is used to express the affine
    update step in the verification problem.

    Parameters
    ----------
    step: Affine
        An instance of the ``Affine`` class to canonicalize to a MIP problem.

    Returns
    -------
    None
    """
    self.steps.append(step)
    self.add_constraint(step.output_mat @ step.output == step.arg)

canonicalize_relu

canonicalize_relu(step: Relu)

Construct the convex-hull linear constraints corresponding to the Rectified Linear Unit (ReLU) operator.

Given a ReLU step: \((\phi(v,x))_i = \mathbf{ReLU}(a^T y) = \max\{a^T y, 0\}.\)

This method derives the convex hull of the region defined by:

\[ \phi = \left\{ (y, w) \in \left [\underline{y}, \overline{y} \right ] \times \mathbb{R} | w = \textbf{ReLU}(a^{T}y) \right\}, \]

It then expresses it as a set of Constraint objects. The convex hull of the region is:

\[ \operatorname{conv}(\Phi) = \left\{ (y, w) \in [\underline{y}, \overline{y}] \times \mathbb{R} \;\middle|\; \begin{cases} w = a^{\mathsf{T}} y, & \ell_J \ge 0, \\[6pt] w = 0, & \ell_J < 0, \\[6pt] (y, w) \in Q, & \text{otherwise} \end{cases} \right\}, \]

with

\[ Q = \left\{ (y, w) \in \mathbb{R}^{d + p + 1} \;\middle|\; \begin{aligned} & w \ge a^{\mathsf{T}} y, \quad y \ge 0, \\[4pt] & w \le \sum_{i \in I} a_i (y_i - \ell_i^{0}) + \frac{\ell_I}{u_o^{0} - \ell_o^{0}} (y_o - \ell_o^{0}), \quad \forall (I, o) \in \mathcal{I} \end{aligned} \right\}, \]

where:

• vector \(l^{0} \in \mathbb{R}^{d}\) with entries \(l_i^{0} = \underline{y}_i \text{ if } a_i \geq 0 \text{ and } \overline{y}_i\) otherwise,
• vector \(u^{0} \in \mathbb{R}^{d}\) with entries \(u_i^{0} = \overline{y}_i \text{ if } a_i \geq 0 \text{ and } \underline{y}_i\) otherwise,
• Given \(\textbf{I} \subseteq \textbf{J} = \left\{1, ..., n\right\}\), the lower bound is defined as \(l_I = \sum_{i \in I} a_i l_i^{0} + \sum_{i \notin I} a_i u_i^{0}\)

Parameters:

Name	Type	Description	Default
step	Relu	An instance of the Relu step to canonicalize.	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/mip.py

def canonicalize_relu(self, step: Relu):
    r"""
    Construct the convex-hull linear constraints corresponding
    to the `Rectified Linear Unit (ReLU)` operator.

    Given a ReLU step: $(\phi(v,x))_i = \mathbf{ReLU}(a^T y) = \max\{a^T y, 0\}.$

    This method derives the convex hull of the region defined by:

    $$
        \phi = \left\{ (y, w) \in \left [\underline{y}, \overline{y} \right ] \times
        \mathbb{R} | w = \textbf{ReLU}(a^{T}y) \right\},
    $$

    It then expresses it as a set of Constraint objects. The convex hull
    of the region is:


    $$
        \operatorname{conv}(\Phi) =
        \left\{
        (y, w) \in [\underline{y}, \overline{y}] \times \mathbb{R}
        \;\middle|\;
        \begin{cases}
        w = a^{\mathsf{T}} y, & \ell_J \ge 0, \\[6pt]
        w = 0, & \ell_J < 0, \\[6pt]
        (y, w) \in Q, & \text{otherwise}
        \end{cases}
        \right\},
    $$

    with

    $$
        Q =
        \left\{
        (y, w) \in \mathbb{R}^{d + p + 1}
        \;\middle|\;
        \begin{aligned}
        & w \ge a^{\mathsf{T}} y, \quad y \ge 0, \\[4pt]
        & w \le \sum_{i \in I} a_i (y_i - \ell_i^{0})
           + \frac{\ell_I}{u_o^{0} - \ell_o^{0}} (y_o - \ell_o^{0}),
           \quad \forall (I, o) \in \mathcal{I}
        \end{aligned}
        \right\},
    $$

    where:

    - vector $l^{0} \in \mathbb{R}^{d}$ with entries $l_i^{0} = \underline{y}_i \text{ if }
    a_i \geq 0 \text{ and } \overline{y}_i$ otherwise,
    - vector $u^{0} \in \mathbb{R}^{d}$ with entries $u_i^{0} = \overline{y}_i \text{ if }
    a_i \geq 0 \text{ and } \underline{y}_i$ otherwise,
    - Given $\textbf{I} \subseteq \textbf{J} = \left\{1, ..., n\right\}$, the lower bound is
    defined as $l_I = \sum_{i \in I} a_i l_i^{0} + \sum_{i \notin I} a_i u_i^{0}$

    Parameters
    ----------
    step: Relu
        An instance of the `Relu` step to canonicalize.

    Returns
    -------
    None
    """
    constraints = self._build_relu_constraints(step)
    self.add_constraints(constraints)
    self.steps.append(step)

canonicalize_saturatedlinear

canonicalize_saturatedlinear(step: SaturatedLinear)

Construct the convex-hull linear constraints corresponding to the saturated linear unit operator.

Given a saturated linear unit operator, defined as:

\[ (\phi(v, b))_i = S_{[b, c]}(a^T y) = \begin{cases} b & a^T y < b, \\\\ a^T y & b \leq a^T y \leq c, \\\\ c & a^T y > c, \end{cases} \quad \text{with } y = (v, b), \text{and } b, c \in \mathbb{R}, \]

This method derives the convex hull of the region defined by:

\[ \Phi = \{ (y, w) \in [\underline{y}, \bar{y}] \times [b, c] \mid w = \mathcal{S}_{[b,c]}(a^\top y) \}. \]

It then expresses it as a set of Constraint objects. The convex hull of the region is:

\[ \operatorname{conv}(\Phi) = \left\{ (y, w) \in [\underline{y}, \bar{y}] \times [b, c] \;\middle|\; \begin{cases} w = b & u_J < b, \\[4pt] w = c & \ell_J > c, \\[4pt] w = a^\top y & b \le \ell_J \le u_J \le c, \\[4pt] (y, w) \in Q & \text{otherwise} \end{cases} \right\}, \]

with

\[ Q = \left\{ (y, w) \in \mathbb{R}^{p + d + 1} \;\middle|\; \begin{aligned} & b \le w \le c, \\[4pt] & \frac{b - c}{b - u_J}(a^\top y - u_J) + c \;\le\; w \;\le\; \frac{c - b}{c - \ell_J}(a^\top y - \ell_J) + b, \\[6pt] & w \le \sum_{i \in I} a_i (y_i - \ell^0_i) + \frac{\ell_I - b}{u^0_o - \ell^0_o}(y_o - \ell^0_o) + b, \qquad \forall (I, o) \in \mathcal{I}^+, \\[6pt] & w \ge \sum_{i \in I} a_i (y_i - u^0_i) + \frac{u_I - c}{\ell^0_o - u^0_o}(y_o - u^0_o) + c, \qquad \forall (I, o) \in \mathcal{I}^- \end{aligned} \right\}, \]

where:

• vector \(l^{0} \in \mathbb{R}^{d}\) with entries \(l_i^{0} = \underline{y}_i \text{ if } a_i \geq 0 \text{ and } \overline{y}_i\) otherwise,
• vector \(u^{0} \in \mathbb{R}^{d}\) with entries \(u_i^{0} = \overline{y}_i \text{ if } a_i \geq 0 \text{ and } \underline{y}_i\) otherwise,
• Given \(\textbf{I} \subseteq \textbf{J} = \left\{1, ..., n\right\}\), the lower bound is defined as \(l_I = \sum_{i \in I} a_i l_i^{0} + \sum_{i \notin I} a_i u_i^{0}\) and the upper bound is \(u_I = \sum_{i \in I} a_i u_i^{0} + \sum_{i \notin I} a_i l_i^{0}\),
• \(\mathcal{I}^+ = \{ (I, o) \in 2^J \times J \mid \ell_I \ge b,\; \ell_{I \cup \{o\}} < b,\; w_I \neq 0 \},\)
• \(\mathcal{I}^- = \{ (I, o) \in 2^J \times J \mid u_I \le c,\; u_{I \cup \{o\}} > c,\; w_I \neq 0 \}.\)

Parameters:

Name	Type	Description	Default
step	SaturatedLinear	An instance of the SaturatedLinear step to canonicalize.	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/mip.py

def canonicalize_saturatedlinear(self, step: SaturatedLinear):
    r"""
    Construct the convex-hull linear constraints corresponding to
    the saturated linear unit operator.

    Given a saturated linear unit operator, defined as:

    $$
        (\phi(v, b))_i = S_{[b, c]}(a^T y) =
        \begin{cases}
        b & a^T y < b, \\\\
        a^T y & b \leq a^T y \leq c, \\\\
        c & a^T y > c,
        \end{cases}
        \quad \text{with } y = (v, b), \text{and }
         b, c \in \mathbb{R},
    $$

    This method derives the convex hull of the region defined by:

    $$
        \Phi = \{ (y, w) \in [\underline{y}, \bar{y}] \times [b, c] \mid w =
        \mathcal{S}_{[b,c]}(a^\top y) \}.
    $$

    It then expresses it as a set of Constraint objects. The convex hull of the
    region is:

    $$
        \operatorname{conv}(\Phi) =
        \left\{
        (y, w) \in [\underline{y}, \bar{y}] \times [b, c]
        \;\middle|\;
        \begin{cases}
        w = b & u_J < b, \\[4pt]
        w = c & \ell_J > c, \\[4pt]
        w = a^\top y & b \le \ell_J \le u_J \le c, \\[4pt]
        (y, w) \in Q & \text{otherwise}
        \end{cases}
        \right\},
    $$

    with

    $$
        Q =
        \left\{
        (y, w) \in \mathbb{R}^{p + d + 1}
        \;\middle|\;
        \begin{aligned}
        & b \le w \le c, \\[4pt]
        & \frac{b - c}{b - u_J}(a^\top y - u_J) + c
        \;\le\;
        w
        \;\le\;
        \frac{c - b}{c - \ell_J}(a^\top y - \ell_J) + b, \\[6pt]
        & w \le \sum_{i \in I} a_i (y_i - \ell^0_i)
        + \frac{\ell_I - b}{u^0_o - \ell^0_o}(y_o - \ell^0_o) + b,
        \qquad \forall (I, o) \in \mathcal{I}^+, \\[6pt]
        & w \ge \sum_{i \in I} a_i (y_i - u^0_i)
        + \frac{u_I - c}{\ell^0_o - u^0_o}(y_o - u^0_o) + c,
        \qquad \forall (I, o) \in \mathcal{I}^-
        \end{aligned}
        \right\},
    $$

    where:

    - vector $l^{0} \in \mathbb{R}^{d}$ with entries $l_i^{0} = \underline{y}_i \text{ if }
    a_i \geq 0 \text{ and } \overline{y}_i$ otherwise,
    - vector $u^{0} \in \mathbb{R}^{d}$ with entries $u_i^{0} = \overline{y}_i \text{ if }
    a_i \geq 0 \text{ and } \underline{y}_i$ otherwise,
    - Given $\textbf{I} \subseteq \textbf{J} = \left\{1, ..., n\right\}$, the lower bound is
    defined as $l_I = \sum_{i \in I} a_i l_i^{0} + \sum_{i \notin I} a_i u_i^{0}$ and the
    upper bound is $u_I = \sum_{i \in I} a_i u_i^{0} + \sum_{i \notin I} a_i l_i^{0}$,
    - $\mathcal{I}^+ = \{ (I, o) \in 2^J \times J \mid \ell_I \ge b,\; \ell_{I \cup \{o\}}
    < b,\; w_I \neq 0 \},$
    - $\mathcal{I}^- = \{ (I, o) \in 2^J \times J \mid u_I \le c,\; u_{I \cup \{o\}} >
    c,\; w_I \neq 0 \}.$

    Parameters
    ----------
    step: SaturatedLinear
        An instance of the `SaturatedLinear` step to canonicalize.

    Returns
    -------
    None
    """
    constraints = []

    for row, (l, u, lb, ub, output, arg) in enumerate(
        zip(step.l, step.u, step.lb, step.ub, step.output, step.arg)
    ):
        if lb >= u:
            constraints.append(output == u)
        elif ub <= l:
            constraints.append(output == l)
        elif lb >= l and ub <= u:
            constraints.append(output == arg)
        else:
            if not step.relax_binary_vars:
                _require_finite_bounds("Saturated-linear", row, lb, ub)
                w1 = LinearExpression.from_int(arg.ndim, is_binary=True)
                w2 = LinearExpression.from_int(arg.ndim, is_binary=True)
                # w1: upper saturation (arg >= u), w2: lower saturation (arg <= l).
                self._step_binaries.setdefault(id(step), {})[row] = {"up": w1, "low": w2}

                if lb < l:
                    constraints.extend(
                        [
                            output <= arg + (ub - lb) * w2,
                            arg >= l + (lb - l) * w2,
                            arg <= l + (ub - l) * (1 - w2),
                        ]
                    )

                    if ub > u:
                        constraints.extend(
                            [
                                output >= u - (u - l) * (1 - w1),
                                output <= l + (u - l) * (1 - w2),
                                output >= l + (lb - l) * w2,
                                w1 + w2 <= 1,
                            ]
                        )
                    else:
                        # Lower-or-middle (lb < l, ub <= u): the upper face is
                        # never active, so only w2 (lower-saturation vs interior)
                        # matters. Pin the saturated branch: w2=1 => output <= l
                        # (with the output-variable bound output >= l this forces
                        # output = l); w2=0 => output <= ub (interior, slack).
                        # Without this the lower-sat output was unpinned and the
                        # MIP admitted off-graph points (mirror of the
                        # soft-threshold left-or-middle bug).
                        constraints.append(output <= l + (ub - l) * (1 - w2))

                if ub > u:
                    constraints.extend(
                        [
                            output >= arg - (ub - lb) * w1,
                            arg >= u + (lb - u) * (1 - w1),
                            arg <= u + (ub - u) * w1,
                        ]
                    )
                    if lb >= l:
                        constraints.append(output >= u - (u - lb) * (1 - w1))

            if lb < l and ub > u:
                constraints.extend(
                    [
                        output <= (u - l) / (u - lb) * (arg - u) + u,
                        output >= (u - l) / (ub - l) * (arg - l) + l,
                    ]
                )

            # Non-strict, so that the boundary cases lb == l and ub == u still pin
            # the output to arg on the unsaturated side.
            elif lb < l and ub <= u:
                constraints.extend(
                    [output >= arg, output <= ((ub - l) / (ub - lb)) * (arg - ub) + ub]
                )

            elif lb >= l and ub > u:
                constraints.extend(
                    [output <= arg, output >= ((u - lb) / (ub - lb) * (arg - lb) + lb)]
                )

    self.add_constraints(constraints)
    self.steps.append(step)

canonicalize_softthreshold

canonicalize_softthreshold(step: SoftThreshold)

Construct the convex-hull linear constraints corresponding to the soft-thresholding operator.

Given a soft-thresholding operator for shrinkage, defined as:

\[ (\phi(v,x))_i=\mathcal{T}_{\lambda}(a^\top y)= \begin{cases} a^\top y-\lambda,& a^\top y>\lambda,\\[6pt] 0,& |a^\top y|\le \lambda,\\[6pt] a^\top y+\lambda,& a^\top y<-\lambda, \end{cases} \quad \text{with } y=(v,x), \]

where \(\mathcal{T}_{\lambda}\) is the soft-thresholding operator with constant \(\lambda>0\), and \(a\in\mathbf{R}^d\).

This method derives the convex hull of the region defined by:

\[ \Phi = \{ (y, w) \in [\underline{y}, \bar{y}] \times [b, c] \mid w = \mathcal{S}_{[b,c]}(a^\top y) \}. \]

It then expresses it as a set of Constraint objects. The convex hull of the region is:

\[ \operatorname{conv}(\Phi) = \left\{ (y, w) \in [\underline{y}, \bar{y}] \times \mathbf{R} \;\middle|\; \begin{cases} w = a^\top y - \lambda & \ell_J > \lambda, \\[4pt] w = a^\top y + \lambda & u_J < - \lambda, \\[4pt] w = 0 & - \lambda \le \ell_J \le u_J \le \lambda, \\[4pt] (y, w) \in Q & \text{otherwise} \end{cases} \right\}, \]

with

\[ Q = \left\{ (y, w) \in \mathbb{R}^{p + d + 1} \;\middle|\; \begin{aligned} & a^\top y - \lambda \le w \le a^\top y + \lambda, \\[4pt] & \frac{l_J + \lambda}{l_J - \lambda}(a^\top y - \lambda) \;\le\; w \;\le\; \frac{u_J - \lambda}{u_J + \lambda}(a^\top y + \lambda), \\[6pt] & w \le \sum_{i \in I} a_i (y_i - \ell^0_i) + \frac{\ell_I - \lambda}{u^0_o - \ell^0_o}(y_o - \ell^0_o), \qquad \forall (I, o) \in \mathcal{I}^+, \\[6pt] & w \ge \sum_{i \in I} a_i (y_i - u^0_i) + \frac{u_I + \lambda}{u^0_o - u^0_o}(y_o - u^0_o), \qquad \forall (I, o) \in \mathcal{I}^- , \end{aligned} \right\}. \]

where:

• vector \(l^{0} \in \mathbb{R}^{d}\) with entries \(l_i^{0} = \underline{y}_i \text{ if } a_i \geq 0 \text{ and } \overline{y}_i\) otherwise,
• vector \(u^{0} \in \mathbb{R}^{d}\) with entries \(u_i^{0} = \overline{y}_i \text{ if } a_i \geq 0 \text{ and } \underline{y}_i\) otherwise,
• Given \(\textbf{I} \subseteq \textbf{J} = \left\{1, ..., n\right\}\), the lower bound is defined as \(l_I = \sum_{i \in I} a_i l_i^{0} + \sum_{i \notin I} a_i u_i^{0}\) and the upper bound is \(u_I = \sum_{i \in I} a_i u_i^{0} + \sum_{i \notin I} a_i l_i^{0}\),
• \(\mathcal{I}^+ = \{ (I, o) \in 2^J \times J \mid \ell_I \ge \lambda,\; \ell_{I \cup \{o\}} < \lambda,\; w_I \neq 0 \},\)
• \(\mathcal{I}^- = \{ (I, o) \in 2^J \times J \mid u_I \le - \lambda,\; u_{I \cup \{o\}} > -\lambda,\; w_I \neq 0 \}.\)
• \(w_I\) is the slice of vector \(w\) over \(I\).

Parameters:

Name	Type	Description	Default
step	SoftThreshold	An instance of the SoftThreshold step to canonicalize.	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/mip.py

def canonicalize_softthreshold(self, step: SoftThreshold):
    r"""
    Construct the convex-hull linear constraints corresponding to the
    soft-thresholding operator.

    Given a soft-thresholding operator for shrinkage, defined as:

    $$
        (\phi(v,x))_i=\mathcal{T}_{\lambda}(a^\top y)=
        \begin{cases}
        a^\top y-\lambda,& a^\top y>\lambda,\\[6pt]
        0,& |a^\top y|\le \lambda,\\[6pt]
        a^\top y+\lambda,& a^\top y<-\lambda,
        \end{cases}
        \quad \text{with } y=(v,x),
    $$

    where $\mathcal{T}_{\lambda}$ is the soft-thresholding operator with constant $\lambda>0$,
    and $a\in\mathbf{R}^d$.

    This method derives the convex hull of the region defined by:

    $$
        \Phi = \{ (y, w) \in [\underline{y}, \bar{y}] \times [b, c]
        \mid w = \mathcal{S}_{[b,c]}(a^\top y) \}.
    $$

    It then expresses it as a set of Constraint objects. The convex hull of the
    region is:

    $$
        \operatorname{conv}(\Phi) =
        \left\{
        (y, w) \in [\underline{y}, \bar{y}] \times \mathbf{R}
        \;\middle|\;
        \begin{cases}
        w = a^\top y - \lambda & \ell_J > \lambda, \\[4pt]
        w = a^\top y  + \lambda & u_J < - \lambda, \\[4pt]
        w = 0 & - \lambda \le \ell_J \le u_J \le \lambda, \\[4pt]
        (y, w) \in Q & \text{otherwise}
        \end{cases}
        \right\},
    $$

    with

    $$
        Q = \left\{
        (y, w) \in \mathbb{R}^{p + d + 1}
        \;\middle|\;
        \begin{aligned}
        & a^\top y - \lambda \le w \le a^\top y + \lambda, \\[4pt]
        & \frac{l_J + \lambda}{l_J - \lambda}(a^\top y - \lambda)
        \;\le\;
        w
        \;\le\;
        \frac{u_J - \lambda}{u_J + \lambda}(a^\top y + \lambda), \\[6pt]
        & w \le \sum_{i \in I} a_i (y_i - \ell^0_i)
        + \frac{\ell_I - \lambda}{u^0_o - \ell^0_o}(y_o - \ell^0_o),
        \qquad \forall (I, o) \in \mathcal{I}^+, \\[6pt]
        & w \ge \sum_{i \in I} a_i (y_i - u^0_i)
        + \frac{u_I + \lambda}{u^0_o - u^0_o}(y_o - u^0_o),
        \qquad \forall (I, o) \in \mathcal{I}^- ,
        \end{aligned}
        \right\}.
    $$

    where:

    - vector $l^{0} \in \mathbb{R}^{d}$ with entries $l_i^{0} = \underline{y}_i \text{ if }
    a_i \geq 0 \text{ and } \overline{y}_i$ otherwise,
    - vector $u^{0} \in \mathbb{R}^{d}$ with entries $u_i^{0} = \overline{y}_i \text{ if }
    a_i \geq 0 \text{ and } \underline{y}_i$ otherwise,
    - Given $\textbf{I} \subseteq \textbf{J} = \left\{1, ..., n\right\}$, the lower bound is
    defined as $l_I = \sum_{i \in I} a_i l_i^{0} + \sum_{i \notin I} a_i u_i^{0}$ and the
    upper bound is $u_I = \sum_{i \in I} a_i u_i^{0} + \sum_{i \notin I} a_i l_i^{0}$,
    - $\mathcal{I}^+ = \{ (I, o) \in 2^J \times J \mid \ell_I \ge \lambda,\; \ell_{I \cup \{o\}}
    < \lambda,\; w_I \neq 0 \},$
    -  $\mathcal{I}^- = \{ (I, o) \in 2^J \times J \mid u_I \le - \lambda,\; u_{I \cup \{o\}} >
    -\lambda,\; w_I \neq 0 \}.$
    - $w_I$ is the slice of vector $w$ over $I$.

    Parameters
    ----------
    step: SoftThreshold
        An instance of the `SoftThreshold` step to canonicalize.

    Returns
    -------
    None
    """
    constraints = []

    i = 0
    for lb, ub, output_lb, output_ub, output, arg in zip(
        step.lb, step.ub, step.output.lb, step.output.ub, step.output, step.arg
    ):
        if lb >= step.lambd:
            constraints.append(output == arg - step.lambd)
        elif ub <= -step.lambd:
            constraints.append(output == arg + step.lambd)
        elif lb >= -step.lambd and ub <= step.lambd:
            constraints.append(output == 0)
        else:
            if not step.relax_binary_vars:
                _require_finite_bounds("Soft-threshold", i, lb, ub)
                w1 = LinearExpression.from_int(arg.ndim, is_binary=True)
                w2 = LinearExpression.from_int(arg.ndim, is_binary=True)
                # w1: right branch (arg >= lambd), w2: left branch (arg <= -lambd).
                self._step_binaries.setdefault(id(step), {})[i] = {"right": w1, "left": w2}

                if ub > step.lambd:
                    step.idx_with_right_binary_vars.add(i)
                    constraints.extend(
                        [
                            arg >= step.lambd + (lb - step.lambd) * (1 - w1),
                            arg <= step.lambd + (ub - step.lambd) * w1,
                            output <= arg - step.lambd + (2 * step.lambd) * (1 - w1),
                        ]
                    )

                    if lb < -step.lambd:
                        constraints.extend(
                            [
                                output <= output_ub * w1,
                                output >= output_lb * w2,
                                w1 + w2 <= 1,
                            ]
                        )

                    # Right-or-middle: the middle (zero) sub-region sits on the
                    # LOWER side, so it is reachable iff lb is inside [-lambd,
                    # lambd]. There w1=0 selects the middle and this pin forces
                    # output <= 0; combined with output >= 0 (line 918) => 0.
                    if -step.lambd <= lb <= step.lambd:
                        constraints.append(output <= output_ub * w1)

                if lb < -step.lambd:
                    step.idx_with_left_binary_vars.add(i)
                    constraints.extend(
                        [
                            arg <= -step.lambd + (ub + step.lambd) * (1 - w2),
                            arg >= -step.lambd + (lb + step.lambd) * w2,
                            output >= arg + step.lambd + (-2 * step.lambd) * (1 - w2),
                        ]
                    )

                    # Left-or-middle: the middle (zero) sub-region sits on the
                    # UPPER side, so it is reachable iff ub is inside [-lambd,
                    # lambd] (guard on ub, mirroring the lb guard above for the
                    # right side). There w2=0 selects the middle and this pin
                    # forces output >= 0; combined with output <= 0 (line 927)
                    # => 0. Guarding on lb here was a bug: this block already
                    # requires lb < -lambd, so an ``lb`` guard is never true and
                    # the middle output was left unpinned, admitting off-graph
                    # points (output in [arg-lambd, 0] when |arg| < lambd).
                    if -step.lambd <= ub <= step.lambd:
                        constraints.append(output >= output_lb * w2)

            constraints.extend(
                softthreshold_envelope(output, arg, lb, ub, output_lb, output_ub, step.lambd)
            )

        i += 1

    self.add_constraints(constraints)
    self.steps.append(step)

copy

copy() -> MIP

Create an independent copy of this MIP canonicalizer.

Returns a new MIP canonicalizer with a copied solver. The copy can be modified without affecting the original.

Returns:

Type	Description
MIP	A new canonicalizer instance with copied state.

Source code in fomtools/canonicalizers/mip.py

def copy(self) -> "MIP":
    """
    Create an independent copy of this MIP canonicalizer.

    Returns a new MIP canonicalizer with a copied solver. The copy
    can be modified without affecting the original.

    Returns
    -------
    MIP
        A new canonicalizer instance with copied state.
    """
    new_canon = MIP.__new__(MIP)
    new_canon.solver_params = dict(self.solver_params) if self.solver_params else None
    new_canon.solver_name = self.solver_name
    new_canon.solver = self.solver.copy()
    new_canon.obbt_handler = self.obbt_handler
    new_canon._objective_spec = self._objective_spec
    new_canon._solved_copy = None
    return new_canon

get_solution_value

get_solution_value(expression: LinearExpression) -> ndarray

Return the solution value for the given LinearExpression.

Parameters:

Name	Type	Description	Default
expression	LinearExpression	The expression whose solution value is to be computed. Each element in its decomposition must have been registered with the solver.	required

Returns:

Type	Description
ndarray[float]	A 1-D array of shape (expression.ndim,) containing the evaluated values at the current model solution.

Raises:

Type	Description
RuntimeError	If no solution is available (solve_model() not called).

Source code in fomtools/canonicalizers/mip.py

def get_solution_value(self, expression: LinearExpression) -> np.ndarray:
    """
    Return the solution value for the given ``LinearExpression``.

    Parameters
    ----------
    expression: LinearExpression
        The expression whose solution value is to be computed. Each element in
        its decomposition must have been registered with the solver.

    Returns
    -------
    np.ndarray[float]
        A 1-D array of shape ``(expression.ndim,)`` containing the
        evaluated values at the current model solution.

    Raises
    ------
    RuntimeError
        If no solution is available (solve_model() not called).
    """
    if self._solved_copy is None or not self._solved_copy.has_solution():
        raise RuntimeError("No feasible incumbent available. Call solve() first.")
    for leaf in expression.decomposition_dict:
        if leaf not in self._solved_copy.vector_var_map:
            if leaf in self.solver.vector_var_map:
                raise RuntimeError(
                    "The expression uses variables added after the last solve; "
                    "call solve() again."
                )
            raise RuntimeError("The expression has variables not registered with this model.")
    return self._solved_copy.get_solution_value(expression)

separate_hull_cuts

separate_hull_cuts(tol: float = 1e-06) -> List[Constraint]

Separate the root LP optimum from the convex hulls of the steps.

Copies the model (which carries every cut added so far), applies the objective, solves the continuous relaxation and returns the most violated hull inequality of every unstable step component. The model itself is not modified; Verifier.solve adds the returned cuts.

Parameters:

Name	Type	Description	Default
tol	float	Minimum violation for a cut to be returned.	1e-06

Returns:

Type	Description
List[Constraint]	The separated cuts.

Source code in fomtools/canonicalizers/mip.py

def separate_hull_cuts(self, tol: float = 1e-6) -> List[Constraint]:
    """
    Separate the root LP optimum from the convex hulls of the steps.

    Copies the model (which carries every cut added so far), applies the
    objective, solves the continuous relaxation and returns the most violated
    hull inequality of every unstable step component. The model itself is not
    modified; ``Verifier.solve`` adds the returned cuts.

    Parameters
    ----------
    tol: float, default 1e-6
        Minimum violation for a cut to be returned.

    Returns
    -------
    List[Constraint]
        The separated cuts.
    """
    relaxed = self.solver.copy()
    if self.solver_params:
        relaxed.set_solver_params(self.solver_params)
    self._apply_objective(relaxed)
    cuts = relaxed.separate_convex_hull_cuts(self.steps, tol=tol)
    self.last_separation_stats = relaxed.last_separation_stats
    return cuts

set_infinity_norm_objective

set_infinity_norm_objective(expr_list, lb_list, ub_list)

Store infinity-norm objective specification for deferred application.

Source code in fomtools/canonicalizers/mip.py

def set_infinity_norm_objective(self, expr_list, lb_list, ub_list):
    """Store infinity-norm objective specification for deferred application."""
    self._objective_spec = ("infinity_norm", expr_list, lb_list, ub_list)

set_one_norm_objective

set_one_norm_objective(expr_list)

Store one-norm objective specification for deferred application.

Source code in fomtools/canonicalizers/mip.py

def set_one_norm_objective(self, expr_list):
    """Store one-norm objective specification for deferred application."""
    self._objective_spec = ("one_norm", expr_list)

set_start_samples

set_start_samples(samples: dict) -> None

Samples of input leaves for the sample-max warm start.

Parameters:

Name	Type	Description	Default
samples	dict	(n, ndim) samples per input leaf (parameters or initial iterates), sharing the same n. Inputs without samples are held at their value if their bounds fix them, and the start is skipped otherwise.	required

Source code in fomtools/canonicalizers/mip.py

def set_start_samples(self, samples: dict) -> None:
    """
    Samples of input leaves for the sample-max warm start.

    Parameters
    ----------
    samples: dict[LinearExpression, np.ndarray]
        ``(n, ndim)`` samples per input leaf (parameters or initial
        iterates), sharing the same ``n``. Inputs without samples are held at
        their value if their bounds fix them, and the start is skipped otherwise.
    """
    self._start_samples = {
        leaf: np.atleast_2d(np.asarray(v, dtype=float)) for leaf, v in samples.items()
    }

set_two_norm_objective

set_two_norm_objective(expr_list, lb_list, ub_list)

Store two-norm objective specification for deferred application.

Source code in fomtools/canonicalizers/mip.py

def set_two_norm_objective(self, expr_list, lb_list, ub_list):
    """Store two-norm objective specification for deferred application."""
    self._objective_spec = ("two_norm", expr_list, lb_list, ub_list)

set_zero_objective

set_zero_objective()

Store zero objective specification for deferred application.

Source code in fomtools/canonicalizers/mip.py

def set_zero_objective(self):
    """Store zero objective specification for deferred application."""
    self._objective_spec = ("zero",)

solve_model

solve_model(warm_start: bool = True, **solver_kwargs) -> float

Optimize the underlying MIP problem.

Creates a copy of the solver, applies the objective to the copy, solves it, and keeps a reference for solution extraction. This prevents objective constraints from accumulating across multiple solve calls. Convex hull cuts are added to the model beforehand by Verifier.solve.

Parameters:

Name	Type	Description	Default
warm_start	bool	Give the solver up to two starts: the inputs of the previous solution's worst case run through the current steps, and the worst sample of set_start_samples. Solvers without MIP starts ignore them.	True
**solver_kwargs		Native solver options forwarded verbatim to the backend (e.g. TimeLimit=10, MIPGap=1e-4 for Gurobi). Applied to the solve copy, so they override construction-time params for this call. A ValueError is raised if the backend rejects an option.	{}

Returns:

Type	Description
float	The finite best upper bound on the maximization objective, including after a time limit. Raises RuntimeError when none is available. Incumbent values and status remain on the solved solver copy.

Source code in fomtools/canonicalizers/mip.py

def solve_model(self, warm_start: bool = True, **solver_kwargs) -> float:
    """
    Optimize the underlying MIP problem.

    Creates a copy of the solver, applies the objective to the copy,
    solves it, and keeps a reference for solution extraction.
    This prevents objective constraints from accumulating across
    multiple solve calls. Convex hull cuts are added to the model
    beforehand by ``Verifier.solve``.

    Parameters
    ----------
    warm_start: bool, default True
        Give the solver up to two starts: the inputs of the previous
        solution's worst case run through the current steps, and the
        worst sample of ``set_start_samples``. Solvers without MIP starts
        ignore them.
    **solver_kwargs
        Native solver options forwarded verbatim to the backend
        (e.g. ``TimeLimit=10``, ``MIPGap=1e-4`` for Gurobi). Applied
        to the solve copy, so they override construction-time
        ``params`` for this call. A ``ValueError`` is raised if the
        backend rejects an option.

    Returns
    -------
    float
        The finite best upper bound on the maximization objective, including
        after a time limit. Raises ``RuntimeError`` when none is available.
        Incumbent values and status remain on the solved solver copy.

    """
    # Create copy to apply objective without modifying original
    self._solved_copy = self.solver.copy()

    # Re-apply construction-time options: HiGHS's copy() uses passModel,
    # which copies the model data but not solver options, so the copy
    # would otherwise reset them to defaults. (Gurobi's copy() preserves
    # them, making this a harmless no-op there.) solver_kwargs are applied
    # last so solve-time options override construction-time params.
    if self.solver_params:
        self._solved_copy.set_solver_params(self.solver_params)

    if solver_kwargs:
        self._solved_copy.set_solver_params(solver_kwargs)

    self._apply_objective(self._solved_copy)
    self.warm_start_stats = None
    if warm_start:
        starts = self._warm_starts()
        if starts:
            self._solved_copy.set_starts(starts)
    self._solved_copy.optimize()
    if self._solved_copy.has_solution():
        self._warm_start_inputs = {
            leaf: self._solved_copy.get_solution_value(leaf) for leaf in self._input_leaves
        }
    return self._solved_copy.get_best_bound()

tighten_objective_bounds

tighten_objective_bounds(pad: float = 1e-06) -> dict | None

Replace the interval bounds of an infinity-norm objective with OBBT bounds.

The infinity-norm encoding takes its big-M constants (the global M and the per-component sign constants) from bounds on each residual component. Any bounds that hold on every feasible point keep the optimum exact, so the interval-propagation bounds set with the objective are intersected with the minimum and maximum of each component over the OBBT LP. The OBBT values are padded outward by pad * max(1, |bound|) to absorb the LP solver's feasibility tolerance, then clipped back into the interval bounds, which are exactly valid.

Does nothing unless the objective is an infinity norm and an OBBT handler is attached.

Parameters:

Name	Type	Description	Default
pad	float	Relative outward padding of the OBBT bounds.	1e-06

Returns:

Type	Description
dict or None	None if nothing was done, otherwise time (seconds), width_ratio (median interval width over tightened width), M_before, M_after and sign_fixed (components whose sign the tightened bounds determine but the interval bounds did not).

Source code in fomtools/canonicalizers/mip.py

def tighten_objective_bounds(self, pad: float = 1e-6) -> dict | None:
    """
    Replace the interval bounds of an infinity-norm objective with OBBT bounds.

    The infinity-norm encoding takes its big-M constants (the global ``M`` and
    the per-component sign constants) from bounds on each residual component.
    Any bounds that hold on every feasible point keep the optimum exact, so the
    interval-propagation bounds set with the objective are intersected with
    the minimum and maximum of each component over the OBBT LP. The OBBT
    values are padded outward by ``pad * max(1, |bound|)`` to absorb the LP
    solver's feasibility tolerance, then clipped back into the interval
    bounds, which are exactly valid.

    Does nothing unless the objective is an infinity norm and an OBBT handler
    is attached.

    Parameters
    ----------
    pad: float, default 1e-6
        Relative outward padding of the OBBT bounds.

    Returns
    -------
    dict or None
        ``None`` if nothing was done, otherwise ``time`` (seconds),
        ``width_ratio`` (median interval width over tightened width),
        ``M_before``, ``M_after`` and ``sign_fixed`` (components whose sign
        the tightened bounds determine but the interval bounds did not).
    """
    obbt_handler = self.obbt_handler
    if obbt_handler is None or self._objective_spec is None:
        return None
    if self._objective_spec[0] != "infinity_norm":
        return None

    start = time.perf_counter()
    _, expr_list, lb_list, ub_list = self._objective_spec
    new_lb, new_ub = [], []
    for expr, lb, ub in zip(expr_list, lb_list, ub_list):
        lb, ub = np.asarray(lb, dtype=float), np.asarray(ub, dtype=float)
        result = obbt_handler.obbt(expr, lb, ub)
        if result is None:
            new_lb.append(lb)
            new_ub.append(ub)
            continue
        t_lb, t_ub = result
        t_lb = np.maximum(t_lb - pad * np.maximum(1.0, np.abs(t_lb)), lb)
        t_ub = np.minimum(t_ub + pad * np.maximum(1.0, np.abs(t_ub)), ub)
        new_lb.append(t_lb)
        new_ub.append(t_ub)
    self._objective_spec = ("infinity_norm", expr_list, new_lb, new_ub)

    def big_m(lbs, ubs):
        return float(np.max(np.abs(np.hstack([np.hstack(lbs), np.hstack(ubs)]))))

    old_lb, old_ub = np.hstack(lb_list), np.hstack(ub_list)
    t_lb, t_ub = np.hstack(new_lb), np.hstack(new_ub)
    old_fixed = (old_lb >= 0) | (old_ub < 0)
    new_fixed = (t_lb >= 0) | (t_ub < 0)
    return dict(
        time=time.perf_counter() - start,
        width_ratio=float(np.median((old_ub - old_lb) / np.maximum(t_ub - t_lb, 1e-12))),
        M_before=big_m(lb_list, ub_list),
        M_after=big_m(new_lb, new_ub),
        sign_fixed=int(np.sum(new_fixed & ~old_fixed)),
    )

update_bounds

update_bounds(expression: LinearExpression, lb: float | ndarray, ub: float | ndarray)

Update the lower and upper bounds of the solver variables corresponding to the given LinearExpression.

Parameters:

Name	Type	Description	Default
expression	LinearExpression	A LinearExpression object whose underlying variables bounds are to be updated.	required
lb	float | ndarray	New lower bound(s).	required
ub	float | ndarray	New upper bound(s).	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/mip.py

def update_bounds(
    self, expression: LinearExpression, lb: float | np.ndarray, ub: float | np.ndarray
):
    """
    Update the lower and upper bounds of the solver variables corresponding
    to the given ``LinearExpression``.

    Parameters
    ----------
    expression: LinearExpression
        A ``LinearExpression`` object whose underlying variables
        bounds are to be updated.

    lb: float or np.ndarray
        New lower bound(s).

    ub: float or np.ndarray
        New upper bound(s).


    Returns
    -------
    None
    """
    self.solver.update_bounds(expression, lb, ub)

OBBT handler

OBBTHandler

Bases: Canonicalizer

Handler for Optimization-Based Bound Tightening (OBBT).

This class provides an interface for tightening variable bounds using optimization. It maintains a separate solver instance that is used to compute tighter lower and upper bounds on variables by solving minimization and maximization problems.

Attributes:

Name	Type	Description
solver	Solver	A ready-to-use instance of a subclass of Solver (e.g., gurobi.Gurobi()).
crossing_tol	float	Largest relative crossing min > max of an OBBT bound pair treated as LP solver noise; such entries are swapped and buffered outward by the same amount. Defaults to Gurobi's default feasibility and optimality tolerances.
crossing_repairs	int	Number of bound entries repaired this way so far.

Source code in fomtools/canonicalizers/obbt_handler.py

class OBBTHandler(Canonicalizer):
    """
    Handler for Optimization-Based Bound Tightening (OBBT).

    This class provides an interface for tightening variable bounds using
    optimization. It maintains a separate solver instance that is used to
    compute tighter lower and upper bounds on variables by solving
    minimization and maximization problems.

    Attributes
    ----------
    solver : Solver
        A ready-to-use instance of a subclass of `Solver`
        (e.g., ``gurobi.Gurobi()``).
    crossing_tol : float
        Largest relative crossing ``min > max`` of an OBBT bound pair treated as
        LP solver noise; such entries are swapped and buffered outward by the
        same amount. Defaults to Gurobi's default feasibility and optimality
        tolerances.
    crossing_repairs : int
        Number of bound entries repaired this way so far.
    """

    def __init__(self, solver: str, solver_params: dict = None, crossing_tol: float = 1e-6):
        """
        Initialize an instance of `OBBTHandler`.

        Parameters
        ----------
        solver : str
            Case-insensitive identifier for the solver to use. Supported
            values (and common aliases) include:
            - ``"gurobi"``, ``"grb"``
            - ``"highs"``
        solver_params : dict or None, optional
            Parameters passed verbatim to the OBBT solver (e.g.
            ``{"OutputFlag": 0}`` for Gurobi).
        crossing_tol : float, default 1e-6
            See the class attribute of the same name.
        """
        super().__init__(solver_params=solver_params, solver=solver)
        self.crossing_tol = crossing_tol
        self.crossing_repairs = 0

    def add_var(self, var: LinearExpression) -> None:
        """
        Add a variable to the OBBT handler instance.

        This method registers a new variable with the OBBT solver so it
        can be used in bound tightening optimization problems.

        Parameters
        ----------
        var : LinearExpression
            Linear expression object to add to the OBBT instance.

        Returns
        -------
        None
        """
        self.solver.add_var(var, var.lb, var.ub)

    def add_constraint(self, constraint: Constraint) -> None:
        """
        Add a constraint to the OBBT handler instance.

        This method registers a constraint with the OBBT solver so it
        is considered during bound tightening optimization.

        Parameters
        ----------
        constraint : Constraint
            Constraint object to add to the OBBT instance.

        Returns
        -------
        None
        """
        self.solver.add_constr(constraint)

    def add_constraints(self, constraints: List[Constraint]) -> None:
        """
        Add a list of constraints to the OBBT handler instance.

        Parameters
        ----------
        constraints : List[Constraint]
            Constraint objects to add to the OBBT instance. Every constraint
            must be binary-free (the OBBT model is a pure linear program).

        Returns
        -------
        None
        """
        self.solver.constraints_to_solver(constraints)

    def update_bounds(
        self, expression: LinearExpression, lb: float | np.ndarray, ub: float | np.ndarray
    ) -> None:
        """
        Update the lower and upper bounds of the solver variables corresponding
        to the given ``LinearExpression``.

        Parameters
        ----------
        expression : LinearExpression
            A ``LinearExpression`` object whose underlying variable
            bounds are to be updated.

        lb : float or np.ndarray
            New lower bound(s).

        ub : float or np.ndarray
            New upper bound(s).

        Returns
        -------
        None
        """
        self.solver.update_bounds(expression, lb, ub)

    def obbt(
        self, linexpr: LinearExpression, lb: np.ndarray, ub: np.ndarray
    ) -> tuple[np.ndarray, np.ndarray] | None:
        """
        Perform Optimization-Based Bound Tightening on a linear expression.

        For each element in the linear expression, this method solves
        minimization and maximization problems to compute potentially
        tighter bounds. The returned bounds are the intersection of the
        input bounds and the computed bounds, padded outward by
        ``1e-6 * max(1, abs(bound))``. This does not certify rounding errors.

        Parameters
        ----------
        linexpr : LinearExpression
            The linear expression whose bounds are to be tightened.

        lb : np.ndarray
            Current lower bounds for each element of the expression.

        ub : np.ndarray
            Current upper bounds for each element of the expression.

        Returns
        -------
        tuple[np.ndarray, np.ndarray] | None
            A tuple ``(new_lb, new_ub)`` containing the tightened lower
            and upper bounds. Returns ``None`` if the optimization failed
            (e.g., the problem is infeasible or unbounded).
        """
        lb_out = np.zeros(linexpr.ndim)
        ub_out = np.zeros(linexpr.ndim)

        for sense in ["minimize", "maximize"]:
            for i, expr in enumerate(linexpr):
                self.solver.set_objective(expr, sense)
                self.solver.optimize()

                if not self.solver.is_optimal():
                    log.info("bound tightening failed: OBBT model not solved to optimality")
                    log.info(f"i = {i}, sense={sense}")
                    return

                if sense == "maximize":
                    ub_out[i] = self.solver["objVal"]
                else:
                    lb_out[i] = self.solver["objVal"]

        # reset objective
        self.solver.set_objective(0)

        # Both the raw LP bounds (min/max solves for the same index) and their
        # intersection with the input bounds can cross by solver noise (e.g.
        # input ub = 0, LP lb = 2e-17), up to the LP tolerances. Swap such
        # entries and buffer them outward by the tolerance so the interval
        # stays valid. Crossings beyond the tolerance raise (genuine bug).
        if not (np.isfinite(lb_out).all() and np.isfinite(ub_out).all()):
            log.info("bound tightening failed: nonfinite LP objective")
            return None
        lb_out, ub_out = self._repair(lb_out, ub_out)
        # Pad the LP bounds outward to cover small LP solver errors, matching the
        # objective-bound padding policy. This is a numerical safeguard, not a
        # directed-rounding certificate.
        lb_out = lb_out - 1e-6 * np.maximum(1.0, np.abs(lb_out))
        ub_out = ub_out + 1e-6 * np.maximum(1.0, np.abs(ub_out))
        lb_new, ub_new = self._repair(np.maximum(lb, lb_out), np.minimum(ub, ub_out))
        # The buffer must not loosen the input bounds (e.g. a ReLU output's
        # lb = 0). The repaired interval always meets [lb, ub], so this clip
        # cannot cross it again.
        return np.maximum(lb_new, lb), np.minimum(ub_new, ub)

    def _repair(self, lb: np.ndarray, ub: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
        """``repair_crossed_bounds`` at the LP tolerance, counting and logging repairs."""
        lb, ub = np.asarray(lb, dtype=float), np.asarray(ub, dtype=float)
        crossed = lb > ub
        if np.any(crossed):
            self.crossing_repairs += int(crossed.sum())
            log.info(
                "OBBT: repaired %d crossed bound(s), largest crossing %.3e",
                int(crossed.sum()),
                float(np.max(lb - ub)),
            )
        return repair_crossed_bounds(lb, ub, tol=self.crossing_tol, buffer=self.crossing_tol)

    # ------------------------------------------------------------------
    # Linear (binary-free) canonicalization of steps into the OBBT LP.
    #
    # These methods mirror the *linear* equality and convex-envelope rows of
    # the MIP canonicalizer (``fomtools/canonicalizers/mip.py``) with all
    # binary-indicator (big-M) rows and Huchette cuts removed. They are an
    # independent reimplementation -- they do not import or call MIP code --
    # so the OBBT model is a pure LP. A small amount of duplication of the row
    # expressions is intentional. Every method must be invoked with the step's
    # *already-tightened* bounds, since the envelope coefficients bake in
    # ``lb``/``ub``.
    # ------------------------------------------------------------------

    def _add_binary_free(self, constraints: List[Constraint]) -> None:
        """Check that the constraints are binary-free, then add them to the LP."""
        if not all(is_binary_free(c) for c in constraints):
            raise ValueError("OBBT handler received a constraint involving a binary variable")
        self.add_constraints(constraints)

    def canonicalize_affine(self, step: Affine) -> None:
        """
        Add the affine equality ``output_mat @ output == arg`` to the OBBT LP.

        ``output_mat`` is a documented attribute of the `Affine` step
        interface; this mirrors ``MIP.canonicalize_affine`` exactly (both rely
        on the same equality), so the two stay consistent if ``Affine`` is
        refactored.
        """
        self._add_binary_free([step.output_mat @ step.output == step.arg])

    def canonicalize_relu(self, step: Relu) -> None:
        """
        Add the binary-free ReLU relaxation rows to the OBBT LP.

        Mirrors ``MIP._build_relu_constraints`` keeping only the equality
        cases and, for the uncertain region, the triangle envelope
        ``{ output >= arg, output <= ub/(ub-lb) * (arg - lb) }``. The big-M rows are
        dropped. Iterates ``proj_indices`` only — for partial-projection
        ReLU the non-proj rows of ``step.output`` share decomposition with
        ``step.arg`` (composite passthrough), so emitting ``output[i] == arg[i]``
        would reduce to a degenerate ``0 == 0`` that the solver rejects.
        """
        constraints = []
        for i in sorted(step.proj_indices):
            _output = step.output[i]
            _arg = step.arg[i]
            if step.ub[i] <= 0:
                constraints.append(_output == 0)
            elif step.lb[i] > 0:
                constraints.append(_output == _arg)
            else:
                constraints.append(_output >= _arg)
                constraints.extend(relu_upper_envelope(_output, _arg, step.lb[i], step.ub[i]))
        self._add_binary_free(constraints)

    def canonicalize_saturatedlinear(self, step: SaturatedLinear) -> None:
        """
        Add the binary-free saturated-linear relaxation rows to the OBBT LP.

        Mirrors ``MIP.canonicalize_saturatedlinear`` keeping only the
        saturated/pass-through equalities and the convex-envelope slope rows;
        all big-M rows are dropped.
        """
        constraints = []
        for l, u, lb, ub, output, arg in zip(
            step.l, step.u, step.lb, step.ub, step.output, step.arg
        ):
            if lb >= u:
                constraints.append(output == u)
            elif ub <= l:
                constraints.append(output == l)
            elif lb >= l and ub <= u:
                constraints.append(output == arg)
            elif lb < l and ub > u:
                constraints.extend(
                    [
                        output <= (u - l) / (u - lb) * (arg - u) + u,
                        output >= (u - l) / (ub - l) * (arg - l) + l,
                    ]
                )
            elif lb < l and ub <= u:
                constraints.extend(
                    [output >= arg, output <= ((ub - l) / (ub - lb)) * (arg - ub) + ub]
                )
            elif lb >= l and ub > u:
                constraints.extend(
                    [output <= arg, output >= ((u - lb) / (ub - lb) * (arg - lb) + lb)]
                )
        self._add_binary_free(constraints)

    def canonicalize_softthreshold(self, step: SoftThreshold) -> None:
        """
        Add the binary-free soft-threshold relaxation rows to the OBBT LP.

        Mirrors ``MIP.canonicalize_softthreshold`` keeping only the fixed-region
        equalities and the convex-envelope rows; all big-M rows are dropped.
        """
        constraints = []
        for lb, ub, output_lb, output_ub, output, arg in zip(
            step.lb, step.ub, step.output.lb, step.output.ub, step.output, step.arg
        ):
            if lb >= step.lambd:
                constraints.append(output == arg - step.lambd)
            elif ub <= -step.lambd:
                constraints.append(output == arg + step.lambd)
            elif lb >= -step.lambd and ub <= step.lambd:
                constraints.append(output == 0)
            else:
                constraints.extend(
                    softthreshold_envelope(output, arg, lb, ub, output_lb, output_ub, step.lambd)
                )
        self._add_binary_free(constraints)

    _CANONICALIZERS: ClassVar[dict] = {
        "affine": canonicalize_affine,
        "relu": canonicalize_relu,
        "saturatedlinear": canonicalize_saturatedlinear,
        "softthreshold": canonicalize_softthreshold,
    }

__init__

__init__(solver: str, solver_params: dict = None, crossing_tol: float = 1e-06)

Parameters:

Name	Type	Description	Default
solver	str	Case-insensitive identifier for the solver to use. Supported values (and common aliases) include: - "gurobi", "grb" - "highs"	required
solver_params	dict or None	Parameters passed verbatim to the OBBT solver (e.g. {"OutputFlag": 0} for Gurobi).	None
crossing_tol	float	See the class attribute of the same name.	1e-6

Source code in fomtools/canonicalizers/obbt_handler.py

def __init__(self, solver: str, solver_params: dict = None, crossing_tol: float = 1e-6):
    """
    Initialize an instance of `OBBTHandler`.

    Parameters
    ----------
    solver : str
        Case-insensitive identifier for the solver to use. Supported
        values (and common aliases) include:
        - ``"gurobi"``, ``"grb"``
        - ``"highs"``
    solver_params : dict or None, optional
        Parameters passed verbatim to the OBBT solver (e.g.
        ``{"OutputFlag": 0}`` for Gurobi).
    crossing_tol : float, default 1e-6
        See the class attribute of the same name.
    """
    super().__init__(solver_params=solver_params, solver=solver)
    self.crossing_tol = crossing_tol
    self.crossing_repairs = 0

add_constraint

add_constraint(constraint: Constraint) -> None

Add a constraint to the OBBT handler instance.

This method registers a constraint with the OBBT solver so it is considered during bound tightening optimization.

Parameters:

Name	Type	Description	Default
constraint	Constraint	Constraint object to add to the OBBT instance.	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/obbt_handler.py

def add_constraint(self, constraint: Constraint) -> None:
    """
    Add a constraint to the OBBT handler instance.

    This method registers a constraint with the OBBT solver so it
    is considered during bound tightening optimization.

    Parameters
    ----------
    constraint : Constraint
        Constraint object to add to the OBBT instance.

    Returns
    -------
    None
    """
    self.solver.add_constr(constraint)

add_constraints

add_constraints(constraints: List[Constraint]) -> None

Add a list of constraints to the OBBT handler instance.

Parameters:

Name	Type	Description	Default
constraints	List[Constraint]	Constraint objects to add to the OBBT instance. Every constraint must be binary-free (the OBBT model is a pure linear program).	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/obbt_handler.py

def add_constraints(self, constraints: List[Constraint]) -> None:
    """
    Add a list of constraints to the OBBT handler instance.

    Parameters
    ----------
    constraints : List[Constraint]
        Constraint objects to add to the OBBT instance. Every constraint
        must be binary-free (the OBBT model is a pure linear program).

    Returns
    -------
    None
    """
    self.solver.constraints_to_solver(constraints)

add_var

add_var(var: LinearExpression) -> None

Add a variable to the OBBT handler instance.

This method registers a new variable with the OBBT solver so it can be used in bound tightening optimization problems.

Parameters:

Name	Type	Description	Default
var	LinearExpression	Linear expression object to add to the OBBT instance.	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/obbt_handler.py

def add_var(self, var: LinearExpression) -> None:
    """
    Add a variable to the OBBT handler instance.

    This method registers a new variable with the OBBT solver so it
    can be used in bound tightening optimization problems.

    Parameters
    ----------
    var : LinearExpression
        Linear expression object to add to the OBBT instance.

    Returns
    -------
    None
    """
    self.solver.add_var(var, var.lb, var.ub)

canonicalize_affine

canonicalize_affine(step: Affine) -> None

Add the affine equality output_mat @ output == arg to the OBBT LP.

output_mat is a documented attribute of the Affine step interface; this mirrors MIP.canonicalize_affine exactly (both rely on the same equality), so the two stay consistent if Affine is refactored.

Source code in fomtools/canonicalizers/obbt_handler.py

def canonicalize_affine(self, step: Affine) -> None:
    """
    Add the affine equality ``output_mat @ output == arg`` to the OBBT LP.

    ``output_mat`` is a documented attribute of the `Affine` step
    interface; this mirrors ``MIP.canonicalize_affine`` exactly (both rely
    on the same equality), so the two stay consistent if ``Affine`` is
    refactored.
    """
    self._add_binary_free([step.output_mat @ step.output == step.arg])

canonicalize_relu

canonicalize_relu(step: Relu) -> None

Add the binary-free ReLU relaxation rows to the OBBT LP.

Mirrors MIP._build_relu_constraints keeping only the equality cases and, for the uncertain region, the triangle envelope { output >= arg, output <= ub/(ub-lb) * (arg - lb) }. The big-M rows are dropped. Iterates proj_indices only — for partial-projection ReLU the non-proj rows of step.output share decomposition with step.arg (composite passthrough), so emitting output[i] == arg[i] would reduce to a degenerate 0 == 0 that the solver rejects.

Source code in fomtools/canonicalizers/obbt_handler.py

def canonicalize_relu(self, step: Relu) -> None:
    """
    Add the binary-free ReLU relaxation rows to the OBBT LP.

    Mirrors ``MIP._build_relu_constraints`` keeping only the equality
    cases and, for the uncertain region, the triangle envelope
    ``{ output >= arg, output <= ub/(ub-lb) * (arg - lb) }``. The big-M rows are
    dropped. Iterates ``proj_indices`` only — for partial-projection
    ReLU the non-proj rows of ``step.output`` share decomposition with
    ``step.arg`` (composite passthrough), so emitting ``output[i] == arg[i]``
    would reduce to a degenerate ``0 == 0`` that the solver rejects.
    """
    constraints = []
    for i in sorted(step.proj_indices):
        _output = step.output[i]
        _arg = step.arg[i]
        if step.ub[i] <= 0:
            constraints.append(_output == 0)
        elif step.lb[i] > 0:
            constraints.append(_output == _arg)
        else:
            constraints.append(_output >= _arg)
            constraints.extend(relu_upper_envelope(_output, _arg, step.lb[i], step.ub[i]))
    self._add_binary_free(constraints)

canonicalize_saturatedlinear

canonicalize_saturatedlinear(step: SaturatedLinear) -> None

Add the binary-free saturated-linear relaxation rows to the OBBT LP.

Mirrors MIP.canonicalize_saturatedlinear keeping only the saturated/pass-through equalities and the convex-envelope slope rows; all big-M rows are dropped.

Source code in fomtools/canonicalizers/obbt_handler.py

def canonicalize_saturatedlinear(self, step: SaturatedLinear) -> None:
    """
    Add the binary-free saturated-linear relaxation rows to the OBBT LP.

    Mirrors ``MIP.canonicalize_saturatedlinear`` keeping only the
    saturated/pass-through equalities and the convex-envelope slope rows;
    all big-M rows are dropped.
    """
    constraints = []
    for l, u, lb, ub, output, arg in zip(
        step.l, step.u, step.lb, step.ub, step.output, step.arg
    ):
        if lb >= u:
            constraints.append(output == u)
        elif ub <= l:
            constraints.append(output == l)
        elif lb >= l and ub <= u:
            constraints.append(output == arg)
        elif lb < l and ub > u:
            constraints.extend(
                [
                    output <= (u - l) / (u - lb) * (arg - u) + u,
                    output >= (u - l) / (ub - l) * (arg - l) + l,
                ]
            )
        elif lb < l and ub <= u:
            constraints.extend(
                [output >= arg, output <= ((ub - l) / (ub - lb)) * (arg - ub) + ub]
            )
        elif lb >= l and ub > u:
            constraints.extend(
                [output <= arg, output >= ((u - lb) / (ub - lb) * (arg - lb) + lb)]
            )
    self._add_binary_free(constraints)

canonicalize_softthreshold

canonicalize_softthreshold(step: SoftThreshold) -> None

Add the binary-free soft-threshold relaxation rows to the OBBT LP.

Mirrors MIP.canonicalize_softthreshold keeping only the fixed-region equalities and the convex-envelope rows; all big-M rows are dropped.

Source code in fomtools/canonicalizers/obbt_handler.py

def canonicalize_softthreshold(self, step: SoftThreshold) -> None:
    """
    Add the binary-free soft-threshold relaxation rows to the OBBT LP.

    Mirrors ``MIP.canonicalize_softthreshold`` keeping only the fixed-region
    equalities and the convex-envelope rows; all big-M rows are dropped.
    """
    constraints = []
    for lb, ub, output_lb, output_ub, output, arg in zip(
        step.lb, step.ub, step.output.lb, step.output.ub, step.output, step.arg
    ):
        if lb >= step.lambd:
            constraints.append(output == arg - step.lambd)
        elif ub <= -step.lambd:
            constraints.append(output == arg + step.lambd)
        elif lb >= -step.lambd and ub <= step.lambd:
            constraints.append(output == 0)
        else:
            constraints.extend(
                softthreshold_envelope(output, arg, lb, ub, output_lb, output_ub, step.lambd)
            )
    self._add_binary_free(constraints)

obbt

obbt(linexpr: LinearExpression, lb: ndarray, ub: ndarray) -> tuple[ndarray, ndarray] | None

Perform Optimization-Based Bound Tightening on a linear expression.

For each element in the linear expression, this method solves minimization and maximization problems to compute potentially tighter bounds. The returned bounds are the intersection of the input bounds and the computed bounds, padded outward by 1e-6 * max(1, abs(bound)). This does not certify rounding errors.

Parameters:

Name	Type	Description	Default
linexpr	LinearExpression	The linear expression whose bounds are to be tightened.	required
lb	ndarray	Current lower bounds for each element of the expression.	required
ub	ndarray	Current upper bounds for each element of the expression.	required

Returns:

Type	Description
tuple[ndarray, ndarray] | None	A tuple (new_lb, new_ub) containing the tightened lower and upper bounds. Returns None if the optimization failed (e.g., the problem is infeasible or unbounded).

Source code in fomtools/canonicalizers/obbt_handler.py

def obbt(
    self, linexpr: LinearExpression, lb: np.ndarray, ub: np.ndarray
) -> tuple[np.ndarray, np.ndarray] | None:
    """
    Perform Optimization-Based Bound Tightening on a linear expression.

    For each element in the linear expression, this method solves
    minimization and maximization problems to compute potentially
    tighter bounds. The returned bounds are the intersection of the
    input bounds and the computed bounds, padded outward by
    ``1e-6 * max(1, abs(bound))``. This does not certify rounding errors.

    Parameters
    ----------
    linexpr : LinearExpression
        The linear expression whose bounds are to be tightened.

    lb : np.ndarray
        Current lower bounds for each element of the expression.

    ub : np.ndarray
        Current upper bounds for each element of the expression.

    Returns
    -------
    tuple[np.ndarray, np.ndarray] | None
        A tuple ``(new_lb, new_ub)`` containing the tightened lower
        and upper bounds. Returns ``None`` if the optimization failed
        (e.g., the problem is infeasible or unbounded).
    """
    lb_out = np.zeros(linexpr.ndim)
    ub_out = np.zeros(linexpr.ndim)

    for sense in ["minimize", "maximize"]:
        for i, expr in enumerate(linexpr):
            self.solver.set_objective(expr, sense)
            self.solver.optimize()

            if not self.solver.is_optimal():
                log.info("bound tightening failed: OBBT model not solved to optimality")
                log.info(f"i = {i}, sense={sense}")
                return

            if sense == "maximize":
                ub_out[i] = self.solver["objVal"]
            else:
                lb_out[i] = self.solver["objVal"]

    # reset objective
    self.solver.set_objective(0)

    # Both the raw LP bounds (min/max solves for the same index) and their
    # intersection with the input bounds can cross by solver noise (e.g.
    # input ub = 0, LP lb = 2e-17), up to the LP tolerances. Swap such
    # entries and buffer them outward by the tolerance so the interval
    # stays valid. Crossings beyond the tolerance raise (genuine bug).
    if not (np.isfinite(lb_out).all() and np.isfinite(ub_out).all()):
        log.info("bound tightening failed: nonfinite LP objective")
        return None
    lb_out, ub_out = self._repair(lb_out, ub_out)
    # Pad the LP bounds outward to cover small LP solver errors, matching the
    # objective-bound padding policy. This is a numerical safeguard, not a
    # directed-rounding certificate.
    lb_out = lb_out - 1e-6 * np.maximum(1.0, np.abs(lb_out))
    ub_out = ub_out + 1e-6 * np.maximum(1.0, np.abs(ub_out))
    lb_new, ub_new = self._repair(np.maximum(lb, lb_out), np.minimum(ub, ub_out))
    # The buffer must not loosen the input bounds (e.g. a ReLU output's
    # lb = 0). The repaired interval always meets [lb, ub], so this clip
    # cannot cross it again.
    return np.maximum(lb_new, lb), np.minimum(ub_new, ub)

update_bounds

update_bounds(expression: LinearExpression, lb: float | ndarray, ub: float | ndarray) -> None

Update the lower and upper bounds of the solver variables corresponding to the given LinearExpression.

Parameters:

Name	Type	Description	Default
expression	LinearExpression	A LinearExpression object whose underlying variable bounds are to be updated.	required
lb	float or ndarray	New lower bound(s).	required
ub	float or ndarray	New upper bound(s).	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/obbt_handler.py

def update_bounds(
    self, expression: LinearExpression, lb: float | np.ndarray, ub: float | np.ndarray
) -> None:
    """
    Update the lower and upper bounds of the solver variables corresponding
    to the given ``LinearExpression``.

    Parameters
    ----------
    expression : LinearExpression
        A ``LinearExpression`` object whose underlying variable
        bounds are to be updated.

    lb : float or np.ndarray
        New lower bound(s).

    ub : float or np.ndarray
        New upper bound(s).

    Returns
    -------
    None
    """
    self.solver.update_bounds(expression, lb, ub)

Simulator

Simulator

Bases: Canonicalizer

An interface for simulating a Verification Problem using CuPy.

Steps are recorded during canonicalization and compiled into efficient batched functions at solve time. This deferred-compilation approach allows an arbitrary sequence of steps to be registered incrementally (matching the MIP/SDP user experience) and then evaluated in a single forward pass over all parameter samples.

Attributes:

Name	Type	Description
param_to_samples	dict[LinearExpression, ndarray]	A dictionary mapping each parameter variable (LinearExpression) to its sample values. Each array should have shape (num_samples, dim), where dim is the dimensionality of the parameter.
iterate_to_samples	dict[LinearExpression, ndarray]	A dictionary mapping each iterate variable (a LinearExpression) to its initial values. Each array should have shape (dim,), representing the initial state for that iterate.
step_descriptors	list[tuple[Step, Callable]]	A list of (step, activation) tuples recorded during canonicalization. Each activation is a callable that applies the step-specific nonlinearity to the assembled arg result.
residual_specs	dict[int, tuple[list[LinearExpression], int | float | str]]	A dictionary mapping step indices to (expr_list, norm_ord) tuples that define how residuals are computed at each step.
device	int	GPU device ID for CuPy execution.

Source code in fomtools/canonicalizers/simulator.py

class Simulator(Canonicalizer):
    """
    An interface for simulating a Verification Problem using CuPy.

    Steps are recorded during canonicalization and compiled into efficient
    batched functions at solve time. This deferred-compilation approach
    allows an arbitrary sequence of steps to be registered incrementally
    (matching the MIP/SDP user experience) and then evaluated in a single
    forward pass over all parameter samples.

    Attributes
    ----------
    param_to_samples : dict[LinearExpression, cp.ndarray]
        A dictionary mapping each parameter variable (LinearExpression)
        to its sample values. Each array should have shape (num_samples, dim),
        where dim is the dimensionality of the parameter.

    iterate_to_samples : dict[LinearExpression, cp.ndarray]
        A dictionary mapping each iterate variable (a LinearExpression) to
        its initial values. Each array should have shape (dim,), representing
        the initial state for that iterate.

    step_descriptors : list[tuple[Step, Callable]]
        A list of (step, activation) tuples recorded during canonicalization.
        Each activation is a callable that applies the step-specific
        nonlinearity to the assembled arg result.

    residual_specs : dict[int, tuple[list[LinearExpression], int | float | str]]
        A dictionary mapping step indices to (expr_list, norm_ord) tuples
        that define how residuals are computed at each step.

    device : int
        GPU device ID for CuPy execution.
    """

    def __init__(self, random_seed=0):
        super(Simulator, self).__init__(solver=None)
        self.param_to_samples = {}
        self.iterate_to_samples = {}

        self._seed_counter = random_seed
        self.step_descriptors = []
        self.residual_specs = {}
        self.user_iterates = {}
        self.device = 0  # GPU device ID

    @staticmethod
    def _sync():
        """Synchronize the CUDA stream if running on GPU, otherwise no-op."""
        if CUDA_AVAILABLE:
            cp.cuda.Stream.null.synchronize()

    @staticmethod
    def _get_device_context(device):
        """Return the active CUDA device context, or a no-op context on CPU."""
        return cp.cuda.Device(device) if CUDA_AVAILABLE else nullcontext()  # no-op on CPU

    @staticmethod
    def _sparse_to_cp(matrix: np.ndarray | spa.spmatrix) -> cp.ndarray | csp.csr_matrix:
        """
        Convert a NumPy or SciPy sparse matrix to a CuPy array
        or cupyx CSR sparse matrix, respectively.

        Parameters
        ----------
        matrix: np.ndarray | scipy.sparse.spmatrix
            The sparse matrix to be converted.

        Returns
        -------
        cp.ndarray | cupyx.scipy.sparse.csr_matrix
            A CuPy array or cupyx CSR sparse matrix containing the same data as
            the input matrix.
        """
        if spa.issparse(matrix):
            return csp.csr_matrix(matrix)
        return cp.array(matrix)

    def _linear_expr_to_cupy(
        self, expr: LinearExpression
    ) -> dict[str, list[tuple[LinearExpression, cp.ndarray | csp.csr_matrix]]]:
        """
        Converts a linear expression to CuPy arrays (dense) or cupyx CSR
        matrices (sparse).

        This function decomposes the provided LinearExpression into its
        constituent coefficients and converts each coefficient to CuPy arrays
        or cupyx CSR matrices. The terms of the expression are grouped
        according to whether their corresponding leaf variable belongs
        to the iterate set or the parameter set.

        Parameters
        ----------
        expr: LinearExpression
            The linear expression to be converted.

        Returns
        -------
        dict[str, list[tuple[LinearExpression, cp.ndarray | csp.csr_matrix]]]
            A dictionary mapping term categories (e.g., iterate or parameter
            terms) to lists of tuples (leaf, coeff_cp), where leaf identifies
            the variable and coeff_cp is the corresponding coefficient
            represented as CuPy arrays or cupyx CSR matrices.

        """
        iterate_coeffs = []
        param_coeffs = []

        for leaf, coeff in expr.decomposition_dict.items():
            coeff_cp = self._sparse_to_cp(coeff)

            if leaf in self.iterate_to_samples:
                iterate_coeffs.append((leaf, coeff_cp))
            elif leaf in self.param_to_samples:
                param_coeffs.append((leaf, coeff_cp))

        return {
            "iterate_coeffs": iterate_coeffs,
            "param_coeffs": param_coeffs,
            "const": cp.array(expr.const, dtype=cp.float32),
        }

    def _next_seed(self) -> int:
        """Generate next random seed for reproducible sampling."""
        seed = self._seed_counter
        self._seed_counter += 1
        return seed

    def add_var(self, var: LinearExpression, **kwargs) -> None:
        """
        Add a variable to the Simulator instance. This method converts the provided
        LinearExpression object to a CuPy array and registers it with the Simulator
        instance so that it can be referenced later.

        Parameters
        ----------
        var: LinearExpression
            Linear expression object to add to the Simulator instance.

        Returns
        -------
        None
        """

        def _register_var(
            _var: LinearExpression,
            var_type: str,
            random_seed: int = 0,
            samples: np.ndarray = None,
            init: np.ndarray = None,
        ) -> None:
            """
            Register a variable to the Simulator instance.

            This method adds the given ``LinearExpression`` object to the appropriate
            internal collection based on its role in the simulation.

            Parameters
            ----------
            _var: LinearExpression
                Linear expression object to add to the Simulator instance.

            var_type : str
                The category of the variable. Must be one of:
                    - "iterate": registers the variable as an iterate.
                    - "param": registers the variable as a parameter.

            random_seed: int, optional
                Random seed for reproducible sampling.

            samples: np.ndarray, optional
                For parameter variables, the sample values of the variable.
                Should have shape (num_samples, dim). Required if the variable is
                a leaf. Ignored for iterate variables.

            init: np.ndarray
                For iterate variables, the initial value. Should have shape (dim,).
                Required if ``var_type`` is ``"iterate"``. Ignored for parameter variables.

            Returns
            -------
            None

            Raises
            ------
            ValueError
                If ``var_type`` is ``"iterate"`` and ``init`` is not provided.

            ValueError
                If ``var_type`` is not one of ``"iterate"`` or ``"param"``.

            ValueError
                If neither samples nor appropriate bounds are provided.

            """
            lb, ub = _var.lb, _var.ub
            rng = np.random.default_rng(random_seed)

            has_bounds = np.all(np.isfinite(lb)) and np.all(np.isfinite(ub))
            has_samples = samples is not None or init is not None

            if has_bounds and has_samples:
                warnings.warn(
                    f"Both bounds and samples are provided for the {var_type}; bounds will be ignored."
                )

            if var_type == "iterate":
                if init is None:
                    if has_bounds:
                        init = rng.uniform(lb, ub, size=(_var.ndim,)).astype(np.float32)
                    else:
                        raise ValueError(
                            "An initial value or finite bounds must be provided when registering an iterate variable."
                        )
                self.iterate_to_samples[_var] = cp.array(init, dtype=cp.float32)

            elif var_type == "param":
                if samples is not None:
                    samples = np.asarray(samples)
                    self.param_to_samples[_var] = cp.array(samples, dtype=cp.float32)
                    if not has_bounds:
                        _var.lb = np.min(samples, axis=0)
                        _var.ub = np.max(samples, axis=0)
                elif has_bounds and np.array_equal(lb, ub):
                    # A user parameter pinned by equal bounds represents a
                    # constant value. Store that value so symbolic expressions
                    # involving the parameter evaluate correctly per sample.
                    self.iterate_to_samples[_var] = cp.array(lb, dtype=cp.float32)
                else:
                    # Step output — register as iterate with zero init so
                    # subsequent steps can reference it via the value table.
                    self.iterate_to_samples[_var] = cp.zeros(_var.ndim, dtype=cp.float32)

            else:
                raise ValueError("Unknown variable type")

        _register_var(var, random_seed=self._next_seed(), **kwargs)

    @staticmethod
    def compute_risk_bound(residuals: cp.ndarray, delta: float = 0.05, epsilon: float = 0.0):
        r"""
        Compute a high-probability bound on the risk using the KL inverse bound.

        Given a tolerance $\epsilon$, a number of iterations $k$ (implicitly given by the last
        column of ``residuals``),  and $N$ sampled problem instances,

        we define the $0$–$1$ error for sample $x$ as

        $$
            e(x) = \mathbf{1}\{ \phi(z^k(x), x) > \epsilon \},
        $$

        which in this implementation corresponds to checking whether the
        final residual exceeds $\epsilon$.

        Let $N$ be the number of samples and let the empirical risk be


        $$
            \hat{r}_S
            =
            \frac{1}{N}
            \sum_{i=1}^{N}
            \mathbf{1}\{ r_i^{(k)} > \epsilon \},
        $$

        where $r_i^{(k)}$ is the final residual of trajectory $i$.

        Then, with probability at least $1 - \delta$, the true risk $r$
        satisfies

        $$
            r
            \;\le\;
            \mathrm{kl}^{-1}
            \!\left(
                \hat{r}_S
                \;\middle|\;
                \frac{\log(2/\delta)}{N}
            \right),
        $$

        where $\mathrm{kl}^{-1}(q \mid c)$ denotes the largest $p \in [q,1]$
        such that

        $$
            \mathrm{KL}(q \,\|\, p)
            \le c,
        $$

        and

        $$
            \mathrm{KL}(q \,\|\, p)
            =
            q \log\!\frac{q}{p}
            +
            (1-q)\log\!\frac{1-q}{1-p}
        $$

        is the binary Kullback–Leibler divergence.

        Parameters
        ----------
        residuals: cp.ndarray
            Residuals of the problem instances.

        delta: float, optional
            Confidence parameter $\delta \in (0,1)$. The bound holds with
            probability at least $1 - \delta$.

        epsilon: float, optional
            Tolerance level $\epsilon$ defining the $0$–$1$ error.

        Returns
        -------
        float
            Upper bound on true risk via KL inverse.

        """

        def _kl_inverse(q, c):
            """
            Compute kl⁻¹(q | c): largest p such that KL(q || p) ≤ c.
            """
            if q >= 1.0:
                return 1.0
            if c == 0:
                return q

            low, high = q, 1.0
            for _ in range(100):
                mid = (low + high) / 2
                if mid == 1.0:
                    kl = float("inf")
                else:
                    kl = q * np.log(q / mid + 1e-10) + (1 - q) * np.log((1 - q) / (1 - mid) + 1e-10)

                if kl < c:
                    low = mid
                else:
                    high = mid
            return high

        N = residuals.shape[0]
        final_residuals = residuals[:, -1]

        errors = (final_residuals > epsilon).astype(cp.float32)
        empirical_risk = cp.mean(errors)

        c = cp.log(2 / delta) / N
        return _kl_inverse(float(empirical_risk), float(c))

    def _prepare_simulation_data(self):
        """
        Prepare stacked arrays and slices for simulation.

        This function packs the iterate initial values and parameter sample
        arrays into contiguous CuPy arrays. It also records slice indices that
        map each original variable (``leaf``) to its corresponding location in
        the concatenated arrays.

        Iterate initial values are concatenated into a single vector ``z0``.
        Parameter samples are concatenated along axis 1 into ``p_samples``.
        The returned slice dictionaries allow reconstruction or indexing of
        the individual variables within these packed arrays.

        Returns
        -------
        dict
            A dictionary containing the concatenated simulation inputs and their
            corresponding slice mappings, where:
                - ``z0`` maps to a CuPy array containing the concatenation of all
                    iterate initial values.
                - ``p_samples`` maps to a CuPy array containing the concatenation (along axis 1)
                    of all parameter sample arrays.
                - ``z_slices`` maps to a dictionary mapping each iterate identifier (leaf)
                    to a slice indicating its location within z0.
                - ``p_slices`` maps to a dictionary mapping each parameter identifier (leaf) to a
                    slice indicating its location within the second dimension of ``p_samples``.

        """

        def stack_with_slices(
            data_dict: dict[LinearExpression, cp.ndarray], axis: int = 0
        ) -> tuple[cp.ndarray, dict[LinearExpression, slice]]:
            """Prepare stacked arrays and slices for simulation."""
            slices = {}
            offset = 0
            for leaf, arr in data_dict.items():
                dim = arr.shape[axis]
                slices[leaf] = slice(offset, offset + dim)
                offset += dim

            if data_dict:
                stacked = cp.concatenate(list(data_dict.values()), axis=axis)
            else:
                stacked = cp.array([])

            return stacked, slices

        z0, z_slices = stack_with_slices(self.iterate_to_samples, axis=0)
        p_samples, p_slices = stack_with_slices(self.param_to_samples, axis=1)

        return {
            "z0": z0,
            "p_samples": p_samples,
            "z_slices": z_slices,
            "p_slices": p_slices,
        }

    def _register_statistics(
        self, residuals: cp.ndarray, delta: float, epsilon: float, total_time: float
    ):
        r"""
        Compute and register summary statistics from simulation residuals.

        Parameters
        ----------
        residuals : cp.ndarray
            Shape (N, K) array of residuals at each iteration.

        delta: float
            Confidence parameter $\delta \in (0,1)$. The bound holds with
            probability at least $1 - \delta$.

        epsilon: float
            Tolerance level $\epsilon$ defining the $0$–$1$ error.

        total_time: float
            Total time taken by the simulation.

        """

        final_residuals = residuals[:, -1]
        n_samples = residuals.shape[0]

        self.summary_statistics = {
            "n_samples": n_samples,
            "n_iterations": residuals.shape[1],
            "final_residuals": final_residuals,
            "mean": float(cp.mean(final_residuals)),
            "std": float(cp.std(final_residuals)),
            "min": float(cp.min(final_residuals)),
            "max": float(cp.max(final_residuals)),
            "median": float(cp.median(final_residuals)),
            "quantiles": {
                0.25: float(cp.quantile(final_residuals, 0.25)),
                0.75: float(cp.quantile(final_residuals, 0.75)),
                0.90: float(cp.quantile(final_residuals, 0.90)),
                0.95: float(cp.quantile(final_residuals, 0.95)),
                0.99: float(cp.quantile(final_residuals, 0.99)),
            },
            "epsilon": epsilon,
            "delta": delta,
            "empirical_risk": float(cp.mean((final_residuals > epsilon).astype(cp.float32))),
            "risk_bound": self.compute_risk_bound(residuals, delta, epsilon),
            "runtime": total_time,
            "time_per_sample": total_time / n_samples,
            "throughput": n_samples / total_time,
        }

    def _to_device(self, arr: np.ndarray | cp.ndarray) -> cp.ndarray | np.ndarray:
        """Move array to the configured device."""
        if CUDA_AVAILABLE:
            with self._get_device_context(self.device):
                return cp.asarray(arr)
        return arr

    def _configure_device(self, device: str) -> None:
        """Configure the CuPy GPU device ID for this instance."""
        if device is None or device == "gpu":
            self.device = 0
        elif device == "cpu":
            warnings.warn(
                "CuPy only supports GPU execution. Defaulting to GPU device 0. "
                "Use NumPy/SciPy directly for CPU-only simulation."
            )
            self.device = 0
        else:
            raise ValueError(f"Unsupported device type: {device}. Expected None, 'cpu', or 'gpu'.")

    @staticmethod
    def _matvec_batch(coeff: cp.ndarray | csp.csr_matrix, x_batch: cp.ndarray) -> cp.ndarray:
        """
        Apply coefficient matrix to a batch of vectors.

        Parameters
        ----------
        coeff: (out_dim, d) # sparse or dense
        x_batch: (N, d) # dense CuPy array

        Returns
        -------
        (N, out_dim) dense CuPy array.
        """
        if csp.issparse(coeff):
            return (coeff @ x_batch.T).T
        return x_batch @ coeff.T

    def _assemble_linear_combination(
        self,
        z_stacked: cp.ndarray,
        p_stacked: cp.ndarray,
        coeffs: dict[str, list[tuple[LinearExpression, cp.ndarray | csp.csr_matrix]]],
        simulation_data: dict[str, Any],
    ) -> cp.ndarray:
        """
        Assemble a batched linear combination of state and parameter slices
        using precomputed coefficients.

        Parameters
        ----------
        z_stacked: cp.ndarray
            Concatenation of all iterate variables. Individual iterate
            values are retrieved using slices stored in ``simulation_data["z_slices"]``.

        p_stacked: cp.ndarray
            Concatenation of all parameter variables. Individual parameter values
            are retrieved using slices stored in ``simulation_data["p_slices"]``.

        coeffs: dict[str, list[tuple[LinearExpression, cp.ndarray | csp.csr_matrix]]]
            A dictionary mapping term categories (e.g., iterate or parameter
            terms) to lists of tuples (leaf, coeff_cp), where leaf identifies
            the variable and coeff_cp is the corresponding coefficient
            represented as CuPy arrays or cupyx CSR matrices.

        simulation_data: dict[str, Any]
             A dictionary containing the concatenated simulation inputs and their
            corresponding slice mappings, where:
                - ``z0`` maps to a CuPy array containing the concatenation of all
                    iterate initial values.
                - ``p_samples`` maps to a CuPy array containing the concatenation (along axis 1)
                    of all parameter sample arrays.
                - ``z_slices`` maps to a dictionary mapping each iterate identifier (leaf)
                    to a slice indicating its location within z0.
                - ``p_slices`` maps to a dictionary mapping each parameter identifier (leaf) to a
                    slice indicating its location within the second dimension of ``p_samples``.

        Returns
        -------
        cp.ndarray
            The value of the linear expression evaluated at the provided
            iterates and parameters.

        """
        N = z_stacked.shape[0]
        result = cp.tile(coeffs["const"], (N, 1))

        for leaf, coeff in coeffs["iterate_coeffs"]:
            z_slice = z_stacked[:, simulation_data["z_slices"][leaf]]
            result += self._matvec_batch(coeff, z_slice)

        for leaf, coeff in coeffs["param_coeffs"]:
            p_slice = p_stacked[:, simulation_data["p_slices"][leaf]]
            result += self._matvec_batch(coeff, p_slice)

        return result

    # ------------------------------------------------------------------
    # Canonicalize methods — record (step, activation) for deferred eval
    # ------------------------------------------------------------------

    def canonicalize_relu(self, step: "Step") -> None:
        """
        Record the provided ReLU step for deferred evaluation at solve time.

        The activation evaluates ReLU on the projected arg coords selected by
        ``step.proj_indices`` and writes them into ``step.output_leaves[0]``.
        For full ReLU, ``proj_indices`` covers every arg row and the fancy
        index is the identity slice. For partial-projection ReLU, non-proj
        rows of ``step.output`` share decomposition with ``step.arg`` and are
        read off the existing arg leaves at evaluation time — no separate
        storage needed.

        Parameters
        ----------
        step: Step
            A step object containing the ReLU operation.
        """
        proj_idx = np.array(sorted(step.proj_indices), dtype=int)

        def activation(result):
            # result shape: (N, arg.ndim).
            # Output shape: (N, len(proj_idx)) — the value of output_leaves[0].
            return cp.maximum(result[:, proj_idx], 0)

        self.step_descriptors.append((step, activation))

    def canonicalize_softthreshold(self, step: "Step") -> None:
        """
        Record the provided soft-threshold step for deferred evaluation at solve time.

        Parameters
        ----------
        step: Step
            A step object containing the soft-threshold operation.
        """
        lambd = step.lambd

        def activation(result):
            return cp.sign(result) * cp.maximum(cp.abs(result) - lambd, 0)

        self.step_descriptors.append((step, activation))

    def canonicalize_saturatedlinear(self, step: "Step") -> None:
        """
        Record the provided saturated linear step for deferred evaluation at solve time.

        Parameters
        ----------
        step: Step
            A step object that represents a saturated linear operation.
        """
        l = step.l.lb if hasattr(step.l, "lb") else step.l
        u = step.u.ub if hasattr(step.u, "ub") else step.u

        def activation(result):
            return cp.clip(result, l, u)

        self.step_descriptors.append((step, activation))

    def canonicalize_affine(self, step: "Step") -> None:
        """
        Record the provided affine step for deferred evaluation at solve time.

        The LU factorization of ``step.output_mat`` is computed eagerly since
        it depends only on a constant matrix.

        Parameters
        ----------
        step: Step
            A step object that represents the affine transformation.
        """
        output_mat_cp = self._sparse_to_cp(step.output_mat)

        if csp.issparse(output_mat_cp):
            # Factor once (the matrix is constant) and solve all N right-hand
            # sides in a single multi-RHS triangular solve. ``arg`` is (N, d);
            # SuperLU.solve accepts a 2-D RHS of shape (d, N) -> (d, N).
            lu = cpspla.splu(output_mat_cp.tocsc())

            def activation(arg):
                return lu.solve(arg.T).T
        else:

            def activation(arg):
                # cp.linalg.solve broadcasts over the columns of the (d, N) RHS,
                # so all samples are solved at once (no per-sample Python loop).
                return cp.linalg.solve(output_mat_cp, arg.T).T

        self.step_descriptors.append((step, activation))

    _CANONICALIZERS = {
        "affine": canonicalize_affine,
        "relu": canonicalize_relu,
        "saturatedlinear": canonicalize_saturatedlinear,
        "softthreshold": canonicalize_softthreshold,
    }

    def register_subproblem_step(
        self, arg: LinearExpression, output_leaf: LinearExpression, activation
    ) -> None:
        """
        Record a subproblem step for deferred evaluation at solve time.

        Used by the step decomposer for proxes/steps that have no ``Step`` class
        on the simulator (projections onto the simplex / ℓ1-ball, the ℓ∞-norm and
        total-variation proxes, polyhedral projections, and inequality QP/LP). The
        ``activation`` is a callable mapping the assembled input batch
        ``(N, arg.ndim)`` to the output batch ``(N, output_leaf.ndim)``; the result
        is written into ``output_leaf``'s slot. See the subproblem activation
        factories defined in this module (e.g. `simplex_activation`,
        `qp_activation`).

        Parameters
        ----------
        arg : LinearExpression
            The assembled per-sample input to the subproblem.
        output_leaf : LinearExpression
            The fresh leaf the activation result is written into.
        activation : Callable
            Maps the input batch to the output batch.
        """
        self.step_descriptors.append((_ActivationStep(arg, output_leaf), activation))

    # ------------------------------------------------------------------
    # Objective methods — store residual specs keyed by step index
    # ------------------------------------------------------------------

    def set_zero_objective(self):
        """Record a zero residual at the most recently registered step."""
        self.residual_specs[len(self.step_descriptors) - 1] = ([], "zero")

    def set_infinity_norm_objective(self, expr_list, lb_list=None, ub_list=None):
        """Record an infinity-norm residual expression at the most recently registered step."""
        self.residual_specs[len(self.step_descriptors) - 1] = (expr_list, np.inf)

    def set_two_norm_objective(self, expr_list, lb_list=None, ub_list=None):
        """Record a squared two-norm residual expression at the most recently registered step."""
        self.residual_specs[len(self.step_descriptors) - 1] = (expr_list, 2)

    def set_one_norm_objective(self, expr_list):
        """Record a one-norm residual expression at the most recently registered step."""
        self.residual_specs[len(self.step_descriptors) - 1] = (expr_list, 1)

    # ------------------------------------------------------------------
    # Compilation — convert recorded steps/residuals to executable fns
    # ------------------------------------------------------------------

    def _compile_steps(self, simulation_data):
        """
        Compile all recorded step descriptors into executable functions.

        Called once at the start of ``solve_model`` after all variables are
        registered. Each step's arg LinearExpression is converted to CuPy
        coefficients, and paired with the step's activation function.

        Parameters
        ----------
        simulation_data : dict
            Output of ``_prepare_simulation_data``.

        Returns
        -------
        list[tuple[Callable, slice, slice, Callable]]
            A list of ``(evaluate_fn, output_slice, prev_slice, compose_output)``
            tuples where ``evaluate_fn(z_stacked, p_stacked) -> cp.ndarray``
            computes the activation output to write into ``output_slice``, and
            ``compose_output(z_stacked, p_stacked) -> cp.ndarray`` evaluates
            the full composite ``step.output`` from the state vector (matters
            for partial-projection ReLU, where the activation only fills
            the fresh proj-leaf slot and the non-proj rows of ``step.output``
            are read off shared arg leaves). ``prev_slice`` points to the
            previous iterate's slice in ``z_stacked`` for the default
            step-0 residual.
        """
        compiled = []
        z_slices = simulation_data["z_slices"]

        # Leaves a step *introduces* (where activation results get written).
        # For most steps this is the full ``output`` leaf; for partial-projection
        # Relu it is the smaller fresh proj-leaf returned by ``output_leaves``.
        # The simulator writes one ``z_new`` slot per step, so we require
        # ``output_leaves`` to be a single-leaf list. If a future step type
        # introduces multiple fresh leaves per call, _compile_steps must be
        # extended to allocate and write a slot per leaf — fail loudly here
        # rather than silently dropping the extra leaves.
        step_leaves = []
        for s, _ in self.step_descriptors:
            leaves = s.output_leaves
            if len(leaves) != 1:
                raise NotImplementedError(
                    f"{type(s).__name__}.output_leaves returned {len(leaves)} leaves; "
                    "Simulator._compile_steps assumes exactly one fresh leaf per step. "
                    "Extend the compile/write loop to handle multi-leaf steps."
                )
            step_leaves.append(leaves[0])

        for i, (step, activation) in enumerate(self.step_descriptors):
            coeffs = self._linear_expr_to_cupy(step.arg)
            # For full ReLU/satlin/etc., ``step.output`` is a leaf and these
            # coeffs amount to ``identity @ z_stacked[:, output_slice]``, so
            # ``compose_output`` simply returns what was just written. For
            # partial-projection ReLU, the composite ``step.output`` mixes
            # the fresh proj-leaf with passthrough rows of ``step.arg``;
            # evaluating this combination yields the full output.
            output_coeffs = self._linear_expr_to_cupy(step.output)
            output_slice = z_slices[step_leaves[i]]

            # Determine previous iterate slice for default residual computation.
            # Step 0: find the first original iterate leaf in the arg decomposition.
            # Step k>0: previous step's composite output is provided by the caller
            # via the cached composed-output from the prior iteration.
            if i == 0:
                prev_slice = None
                for leaf in step.arg.decomposition_dict:
                    if leaf in z_slices and leaf not in set(step_leaves):
                        prev_slice = z_slices[leaf]
                        break
                if prev_slice is None:
                    prev_slice = output_slice
            else:
                prev_slice = z_slices[step_leaves[i - 1]]

            def _make_fn(_coeffs=coeffs, _activation=activation):
                def evaluate_fn(z_stacked, p_stacked):
                    result = self._assemble_linear_combination(
                        z_stacked, p_stacked, _coeffs, simulation_data
                    )
                    return _activation(result)

                return evaluate_fn

            def _make_composer(_output_coeffs=output_coeffs):
                def compose_output(z_stacked, p_stacked):
                    return self._assemble_linear_combination(
                        z_stacked, p_stacked, _output_coeffs, simulation_data
                    )

                return compose_output

            compiled.append((_make_fn(), output_slice, prev_slice, _make_composer()))
        return compiled

    def record_iterate(self, expression):
        """Record a public operation's output after all its internal steps."""
        self.user_iterates[len(self.step_descriptors) - 1] = expression

    def _compile_residuals(self, simulation_data, specs=None):
        """
        Compile all recorded residual specs into executable functions.

        Parameters
        ----------
        simulation_data : dict
            Output of ``_prepare_simulation_data``.

        Returns
        -------
        dict[int, Callable]
            A dictionary mapping step index to a callable
            ``residual_fn(z_stacked, p_stacked) -> cp.ndarray`` of shape ``(N,)``.
        """
        compiled = {}
        for step_idx, (expr_list, norm_ord) in (
            self.residual_specs if specs is None else specs
        ).items():
            if norm_ord == "zero":

                def _make_zero_fn():
                    def residual_fn(z_stacked, p_stacked):
                        return cp.zeros(z_stacked.shape[0], dtype=cp.float32)

                    return residual_fn

                compiled[step_idx] = _make_zero_fn()
            else:
                coeffs_list = [self._linear_expr_to_cupy(expr) for expr in expr_list]

                def _make_fn(_coeffs_list=coeffs_list, _norm_ord=norm_ord):
                    def residual_fn(z_stacked, p_stacked):
                        values = [
                            self._assemble_linear_combination(
                                z_stacked, p_stacked, c, simulation_data
                            )
                            for c in _coeffs_list
                        ]
                        concat = cp.concatenate(values, axis=1)

                        if _norm_ord == np.inf:
                            return cp.max(cp.abs(concat), axis=1)
                        elif _norm_ord == 2:
                            return cp.sum(concat**2, axis=1)
                        elif _norm_ord == 1:
                            return cp.sum(cp.abs(concat), axis=1)

                    return residual_fn

                compiled[step_idx] = _make_fn()
        return compiled

    # ------------------------------------------------------------------
    # Solve
    # ------------------------------------------------------------------

    def solve_model(
        self, delta: float = 0.05, epsilon: float = 0.0, device: str = None
    ) -> cp.ndarray:
        """
        Run a batched simulation of all registered steps.

        Each call re-simulates from scratch: the value table is rebuilt,
        all steps are compiled and executed in sequence, and residuals are
        computed. This method can be called repeatedly (e.g. inside the
        user's iteration loop) as new steps are registered.

        Parameters
        ----------
        delta: float, optional
            Confidence parameter for the risk bound.

        epsilon: float, optional
            Tolerance level defining the 0–1 error.

        device: str, optional
            Device to run the simulation on (``None``, ``"cpu"``, or ``"gpu"``).

        Returns
        -------
        residuals: cp.ndarray
            A 2D array of shape ``(num_samples, n_residuals)`` containing
            the residual at each step for each parameter sample.

        Raises
        ------
        ValueError
            If no steps have been registered.
        """
        if not self.step_descriptors:
            raise ValueError(
                "No steps have been registered with this simulator instance. "
                "Call vp.step.*() before solve()."
            )

        self._configure_device(device)
        simulation_data = self._prepare_simulation_data()
        compiled_steps = self._compile_steps(simulation_data)

        specs = self.residual_specs
        if not specs:
            specs = {}
            previous = None
            for idx, expr in self.user_iterates.items():
                if previous is not None and idx >= 0:
                    if expr.ndim != previous.ndim:
                        raise ValueError(
                            "Default residuals require equal-sized user iterates; set an explicit objective."
                        )
                    specs[idx] = ([expr - previous], np.inf)
                previous = expr
            if not specs:
                raise ValueError(
                    "Default residuals require an initial iterate; call Verifier.iterate or set an explicit objective."
                )
        compiled_residuals = self._compile_residuals(simulation_data, specs)
        n_residuals = len(compiled_residuals)

        with self._get_device_context(self.device):
            z0 = self._to_device(simulation_data["z0"])
            p_samples = self._to_device(simulation_data["p_samples"])
            N = p_samples.shape[0]

            def _simulate_batch(z0: cp.ndarray, p_samples: cp.ndarray) -> cp.ndarray:
                """Simulate all registered steps for all N samples in parallel."""
                z_batch = cp.tile(z0, (N, 1))
                residuals = cp.empty((N, n_residuals), dtype=cp.float32)
                residual_col = 0

                for step_idx, (
                    evaluate_fn,
                    output_slice,
                    prev_slice,
                    compose_output,
                ) in enumerate(compiled_steps):
                    z_new = evaluate_fn(z_batch, p_samples)
                    z_batch[:, output_slice] = z_new

                    if step_idx in compiled_residuals:
                        residuals[:, residual_col] = compiled_residuals[step_idx](
                            z_batch, p_samples
                        )
                        residual_col += 1

                return residuals

            # Warm-up: trigger CuPy's lazy GPU kernel compilation so it is excluded
            # from the timed run. Only relevant on GPU — CLARABEL and the CPU-side
            # activations (TV/polyhedral/QP/LP loops) have no lazy-compile phase, so
            # warming up there would just double the solve time for no benefit.
            if CUDA_AVAILABLE:
                _simulate_batch(z0, p_samples)
                self._sync()

            # timed execution
            start = time.perf_counter()
            residuals = _simulate_batch(z0, p_samples)
            self._sync()
            elapsed = time.perf_counter() - start

        self._register_statistics(residuals, delta, epsilon, elapsed)
        return residuals

add_var

add_var(var: LinearExpression, **kwargs) -> None

Add a variable to the Simulator instance. This method converts the provided LinearExpression object to a CuPy array and registers it with the Simulator instance so that it can be referenced later.

Parameters:

Name	Type	Description	Default
var	LinearExpression	Linear expression object to add to the Simulator instance.	required

Returns:

Type	Description
None

Source code in fomtools/canonicalizers/simulator.py

def add_var(self, var: LinearExpression, **kwargs) -> None:
    """
    Add a variable to the Simulator instance. This method converts the provided
    LinearExpression object to a CuPy array and registers it with the Simulator
    instance so that it can be referenced later.

    Parameters
    ----------
    var: LinearExpression
        Linear expression object to add to the Simulator instance.

    Returns
    -------
    None
    """

    def _register_var(
        _var: LinearExpression,
        var_type: str,
        random_seed: int = 0,
        samples: np.ndarray = None,
        init: np.ndarray = None,
    ) -> None:
        """
        Register a variable to the Simulator instance.

        This method adds the given ``LinearExpression`` object to the appropriate
        internal collection based on its role in the simulation.

        Parameters
        ----------
        _var: LinearExpression
            Linear expression object to add to the Simulator instance.

        var_type : str
            The category of the variable. Must be one of:
                - "iterate": registers the variable as an iterate.
                - "param": registers the variable as a parameter.

        random_seed: int, optional
            Random seed for reproducible sampling.

        samples: np.ndarray, optional
            For parameter variables, the sample values of the variable.
            Should have shape (num_samples, dim). Required if the variable is
            a leaf. Ignored for iterate variables.

        init: np.ndarray
            For iterate variables, the initial value. Should have shape (dim,).
            Required if ``var_type`` is ``"iterate"``. Ignored for parameter variables.

        Returns
        -------
        None

        Raises
        ------
        ValueError
            If ``var_type`` is ``"iterate"`` and ``init`` is not provided.

        ValueError
            If ``var_type`` is not one of ``"iterate"`` or ``"param"``.

        ValueError
            If neither samples nor appropriate bounds are provided.

        """
        lb, ub = _var.lb, _var.ub
        rng = np.random.default_rng(random_seed)

        has_bounds = np.all(np.isfinite(lb)) and np.all(np.isfinite(ub))
        has_samples = samples is not None or init is not None

        if has_bounds and has_samples:
            warnings.warn(
                f"Both bounds and samples are provided for the {var_type}; bounds will be ignored."
            )

        if var_type == "iterate":
            if init is None:
                if has_bounds:
                    init = rng.uniform(lb, ub, size=(_var.ndim,)).astype(np.float32)
                else:
                    raise ValueError(
                        "An initial value or finite bounds must be provided when registering an iterate variable."
                    )
            self.iterate_to_samples[_var] = cp.array(init, dtype=cp.float32)

        elif var_type == "param":
            if samples is not None:
                samples = np.asarray(samples)
                self.param_to_samples[_var] = cp.array(samples, dtype=cp.float32)
                if not has_bounds:
                    _var.lb = np.min(samples, axis=0)
                    _var.ub = np.max(samples, axis=0)
            elif has_bounds and np.array_equal(lb, ub):
                # A user parameter pinned by equal bounds represents a
                # constant value. Store that value so symbolic expressions
                # involving the parameter evaluate correctly per sample.
                self.iterate_to_samples[_var] = cp.array(lb, dtype=cp.float32)
            else:
                # Step output — register as iterate with zero init so
                # subsequent steps can reference it via the value table.
                self.iterate_to_samples[_var] = cp.zeros(_var.ndim, dtype=cp.float32)

        else:
            raise ValueError("Unknown variable type")

    _register_var(var, random_seed=self._next_seed(), **kwargs)

canonicalize_affine

canonicalize_affine(step: Step) -> None

Record the provided affine step for deferred evaluation at solve time.

The LU factorization of step.output_mat is computed eagerly since it depends only on a constant matrix.

Parameters:

Name	Type	Description	Default
step	Step	A step object that represents the affine transformation.	required

Source code in fomtools/canonicalizers/simulator.py

def canonicalize_affine(self, step: "Step") -> None:
    """
    Record the provided affine step for deferred evaluation at solve time.

    The LU factorization of ``step.output_mat`` is computed eagerly since
    it depends only on a constant matrix.

    Parameters
    ----------
    step: Step
        A step object that represents the affine transformation.
    """
    output_mat_cp = self._sparse_to_cp(step.output_mat)

    if csp.issparse(output_mat_cp):
        # Factor once (the matrix is constant) and solve all N right-hand
        # sides in a single multi-RHS triangular solve. ``arg`` is (N, d);
        # SuperLU.solve accepts a 2-D RHS of shape (d, N) -> (d, N).
        lu = cpspla.splu(output_mat_cp.tocsc())

        def activation(arg):
            return lu.solve(arg.T).T
    else:

        def activation(arg):
            # cp.linalg.solve broadcasts over the columns of the (d, N) RHS,
            # so all samples are solved at once (no per-sample Python loop).
            return cp.linalg.solve(output_mat_cp, arg.T).T

    self.step_descriptors.append((step, activation))

canonicalize_relu

canonicalize_relu(step: Step) -> None

Record the provided ReLU step for deferred evaluation at solve time.

The activation evaluates ReLU on the projected arg coords selected by step.proj_indices and writes them into step.output_leaves[0]. For full ReLU, proj_indices covers every arg row and the fancy index is the identity slice. For partial-projection ReLU, non-proj rows of step.output share decomposition with step.arg and are read off the existing arg leaves at evaluation time — no separate storage needed.

Parameters:

Name	Type	Description	Default
step	Step	A step object containing the ReLU operation.	required

Source code in fomtools/canonicalizers/simulator.py

def canonicalize_relu(self, step: "Step") -> None:
    """
    Record the provided ReLU step for deferred evaluation at solve time.

    The activation evaluates ReLU on the projected arg coords selected by
    ``step.proj_indices`` and writes them into ``step.output_leaves[0]``.
    For full ReLU, ``proj_indices`` covers every arg row and the fancy
    index is the identity slice. For partial-projection ReLU, non-proj
    rows of ``step.output`` share decomposition with ``step.arg`` and are
    read off the existing arg leaves at evaluation time — no separate
    storage needed.

    Parameters
    ----------
    step: Step
        A step object containing the ReLU operation.
    """
    proj_idx = np.array(sorted(step.proj_indices), dtype=int)

    def activation(result):
        # result shape: (N, arg.ndim).
        # Output shape: (N, len(proj_idx)) — the value of output_leaves[0].
        return cp.maximum(result[:, proj_idx], 0)

    self.step_descriptors.append((step, activation))

canonicalize_saturatedlinear

canonicalize_saturatedlinear(step: Step) -> None

Record the provided saturated linear step for deferred evaluation at solve time.

Parameters:

Name	Type	Description	Default
step	Step	A step object that represents a saturated linear operation.	required

Source code in fomtools/canonicalizers/simulator.py

def canonicalize_saturatedlinear(self, step: "Step") -> None:
    """
    Record the provided saturated linear step for deferred evaluation at solve time.

    Parameters
    ----------
    step: Step
        A step object that represents a saturated linear operation.
    """
    l = step.l.lb if hasattr(step.l, "lb") else step.l
    u = step.u.ub if hasattr(step.u, "ub") else step.u

    def activation(result):
        return cp.clip(result, l, u)

    self.step_descriptors.append((step, activation))

canonicalize_softthreshold

canonicalize_softthreshold(step: Step) -> None

Record the provided soft-threshold step for deferred evaluation at solve time.

Parameters:

Name	Type	Description	Default
step	Step	A step object containing the soft-threshold operation.	required

Source code in fomtools/canonicalizers/simulator.py

def canonicalize_softthreshold(self, step: "Step") -> None:
    """
    Record the provided soft-threshold step for deferred evaluation at solve time.

    Parameters
    ----------
    step: Step
        A step object containing the soft-threshold operation.
    """
    lambd = step.lambd

    def activation(result):
        return cp.sign(result) * cp.maximum(cp.abs(result) - lambd, 0)

    self.step_descriptors.append((step, activation))

compute_risk_bound staticmethod

compute_risk_bound(residuals: ndarray, delta: float = 0.05, epsilon: float = 0.0)

Compute a high-probability bound on the risk using the KL inverse bound.

Given a tolerance \(\epsilon\), a number of iterations \(k\) (implicitly given by the last column of residuals), and \(N\) sampled problem instances,

we define the \(0\)–\(1\) error for sample \(x\) as

\[ e(x) = \mathbf{1}\{ \phi(z^k(x), x) > \epsilon \}, \]

which in this implementation corresponds to checking whether the final residual exceeds \(\epsilon\).

Let \(N\) be the number of samples and let the empirical risk be

\[ \hat{r}_S = \frac{1}{N} \sum_{i=1}^{N} \mathbf{1}\{ r_i^{(k)} > \epsilon \}, \]

where \(r_i^{(k)}\) is the final residual of trajectory \(i\).

Then, with probability at least \(1 - \delta\), the true risk \(r\) satisfies

\[ r \;\le\; \mathrm{kl}^{-1} \!\left( \hat{r}_S \;\middle|\; \frac{\log(2/\delta)}{N} \right), \]

where \(\mathrm{kl}^{-1}(q \mid c)\) denotes the largest \(p \in [q,1]\) such that

\[ \mathrm{KL}(q \,\|\, p) \le c, \]

and

\[ \mathrm{KL}(q \,\|\, p) = q \log\!\frac{q}{p} + (1-q)\log\!\frac{1-q}{1-p} \]

is the binary Kullback–Leibler divergence.

Parameters:

Name	Type	Description	Default
residuals	ndarray	Residuals of the problem instances.	required
delta	float	Confidence parameter \(\delta \in (0,1)\). The bound holds with probability at least \(1 - \delta\).	0.05
epsilon	float	Tolerance level \(\epsilon\) defining the \(0\)–\(1\) error.	0.0

Returns:

Type	Description
float	Upper bound on true risk via KL inverse.

Source code in fomtools/canonicalizers/simulator.py

@staticmethod
def compute_risk_bound(residuals: cp.ndarray, delta: float = 0.05, epsilon: float = 0.0):
    r"""
    Compute a high-probability bound on the risk using the KL inverse bound.

    Given a tolerance $\epsilon$, a number of iterations $k$ (implicitly given by the last
    column of ``residuals``),  and $N$ sampled problem instances,

    we define the $0$–$1$ error for sample $x$ as

    $$
        e(x) = \mathbf{1}\{ \phi(z^k(x), x) > \epsilon \},
    $$

    which in this implementation corresponds to checking whether the
    final residual exceeds $\epsilon$.

    Let $N$ be the number of samples and let the empirical risk be


    $$
        \hat{r}_S
        =
        \frac{1}{N}
        \sum_{i=1}^{N}
        \mathbf{1}\{ r_i^{(k)} > \epsilon \},
    $$

    where $r_i^{(k)}$ is the final residual of trajectory $i$.

    Then, with probability at least $1 - \delta$, the true risk $r$
    satisfies

    $$
        r
        \;\le\;
        \mathrm{kl}^{-1}
        \!\left(
            \hat{r}_S
            \;\middle|\;
            \frac{\log(2/\delta)}{N}
        \right),
    $$

    where $\mathrm{kl}^{-1}(q \mid c)$ denotes the largest $p \in [q,1]$
    such that

    $$
        \mathrm{KL}(q \,\|\, p)
        \le c,
    $$

    and

    $$
        \mathrm{KL}(q \,\|\, p)
        =
        q \log\!\frac{q}{p}
        +
        (1-q)\log\!\frac{1-q}{1-p}
    $$

    is the binary Kullback–Leibler divergence.

    Parameters
    ----------
    residuals: cp.ndarray
        Residuals of the problem instances.

    delta: float, optional
        Confidence parameter $\delta \in (0,1)$. The bound holds with
        probability at least $1 - \delta$.

    epsilon: float, optional
        Tolerance level $\epsilon$ defining the $0$–$1$ error.

    Returns
    -------
    float
        Upper bound on true risk via KL inverse.

    """

    def _kl_inverse(q, c):
        """
        Compute kl⁻¹(q | c): largest p such that KL(q || p) ≤ c.
        """
        if q >= 1.0:
            return 1.0
        if c == 0:
            return q

        low, high = q, 1.0
        for _ in range(100):
            mid = (low + high) / 2
            if mid == 1.0:
                kl = float("inf")
            else:
                kl = q * np.log(q / mid + 1e-10) + (1 - q) * np.log((1 - q) / (1 - mid) + 1e-10)

            if kl < c:
                low = mid
            else:
                high = mid
        return high

    N = residuals.shape[0]
    final_residuals = residuals[:, -1]

    errors = (final_residuals > epsilon).astype(cp.float32)
    empirical_risk = cp.mean(errors)

    c = cp.log(2 / delta) / N
    return _kl_inverse(float(empirical_risk), float(c))

record_iterate

record_iterate(expression)

Record a public operation's output after all its internal steps.

Source code in fomtools/canonicalizers/simulator.py

def record_iterate(self, expression):
    """Record a public operation's output after all its internal steps."""
    self.user_iterates[len(self.step_descriptors) - 1] = expression

register_subproblem_step

register_subproblem_step(arg: LinearExpression, output_leaf: LinearExpression, activation) -> None

Record a subproblem step for deferred evaluation at solve time.

Used by the step decomposer for proxes/steps that have no Step class on the simulator (projections onto the simplex / ℓ1-ball, the ℓ∞-norm and total-variation proxes, polyhedral projections, and inequality QP/LP). The activation is a callable mapping the assembled input batch (N, arg.ndim) to the output batch (N, output_leaf.ndim); the result is written into output_leaf's slot. See the subproblem activation factories defined in this module (e.g. simplex_activation, qp_activation).

Parameters:

Name	Type	Description	Default
arg	LinearExpression	The assembled per-sample input to the subproblem.	required
output_leaf	LinearExpression	The fresh leaf the activation result is written into.	required
activation	Callable	Maps the input batch to the output batch.	required

Source code in fomtools/canonicalizers/simulator.py

def register_subproblem_step(
    self, arg: LinearExpression, output_leaf: LinearExpression, activation
) -> None:
    """
    Record a subproblem step for deferred evaluation at solve time.

    Used by the step decomposer for proxes/steps that have no ``Step`` class
    on the simulator (projections onto the simplex / ℓ1-ball, the ℓ∞-norm and
    total-variation proxes, polyhedral projections, and inequality QP/LP). The
    ``activation`` is a callable mapping the assembled input batch
    ``(N, arg.ndim)`` to the output batch ``(N, output_leaf.ndim)``; the result
    is written into ``output_leaf``'s slot. See the subproblem activation
    factories defined in this module (e.g. `simplex_activation`,
    `qp_activation`).

    Parameters
    ----------
    arg : LinearExpression
        The assembled per-sample input to the subproblem.
    output_leaf : LinearExpression
        The fresh leaf the activation result is written into.
    activation : Callable
        Maps the input batch to the output batch.
    """
    self.step_descriptors.append((_ActivationStep(arg, output_leaf), activation))

set_infinity_norm_objective

set_infinity_norm_objective(expr_list, lb_list=None, ub_list=None)

Record an infinity-norm residual expression at the most recently registered step.

Source code in fomtools/canonicalizers/simulator.py

def set_infinity_norm_objective(self, expr_list, lb_list=None, ub_list=None):
    """Record an infinity-norm residual expression at the most recently registered step."""
    self.residual_specs[len(self.step_descriptors) - 1] = (expr_list, np.inf)

set_one_norm_objective

set_one_norm_objective(expr_list)

Record a one-norm residual expression at the most recently registered step.

Source code in fomtools/canonicalizers/simulator.py

def set_one_norm_objective(self, expr_list):
    """Record a one-norm residual expression at the most recently registered step."""
    self.residual_specs[len(self.step_descriptors) - 1] = (expr_list, 1)

set_two_norm_objective

set_two_norm_objective(expr_list, lb_list=None, ub_list=None)

Record a squared two-norm residual expression at the most recently registered step.

Source code in fomtools/canonicalizers/simulator.py

def set_two_norm_objective(self, expr_list, lb_list=None, ub_list=None):
    """Record a squared two-norm residual expression at the most recently registered step."""
    self.residual_specs[len(self.step_descriptors) - 1] = (expr_list, 2)

set_zero_objective

set_zero_objective()

Record a zero residual at the most recently registered step.

Source code in fomtools/canonicalizers/simulator.py

def set_zero_objective(self):
    """Record a zero residual at the most recently registered step."""
    self.residual_specs[len(self.step_descriptors) - 1] = ([], "zero")

solve_model

solve_model(delta: float = 0.05, epsilon: float = 0.0, device: str = None) -> ndarray

Run a batched simulation of all registered steps.

Each call re-simulates from scratch: the value table is rebuilt, all steps are compiled and executed in sequence, and residuals are computed. This method can be called repeatedly (e.g. inside the user's iteration loop) as new steps are registered.

Parameters:

Name	Type	Description	Default
delta	float	Confidence parameter for the risk bound.	0.05
epsilon	float	Tolerance level defining the 0–1 error.	0.0
device	str	Device to run the simulation on (None, "cpu", or "gpu").	None

Returns:

Name	Type	Description
residuals	ndarray	A 2D array of shape (num_samples, n_residuals) containing the residual at each step for each parameter sample.

Raises:

Type	Description
ValueError	If no steps have been registered.

Source code in fomtools/canonicalizers/simulator.py

def solve_model(
    self, delta: float = 0.05, epsilon: float = 0.0, device: str = None
) -> cp.ndarray:
    """
    Run a batched simulation of all registered steps.

    Each call re-simulates from scratch: the value table is rebuilt,
    all steps are compiled and executed in sequence, and residuals are
    computed. This method can be called repeatedly (e.g. inside the
    user's iteration loop) as new steps are registered.

    Parameters
    ----------
    delta: float, optional
        Confidence parameter for the risk bound.

    epsilon: float, optional
        Tolerance level defining the 0–1 error.

    device: str, optional
        Device to run the simulation on (``None``, ``"cpu"``, or ``"gpu"``).

    Returns
    -------
    residuals: cp.ndarray
        A 2D array of shape ``(num_samples, n_residuals)`` containing
        the residual at each step for each parameter sample.

    Raises
    ------
    ValueError
        If no steps have been registered.
    """
    if not self.step_descriptors:
        raise ValueError(
            "No steps have been registered with this simulator instance. "
            "Call vp.step.*() before solve()."
        )

    self._configure_device(device)
    simulation_data = self._prepare_simulation_data()
    compiled_steps = self._compile_steps(simulation_data)

    specs = self.residual_specs
    if not specs:
        specs = {}
        previous = None
        for idx, expr in self.user_iterates.items():
            if previous is not None and idx >= 0:
                if expr.ndim != previous.ndim:
                    raise ValueError(
                        "Default residuals require equal-sized user iterates; set an explicit objective."
                    )
                specs[idx] = ([expr - previous], np.inf)
            previous = expr
        if not specs:
            raise ValueError(
                "Default residuals require an initial iterate; call Verifier.iterate or set an explicit objective."
            )
    compiled_residuals = self._compile_residuals(simulation_data, specs)
    n_residuals = len(compiled_residuals)

    with self._get_device_context(self.device):
        z0 = self._to_device(simulation_data["z0"])
        p_samples = self._to_device(simulation_data["p_samples"])
        N = p_samples.shape[0]

        def _simulate_batch(z0: cp.ndarray, p_samples: cp.ndarray) -> cp.ndarray:
            """Simulate all registered steps for all N samples in parallel."""
            z_batch = cp.tile(z0, (N, 1))
            residuals = cp.empty((N, n_residuals), dtype=cp.float32)
            residual_col = 0

            for step_idx, (
                evaluate_fn,
                output_slice,
                prev_slice,
                compose_output,
            ) in enumerate(compiled_steps):
                z_new = evaluate_fn(z_batch, p_samples)
                z_batch[:, output_slice] = z_new

                if step_idx in compiled_residuals:
                    residuals[:, residual_col] = compiled_residuals[step_idx](
                        z_batch, p_samples
                    )
                    residual_col += 1

            return residuals

        # Warm-up: trigger CuPy's lazy GPU kernel compilation so it is excluded
        # from the timed run. Only relevant on GPU — CLARABEL and the CPU-side
        # activations (TV/polyhedral/QP/LP loops) have no lazy-compile phase, so
        # warming up there would just double the solve time for no benefit.
        if CUDA_AVAILABLE:
            _simulate_batch(z0, p_samples)
            self._sync()

        # timed execution
        start = time.perf_counter()
        residuals = _simulate_batch(z0, p_samples)
        self._sync()
        elapsed = time.perf_counter() - start

    self._register_statistics(residuals, delta, epsilon, elapsed)
    return residuals

condat_tv_1d

condat_tv_1d(y: ndarray, lam: float) -> ndarray

1-D total-variation prox via Condat's O(n) direct algorithm.

Computes argmin_x 1/2 ||x - y||^2 + lam * sum_i |x_{i+1} - x_i| exactly in a single forward pass (Condat, "A Direct Algorithm for 1-D Total Variation Denoising", 2013). lam <= 0 or length < 2 returns a copy of y.

Parameters:

Name	Type	Description	Default
y	(n,) np.ndarray	Input signal.	required
lam	float	Total-variation weight.	required

Returns:

Type	Description
(n,) np.ndarray	The proximal point.

Source code in fomtools/canonicalizers/simulator.py

def condat_tv_1d(y: np.ndarray, lam: float) -> np.ndarray:
    r"""
    1-D total-variation prox via Condat's O(n) direct algorithm.

    Computes ``argmin_x 1/2 ||x - y||^2 + lam * sum_i |x_{i+1} - x_i|`` exactly in a
    single forward pass (Condat, "A Direct Algorithm for 1-D Total Variation
    Denoising", 2013). ``lam <= 0`` or length ``< 2`` returns a copy of ``y``.

    Parameters
    ----------
    y : (n,) np.ndarray
        Input signal.
    lam : float
        Total-variation weight.

    Returns
    -------
    (n,) np.ndarray
        The proximal point.
    """
    y = np.asarray(y, dtype=float)
    n = y.shape[0]
    if n < 2 or lam <= 0.0:
        return y.copy()

    x = np.empty(n, dtype=float)
    k = 0  # start index of the current segment under construction
    k0 = 0  # current sample being processed
    umin = lam
    umax = -lam
    vmin = y[0] - lam
    vmax = y[0] + lam
    kplus = 0  # last index where the upper slack saturated
    kminus = 0  # last index where the lower slack saturated

    while True:
        if k0 == n - 1:
            # Reached the end of the signal: flush the remaining segment.
            #
            # Each flush branch below resets the tracker/envelope of its OWN side but
            # leaves the opposite one (kplus/vmax after a umin flush, kminus/vmin after
            # a umax flush) stale. This is safe: the flush sets its own slack to +/-lam,
            # so on re-entry the same-side test fails and — by the algorithm's envelope
            # invariant — the opposite flush is never taken with a stale tracker (an
            # empty x[k:kplus+1] write with kplus < k cannot occur). Verified by
            # exhaustive search over short integer signals; matches Condat's reference,
            # which likewise does not reset the opposite side here.
            if umin < 0.0:
                x[k : kminus + 1] = vmin
                k = kminus = kminus + 1
                if k >= n:
                    # The flush filled x[: n]; the new segment would start past the
                    # end, so guard the y[k] read and finish.
                    return x
                k0 = k
                vmin = y[k]
                umin = lam
                umax = y[k] + lam - vmax
            elif umax > 0.0:
                x[k : kplus + 1] = vmax
                k = kplus = kplus + 1
                if k >= n:
                    return x
                k0 = k
                vmax = y[k]
                umax = -lam
                umin = y[k] - lam - vmin
            else:
                x[k:n] = vmin + umin / (k0 - k + 1)
                return x
        else:
            if y[k0 + 1] + umin < vmin - lam:
                # negative jump
                x[k : kminus + 1] = vmin
                k = k0 = kplus = kminus = kminus + 1
                vmin = y[k]
                vmax = y[k] + 2 * lam
                umin = lam
                umax = -lam
            elif y[k0 + 1] + umax > vmax + lam:
                # positive jump
                x[k : kplus + 1] = vmax
                k = k0 = kplus = kminus = kplus + 1
                vmax = y[k]
                vmin = y[k] - 2 * lam
                umin = lam
                umax = -lam
            else:
                # no jump yet: advance and update the slacks
                k0 += 1
                umin += y[k0] - vmin
                umax += y[k0] - vmax
                if umin >= lam:
                    vmin += (umin - lam) / (k0 - k + 1)
                    umin = lam
                    kminus = k0
                if umax <= -lam:
                    vmax += (umax + lam) / (k0 - k + 1)
                    umax = -lam
                    kplus = k0

l1_ball_activation

l1_ball_activation(r: float)

Activation projecting each sample onto the ℓ1-ball of radius r.

Source code in fomtools/canonicalizers/simulator.py

def l1_ball_activation(r: float):
    """Activation projecting each sample onto the ℓ1-ball of radius ``r``."""
    return lambda V: project_l1_ball_batch(V, r)

norm_inf_activation

norm_inf_activation(radius: float)

Activation for the ℓ∞-norm prox via Moreau: v - proj_{||.||_1 <= radius}(v).

Source code in fomtools/canonicalizers/simulator.py

def norm_inf_activation(radius: float):
    """Activation for the ℓ∞-norm prox via Moreau: ``v - proj_{||.||_1 <= radius}(v)``."""
    return lambda V: V - project_l1_ball_batch(V, radius)

polyhedral_activation

polyhedral_activation(A: ndarray, b, n: int, m: int, b_is_param: bool)

Activation projecting each sample onto {u : A u <= b} with cvxpy.

Builds a single parameterized (DPP) projection problem and solves it per sample. The input batch has the point in its first n columns; when b is a parameter (b_is_param), the next m columns carry the per-sample b.

Parameters:

Name	Type	Description	Default
A	(m, n) np.ndarray	Inequality matrix.	required
b	(m,) np.ndarray or None	Fixed right-hand side when b is not a parameter.	required
n	int	Point and constraint dimensions.	required
m	int	Point and constraint dimensions.	required
b_is_param	bool	Whether b varies per sample (stacked into the input batch).	required

Notes

The per-sample loop runs on host (cvxpy/CLARABEL has no GPU backend); on a GPU run each call moves the batch device→host and the result back (see _to_numpy).

Source code in fomtools/canonicalizers/simulator.py

def polyhedral_activation(A: np.ndarray, b, n: int, m: int, b_is_param: bool):
    r"""
    Activation projecting each sample onto ``{u : A u <= b}`` with cvxpy.

    Builds a single parameterized (DPP) projection problem and solves it per sample.
    The input batch has the point in its first ``n`` columns; when ``b`` is a
    parameter (``b_is_param``), the next ``m`` columns carry the per-sample ``b``.

    Parameters
    ----------
    A : (m, n) np.ndarray
        Inequality matrix.
    b : (m,) np.ndarray or None
        Fixed right-hand side when ``b`` is not a parameter.
    n, m : int
        Point and constraint dimensions.
    b_is_param : bool
        Whether ``b`` varies per sample (stacked into the input batch).

    Notes
    -----
    The per-sample loop runs on host (cvxpy/CLARABEL has no GPU backend); on a GPU run
    each call moves the batch device→host and the result back (see `_to_numpy`).
    """
    import cvxpy as cp_solver

    u = cp_solver.Variable(n)
    x_par = cp_solver.Parameter(n)
    if b_is_param:
        b_par = cp_solver.Parameter(m)
        constraints = [A @ u <= b_par]
    else:
        b_par = None
        constraints = [A @ u <= np.asarray(b, dtype=float)]
    prob = cp_solver.Problem(cp_solver.Minimize(cp_solver.sum_squares(u - x_par)), constraints)

    def activation(V):
        Vnp = _to_numpy(V)
        out = np.empty((Vnp.shape[0], n), dtype=float)
        for i in range(Vnp.shape[0]):
            x_par.value = Vnp[i, :n]
            if b_is_param:
                b_par.value = Vnp[i, n : n + m]
            prob.solve(solver=cp_solver.CLARABEL)
            out[i] = u.value if u.value is not None else np.nan
        _warn_if_infeasible(out, "IndPolyhedral projection (simulator)")
        return _to_like(out, V)

    return activation

project_l1_ball_batch

project_l1_ball_batch(V, r: float)

Euclidean projection of each row of V onto {u : ||u||_1 <= r}.

Rows already inside the ball are returned unchanged; rows outside are projected via sign(V) * project_simplex_batch(|V|, r). No Python loop.

Parameters:

Name	Type	Description	Default
V	(N, n) array (numpy or cupy)	Points to project.	required
r	float	Ball radius (>= 0).	required

Returns:

Type	Description
(N, n) array	The row-wise projections.

Source code in fomtools/canonicalizers/simulator.py

def project_l1_ball_batch(V, r: float):
    r"""
    Euclidean projection of each row of ``V`` onto ``{u : ||u||_1 <= r}``.

    Rows already inside the ball are returned unchanged; rows outside are
    projected via ``sign(V) * project_simplex_batch(|V|, r)``. No Python loop.

    Parameters
    ----------
    V : (N, n) array (numpy or cupy)
        Points to project.
    r : float
        Ball radius (``>= 0``).

    Returns
    -------
    (N, n) array
        The row-wise projections.
    """
    xp = _xp(V)
    if r <= 0.0:
        return xp.zeros_like(V)
    absV = xp.abs(V)
    over = absV.sum(axis=1) > r
    w = xp.sign(V) * project_simplex_batch(absV, r)
    return xp.where(over[:, None], w, V)

project_simplex_batch

project_simplex_batch(V, a: float)

Euclidean projection of each row of V onto {u >= 0, 1^T u = a}.

Vectorized Held–Wolfe / Duchi (2008) sort algorithm: for a > 0 the threshold theta is found per row from the descending sort and cumulative sums, and the projection is max(V - theta, 0). No Python loop over the batch.

Parameters:

Name	Type	Description	Default
V	(N, n) array (numpy or cupy)	Points to project (one per row).	required
a	float	Positive simplex sum.	required

Returns:

Type	Description
(N, n) array	The row-wise projections.

Source code in fomtools/canonicalizers/simulator.py

def project_simplex_batch(V, a: float):
    r"""
    Euclidean projection of each row of ``V`` onto ``{u >= 0, 1^T u = a}``.

    Vectorized Held–Wolfe / Duchi (2008) sort algorithm: for ``a > 0`` the
    threshold ``theta`` is found per row from the descending sort and cumulative
    sums, and the projection is ``max(V - theta, 0)``. No Python loop over the
    batch.

    Parameters
    ----------
    V : (N, n) array (numpy or cupy)
        Points to project (one per row).
    a : float
        Positive simplex sum.

    Returns
    -------
    (N, n) array
        The row-wise projections.
    """
    xp = _xp(V)
    N, n = V.shape
    U = xp.flip(xp.sort(V, axis=1), axis=1)  # each row sorted descending
    cssv = xp.cumsum(U, axis=1) - a
    ind = xp.arange(1, n + 1)
    rho = xp.sum((U - cssv / ind) > 0, axis=1)  # >= 1 since a > 0
    theta = cssv[xp.arange(N), rho - 1] / rho
    return xp.maximum(V - theta[:, None], 0.0)

qp_activation

qp_activation(P, A, C, n: int, p: int, m: int, is_lp: bool)

Activation solving a QP/LP per sample with cvxpy.

Builds a single parameterized (DPP) problem minimize 0.5 x^T P x + q^T x s.t. A x = b, C x <= d (the quadratic term is dropped when is_lp) and solves it per sample. The input batch stacks the present parametric blocks in the order [q (n) | b (p) | d (m)].

Parameters:

Name	Type	Description	Default
P	(n, n) array or None	Quadratic term (ignored when is_lp).	required
A	(p, n) array or None	Equality matrix (None if absent).	required
C	(m, n) array or None	Inequality matrix (None if absent).	required
n	int	Primal / equality / inequality dimensions.	required
p	int	Primal / equality / inequality dimensions.	required
m	int	Primal / equality / inequality dimensions.	required
is_lp	bool	Drop the quadratic term (linear program).	required

Notes

The per-sample loop runs on host (cvxpy/CLARABEL has no GPU backend); on a GPU run each call moves the batch device→host and the result back (see _to_numpy).

Source code in fomtools/canonicalizers/simulator.py

def qp_activation(P, A, C, n: int, p: int, m: int, is_lp: bool):
    r"""
    Activation solving a QP/LP per sample with cvxpy.

    Builds a single parameterized (DPP) problem ``minimize 0.5 x^T P x + q^T x s.t.
    A x = b, C x <= d`` (the quadratic term is dropped when ``is_lp``) and solves it
    per sample. The input batch stacks the present parametric blocks in the order
    ``[q (n) | b (p) | d (m)]``.

    Parameters
    ----------
    P : (n, n) array or None
        Quadratic term (ignored when ``is_lp``).
    A : (p, n) array or None
        Equality matrix (``None`` if absent).
    C : (m, n) array or None
        Inequality matrix (``None`` if absent).
    n, p, m : int
        Primal / equality / inequality dimensions.
    is_lp : bool
        Drop the quadratic term (linear program).

    Notes
    -----
    The per-sample loop runs on host (cvxpy/CLARABEL has no GPU backend); on a GPU run
    each call moves the batch device→host and the result back (see `_to_numpy`).
    """
    import cvxpy as cp_solver

    x = cp_solver.Variable(n)
    q_par = cp_solver.Parameter(n)
    objective = q_par @ x
    if not is_lp:
        P_dense = P.toarray() if spa.issparse(P) else np.asarray(P, dtype=float)
        objective = 0.5 * cp_solver.quad_form(x, cp_solver.psd_wrap(P_dense)) + objective

    constraints = []
    b_par = d_par = None
    if A is not None:
        b_par = cp_solver.Parameter(p)
        A_mat = A.toarray() if spa.issparse(A) else np.asarray(A, dtype=float)
        constraints.append(A_mat @ x == b_par)
    if C is not None:
        d_par = cp_solver.Parameter(m)
        C_mat = C.toarray() if spa.issparse(C) else np.asarray(C, dtype=float)
        constraints.append(C_mat @ x <= d_par)
    prob = cp_solver.Problem(cp_solver.Minimize(objective), constraints)

    def activation(V):
        Vnp = _to_numpy(V)
        out = np.empty((Vnp.shape[0], n), dtype=float)
        for i in range(Vnp.shape[0]):
            row = Vnp[i]
            q_par.value = row[:n]
            off = n
            if A is not None:
                b_par.value = row[off : off + p]
                off += p
            if C is not None:
                d_par.value = row[off : off + m]
            prob.solve(solver=cp_solver.CLARABEL)
            out[i] = x.value if x.value is not None else np.nan
        _warn_if_infeasible(out, "QP/LP solve (simulator)")
        return _to_like(out, V)

    return activation

simplex_activation

simplex_activation(a: float)

Activation projecting each sample onto the scaled simplex.

Source code in fomtools/canonicalizers/simulator.py

def simplex_activation(a: float):
    """Activation projecting each sample onto the scaled simplex."""
    return lambda V: project_simplex_batch(V, a)

tv_activation

tv_activation(lam: float)

Activation applying Condat's TV prox to each sample (looped on host).

Host-side loop: on a GPU run the batch is moved device→host and the result back (see _to_numpy).

Source code in fomtools/canonicalizers/simulator.py

def tv_activation(lam: float):
    """Activation applying Condat's TV prox to each sample (looped on host).

    Host-side loop: on a GPU run the batch is moved device→host and the result back
    (see `_to_numpy`).
    """

    def activation(V):
        Vnp = _to_numpy(V)
        out = np.stack([condat_tv_1d(row, lam) for row in Vnp])
        return _to_like(out, V)

    return activation

SDP canonicalizer

SDP

Bases: Canonicalizer

Semidefinite programming canonicalizer for verification problems.

This class converts verification problems involving ReLU and other piecewise linear operations into semidefinite programming (SDP) formulations. It builds the SDP problem incrementally by adding variables and constraints, then solves using Clarabel or CVXPY.

The SDP uses a lifted matrix variable X where X[0,0] = 1 (homogenization). Variables are embedded in the diagonal blocks, and constraints are encoded as linear matrix inequalities.

Attributes:

Name	Type	Description
solver_name	str	Name of the SDP solver backend ('clarabel', 'cvxpy', 'cuclarabel').
range_marker	int	Current dimension of the SDP matrix variable.
var_idx_map	dict	Mapping from LinearExpression to (start_idx, end_idx) in SDP matrix.
Aeq_matrices	list[csc_matrix]	Equality constraint matrices.
beq_consts	list[float]	Equality constraint right-hand sides.
Aineq_matrices	list[csc_matrix]	Inequality constraint matrices.
bineq_upperbounds	list[float]	Inequality constraint upper bounds.
psd_cone_indices	list[set]	Indices defining PSD cone constraints.
bound_constraint_indices	dict	Mapping from variable to its bound constraint indices.
rlt_cuts	bool	If True, add RLT (Reformulation-Linearization Technique) cuts between off-diagonal pairs of SDP indices that co-exist in some psd_cone_indices entry. Each pair yields 4 McCormick inequalities. Cuts are lazily rebuilt when bounds, cones, or variables change. See paper §4.5 eq 18.
triangle_cuts	bool	If True, add lifted triangle cuts for every ReLU step's mixed-bound output rows. Each cut multiplies the per-output triangle inequality by a nonneg bound multiplier, yielding a quadratic-in-X inequality. See paper §4.5 eq 25/26.
C_obj	csc_matrix	Objective matrix for maximization.

Source code in fomtools/canonicalizers/sdp.py

class SDP(Canonicalizer):
    """
    Semidefinite programming canonicalizer for verification problems.

    This class converts verification problems involving ReLU and other piecewise
    linear operations into semidefinite programming (SDP) formulations. It builds
    the SDP problem incrementally by adding variables and constraints, then solves
    using Clarabel or CVXPY.

    The SDP uses a lifted matrix variable X where X[0,0] = 1 (homogenization).
    Variables are embedded in the diagonal blocks, and constraints are encoded
    as linear matrix inequalities.

    Attributes
    ----------
    solver_name : str
        Name of the SDP solver backend ('clarabel', 'cvxpy', 'cuclarabel').
    range_marker : int
        Current dimension of the SDP matrix variable.
    var_idx_map : dict
        Mapping from LinearExpression to (start_idx, end_idx) in SDP matrix.
    Aeq_matrices : list[scipy.sparse.csc_matrix]
        Equality constraint matrices.
    beq_consts : list[float]
        Equality constraint right-hand sides.
    Aineq_matrices : list[scipy.sparse.csc_matrix]
        Inequality constraint matrices.
    bineq_upperbounds : list[float]
        Inequality constraint upper bounds.
    psd_cone_indices : list[set]
        Indices defining PSD cone constraints.
    bound_constraint_indices : dict
        Mapping from variable to its bound constraint indices.
    rlt_cuts : bool
        If True, add RLT (Reformulation-Linearization Technique) cuts
        between off-diagonal pairs of SDP indices that co-exist in some
        ``psd_cone_indices`` entry. Each pair yields 4 McCormick
        inequalities. Cuts are lazily rebuilt when bounds, cones, or
        variables change. See paper §4.5 eq 18.
    triangle_cuts : bool
        If True, add lifted triangle cuts for every ReLU step's
        mixed-bound output rows. Each cut multiplies the per-output
        triangle inequality by a nonneg bound multiplier, yielding a
        quadratic-in-X inequality. See paper §4.5 eq 25/26.
    C_obj : scipy.sparse.csc_matrix
        Objective matrix for maximization.
    """

    def __init__(
        self,
        solver_params=None,
        solver="clarabel",
        rlt_cuts=False,
        triangle_cuts=False,
        ineq_product_cuts=True,
        ineq_box_cuts=True,
        ineq_square_cuts=True,
        relu_envelope_cuts=False,
        rlt_all_pairs=False,
        full_psd_cone=False,
        cross_cut=False,
        subgrad_mono_cut=False,
    ):
        """
        Initialize the SDP canonicalizer.

        Parameters
        ----------
        solver_params : dict, optional
            Dictionary of solver-specific parameters to pass to the backend.
        solver : str, default='clarabel'
            Name of the SDP solver to use. Options: 'clarabel', 'cvxpy',
            'cuclarabel'.
        rlt_cuts : bool, default=False
            If True, add RLT (Reformulation-Linearization Technique) cuts
            between cone-mate SDP indices to tighten the relaxation.
        triangle_cuts : bool, default=False
            If True, add lifted triangle cuts for ReLU steps with mixed-bound
            output dimensions to tighten the relaxation.
        ineq_product_cuts : bool, default=True
            For each inequality constraint ``Ax + b <= 0``, add the off-diagonal
            products ``g_i g_j >= 0`` (valid since each ``g_i <= 0``).
        ineq_box_cuts : bool, default=True
            For each inequality constraint, add the RLT products of the constraint
            with the variable box bounds (``g_i (x_p - l_p) <= 0`` and
            ``g_i (u_p - x_p) <= 0``).
        ineq_square_cuts : bool, default=True
            For each inequality constraint, add the self-square upper bound
            ``g_i^2 <= g_i_lower * g_i`` using a propagated lower bound on ``g_i``.
        relu_envelope_cuts : bool, default=False
            If True, add the linear ReLU convex-hull inequalities on the
            first-moment (vector) variables: the lower envelope ``y_i >= a_i^T x
            + b_i`` for every projected row and, on mixed-bound rows
            (``l_i < 0 < u_i``), the upper chord ``y_i <= E_ii (a_i^T x + b_i -
            l_i)`` with ``E_ii = u_i / (u_i - l_i)``. The ``y_i >= 0`` line is
            omitted because the y-leaf box bound plus the PSD constraint already
            imply it.
        rlt_all_pairs : bool, default=False
            If True, RLT *every* pair of variables (not just cone-mates) by
            registering a cheap ``{0, p, q}`` 3x3 PSD minor for each pair so the
            all-pairs McCormick cuts actually bind, then firing ``_build_rlt_cuts``
            on the resulting (now all-pairs) cone structure. This is the middle
            tier between cone-mate-only ``rlt_cuts`` and the dense
            ``full_psd_cone``: ``baseline >= rlt_cuts >= rlt_all_pairs >=
            full_psd_cone + rlt_cuts``. Pairs already inside a larger cone add no
            new minor (``add_psd_cone_indices`` drops subsumed cones), so it only
            couples genuinely-uncoupled pairs. Implies RLT cut emission on its own
            (no need to also set ``rlt_cuts``).
        full_psd_cone : bool, default=False
            If True, union all PSD cones into a single full moment-matrix block
            at solve time. Combined with ``rlt_cuts=True`` this yields the full
            Shor + RLT relaxation (RLT then fires on every variable pair, not
            just cone-mates). Tightest but most expensive configuration.
        cross_cut : bool, default=False
            If True, add the PEP "cross" interpolation cuts ``z_a r_b <= 0`` over
            ordered pairs of ReLU output components, where ``z_a >= 0`` is a ReLU
            output and ``r_b = v_b - z_b <= 0`` is another ReLU's residual (valid
            for every orthant projection). Couples *different* steps, which the
            per-step encoding does not. Each emitted cut self-registers the minimal
            PSD cone over the indices it touches so it binds without
            ``full_psd_cone`` (Clarabel handles many small cones efficiently; the
            ``add_psd_cone_indices`` antichain subsumes a cone already covered by a
            larger one). For saturated-linear (box projection) steps this flag also
            emits the box normal-cone cuts (per coordinate ``j``: ``r_j(z_j-l_j)>=0``,
            ``r_j(z_j-u_j)>=0`` within a step, and ``r_j^b(z_j^a-z_j^b)<=0`` across
            same-box steps), via `_build_box_normalcone_cuts`.
        subgrad_mono_cut : bool, default=False
            If True, add the PEP subgradient-monotonicity cuts
            ``<s^i/tau_i - s^j/tau_j, z^i> >= 0`` over ordered pairs of registered
            prox steps, with ``s = v - z`` and ``tau`` the prox threshold
            (``tau = 1`` for ReLU/orthant, ``tau = lambda`` for soft-threshold).
            Normalizing by ``tau`` places the subgradient in ``d||.||_1`` so steps
            of the same (homogeneous) function are comparable. On the orthant this
            reduces to the cross cut; on the soft-threshold (ell-1 prox) it is a
            genuinely distinct coupling. Self-registers the PSD cone over the leaves
            each cut couples (without needing ``full_psd_cone``); note that for a
            composite prox (e.g. soft-threshold ``z = relu_pos - relu_neg``) this
            cone spans several index ranges, so it is not as tight as the cross
            cut's truly-minimal ``{0, ya, yb} U x_b``. For saturated-linear (box
            projection) steps this flag also
            emits the box firm-nonexpansiveness/monotonicity cuts (per coordinate:
            ``(z_j^a-z_j^b)(r_j^a-r_j^b)>=0`` across same-box steps), via
            `_build_box_mono_cuts`.
        """
        super(SDP, self).__init__(solver_params, solver)
        self.solver_name = solver.lower()  # Store for dispatch

        # cuClarabel cannot yet apply custom solver options (its Julia settings
        # are hardcoded). Reject construction-time params eagerly here rather
        # than surprising the user with a failure only at solve time.
        if self.solver_name == "cuclarabel" and self.solver_params:
            raise ValueError(
                "cuClarabel does not support custom solver options yet; "
                f"remove params={self.solver_params} from the Verifier."
            )
        self.range_marker = 1
        self.var_idx_map = {}

        self.Aeq_matrices = []
        self.beq_consts = []
        self.Aineq_matrices = []
        self.bineq_upperbounds = []
        self.psd_cone_indices = []
        self.bound_constraint_indices = {}  # var -> (start_idx, end_idx)

        self.rlt_cuts = rlt_cuts
        self.triangle_cuts = triangle_cuts
        self.ineq_product_cuts = ineq_product_cuts
        self.ineq_box_cuts = ineq_box_cuts
        self.ineq_square_cuts = ineq_square_cuts
        self.relu_envelope_cuts = relu_envelope_cuts
        self.rlt_all_pairs = rlt_all_pairs
        self.full_psd_cone = full_psd_cone
        self.cross_cut = cross_cut
        self.subgrad_mono_cut = subgrad_mono_cut
        self._cuts_dirty = True
        self._cut_ineq_matrices = []
        self._cut_bineq_upperbounds = []
        self._relu_steps = []  # registry for triangle / envelope / cross cuts
        self._prox_steps = []  # registry for subgradient-monotonicity cuts
        self._box_prox_steps = []  # registry for satlin (box projection) PEP cuts
        self._ineq_constraints = []  # (constraint, add_product_cuts) for inequality cuts

        self.C_obj = None
        self._objective_spec = None
        self._solved_copy = None
        self._initialize_corner_constraint()

    def _initialize_corner_constraint(self) -> None:
        """
        Add the homogenization constraint X[0,0] = 1.

        This constraint ensures the lifted SDP formulation is properly
        normalized. It is added automatically during initialization.
        """
        curr_prob_dim = self.get_sdp_var_dim()
        new_A = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
        new_A[0, 0] = 1
        self.Aeq_matrices.append(symmetrize(new_A.tocsc()))
        self.beq_consts.append(1)

    def _pad_existing_A_matrices(self) -> None:
        """
        Expand existing constraint matrices to accommodate new variables.

        When new variables are added, the SDP matrix dimension increases.
        This method pads all existing equality and inequality constraint
        matrices with zeros to match the new dimension.
        """
        new_dim = self.get_sdp_var_dim()
        new_Aeq_matrices = [efficient_csc_pad(A, (new_dim, new_dim)) for A in self.Aeq_matrices]
        new_Aineq_matrices = [efficient_csc_pad(A, (new_dim, new_dim)) for A in self.Aineq_matrices]
        self.Aeq_matrices = new_Aeq_matrices
        self.Aineq_matrices = new_Aineq_matrices

    def add_var(self, var: LinearExpression, **kwargs) -> None:
        """
        Add a variable to the SDP with bound constraints.

        Embeds the variable in the SDP matrix diagonal and adds quadratic
        bound constraints of the form (x - l)(x - u) <= 0 for each
        component, which encode the box constraints l <= x <= u.

        Parameters
        ----------
        var : LinearExpression
            The variable to add. Must have finite bounds in ``var.lb``
            and ``var.ub``.

        Raises
        ------
        NotImplementedError
            If any bound is infinite.
        """
        lb, ub = var.lb, var.ub
        dim = var.ndim
        bounds = (self.range_marker, self.range_marker + dim)
        self.range_marker += dim
        self.var_idx_map[var] = bounds

        # whenever a new var is added, need to pad existing A matrices
        self._pad_existing_A_matrices()

        curr_prob_dim = self.get_sdp_var_dim()
        constraint_start = len(self.Aineq_matrices)

        for i, l, u in zip(range(bounds[0], bounds[1]), lb, ub):
            if np.isinf(l) or np.isinf(u):
                raise NotImplementedError("Infinite bounds not yet supported in SDP canonicalizer")
            new_A = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
            new_A[i, i] = 1
            new_A[i, 0] = -(l + u)

            new_A = new_A.tocsc()
            new_A = symmetrize(new_A)
            self.Aineq_matrices.append(new_A)
            self.bineq_upperbounds.append(-u * l)

        constraint_end = len(self.Aineq_matrices)
        self.bound_constraint_indices[var] = (constraint_start, constraint_end)
        self._cuts_dirty = True

    def get_sdp_var_dim(self) -> int:
        """
        Get the current dimension of the SDP matrix variable.

        Returns
        -------
        int
            The current size of the symmetric matrix variable X.
        """
        return self.range_marker

    def update_bounds(
        self, expression: LinearExpression, lb: float | np.ndarray, ub: float | np.ndarray
    ) -> None:
        """
        Update bounds for a variable by regenerating its constraint matrices.

        Modifies the existing inequality constraint matrices for the given
        variable to reflect new lower and upper bounds. This is used during
        bound tightening procedures.

        Parameters
        ----------
        expression : LinearExpression
            The variable whose bounds are being updated. Must already be
            registered via ``add_var``.
        lb : np.ndarray
            New lower bounds. Must have length equal to ``expression.ndim``.
        ub : np.ndarray
            New upper bounds. Must have length equal to ``expression.ndim``.

        Raises
        ------
        ValueError
            If the expression is not found in the canonicalizer, or if
            the bounds do not have the expected length.
        NotImplementedError
            If any new bound is infinite.
        """
        if expression not in self.bound_constraint_indices:
            raise ValueError("Expression not found in SDP canonicalizer")

        constraint_start, constraint_end = self.bound_constraint_indices[expression]
        sdp_start, sdp_end = self.var_idx_map[expression]
        curr_prob_dim = self.get_sdp_var_dim()

        lb = np.atleast_1d(lb)
        ub = np.atleast_1d(ub)

        expected_len = sdp_end - sdp_start
        if len(lb) != expected_len:
            raise ValueError(f"Expected lb of length {expected_len}, got {len(lb)}")
        if len(ub) != expected_len:
            raise ValueError(f"Expected ub of length {expected_len}, got {len(ub)}")

        for j, (i, l, u) in enumerate(zip(range(sdp_start, sdp_end), lb, ub)):
            if np.isinf(l) or np.isinf(u):
                raise NotImplementedError("Infinite bounds not yet supported in SDP canonicalizer")

            new_A = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
            new_A[i, i] = 1
            new_A[i, 0] = -(l + u)

            constraint_idx = constraint_start + j
            self.Aineq_matrices[constraint_idx] = symmetrize(new_A.tocsc())
            self.bineq_upperbounds[constraint_idx] = -u * l

        self._cuts_dirty = True

    def add_param_stack_var(self, param_stack_var, param_list):
        """
        Stack parameter variables (not implemented for SDP).

        Raises
        ------
        NotImplementedError
            This method is not supported in the SDP canonicalizer.
        """
        raise NotImplementedError

    def copy(self) -> "SDP":
        """
        Create an independent copy of this SDP canonicalizer.

        Returns a new SDP canonicalizer with deep copies of all mutable
        state (constraint matrices, cone indices, etc.). The copy can be
        modified without affecting the original.

        Returns
        -------
        SDP
            A new canonicalizer instance with copied state.
        """
        new_canon = SDP.__new__(SDP)
        new_canon.solver_params = dict(self.solver_params) if self.solver_params else None
        new_canon.solver_name = self.solver_name
        new_canon.solver = None  # SDP solver is instantiated at solve time
        new_canon.range_marker = self.range_marker
        new_canon.var_idx_map = self.var_idx_map.copy()  # shallow copy ok (immutable values)

        # Deep copy mutable arrays/lists
        new_canon.Aeq_matrices = [m.copy() for m in self.Aeq_matrices]
        new_canon.beq_consts = self.beq_consts.copy()
        new_canon.Aineq_matrices = [m.copy() for m in self.Aineq_matrices]
        new_canon.bineq_upperbounds = self.bineq_upperbounds.copy()
        new_canon.psd_cone_indices = [s.copy() for s in self.psd_cone_indices]
        new_canon.bound_constraint_indices = self.bound_constraint_indices.copy()
        new_canon.rlt_cuts = self.rlt_cuts
        new_canon.triangle_cuts = self.triangle_cuts
        new_canon.ineq_product_cuts = self.ineq_product_cuts
        new_canon.ineq_box_cuts = self.ineq_box_cuts
        new_canon.ineq_square_cuts = self.ineq_square_cuts
        new_canon.relu_envelope_cuts = self.relu_envelope_cuts
        new_canon.rlt_all_pairs = self.rlt_all_pairs
        new_canon.full_psd_cone = self.full_psd_cone
        new_canon.cross_cut = self.cross_cut
        new_canon.subgrad_mono_cut = self.subgrad_mono_cut
        # Cuts get rebuilt from current state on the copy's first solve.
        new_canon._cuts_dirty = True
        new_canon._cut_ineq_matrices = []
        new_canon._cut_bineq_upperbounds = []
        new_canon._relu_steps = list(self._relu_steps)  # shallow: Relu objs are shared
        new_canon._prox_steps = list(self._prox_steps)  # shallow: exprs/scalars immutable
        new_canon._box_prox_steps = list(self._box_prox_steps)  # shallow: exprs/arrays immutable
        new_canon._ineq_constraints = list(self._ineq_constraints)  # shallow: (constr, flag) tuples
        new_canon.C_obj = self.C_obj.copy() if self.C_obj is not None else None
        new_canon.obbt_handler = self.obbt_handler
        new_canon._objective_spec = self._objective_spec
        new_canon._solved_copy = None

        return new_canon

    def add_constraint(self, constraint: Constraint) -> None:
        """
        Add a constraint to the SDP instance.

        Dispatches to the appropriate implementation based on the constraint
        operator type (equality or inequality).

        Parameters
        ----------
        constraint : Constraint
            Constraint object to add to the SDP instance.
        """
        match constraint.operator:
            case "=":
                self._add_equality_constraint_impl(constraint)
            case "<=":
                self._add_inequality_constraint_impl(constraint)
            case "complementarity":
                self._add_complementarity_constraint_impl(constraint)
            case "sum_squares":
                self._add_sum_squares_constraint_impl(constraint)
            case _:
                raise ValueError(f"Unknown constraint operator: {constraint.operator}")

    def _add_equality_constraint_impl(self, constraint: Constraint) -> None:
        """
        Add an equality constraint and its quadratic strengthening.

        For Ax + b = 0, adds:
        1. Linear constraint in lifted form
        2. Quadratic constraint: diag([b|A] X [b|A]^T) = 0

        With the PSD constraint, the diagonal being zero implies the full
        quadratic form is zero. Note this is already the full equality-RLT
        closure within the cone: ``c_i^T X c_i = 0`` with ``X ⪰ 0`` forces the
        whole column ``X c_i = 0``, i.e. every product ``h_i(x) x_p = 0`` for
        cone-mate ``p`` — so no separate equality-RLT cuts are needed.

        Parameters
        ----------
        constraint : Constraint
            An equality constraint (operator "=").
        """
        A_mat, x_idx_bounds = self.linexpr_to_Amat_and_bounds(constraint.lhs)
        b_vec = constraint.lhs.const

        m, n = A_mat.shape
        curr_prob_dim = self.get_sdp_var_dim()

        # Linear: X[0, x_indices] (row=True, then symmetrize handles X[x, 0])
        x_idx_ranges_1d = process_1d_ranges(x_idx_bounds, row=True)

        # Quadratic: X[[0, x], [0, x]] for the outer product
        all_idx_bounds = [(0, 1)] + x_idx_bounds
        all_idx_ranges_2d = process_2d_ranges(all_idx_bounds, all_idx_bounds)

        for i in range(m):
            # Linear: A_mat[i, :] @ X[x, 0] = -b[i]
            M_lin = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
            M_lin[x_idx_ranges_1d] = A_mat[i, :]
            self.Aeq_matrices.append(symmetrize(M_lin.tocsc()))
            self.beq_consts.append(-b_vec[i])

            # Quadratic: c_i^T X c_i = 0 where c_i = [b_i, A_i]
            c_i = np.concatenate([[b_vec[i]], np.asarray(A_mat[i, :]).flatten()])
            M_quad = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
            M_quad[all_idx_ranges_2d] = np.outer(c_i, c_i)
            self.Aeq_matrices.append(symmetrize(M_quad.tocsc()))
            self.beq_consts.append(0)

        # PSD cone for {0} ∪ x_indices (also marks _cuts_dirty via add_psd_cone_indices)
        self.add_psd_cone_indices(x_idx_bounds)

    def _add_inequality_constraint_impl(
        self, constraint: Constraint, add_product_cuts: bool = True
    ) -> None:
        """
        Add an inequality constraint ``Ax + b <= 0`` and register it for RLT cuts.

        Adds the base linear constraint ``a_i^T x <= -b_i`` per row (lifted into
        ``Aineq_matrices`` / ``bineq_upperbounds``) and a PSD cone over ``{0} ∪ x``
        so the quadratic block is constrained (matching the equality impl). The
        constraint is recorded in ``self._ineq_constraints`` so the lazy cut builder
        can emit the optional RLT strengthenings gated by the ``ineq_*_cuts`` flags;
        those are (re)built in ``_rebuild_cuts_if_dirty``.

        Parameters
        ----------
        constraint : Constraint
            An inequality constraint (operator "<=").
        add_product_cuts : bool, default True
            Whether the **product cuts** (``g_i g_j >= 0`` for ``i < j``, the
            ``ineq_product_cuts`` family) may be emitted for this constraint. The box
            and square cut families are always eligible (subject to their global flags);
            only the cross-product family is gated here.

            Set ``False`` for the **nonnegativity sides of a complementarity constraint**
            (``e1 >= 0`` / ``e2 >= 0`` emitted by
            `_add_complementarity_constraint_impl`). Rationale:

            - The product family is ``O(m^2)`` per constraint — the dominant cut cost —
              and each complementarity adds two such sides, so leaving it on multiplies
              the cut count for multi-complementarity proxes (``IndBallL1``, ``TV1D``).
            - Empirically these product cuts add **no** tightening on those sides: the box
              and square cuts already drive the relaxation to the same value (e.g.
              ``IndBallL1`` stays exact at a fixed point with product cuts off but box/
              square on). Keeping box/square is what preserves tightness.
            - For ``IndPolyhedral`` the ``e2 = b - A u >= 0`` side also duplicates the
              explicitly-posted ``A u <= b``; dropping its product cuts removes that
              duplication too.

            The genuine problem inequalities (e.g. the explicit ``A u <= b``) are added
            with the default ``True`` and keep generating product cuts — those *do*
            tighten (the ``IndPolyhedral`` 4.0 -> 0.25 test relies on them), so the family
            is gated per-constraint rather than globally.
        """
        A_mat, x_idx_bounds = self.linexpr_to_Amat_and_bounds(constraint.lhs)
        b_vec = constraint.lhs.const

        m, _n = A_mat.shape
        curr_prob_dim = self.get_sdp_var_dim()

        # Linear: X[0, x_indices] (row=True, then symmetrize handles X[x, 0]).
        x_idx_ranges_1d = process_1d_ranges(x_idx_bounds, row=True)

        for i in range(m):
            # a_i^T x <= -b_i
            M_lin = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
            M_lin[x_idx_ranges_1d] = A_mat[i, :]
            self.Aineq_matrices.append(symmetrize(M_lin.tocsc()))
            self.bineq_upperbounds.append(-b_vec[i])

        # PSD cone for {0} ∪ x_indices (also marks _cuts_dirty via add_psd_cone_indices).
        self.add_psd_cone_indices(x_idx_bounds)

        # Register for lazy cut generation. The per-constraint ``add_product_cuts`` flag is
        # stored alongside so the builder can skip only the (expensive) product family for
        # complementarity nonnegativity sides while still emitting box/square cuts.
        self._ineq_constraints.append((constraint, add_product_cuts))
        self._cuts_dirty = True

    def _add_sum_squares_constraint_impl(self, constraint) -> None:
        r"""
        Add a sum-of-squares constraint ``sum_i (a_i^T x + b_i)^2  op  r^2``.

        With ``C = [b; A]^T [b; A]`` the squared two-norm is linear in the lifted matrix:
        ``sum_i (a_i^T x + b_i)^2 = tr(C X)`` (since ``X[0, 0] = 1``). Each sense becomes a
        single linear constraint on ``X``:

        - ``"<="`` : ``tr(C X) <= r^2``      -> ``Aineq``  with ub ``r^2``
        - ``">="`` : ``tr(C X) >= r^2``      -> ``Aineq``  on ``-C`` with ub ``-r^2``
        - ``"=="`` : ``tr(C X)  = r^2``      -> ``Aeq``    with rhs ``r^2``

        All senses are valid relaxations: the rank-1 lift ``X = [1; x][1; x]^T`` attains
        ``tr(C X) = ||A x + b||^2`` exactly, so no feasible point is cut (for ``>=`` / ``==``
        the PSD relaxation is merely looser, never invalid). A PSD cone over ``{0} U x`` is
        registered so the quadratic block is coupled, matching the (in)equality impls.

        Parameters
        ----------
        constraint : QuadraticConstraint
            A sum-of-squares constraint (operator ``"sum_squares"``).
        """
        A_mat, idx_bounds = self.linexpr_to_Amat_and_bounds(constraint.expr)
        b_vec = constraint.expr.const.reshape((-1, 1))
        curr_prob_dim = self.get_sdp_var_dim()

        # Build C = [b; A]^T [b; A] in lifted coordinates, mirroring _apply_two_norm_objective.
        C_mat = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
        C_mat[process_2d_ranges(idx_bounds, idx_bounds)] = A_mat.T @ A_mat
        C_mat[process_1d_ranges(idx_bounds, row=True)] = b_vec.T @ A_mat
        C_mat[process_1d_ranges(idx_bounds, row=False)] = A_mat.T @ b_vec
        C_mat[0, 0] = float(b_vec.flatten() @ b_vec.flatten())  # b^T b (scalar)
        C = symmetrize(C_mat.tocsc())

        r_sq = constraint.radius_sq
        match constraint.sense:
            case "<=":
                self.Aineq_matrices.append(C)
                self.bineq_upperbounds.append(r_sq)
            case ">=":
                self.Aineq_matrices.append(-C)
                self.bineq_upperbounds.append(-r_sq)
            case "==":
                self.Aeq_matrices.append(C)
                self.beq_consts.append(r_sq)

        # PSD cone for {0} U x_indices (also marks _cuts_dirty via add_psd_cone_indices).
        self.add_psd_cone_indices(idx_bounds)

    def _add_complementarity_constraint_impl(self, constraint) -> None:
        r"""
        Encode elementwise complementarity ``e1 ⊥ e2`` (both ``>= 0``) in the lifted SDP.

        With homogenized coefficient vectors ``c1_i`` / ``c2_i`` (so ``e1_i = c1_i^T w``,
        ``e2_i = c2_i^T w``), the product ``e1_i e2_i = c1_i^T X c2_i`` is linear in ``X``.
        For each row ``i`` this adds the equality ``<sym(c1_i c2_i^T), X> = 0`` (the
        cross-product generalization of the equality strengthening in
        `_add_equality_constraint_impl`), the nonnegativities ``e1 >= 0`` and
        ``e2 >= 0`` (via `_add_inequality_constraint_impl`), and a PSD cone over
        ``{0} ∪ vars(e1) ∪ vars(e2)`` so the bilinear entries are constrained.

        Parameters
        ----------
        constraint : ComplementarityConstraint
            Carries ``expr1`` / ``expr2`` (each a length-``ndim`` LinearExpression).
        """
        e1, e2 = constraint.expr1, constraint.expr2
        A1, idx1 = self.linexpr_to_Amat_and_bounds(e1)
        A2, idx2 = self.linexpr_to_Amat_and_bounds(e2)
        b1 = np.asarray(e1.const).flatten()
        b2 = np.asarray(e2.const).flatten()

        # The compact outer-product scatter assumes the two expressions reference
        # disjoint variables (true for λ ⊥ (b−Au): λ and u are distinct leaves).
        # Overlapping leaves would double-index the block; guard rather than mis-encode.
        # Compare the underlying leaf variables directly (decomposition_dict keys), not the
        # (start, end) index ranges.
        if set(e1.decomposition_dict) & set(e2.decomposition_dict):
            raise NotImplementedError(
                "SDP complementarity between expressions sharing variables is not supported."
            )

        n1 = sum(e - s for (s, e) in idx1)
        n2 = sum(e - s for (s, e) in idx2)
        curr = self.get_sdp_var_dim()
        # Compact layout over [const, e1 vars, e2 vars].
        all_idx_bounds = [(0, 1)] + idx1 + idx2
        idx_2d = process_2d_ranges(all_idx_bounds, all_idx_bounds)

        for i in range(constraint.ndim):
            c1 = np.concatenate([[b1[i]], np.asarray(A1[i, :]).flatten(), np.zeros(n2)])
            c2 = np.concatenate([[b2[i]], np.zeros(n1), np.asarray(A2[i, :]).flatten()])
            M = spa.lil_matrix((curr, curr))
            M[idx_2d] = np.outer(c1, c2)
            self.Aeq_matrices.append(symmetrize(M.tocsc()))
            self.beq_consts.append(0)

        # Nonnegativity of both sides (``e1 >= 0`` -> ``-e1 <= 0``). Skip the (O(m^2),
        # tightening-neutral) product-cut family on these sides; box/square cuts still
        # apply and are what preserve tightness (see ``add_product_cuts``).
        self._add_inequality_constraint_impl(e1 >= 0, add_product_cuts=False)
        self._add_inequality_constraint_impl(e2 >= 0, add_product_cuts=False)

        # Constrain the bilinear entries.
        self.add_psd_cone_indices(idx1 + idx2)

    def register_prox_step(
        self, z: LinearExpression, v: LinearExpression, tau: float, *, func: str
    ) -> None:
        """
        Register a prox step for the subgradient-monotonicity cut.

        Records ``(z, v, tau, func)`` where ``z`` is the prox output expression,
        ``v`` is the prox input expression (so the subgradient is ``s = v - z``),
        ``tau`` is the prox threshold (``tau = 1`` for orthant projection / ReLU,
        ``tau = lambda`` for soft-thresholding), and ``func`` names the sublinear
        function the step is the prox of. The subgradient cut builder
        (`_build_subgrad_mono_cuts`) consumes this registry and pairs only
        steps with the same ``func`` (the cut is invalid across functions).

        Parameters
        ----------
        z : LinearExpression
            Prox output. May be composite (e.g. ``relu_pos - relu_neg`` for
            soft-thresholding); leaves must already be registered in
            ``var_idx_map`` at call time.
        v : LinearExpression
            Prox input (``step.arg``).
        tau : float
            Prox threshold used to normalize the subgradient.
        func : str
            Identifier of the sublinear function (``"nonneg_orthant"`` for ReLU,
            ``"l1"`` for soft-thresholding).
        """
        self._prox_steps.append({"z": z, "v": v, "tau": float(tau), "func": func})
        self._cuts_dirty = True

    def register_box_prox_step(self, z: LinearExpression, v: LinearExpression, l, u) -> None:
        """
        Register a saturated-linear (box projection) step for the PEP box cuts.

        Records ``(z, v, l, u)`` where ``z = clip(v, l, u)`` is the composite
        projection output (``relu(v - l) - relu(v - u) + l`` in the SDP lift), ``v``
        is the input expression (``step.arg``; residual ``r = v - z``), and ``l`` /
        ``u`` are the box bounds (constant arrays or ``LinearExpression`` if
        parametric). Consumed by `_build_box_normalcone_cuts` (under
        ``cross_cut``) and `_build_box_mono_cuts` (under ``subgrad_mono_cut``).

        Parameters
        ----------
        z : LinearExpression
            Box-projection output (composite); leaves already in ``var_idx_map``.
        v : LinearExpression
            Box-projection input (``step.arg``).
        l, u : np.ndarray | LinearExpression
            Lower/upper box bounds (per the operator's ``l`` / ``u``).
        """
        self._box_prox_steps.append({"z": z, "v": v, "l": l, "u": u})
        self._cuts_dirty = True

    def add_psd_cone_indices(self, new_indices):
        """
        Register a new PSD cone constraint on a submatrix.

        Adds the constraint that the submatrix of X indexed by ``new_indices``
        (including the homogenization index 0) is positive semidefinite.

        This method maintains an antichain: no stored set is a subset of another.
        If a matrix X is PSD, any principal submatrix is also PSD, so subset
        constraints are redundant.

        Parameters
        ----------
        new_indices : list[tuple[int, int]]
            List of (start, end) index ranges defining the submatrix.
            Index 0 is automatically included for homogenization.
        """
        new_idx_set = range_to_indices(new_indices)
        new_idx_set.add(0)

        # Check if new set is contained in any existing set (redundant)
        for existing_set in self.psd_cone_indices:
            if new_idx_set <= existing_set:
                return

        # Filter out existing sets that are strict subsets of the new set
        self.psd_cone_indices = [s for s in self.psd_cone_indices if not s < new_idx_set]

        self.psd_cone_indices.append(new_idx_set)
        self._cuts_dirty = True

    def canonicalize_relu(self, step: Relu) -> None:
        """
        Encode a ReLU step as SDP constraints.

        Converts the relationship y = ReLU(Ax + b) into equality constraints
        on the SDP matrix variable and registers the appropriate PSD cone
        constraint for the lifted formulation.

        Operates uniformly over ``step.proj_indices`` and binds equalities to
        ``step.output_leaves[0]``. For full ReLU, ``proj_indices`` covers every
        arg row and ``output_leaves[0]`` is the full output leaf — equivalent to a
        standard full-output encoding. For partial-projection ReLU, only the
        projected rows get lifted equalities and ``output_leaves[0]`` is the
        smaller fresh proj-leaf; the non-projected rows are structurally
        fused into shared arg leaves at composition time, so no equalities
        and no SDP rows are emitted for them.

        Parameters
        ----------
        step : Relu
            The ReLU step containing input expression (arg) and output
            variable (output).
        """
        arg = step.arg
        A_mat, x_idx_bounds = self.linexpr_to_Amat_and_bounds(arg)
        b_vec = arg.const

        y_leaf = step.output_leaves[0]
        proj_list = sorted(step.proj_indices)

        y_idx_bounds = [self.var_idx_map[y_leaf]]
        relu_psd_cone_indices = y_idx_bounds + x_idx_bounds

        y_idx_ranges_2d = process_2d_ranges(y_idx_bounds, y_idx_bounds)
        y_idx_ranges_1d = process_1d_ranges(y_idx_bounds)
        xy_idx_ranges = process_2d_ranges(x_idx_bounds, y_idx_bounds)
        I = np.eye(y_leaf.ndim)

        curr_prob_dim = self.get_sdp_var_dim()
        for j, i in enumerate(proj_list):
            # j indexes y_leaf (size = len(proj_list));
            # i indexes arg row in A_mat / b_vec (size = arg.ndim).
            Ij = I[j]
            new_A = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
            new_A[y_idx_ranges_2d] = np.outer(Ij, Ij)
            new_A[xy_idx_ranges] = -np.outer(A_mat[i], Ij)
            new_A[y_idx_ranges_1d] = -b_vec[i] * Ij

            self.Aeq_matrices.append(symmetrize(new_A.tocsc()))
            self.beq_consts.append(0)

        self.add_psd_cone_indices(relu_psd_cone_indices)
        self._relu_steps.append(step)

        # Register the ReLU as a prox step (orthant projection, tau=1) for the
        # subgradient-monotonicity cut — unless this ReLU is an internal piece of
        # a soft-threshold decomposition, in which case the composite prox is
        # registered by the decomposer instead (see SDPStepDecomposer). The ReLU
        # still joins ``_relu_steps`` above so the cross cut sees every node.
        if not getattr(step, "_skip_prox_registration", False):
            self.register_prox_step(z=step.output, v=step.arg, tau=1.0, func="nonneg_orthant")

        self._cuts_dirty = True

    def canonicalize_affine(self, step: Affine) -> None:
        """
        Encode an Affine step as a linear equality constraint.

        The Affine step asserts ``step.output_mat @ step.output == step.arg``;
        this is the same equality MIP encodes (``mip.py:404``). On SDP it
        flows through ``add_constraint`` → ``_add_equality_constraint_impl``,
        which emits the linear matrix, the quadratic strengthening, and
        the PSD cone membership for the involved leaves.

        Parameters
        ----------
        step : Affine
            The affine step to canonicalize.
        """
        self.add_constraint(step.output_mat @ step.output == step.arg)

    _CANONICALIZERS = {
        "affine": canonicalize_affine,
        "relu": canonicalize_relu,
    }

    def linexpr_to_Amat_and_bounds(self, expr: LinearExpression):
        """
        Extract the coefficient matrix and SDP indices from a LinearExpression.

        Converts a LinearExpression of the form Ax + b into the coefficient
        matrix A and the index ranges where the corresponding variables are
        located in the SDP matrix.

        Parameters
        ----------
        expr : LinearExpression
            The expression to decompose.

        Returns
        -------
        tuple[np.ndarray, list[tuple[int, int]]]
            A tuple containing:
            - The horizontally stacked coefficient matrix.
            - List of (start, end) index bounds for each variable block.
        """
        all_A = []
        idx_bounds = []
        for leaf_var, mat in expr.decomposition_dict.items():
            all_A.append(mat.todense() if spa.issparse(mat) else mat)
            idx_bounds.append(self.var_idx_map[leaf_var])

        return np.hstack(all_A), idx_bounds

    def set_two_norm_objective(self, expr_list, lb_list=None, ub_list=None) -> None:
        """
        Store two-norm objective specification for deferred application.

        The objective will be applied to a copy of the canonicalizer at
        solve time.

        Parameters
        ----------
        expr_list : list[LinearExpression]
            List of expressions whose squared norms are summed in the
            objective.
        lb_list : list[np.ndarray], optional
            Lower bounds for each expression (unused in SDP, for interface consistency).
        ub_list : list[np.ndarray], optional
            Upper bounds for each expression (unused in SDP, for interface consistency).
        """
        self._objective_spec = ("two_norm", expr_list)

    def _apply_two_norm_objective(self, canon_copy, expr_list) -> None:
        """
        Apply two-norm objective to a canonicalizer copy.

        Configures the SDP objective matrix C to compute
        sum_i ||A_i x + b_i||_2^2 = sum_i tr(Q_i @ X) for appropriate
        quadratic forms Q_i derived from each expression.

        Also adds a PSD cone constraint covering all variables involved
        in the objective to ensure cross-terms are properly constrained.

        Parameters
        ----------
        canon_copy : SDP
            The canonicalizer copy to apply the objective to.
        expr_list : list[LinearExpression]
            List of expressions whose squared norms are summed in the
            objective.
        """
        curr_prob_dim = canon_copy.get_sdp_var_dim()
        overall_C_mat = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
        all_idx_bounds = []

        for expr in expr_list:
            C_mat = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
            A_mat, idx_bounds = canon_copy.linexpr_to_Amat_and_bounds(expr)
            b_vec = expr.const.reshape((-1, 1))

            # Collect all index bounds for the combined PSD cone
            all_idx_bounds.extend(idx_bounds)

            ATA = A_mat.T @ A_mat
            bTA = b_vec.T @ A_mat
            ATb = A_mat.T @ b_vec
            bbT = b_vec.T @ b_vec

            idx_ranges_2d = process_2d_ranges(idx_bounds, idx_bounds)
            idx_ranges_1d_row = process_1d_ranges(idx_bounds, row=True)
            idx_ranges_1d_col = process_1d_ranges(idx_bounds, row=False)

            C_mat[idx_ranges_2d] = ATA
            C_mat[idx_ranges_1d_row] = bTA
            C_mat[idx_ranges_1d_col] = ATb
            C_mat[0, 0] = np.trace(bbT)

            overall_C_mat += symmetrize(C_mat)

        # Add PSD cone covering all variables in the objective
        # This ensures cross-terms between different variables are constrained
        canon_copy.add_psd_cone_indices(all_idx_bounds)

        canon_copy.C_obj = overall_C_mat

    def set_infinity_norm_objective(self, expr_list, lb=None, ub=None) -> None:
        """
        Store infinity-norm objective specification for deferred application.

        Uses the identities:
        - abs(x) = relu(x) + relu(-x)
        - max(a, b) = a + relu(b - a)

        Parameters
        ----------
        expr_list : list[LinearExpression]
            List of expressions whose infinity-norm maximum is the objective.
        lb, ub : optional
            Unused (kept for interface compatibility with MIP).
        """
        self._objective_spec = ("infinity_norm", expr_list)

    def set_one_norm_objective(self, expr_list) -> None:
        """
        Store one-norm objective specification for deferred application.

        The objective will be applied to a copy of the canonicalizer at
        solve time using the identity: ||x||_1 = 1^T @ (relu(x) + relu(-x)).

        Parameters
        ----------
        expr_list : list[LinearExpression]
            List of expressions whose one-norms are summed in the objective.
        """
        self._objective_spec = ("one_norm", expr_list)

    def _apply_one_norm_objective(self, canon_copy, expr_list) -> None:
        """
        Apply one-norm objective to a canonicalizer copy.

        Uses the identity: ||x||_1 = 1^T @ (relu(x) + relu(-x))

        Creates auxiliary ReLU variables, adds their constraints, and sets
        up a linear objective to sum the ReLU outputs.

        Parameters
        ----------
        canon_copy : SDP
            The canonicalizer copy to apply the objective to.
        expr_list : list[LinearExpression]
            List of expressions whose one-norms are summed in the objective.
        """
        relu_output_vars = []

        for expr in expr_list:
            relu_output_vars.append(self._create_relu_step(canon_copy, expr))
            relu_output_vars.append(self._create_relu_step(canon_copy, -expr))

        # Build linear objective: maximize sum of all relu outputs
        # For lifted SDP with X = [1, x]^T @ [1, x], linear term c^T x
        # is encoded as C[0, idx] = c/2, C[idx, 0] = c/2
        curr_prob_dim = canon_copy.get_sdp_var_dim()
        C_mat = spa.lil_matrix((curr_prob_dim, curr_prob_dim))

        all_idx_bounds = []
        for output_var in relu_output_vars:
            idx_bounds = canon_copy.var_idx_map[output_var]
            all_idx_bounds.append(idx_bounds)
            start, end = idx_bounds
            # Coefficient 1 for each component, divided by 2 for symmetry
            C_mat[0, start:end] = 0.5
            C_mat[start:end, 0] = 0.5

        # Add PSD cone covering all relu output variables
        canon_copy.add_psd_cone_indices(all_idx_bounds)

        canon_copy.C_obj = C_mat.tocsc()

    def _apply_infinity_norm_objective(self, canon_copy, expr_list) -> None:
        """
        Apply infinity-norm objective using ReLU decomposition.

        Computes ||x||_∞ = max_i |x_i| via:
        - abs(x_i) = relu(x_i) + relu(-x_i)
        - max(M, abs_i) = M + relu(abs_i - M)

        Total ReLUs: 3n - 1 (2n for absolute values, n-1 for max operations)

        Parameters
        ----------
        canon_copy : SDP
            The canonicalizer copy to apply the objective to.
        expr_list : list[LinearExpression]
            List of expressions whose infinity-norm maximum is the objective.
        """
        all_idx_bounds = []
        M_current = None  # Running maximum (scalar LinearExpression)

        for expr in expr_list:
            for i in range(expr.ndim):
                x_i = expr[i]  # Scalar slice

                # abs(x_i) = relu(x_i) + relu(-x_i)
                relu_pos = self._create_relu_step(canon_copy, x_i)
                relu_neg = self._create_relu_step(canon_copy, -x_i)
                abs_i = relu_pos + relu_neg

                all_idx_bounds.append(canon_copy.var_idx_map[relu_pos])
                all_idx_bounds.append(canon_copy.var_idx_map[relu_neg])

                if M_current is None:
                    M_current = abs_i
                else:
                    # max(M, abs_i) = M + relu(abs_i - M)
                    diff = abs_i - M_current
                    relu_max = self._create_relu_step(canon_copy, diff)
                    all_idx_bounds.append(canon_copy.var_idx_map[relu_max])
                    M_current = M_current + relu_max

        # Build objective matrix for M_current (scalar linear expression)
        curr_prob_dim = canon_copy.get_sdp_var_dim()
        C_mat = spa.lil_matrix((curr_prob_dim, curr_prob_dim))

        A_mat, idx_bounds = canon_copy.linexpr_to_Amat_and_bounds(M_current)
        b_vec = M_current.const

        # Linear c^T x encoded as C[0, idx] = c/2, C[idx, 0] = c/2
        idx_ranges_1d_row = process_1d_ranges(idx_bounds, row=True)
        idx_ranges_1d_col = process_1d_ranges(idx_bounds, row=False)

        C_mat[idx_ranges_1d_row] = 0.5 * A_mat
        C_mat[idx_ranges_1d_col] = 0.5 * A_mat.T
        C_mat[0, 0] = float(b_vec.sum())

        canon_copy.add_psd_cone_indices(all_idx_bounds)
        canon_copy.C_obj = C_mat.tocsc()

    def _create_relu_step(self, canon_copy: "SDP", expr: LinearExpression) -> LinearExpression:
        """
        Create a ReLU step and register it with the canonicalizer.

        Parameters
        ----------
        canon_copy : SDP
            The canonicalizer copy to register the ReLU with.
        expr : LinearExpression
            The input expression for the ReLU.

        Returns
        -------
        LinearExpression
            The output (output) of the ReLU step.
        """
        step = Relu(arg=expr)
        step.update_bounds(*canon_copy.get_expression_bounds(expr))
        for leaf in step.output_leaves:
            canon_copy.add_var(leaf)
        canon_copy.canonicalize_relu(step)
        return step.output

    def set_zero_objective(self):
        """
        Set objective to zero (not yet implemented for SDP).

        Raises
        ------
        NotImplementedError
            This method is not yet implemented for the SDP canonicalizer.
        """
        raise NotImplementedError(
            "set_zero_objective is not yet implemented for the SDP canonicalizer."
        )

    def get_solution_value(self, expression: LinearExpression) -> np.ndarray:
        """
        Get the solution value for an expression (not yet implemented for SDP).

        Parameters
        ----------
        expression : LinearExpression
            The expression to evaluate.

        Returns
        -------
        np.ndarray
            The solution values.

        Raises
        ------
        NotImplementedError
            This method is not yet implemented for the SDP canonicalizer.
        """
        raise NotImplementedError(
            "get_solution_value is not yet implemented for the SDP canonicalizer. "
            "SDP solves a relaxation and does not provide primal variable values."
        )

    def add_theory_cut(
        self,
        margin: float,
        target_expr: LinearExpression,
        lb: float | np.ndarray,
        ub: float | np.ndarray,
    ):
        """
        Add theory cut constraints (not yet implemented for SDP).

        Parameters
        ----------
        margin : float
            Relaxation margin for the bounds.
        target_expr : LinearExpression
            The expression whose bounds are to be relaxed.
        lb : float | np.ndarray
            Lower bound(s) for target_expr.
        ub : float | np.ndarray
            Upper bound(s) for target_expr.

        Raises
        ------
        NotImplementedError
            This method is not yet implemented for the SDP canonicalizer.
        """
        raise NotImplementedError(
            "add_theory_cut is not yet implemented for the SDP canonicalizer."
        )

    def _build_sdp_idx_bounds(self) -> dict:
        """
        Build a reverse map ``sdp_idx -> (lb_scalar, ub_scalar)`` from the
        current state of ``var_idx_map``. Used by RLT cut construction to
        look up the per-index bounds without scanning leaves repeatedly.

        Returns
        -------
        dict[int, tuple[float, float]]
            Mapping from each SDP matrix index (excluding 0) to its current
            (lower, upper) bound pair.
        """
        sdp_idx_bounds = {}
        for leaf, (start, end) in self.var_idx_map.items():
            for offset, sdp_idx in enumerate(range(start, end)):
                sdp_idx_bounds[sdp_idx] = (float(leaf.lb[offset]), float(leaf.ub[offset]))
        return sdp_idx_bounds

    def _rebuild_cuts_if_dirty(self) -> None:
        """
        Rebuild the cut inequality block if any cut-affecting state has
        changed since the last rebuild.

        Triggered by ``add_var``, ``canonicalize_relu``, ``add_psd_cone_indices``,
        ``_add_equality_constraint_impl``, and ``update_bounds``. Cuts live in
        ``_cut_ineq_matrices`` / ``_cut_bineq_upperbounds``, which are
        concatenated with the regular ineq block at solve time.

        When all cut flags are False (or there are no registered inequality
        constraints) this is effectively a no-op (cut lists stay empty).
        """
        if not self._cuts_dirty:
            return

        # Option A: collapse the antichain of cones into one full moment-matrix
        # block so that ``_build_rlt_cuts`` (which keys off cone membership)
        # fires on every variable pair. Solve always runs on ``self.copy()``
        # (see ``solve_model``), so mutating the cone here is local to the copy.
        if self.full_psd_cone:
            full_cone = set(range(self.get_sdp_var_dim()))
            # Guard on the exact target, not the cone count: a single cone that is
            # not the full moment matrix (e.g. iterates merged but some auxiliary
            # variables still uncoupled) must still be unioned.
            if self.psd_cone_indices != [full_cone]:
                self.psd_cone_indices = [full_cone]
        elif self.rlt_all_pairs:
            # Middle tier: couple every pair via a cheap {0, p, q} 3x3 PSD minor
            # so the all-pairs McCormick cuts bind. add_psd_cone_indices keeps the
            # antichain, so a pair already inside a larger cone adds no new minor.
            # (Each call sets _cuts_dirty = True; that churn is harmless — it is
            # overridden by _cuts_dirty = False at the end of this method.)
            for p, q in combinations(range(1, self.get_sdp_var_dim()), 2):
                self.add_psd_cone_indices([(p, p + 1), (q, q + 1)])

        self._cut_ineq_matrices = []
        self._cut_bineq_upperbounds = []

        if self.rlt_cuts or self.rlt_all_pairs or self.triangle_cuts:
            sdp_idx_bounds = self._build_sdp_idx_bounds()

            if self.rlt_cuts or self.rlt_all_pairs:
                self._build_rlt_cuts(sdp_idx_bounds)

            if self.triangle_cuts:
                self._build_triangle_cuts()

        if self.relu_envelope_cuts:
            self._build_relu_envelope_cuts()

        # PEP cross-step interpolation cuts. Each builder self-registers the
        # minimal PSD cone for the indices it touches so the cross-step products
        # bind without ``full_psd_cone`` (the antichain subsumes a cone already
        # covered by a larger one). The cone additions set ``_cuts_dirty = True``;
        # that churn is harmless — it is overridden by ``_cuts_dirty = False`` at
        # the end of this method (same pattern as the ``rlt_all_pairs`` branch).
        if self.cross_cut:
            self._build_cross_cuts()
            self._build_box_normalcone_cuts()

        if self.subgrad_mono_cut:
            self._build_subgrad_mono_cuts()
            self._build_box_mono_cuts()

        if self._ineq_constraints and (
            self.ineq_product_cuts or self.ineq_box_cuts or self.ineq_square_cuts
        ):
            self._build_ineq_cuts()

        self._cuts_dirty = False

    def _build_ineq_cuts(self) -> None:
        r"""
        Append RLT-style strengthening cuts for every registered inequality
        constraint ``Ax + b <= 0``.

        For each row ``g_i = a_i^T x + b_i <= 0`` with homogenized coefficient
        vector ``c_i = [b_i; a_i]`` (so ``g_i = c_i^T w`` and
        ``g_i g_j = c_i^T X c_j`` is linear in the lifted ``X``), emit the cuts
        enabled by the ``ineq_*_cuts`` flags:

        - ``ineq_product_cuts``: ``g_i g_j >= 0`` for ``i < j`` (the diagonal
          ``g_i^2 >= 0`` is implied by ``X ⪰ 0`` and is skipped).
        - ``ineq_box_cuts``: ``g_i (x_p - l_p) <= 0`` and ``g_i (u_p - x_p) <= 0``
          for each variable index ``p`` appearing in the constraint.
        - ``ineq_square_cuts``: ``g_i^2 <= g_i_lower * g_i`` using a propagated
          lower bound ``g_i_lower`` on ``g_i`` (analog of the per-variable box
          product in `add_var`, generalized from ``e_i`` to ``c_i``).

        Bounds are read fresh from ``_build_sdp_idx_bounds`` / ``bound_propagation``
        each rebuild so the cuts track leaf tightening (cf. ``_build_triangle_cuts``).
        Cuts are appended to ``_cut_ineq_matrices`` / ``_cut_bineq_upperbounds`` in
        the ``<A, X> <= b`` convention; a ``>= 0`` cut is pushed as ``-sym(M) <= 0``.
        """
        sdp_idx_bounds = self._build_sdp_idx_bounds()
        n_dim = self.get_sdp_var_dim()

        for constraint, product_cuts_ok in self._ineq_constraints:
            A_mat, x_idx_bounds = self.linexpr_to_Amat_and_bounds(constraint.lhs)
            b_vec = np.asarray(constraint.lhs.const).flatten()
            m = A_mat.shape[0]

            # Compact index layout over {0} ∪ x for outer-product scatter.
            all_idx_bounds = [(0, 1)] + x_idx_bounds
            idx_2d = process_2d_ranges(all_idx_bounds, all_idx_bounds)
            # SDP indices of the x-block in compact order (compact pos k -> k+1).
            x_sdp_indices = [idx for (s, e) in x_idx_bounds for idx in range(s, e)]

            # c_i = [b_i, a_i] in compact ordering matching all_idx_bounds.
            c_list = [
                np.concatenate([[b_vec[i]], np.asarray(A_mat[i, :]).flatten()]) for i in range(m)
            ]

            if self.ineq_square_cuts:
                # g_i_lower <= g_i <= 0; use the propagated lower bound on g.
                g_lb, _g_ub = constraint.lhs.bound_propagation()

            for i in range(m):
                c_i = c_list[i]

                # Cut (3): g_i^2 <= g_i_lower * g_i  ->  c_i c_i^T - g_lb*(c_i e0^T) <= 0.
                if self.ineq_square_cuts:
                    Mc = np.outer(c_i, c_i)
                    Mc[:, 0] -= float(g_lb[i]) * c_i
                    M = spa.lil_matrix((n_dim, n_dim))
                    M[idx_2d] = Mc
                    self._cut_ineq_matrices.append(symmetrize(M.tocsc()))
                    self._cut_bineq_upperbounds.append(0.0)

                # Cut (1): g_i g_j >= 0 for i < j  ->  -sym(c_i c_j^T) <= 0.
                # Per-constraint ``product_cuts_ok`` skips this O(m^2) family for
                # complementarity nonnegativity sides (see ``add_product_cuts``).
                if self.ineq_product_cuts and product_cuts_ok:
                    for j in range(i + 1, m):
                        Mc = np.outer(c_i, c_list[j])
                        M = spa.lil_matrix((n_dim, n_dim))
                        M[idx_2d] = Mc
                        self._cut_ineq_matrices.append(-symmetrize(M.tocsc()))
                        self._cut_bineq_upperbounds.append(0.0)

                # Cut (2): g_i (x_p - l_p) <= 0 and g_i (u_p - x_p) <= 0.
                if self.ineq_box_cuts:
                    for k, p in enumerate(x_sdp_indices):
                        l_p, u_p = sdp_idx_bounds[p]
                        # compact position of x index p is k + 1
                        d_lower = np.zeros_like(c_i)
                        d_lower[k + 1] = 1.0
                        d_lower[0] = -l_p
                        d_upper = np.zeros_like(c_i)
                        d_upper[k + 1] = -1.0
                        d_upper[0] = u_p
                        for d in (d_lower, d_upper):
                            M = spa.lil_matrix((n_dim, n_dim))
                            M[idx_2d] = np.outer(c_i, d)
                            self._cut_ineq_matrices.append(symmetrize(M.tocsc()))
                            self._cut_bineq_upperbounds.append(0.0)

    def _build_rlt_cuts(self, sdp_idx_bounds: dict) -> None:
        """
        Append RLT cuts for off-diagonal cone-mate pairs.

        For every pair ``(p, q)`` of distinct, nonzero SDP indices that
        co-exist in some entry of ``psd_cone_indices``, emit the 4
        McCormick inequalities (paper §4.5 eq 18). Each pair is processed
        at most once even if it appears in multiple cones.

        Parameters
        ----------
        sdp_idx_bounds : dict[int, tuple[float, float]]
            Mapping from SDP index to its current ``(lb, ub)`` pair.
        """
        added_pairs = set()
        for cone_set in self.psd_cone_indices:
            sorted_indices = sorted(i for i in cone_set if i != 0)
            for p, q in combinations(sorted_indices, 2):
                if (p, q) in added_pairs:
                    continue
                added_pairs.add((p, q))
                l_p, u_p = sdp_idx_bounds[p]
                l_q, u_q = sdp_idx_bounds[q]
                self._build_rlt_cuts_for_pair(p, q, l_p, u_p, l_q, u_q)

    def _build_rlt_cuts_for_pair(
        self,
        p: int,
        q: int,
        l_p: float,
        u_p: float,
        l_q: float,
        u_q: float,
    ) -> None:
        """
        Append the 4 McCormick RLT inequalities for an off-diagonal pair.

        Each unsymmetrized matrix ``M`` has at most 3 nonzero entries:
        the off-diagonal ``[p, q]`` slot and the two linear slots
        ``[0, p]`` and ``[0, q]``. After ``symmetrize`` and a trace inner
        product against the symmetric lifted variable ``X``, the
        coefficient on ``X[p, q]`` is whatever was written at ``[p, q]``
        and the coefficients on ``X[0, p]`` / ``X[0, q]`` are whatever
        was written at those slots.

        Parameters
        ----------
        p, q : int
            Distinct SDP indices, both ``> 0``.
        l_p, u_p, l_q, u_q : float
            Lower/upper bounds at indices ``p`` and ``q``.
        """
        n = self.get_sdp_var_dim()

        # Cut 1: (X_p - l_p)(X_q - l_q) >= 0
        #   -> -X[p,q] + l_q X[0,p] + l_p X[0,q] <= l_p l_q
        M1 = spa.lil_matrix((n, n))
        M1[p, q] = -1.0
        M1[0, p] = l_q
        M1[0, q] = l_p
        self._cut_ineq_matrices.append(symmetrize(M1.tocsc()))
        self._cut_bineq_upperbounds.append(l_p * l_q)

        # Cut 2: (u_p - X_p)(u_q - X_q) >= 0
        #   -> -X[p,q] + u_q X[0,p] + u_p X[0,q] <= u_p u_q
        M2 = spa.lil_matrix((n, n))
        M2[p, q] = -1.0
        M2[0, p] = u_q
        M2[0, q] = u_p
        self._cut_ineq_matrices.append(symmetrize(M2.tocsc()))
        self._cut_bineq_upperbounds.append(u_p * u_q)

        # Cut 3: (X_p - l_p)(u_q - X_q) >= 0
        #   -> X[p,q] - u_q X[0,p] - l_p X[0,q] <= -l_p u_q
        M3 = spa.lil_matrix((n, n))
        M3[p, q] = 1.0
        M3[0, p] = -u_q
        M3[0, q] = -l_p
        self._cut_ineq_matrices.append(symmetrize(M3.tocsc()))
        self._cut_bineq_upperbounds.append(-l_p * u_q)

        # Cut 4: (u_p - X_p)(X_q - l_q) >= 0
        #   -> X[p,q] - l_q X[0,p] - u_p X[0,q] <= -u_p l_q
        M4 = spa.lil_matrix((n, n))
        M4[p, q] = 1.0
        M4[0, p] = -l_q
        M4[0, q] = -u_p
        self._cut_ineq_matrices.append(symmetrize(M4.tocsc()))
        self._cut_bineq_upperbounds.append(-u_p * l_q)

    @staticmethod
    def _relu_arg_bounds(step: Relu) -> tuple[np.ndarray, np.ndarray]:
        """
        Bounds on a ReLU step's input used by the ReLU cut builders.

        Intersects the step's cached input bounds ``step.lb``/``step.ub`` (which
        carry any OBBT tightening applied in ``_process_primitive``) with a fresh
        ``step.arg.bound_propagation()`` (which reflects leaf tightening since the
        step was canonicalized). Both are valid, so the intersection is too; tiny
        solver-noise crossings are repaired.
        """
        prop_lb, prop_ub = step.arg.bound_propagation()
        d = step.arg.ndim
        lb = np.maximum(np.broadcast_to(np.asarray(step.lb, dtype=float), d), prop_lb)
        ub = np.minimum(np.broadcast_to(np.asarray(step.ub, dtype=float), d), prop_ub)
        return repair_crossed_bounds(lb, ub)

    def _build_triangle_cuts(self) -> None:
        """
        Append lifted triangle cuts for every registered ReLU step.

        For each step, iterate over output index pairs ``(i, j)``: emit one
        cut whenever row ``i`` has mixed bounds (``arg_lb[i] < 0 <
        arg_ub[i]``) and column ``j`` has finite upper bound. Bounds come
        from `_relu_arg_bounds`: the OBBT-tightened ``step.lb``/``step.ub``
        intersected with a fresh ``step.arg.bound_propagation()`` (which
        reflects any leaf tightening since ``canonicalize_relu`` ran). See
        paper §4.5 eq 25/26 for the derivation.
        """
        for step in self._relu_steps:
            arg = step.arg
            arg_lb, arg_ub = self._relu_arg_bounds(step)
            A_mat, x_idx_bounds = self.linexpr_to_Amat_and_bounds(arg)
            b_vec = np.asarray(arg.const)

            # Map each arg row to its position in the fresh y-leaf. For full
            # relu this is the identity dict (proj_indices spans arg.ndim);
            # for partial relu it maps only projected arg rows and skips the
            # rest (handled by the ``if i not in rhs_to_y_pos`` guard below).
            y_leaf = step.output_leaves[0]
            rhs_to_y_pos = {i: j for j, i in enumerate(sorted(step.proj_indices))}
            y_idx_bounds = [self.var_idx_map[y_leaf]]

            d = arg.ndim
            for i in range(d):
                if i not in rhs_to_y_pos:
                    continue
                if arg_lb[i] >= 0 or arg_ub[i] <= 0:
                    continue
                for j in range(d):
                    if not np.isfinite(arg_ub[j]):
                        continue
                    self._build_triangle_cut_for_pair(
                        i,
                        j,
                        A_mat,
                        b_vec,
                        x_idx_bounds,
                        y_idx_bounds,
                        arg_lb,
                        arg_ub,
                        y_pos=rhs_to_y_pos[i],
                    )

    def _build_triangle_cut_for_pair(
        self,
        i: int,
        j: int,
        A_mat: np.ndarray,
        b_vec: np.ndarray,
        x_idx_bounds: list,
        y_idx_bounds: list,
        arg_lb: np.ndarray,
        arg_ub: np.ndarray,
        y_pos: int | None = None,
    ) -> None:
        """
        Append the lifted triangle cut for output pair ``(i, j)`` of a ReLU
        step.

        Derivation:
        ``(E_ii (a_i^T x + b_i) + c_i − y_i)(u_j − a_j^T x − b_j) ≥ 0``
        where ``E_ii = u_i / (u_i − l_i)``, ``c_i = −E_ii · l_i``, and
        ``(l_i, u_i, u_j)`` are taken from ``arg_lb`` / ``arg_ub``, the bounds
        of the ReLU's input expression from `_relu_arg_bounds` (OBBT
        bounds intersected with fresh propagation) — see ``_build_triangle_cuts``.

        Expanding gives 9 product groups that land in the lifted-matrix
        slots:
        - xx-block: coefficient ``−E_ii · outer(a_i, a_j)``
        - y_i ⊗ x row: coefficient ``+a_j``
        - y_i linear (``[y_i_idx, 0]``): ``(b_j − u_j)``
        - x linear (``[0, x]``): ``E_ii u_j a_i − E_ii b_j a_i − E_ii b_i a_j
          − c_i a_j``
        - constant (``[0, 0]``): ``E_ii b_i u_j − E_ii b_i b_j + c_i u_j
          − c_i b_j``

        The cut is ``tr(symmetrize(N) X) ≥ 0``, stored as
        ``tr(−symmetrize(N) X) ≤ 0``.
        """
        n_dim = self.get_sdp_var_dim()

        lb_i = arg_lb[i]
        ub_i = arg_ub[i]
        ub_j = arg_ub[j]
        E_ii = ub_i / (ub_i - lb_i)
        c_i = -E_ii * lb_i

        a_i = np.asarray(A_mat[i, :]).flatten()
        a_j = np.asarray(A_mat[j, :]).flatten()
        b_i = float(b_vec[i])
        b_j = float(b_vec[j])

        y_start, _y_end = y_idx_bounds[0]
        # ``y_pos`` is the row of the (possibly smaller) fresh leaf that
        # corresponds to arg row ``i``. For full-projection it equals ``i``.
        y_i_idx = y_start + (y_pos if y_pos is not None else i)

        N = spa.lil_matrix((n_dim, n_dim))

        # xx-block: -E_ii * outer(a_i, a_j) at process_2d_ranges(x, x)
        xx_idx = process_2d_ranges(x_idx_bounds, x_idx_bounds)
        N[xx_idx] = -E_ii * np.outer(a_i, a_j)

        # y_i ⊗ x row: +a_j at row y_i_idx, cols x-concat
        yi_x_idx = process_2d_ranges([(y_i_idx, y_i_idx + 1)], x_idx_bounds)
        N[yi_x_idx] = a_j

        # y_i linear: (b_j - ub_j) at [y_i_idx, 0]
        N[y_i_idx, 0] = b_j - ub_j

        # x linear at row 0
        lin = E_ii * ub_j * a_i - E_ii * b_j * a_i - E_ii * b_i * a_j - c_i * a_j
        x_row0_idx = process_1d_ranges(x_idx_bounds, row=True)
        N[x_row0_idx] = lin

        # Constant at [0, 0]
        const = E_ii * b_i * ub_j - E_ii * b_i * b_j + c_i * ub_j - c_i * b_j
        N[0, 0] = const

        # Cut: tr(symmetrize(N) X) >= 0  =>  push -symmetrize(N) with ub = 0
        N_sym = symmetrize(N.tocsc())
        self._cut_ineq_matrices.append(-N_sym)
        self._cut_bineq_upperbounds.append(0.0)

    def _build_relu_envelope_cuts(self) -> None:
        r"""
        Append the linear ReLU convex-hull inequalities on the first-moment
        (vector) variables for every registered ReLU step.

        Unlike the *lifted* triangle cut (a quadratic-in-X product), these are
        linear in the first moments ``X[0, .]`` and so restrict the relaxation
        in a way the lifted equality ``y^2 = (a^T x + b) y`` plus PSD does not.
        For each projected arg row ``i`` with ``hat_x_i = a_i^T x + b_i``:

        - Lower envelope ``y_i >= hat_x_i`` (valid for every row, since
          ``ReLU(t) >= t``).
        - Upper chord ``y_i <= E_ii (hat_x_i - l_i)`` with
          ``E_ii = u_i / (u_i - l_i)``, emitted only on mixed-bound rows
          (``l_i < 0 < u_i``, finite ``u_i``) where the secant of ``max(0, .)``
          through ``(l_i, 0)`` and ``(u_i, u_i)`` dominates the true map. On a
          passthrough row (``l_i >= 0``) this chord would cut the true point, so
          it is skipped (gated exactly like the triangle cut).

        ``y_i >= 0`` is intentionally omitted: the y-leaf box bound (lb >= 0)
        together with the PSD ``2x2`` minor already implies it.

        Bounds come from `_relu_arg_bounds` (OBBT-tightened ``step.lb``/
        ``step.ub`` intersected with fresh propagation; parity with
        ``_build_triangle_cuts``).
        """
        n_dim = self.get_sdp_var_dim()
        for step in self._relu_steps:
            arg = step.arg
            arg_lb, arg_ub = self._relu_arg_bounds(step)
            A_mat, x_idx_bounds = self.linexpr_to_Amat_and_bounds(arg)
            b_vec = np.asarray(arg.const)

            y_leaf = step.output_leaves[0]
            rhs_to_y_pos = {i: j for j, i in enumerate(sorted(step.proj_indices))}
            y_start, _y_end = self.var_idx_map[y_leaf]
            x_row0_idx = process_1d_ranges(x_idx_bounds, row=True)

            for i in range(arg.ndim):
                if i not in rhs_to_y_pos:
                    continue
                a_i = np.asarray(A_mat[i, :]).flatten()
                b_i = float(b_vec[i])
                y_i_idx = y_start + rhs_to_y_pos[i]

                # Lower envelope: y_i - (a_i^T x + b_i) >= 0.
                N = spa.lil_matrix((n_dim, n_dim))
                N[y_i_idx, 0] = 1.0
                N[x_row0_idx] = -a_i
                N[0, 0] = -b_i
                self._cut_ineq_matrices.append(-symmetrize(N.tocsc()))
                self._cut_bineq_upperbounds.append(0.0)

                # Upper chord (mixed rows only): E_ii (a_i^T x + b_i - l_i) - y_i >= 0.
                lb_i = arg_lb[i]
                ub_i = arg_ub[i]
                if lb_i >= 0 or ub_i <= 0 or not np.isfinite(ub_i):
                    continue
                E_ii = ub_i / (ub_i - lb_i)
                N = spa.lil_matrix((n_dim, n_dim))
                N[x_row0_idx] = E_ii * a_i
                N[0, 0] = E_ii * b_i - E_ii * lb_i
                N[y_i_idx, 0] = -1.0
                self._cut_ineq_matrices.append(-symmetrize(N.tocsc()))
                self._cut_bineq_upperbounds.append(0.0)

    def _relu_output_components(self) -> list[dict]:
        """
        Flatten all registered ReLU steps into per-output-component records.

        Each record describes one projected ReLU output ``z = relu(v)`` at a
        single coordinate:

        - ``y_idx``  : its SDP matrix index (used for the ``z`` factor),
        - ``a``      : coefficient row of the affine input ``v = a^T x + beta``,
        - ``beta``   : constant of ``v`` at this row,
        - ``x_bounds``: the input leaves' SDP index ranges (parallel to ``a``).

        Used by `_build_cross_cuts`. The arg coefficients are read via
        ``linexpr_to_Amat_and_bounds`` once per step.
        """
        comps = []
        for step in self._relu_steps:
            A_mat, x_idx_bounds = self.linexpr_to_Amat_and_bounds(step.arg)
            b_vec = np.asarray(step.arg.const)
            y_leaf = step.output_leaves[0]
            y_start, _y_end = self.var_idx_map[y_leaf]
            rhs_to_y_pos = {i: j for j, i in enumerate(sorted(step.proj_indices))}
            for rb, y_pos in rhs_to_y_pos.items():
                comps.append(
                    {
                        "y_idx": y_start + y_pos,
                        "a": np.asarray(A_mat[rb, :]).flatten(),
                        "beta": float(b_vec[rb]),
                        "x_bounds": x_idx_bounds,
                    }
                )
        return comps

    def _build_cross_cuts(self) -> None:
        r"""
        Append the PEP "cross" interpolation cuts ``z_a r_b <= 0``.

        For any two ReLU output components ``a`` and ``b`` (across all registered
        ReLU steps), ``z_a >= 0`` and the residual ``r_b = v_b - z_b <= 0``, so
        ``z_a r_b <= 0`` is valid (PDF eq 3; for ReLU = orthant projection it is
        the indicator-function specialization of the subgradient inequality). It
        couples *different* steps, which the per-step encoding does not. With
        ``r_b = v_b - z_b`` and ``v_b = a_b^T x + beta_b`` this is linear in ``X``:

        ``z_a r_b = sum_p a_b[p] X[ya, x_p] + beta_b X[0, ya] - X[ya, yb] <= 0``.

        The diagonal ``a == b`` (``ya == yb``) is the complementarity identity
        ``z_a r_a = 0`` already encoded exactly by `canonicalize_relu`, so it
        is skipped. Each cut registers the minimal PSD cone ``{0, ya, yb} U x_b``
        so its second-moment entries bind without ``full_psd_cone``.

        TODO(scaling): this emits O(N^2) cuts and cones, where ``N`` is the total
        number of scalar ReLU outputs across the unrolled network, so cost grows
        quadratically with depth. Fine at current scales (a few seconds for the
        tested instances); large networks will want a budget/cap or per-step
        batching of the (a, b) pairs.
        """
        n_dim = self.get_sdp_var_dim()
        comps = self._relu_output_components()

        for comp_a in comps:
            ya_idx = comp_a["y_idx"]
            for comp_b in comps:
                yb_idx = comp_b["y_idx"]
                if ya_idx == yb_idx:
                    # Complementarity diagonal z_a r_a = 0 (already exact); skip.
                    continue

                a_b = comp_b["a"]
                beta_b = comp_b["beta"]
                xb_idx_bounds = comp_b["x_bounds"]

                # Couple {0, ya, yb} U x_b so X[ya, yb] and X[ya, x_p] are
                # constrained by a PSD minor (else the cut is vacuous).
                self.add_psd_cone_indices(
                    [(ya_idx, ya_idx + 1), (yb_idx, yb_idx + 1)] + xb_idx_bounds
                )

                M = spa.lil_matrix((n_dim, n_dim))
                # z_a * (a_b^T x): single y-row times the x-concat columns.
                M[process_2d_ranges([(ya_idx, ya_idx + 1)], xb_idx_bounds)] = a_b
                # beta_b * z_a (linear).
                M[ya_idx, 0] = beta_b
                # - z_a z_b (second moment).
                M[ya_idx, yb_idx] = -1.0

                # Cut is already <= 0: push +symmetrize(M) with ub 0 (no negation).
                self._cut_ineq_matrices.append(symmetrize(M.tocsc()))
                self._cut_bineq_upperbounds.append(0.0)

    def _cross_innerprod_block(self, expr1: LinearExpression, expr2: LinearExpression):
        r"""
        Build the unsymmetrized lifted block for the inner product
        ``<expr1, expr2> = <E w + e, F w + f>``.

        Expanding ``(E w + e)^T (F w + f) = w^T E^T F w + e^T F w + f^T E w +
        e^T f`` and reading each monomial off the lifted ``X`` (``X[p, q]`` the
        second moment, ``X[0, p]`` the first moment, ``X[0, 0] = 1``):

        - ``E^T F``    -> ``M[idxE, idxF]``  (the second-moment block),
        - ``f^T E``    -> ``X[0, idxE]``     (first-moment row),
        - ``e^T F``    -> ``X[0, idxF]``     (first-moment row),
        - ``e^T f``    -> ``M[0, 0]``.

        Generalizes `_apply_two_norm_objective` (the self case
        ``<Ax+b, Ax+b>``) to two *different* expressions. The two first-moment
        contributions are accumulated **additively** over the union of their SDP
        indices and scattered once, so a leaf shared by ``expr1`` and ``expr2``
        does not get one write clobbered by the other. The caller symmetrizes.

        Returns
        -------
        scipy.sparse.lil_matrix
            The unsymmetrized block ``M`` with ``tr(M X) = <expr1, expr2>``.
        """
        n_dim = self.get_sdp_var_dim()
        E_mat, idxE = self.linexpr_to_Amat_and_bounds(expr1)
        F_mat, idxF = self.linexpr_to_Amat_and_bounds(expr2)
        e = np.asarray(expr1.const).reshape(-1, 1)
        f = np.asarray(expr2.const).reshape(-1, 1)

        M = spa.lil_matrix((n_dim, n_dim))

        # Second-moment block E^T F over (idxE rows, idxF cols).
        M[process_2d_ranges(idxE, idxF)] = E_mat.T @ F_mat

        # First moments: accumulate f^T E (on idxE) and e^T F (on idxF) additively
        # into a single dense row over the union of SDP indices, then scatter once.
        # asarray guards against np.matrix (linexpr_to_Amat_and_bounds may return a
        # matrix), whose .flatten() stays 2-D and breaks integer indexing below.
        fTE = np.asarray(f.T @ E_mat).flatten()  # length = sum of idxE block widths
        eTF = np.asarray(e.T @ F_mat).flatten()  # length = sum of idxF block widths
        row0 = {}
        for bounds, vec in ((idxE, fTE), (idxF, eTF)):
            offset = 0
            for s, end in bounds:
                for col in range(s, end):
                    row0[col] = row0.get(col, 0.0) + vec[offset]
                    offset += 1
        for col, val in row0.items():
            M[0, col] = val

        M[0, 0] = float(np.dot(np.asarray(e).flatten(), np.asarray(f).flatten()))
        return M

    def _build_subgrad_mono_cuts(self) -> None:
        r"""
        Append the PEP subgradient-monotonicity cuts
        ``<s^i/tau_i - s^j/tau_j, z^i> >= 0`` over ordered prox pairs.

        For prox steps ``i, j`` with output ``z``, input ``v``, threshold ``tau``
        and subgradient ``s = v - z`` (PDF eq 4, normalized by ``tau`` so the
        subgradient lives in the subdifferential of the same homogeneous
        function). Expanding,

        ``<s^i/tau_i - s^j/tau_j, z^i>
            = (1/tau_i)[<v^i, z^i> - <z^i, z^i>]
            - (1/tau_j)[<v^j, z^i> - <z^j, z^i>]``,

        each term a lifted inner product built by `_cross_innerprod_block`.
        On the orthant (``tau = 1``, ``z`` a ReLU output) this reduces to the
        cross cut; on the soft-threshold (``tau = lambda``, ``z`` composite) it is
        a distinct coupling. Each cut registers a PSD cone over the indices of
        ``z^i, v^i, z^j, v^j`` so the cross-step moments bind (this is the smallest
        cone that couples the involved leaves, but for composite proxes it spans
        several index ranges, so it is not as small as the cross cut's cone).

        The cut is valid only when ``partial f_j(0) subseteq partial f_i(0)``, so
        pairs are restricted to proxes of the same function (``func``) and the
        same dimension. Skipping mismatched-dimension orthant pairs loses nothing:
        there the cut is a sum of per-coordinate cross cuts (``cross_cut``).
        """
        prox = self._prox_steps
        for i, pi in enumerate(prox):
            zi, vi, tau_i = pi["z"], pi["v"], pi["tau"]
            if tau_i < 1e-9:
                continue
            for j, pj in enumerate(prox):
                if i == j:
                    continue
                zj, vj, tau_j = pj["z"], pj["v"], pj["tau"]
                if tau_j < 1e-9:
                    continue
                # Same sublinear function and dimension only (else the cut is invalid).
                if pi["func"] != pj["func"] or zi.ndim != zj.ndim:
                    continue

                # Couple all involved leaves so the cross-step moments bind.
                _, idx_zi = self.linexpr_to_Amat_and_bounds(zi)
                _, idx_vi = self.linexpr_to_Amat_and_bounds(vi)
                _, idx_zj = self.linexpr_to_Amat_and_bounds(zj)
                _, idx_vj = self.linexpr_to_Amat_and_bounds(vj)
                self.add_psd_cone_indices(idx_zi + idx_vi + idx_zj + idx_vj)

                # Accumulate the four (unsymmetrized) inner-product blocks.
                M = (
                    (1.0 / tau_i) * self._cross_innerprod_block(vi, zi)
                    - (1.0 / tau_i) * self._cross_innerprod_block(zi, zi)
                    - (1.0 / tau_j) * self._cross_innerprod_block(vj, zi)
                    + (1.0 / tau_j) * self._cross_innerprod_block(zj, zi)
                )

                # Cut is >= 0: push -symmetrize(M) with ub 0.
                self._cut_ineq_matrices.append(-symmetrize(M.tocsc()))
                self._cut_bineq_upperbounds.append(0.0)

    @staticmethod
    def _box_bound_scalar(bound, j):
        """Scalar j-th box bound: a float for constant arrays, or the j-th slice
        (a length-1 LinearExpression) when the bound is parametric."""
        if isinstance(bound, LinearExpression):
            return bound[j]
        return float(np.atleast_1d(bound)[j])

    @staticmethod
    def _box_bound_eq(x, y) -> bool:
        """Whether two box bounds are 'the same' for cross-step pairing: identity
        for parametric (LinearExpression) bounds, value-equality for constants."""
        if isinstance(x, LinearExpression) or isinstance(y, LinearExpression):
            return x is y
        return np.array_equal(np.atleast_1d(x), np.atleast_1d(y))

    def _same_box(self, a: dict, b: dict) -> bool:
        """Two box-prox steps project onto the same box ``[l, u]``."""
        return self._box_bound_eq(a["l"], b["l"]) and self._box_bound_eq(a["u"], b["u"])

    def _emit_scalar_product_cut(self, e1: LinearExpression, e2: LinearExpression, sense: str):
        r"""
        Emit the per-component cut ``e1 . e2  {<=,>=}  0`` where ``e1``, ``e2`` are
        scalar (length-1) affine expressions. ``tr(M X) = e1 . e2`` via
        `_cross_innerprod_block` (inner product of 1-vectors = the scalar
        product). Registers the minimal PSD cone over the involved leaves so the
        bilinear term binds, then pushes ``+sym`` (for ``<=``) or ``-sym`` (for
        ``>=``) with ub 0 — the standard convention.
        """
        _, idx1 = self.linexpr_to_Amat_and_bounds(e1)
        _, idx2 = self.linexpr_to_Amat_and_bounds(e2)
        self.add_psd_cone_indices(idx1 + idx2)
        M = self._cross_innerprod_block(e1, e2)
        sym = symmetrize(M.tocsc())
        self._cut_ineq_matrices.append(sym if sense == "<=" else -sym)
        self._cut_bineq_upperbounds.append(0.0)

    def _build_box_normalcone_cuts(self) -> None:
        r"""
        Append the PEP box-projection normal-cone cuts for satlin steps (the box
        analog of the ReLU cross cut; emitted under ``cross_cut``).

        For a box projection ``z_j = clip(v_j, l_j, u_j)`` with residual
        ``r_j = v_j - z_j`` (``r_j in N_{[l_j,u_j]}(z_j)``):

        - **C1/C2 (within step):** ``r_j (z_j - l_j) >= 0`` and ``r_j (z_j - u_j) >= 0``
          (the residual vs the two box corners).
        - **C3 (cross step, same coord, same box):** ``r_j^b (z_j^a - z_j^b) <= 0``
          for any other box-projection output ``z^a`` onto the same box (since
          ``z_j^a in [l_j, u_j]`` and ``r_j^b`` is in the normal cone at ``z_j^b``).

        Cuts are **per-coordinate** (the box ``z``/``r`` are not sign-definite, so
        cross-coordinate products are NOT valid — unlike the ReLU cone cross cut).
        """
        boxes = self._box_prox_steps
        for b_idx, b in enumerate(boxes):
            zb, vb = b["z"], b["v"]
            rb = vb - zb
            n = zb.ndim
            for j in range(n):
                rbj = rb[j]
                lj = self._box_bound_scalar(b["l"], j)
                uj = self._box_bound_scalar(b["u"], j)
                # C1: r_j (z_j - l_j) >= 0 ; C2: r_j (z_j - u_j) >= 0
                self._emit_scalar_product_cut(rbj, zb[j] - lj, ">=")
                self._emit_scalar_product_cut(rbj, zb[j] - uj, ">=")

            # C3: r_j^b (z_j^a - z_j^b) <= 0 over ordered same-box pairs.
            for a_idx, a in enumerate(boxes):
                if a_idx == b_idx or not self._same_box(a, b):
                    continue
                za = a["z"]
                for j in range(n):
                    self._emit_scalar_product_cut(rb[j], za[j] - zb[j], "<=")

    def _build_box_mono_cuts(self) -> None:
        r"""
        Append the PEP box-projection firm-nonexpansiveness / monotonicity cuts for
        satlin steps (the box analog of the subgradient-monotonicity cut; emitted
        under ``subgrad_mono_cut``).

        For two box projections onto the **same** box, per coordinate ``j``:
        ``(z_j^a - z_j^b)(r_j^a - r_j^b) >= 0`` with ``r = v - z`` — equivalently
        the firm-nonexpansiveness inequality ``<z^a-z^b, v^a-v^b> >= ||z^a-z^b||^2``
        read per coordinate. Valid because the scalar clip and its residual are both
        monotone nondecreasing. Unordered pairs (the cut is symmetric in ``a, b``).
        """
        boxes = self._box_prox_steps
        for a_idx, b_idx in combinations(range(len(boxes)), 2):
            a, b = boxes[a_idx], boxes[b_idx]
            if not self._same_box(a, b):
                continue
            za, va = a["z"], a["v"]
            zb, vb = b["z"], b["v"]
            ra, rb = va - za, vb - zb
            for j in range(za.ndim):
                self._emit_scalar_product_cut(za[j] - zb[j], ra[j] - rb[j], ">=")

    def solve_model(self, steps=None, convex_hull_cuts=False, **solver_kwargs):
        """Solve the SDP problem using the configured solver.

        Creates a copy of the canonicalizer, applies the objective to the copy,
        solves it, and keeps a reference for future use. This prevents objective
        constraints from accumulating across multiple solve calls.

        Parameters
        ----------
        steps : list, optional
            Unused by SDP, accepted for API compatibility.
        convex_hull_cuts : bool, optional
            Unused by SDP, accepted for API compatibility.
        **solver_kwargs
            Native solver options forwarded verbatim to the backend
            (e.g. ``max_iter=500``, ``tol_gap_abs=1e-9`` for
            Clarabel/cvxpy). They override construction-time ``params``
            for this call. A ``ValueError`` is raised if the backend
            rejects an option.

        Returns
        -------
        float
            The objective value.
        """
        # Create copy to apply objective without modifying original
        self._solved_copy = self.copy()

        # Apply objective to the copy
        if self._objective_spec is not None:
            obj_type = self._objective_spec[0]
            match obj_type:
                case "two_norm":
                    _, expr_list = self._objective_spec
                    self._apply_two_norm_objective(self._solved_copy, expr_list)
                case "one_norm":
                    _, expr_list = self._objective_spec
                    self._apply_one_norm_objective(self._solved_copy, expr_list)
                case "infinity_norm":
                    _, expr_list = self._objective_spec
                    self._apply_infinity_norm_objective(self._solved_copy, expr_list)

        # Solve using the copy's data
        return self._solved_copy._solve_internal(**solver_kwargs)

    def _solve_internal(self, **solver_kwargs):
        """
        Actually solve using this canonicalizer's data.

        This is the internal solve method called on the copy after
        the objective has been applied.

        Parameters
        ----------
        **solver_kwargs
            Native solver options forwarded verbatim to the backend.

        Returns
        -------
        float
            The objective value.
        """
        match self.solver_name:
            case "clarabel":
                return self._solve_via_clarabel_internal(**solver_kwargs)
            case "cvxpy":
                return self._solve_via_cvxpy_internal(**solver_kwargs)
            case "cuclarabel":
                return self._solve_via_cuclarabel_internal(**solver_kwargs)
            case _:
                raise ValueError(f"Unknown SDP solver: {self.solver_name}")

    def _ensure_objective_applied(self):
        """
        Ensure the objective has been applied to this canonicalizer.

        If the objective specification is stored but not yet applied,
        apply it now. This is used for backward compatibility with
        direct solver calls.
        """
        if self._objective_spec is not None and self.C_obj is None:
            obj_type = self._objective_spec[0]
            match obj_type:
                case "two_norm":
                    _, expr_list = self._objective_spec
                    self._apply_two_norm_objective(self, expr_list)
                case "one_norm":
                    _, expr_list = self._objective_spec
                    self._apply_one_norm_objective(self, expr_list)
                case "infinity_norm":
                    _, expr_list = self._objective_spec
                    self._apply_infinity_norm_objective(self, expr_list)

    def solve_via_cvxpy(self, **solver_kwargs):
        """
        Solve the SDP using the CVXPY modeling interface.

        Constructs a CVXPY problem from the stored constraint matrices
        and solves it using the Clarabel backend via CVXPY.

        Parameters
        ----------
        **solver_kwargs
            Native solver options forwarded verbatim to ``Problem.solve``.

        Returns
        -------
        float
            The optimal objective value.
        """
        self._ensure_objective_applied()
        return self._solve_via_cvxpy_internal(**solver_kwargs)

    def _solve_via_cvxpy_internal(self, **solver_kwargs):
        """Internal CVXPY solve without checking objective."""
        self._rebuild_cuts_if_dirty()
        Aineq_all = self.Aineq_matrices + self._cut_ineq_matrices
        bineq_all = self.bineq_upperbounds + self._cut_bineq_upperbounds

        dim = self.get_sdp_var_dim()
        X = cp.Variable((dim, dim), symmetric=True)
        obj = cp.Maximize(cp.trace(self.C_obj @ X))
        constraints = []
        for A, b in zip(self.Aeq_matrices, self.beq_consts):
            constraints += [
                cp.trace(A @ X) == b,
            ]
        for A, b in zip(Aineq_all, bineq_all):
            constraints += [
                cp.trace(A @ X) <= b,
            ]
        # Add PSD constraints for each structured cone (matching Clarabel path)
        for cone_indices in self.psd_cone_indices:
            idx = sorted(cone_indices)
            constraints += [X[np.ix_(idx, idx)] >> 0]
        self.last_problem = prob = cp.Problem(obj, constraints)
        # `solver` is fixed to CLARABEL below (it is a named arg of cvxpy's
        # Problem.solve, so a user `solver=` kwarg would collide there);
        # solve-time options override construction-time params.
        merged = {**(self.solver_params or {}), **solver_kwargs}
        try:
            res = prob.solve(solver=cp.CLARABEL, **merged)
        except TypeError as e:
            raise ValueError(f"Invalid cvxpy/Clarabel solver option in {merged}: {e}") from e
        except cp.error.SolverError as e:
            raise RuntimeError(f"CVXPY/Clarabel failed to solve the SDP: {e}") from e
        if (
            prob.status not in (cp.OPTIMAL, cp.OPTIMAL_INACCURATE)
            or res is None
            or not np.isfinite(res)
        ):
            raise RuntimeError(f"CVXPY terminated with {prob.status}; no accepted SDP bound.")
        if prob.status == cp.OPTIMAL_INACCURATE:
            warnings.warn(
                "CVXPY terminated optimal_inaccurate; the SDP bound holds only to the "
                "solver's reduced tolerances.",
                stacklevel=2,
            )
        # For this linear maximization SDP, the dual objective is b^T lambda.
        # PSD constraints have zero right-hand sides. Keep the Problem for
        # status, primal, and dual inspection; acceptance is numerical.
        rhs = list(self.beq_consts) + list(bineq_all)
        if any(c.dual_value is None for c in constraints[: len(rhs)]):
            raise RuntimeError("CVXPY returned no SDP dual solution.")
        bound = float(sum(b * float(c.dual_value) for b, c in zip(rhs, constraints)))
        if not np.isfinite(bound):
            raise RuntimeError("CVXPY returned a nonfinite SDP dual objective.")
        return bound

    def solve_via_clarabel(self, **solver_kwargs):
        """
        Solve the SDP using the native Clarabel interface.

        Directly invokes the Clarabel solver without going through CVXPY,
        which can be more efficient for large problems.

        Parameters
        ----------
        **solver_kwargs
            Native Clarabel settings forwarded verbatim to the solver.

        Returns
        -------
        float
            The optimal objective value.
        """
        self._ensure_objective_applied()
        return self._solve_via_clarabel_internal(**solver_kwargs)

    def _solve_via_clarabel_internal(self, **solver_kwargs):
        """Internal Clarabel solve without checking objective."""
        self._rebuild_cuts_if_dirty()
        Aineq_all = self.Aineq_matrices + self._cut_ineq_matrices
        bineq_all = self.bineq_upperbounds + self._cut_bineq_upperbounds

        from fomtools.solvers.clarabel import ClarabelSolver

        solver = ClarabelSolver(solver_params=self.solver_params)
        solver.setup_clarabel_data(
            self.get_sdp_var_dim(),
            self.C_obj,
            self.Aeq_matrices,
            self.beq_consts,
            Aineq_all,
            bineq_all,
            self.psd_cone_indices,
        )
        self.last_solver = solver
        return solver.solve(**solver_kwargs)

    def solve_via_cuclarabel(self, **solver_kwargs):
        """
        Solve the SDP using GPU-accelerated Clarabel.

        Uses the CUDA-enabled Clarabel solver for GPU acceleration.
        Requires CuPy and Julia with CUDA.jl and Clarabel.jl.

        Parameters
        ----------
        **solver_kwargs
            Native Clarabel settings forwarded verbatim to the solver.

        Returns
        -------
        float
            The optimal objective value.

        Raises
        ------
        ValueError
            If CUDA dependencies (cupy) are not available.
        """
        self._ensure_objective_applied()
        return self._solve_via_cuclarabel_internal(**solver_kwargs)

    def _solve_via_cuclarabel_internal(self, **solver_kwargs):
        """Internal CuClarabel solve without checking objective."""
        self._rebuild_cuts_if_dirty()
        Aineq_all = self.Aineq_matrices + self._cut_ineq_matrices
        bineq_all = self.bineq_upperbounds + self._cut_bineq_upperbounds

        from fomtools.solvers.cuclarabel import CuClarabelSolver

        solver = CuClarabelSolver(solver_params=self.solver_params)
        solver.setup_clarabel_data(
            self.get_sdp_var_dim(),
            self.C_obj,
            self.Aeq_matrices,
            self.beq_consts,
            Aineq_all,
            bineq_all,
            self.psd_cone_indices,
        )
        self.last_solver = solver
        return solver.solve(**solver_kwargs)

__init__

__init__(solver_params=None, solver='clarabel', rlt_cuts=False, triangle_cuts=False, ineq_product_cuts=True, ineq_box_cuts=True, ineq_square_cuts=True, relu_envelope_cuts=False, rlt_all_pairs=False, full_psd_cone=False, cross_cut=False, subgrad_mono_cut=False)

Parameters:

Name	Type	Description	Default
solver_params	dict	Dictionary of solver-specific parameters to pass to the backend.	None
solver	str	Name of the SDP solver to use. Options: 'clarabel', 'cvxpy', 'cuclarabel'.	'clarabel'
rlt_cuts	bool	If True, add RLT (Reformulation-Linearization Technique) cuts between cone-mate SDP indices to tighten the relaxation.	False
triangle_cuts	bool	If True, add lifted triangle cuts for ReLU steps with mixed-bound output dimensions to tighten the relaxation.	False
ineq_product_cuts	bool	For each inequality constraint Ax + b <= 0, add the off-diagonal products g_i g_j >= 0 (valid since each g_i <= 0).	True
ineq_box_cuts	bool	For each inequality constraint, add the RLT products of the constraint with the variable box bounds (g_i (x_p - l_p) <= 0 and g_i (u_p - x_p) <= 0).	True
ineq_square_cuts	bool	For each inequality constraint, add the self-square upper bound g_i^2 <= g_i_lower * g_i using a propagated lower bound on g_i.	True
relu_envelope_cuts	bool	If True, add the linear ReLU convex-hull inequalities on the first-moment (vector) variables: the lower envelope y_i >= a_i^T x + b_i for every projected row and, on mixed-bound rows (l_i < 0 < u_i), the upper chord y_i <= E_ii (a_i^T x + b_i - l_i) with E_ii = u_i / (u_i - l_i). The y_i >= 0 line is omitted because the y-leaf box bound plus the PSD constraint already imply it.	False
rlt_all_pairs	bool	If True, RLT every pair of variables (not just cone-mates) by registering a cheap {0, p, q} 3x3 PSD minor for each pair so the all-pairs McCormick cuts actually bind, then firing _build_rlt_cuts on the resulting (now all-pairs) cone structure. This is the middle tier between cone-mate-only rlt_cuts and the dense full_psd_cone: baseline >= rlt_cuts >= rlt_all_pairs >= full_psd_cone + rlt_cuts. Pairs already inside a larger cone add no new minor (add_psd_cone_indices drops subsumed cones), so it only couples genuinely-uncoupled pairs. Implies RLT cut emission on its own (no need to also set rlt_cuts).	False
full_psd_cone	bool	If True, union all PSD cones into a single full moment-matrix block at solve time. Combined with rlt_cuts=True this yields the full Shor + RLT relaxation (RLT then fires on every variable pair, not just cone-mates). Tightest but most expensive configuration.	False
cross_cut	bool	If True, add the PEP "cross" interpolation cuts z_a r_b <= 0 over ordered pairs of ReLU output components, where z_a >= 0 is a ReLU output and r_b = v_b - z_b <= 0 is another ReLU's residual (valid for every orthant projection). Couples different steps, which the per-step encoding does not. Each emitted cut self-registers the minimal PSD cone over the indices it touches so it binds without full_psd_cone (Clarabel handles many small cones efficiently; the add_psd_cone_indices antichain subsumes a cone already covered by a larger one). For saturated-linear (box projection) steps this flag also emits the box normal-cone cuts (per coordinate j: r_j(z_j-l_j)>=0, r_j(z_j-u_j)>=0 within a step, and r_j^b(z_j^a-z_j^b)<=0 across same-box steps), via _build_box_normalcone_cuts.	False
subgrad_mono_cut	bool	If True, add the PEP subgradient-monotonicity cuts <s^i/tau_i - s^j/tau_j, z^i> >= 0 over ordered pairs of registered prox steps, with s = v - z and tau the prox threshold (tau = 1 for ReLU/orthant, tau = lambda for soft-threshold). Normalizing by tau places the subgradient in d||.||_1 so steps of the same (homogeneous) function are comparable. On the orthant this reduces to the cross cut; on the soft-threshold (ell-1 prox) it is a genuinely distinct coupling. Self-registers the PSD cone over the leaves each cut couples (without needing full_psd_cone); note that for a composite prox (e.g. soft-threshold z = relu_pos - relu_neg) this cone spans several index ranges, so it is not as tight as the cross cut's truly-minimal {0, ya, yb} U x_b. For saturated-linear (box projection) steps this flag also emits the box firm-nonexpansiveness/monotonicity cuts (per coordinate: (z_j^a-z_j^b)(r_j^a-r_j^b)>=0 across same-box steps), via _build_box_mono_cuts.	False

Source code in fomtools/canonicalizers/sdp.py

def __init__(
    self,
    solver_params=None,
    solver="clarabel",
    rlt_cuts=False,
    triangle_cuts=False,
    ineq_product_cuts=True,
    ineq_box_cuts=True,
    ineq_square_cuts=True,
    relu_envelope_cuts=False,
    rlt_all_pairs=False,
    full_psd_cone=False,
    cross_cut=False,
    subgrad_mono_cut=False,
):
    """
    Initialize the SDP canonicalizer.

    Parameters
    ----------
    solver_params : dict, optional
        Dictionary of solver-specific parameters to pass to the backend.
    solver : str, default='clarabel'
        Name of the SDP solver to use. Options: 'clarabel', 'cvxpy',
        'cuclarabel'.
    rlt_cuts : bool, default=False
        If True, add RLT (Reformulation-Linearization Technique) cuts
        between cone-mate SDP indices to tighten the relaxation.
    triangle_cuts : bool, default=False
        If True, add lifted triangle cuts for ReLU steps with mixed-bound
        output dimensions to tighten the relaxation.
    ineq_product_cuts : bool, default=True
        For each inequality constraint ``Ax + b <= 0``, add the off-diagonal
        products ``g_i g_j >= 0`` (valid since each ``g_i <= 0``).
    ineq_box_cuts : bool, default=True
        For each inequality constraint, add the RLT products of the constraint
        with the variable box bounds (``g_i (x_p - l_p) <= 0`` and
        ``g_i (u_p - x_p) <= 0``).
    ineq_square_cuts : bool, default=True
        For each inequality constraint, add the self-square upper bound
        ``g_i^2 <= g_i_lower * g_i`` using a propagated lower bound on ``g_i``.
    relu_envelope_cuts : bool, default=False
        If True, add the linear ReLU convex-hull inequalities on the
        first-moment (vector) variables: the lower envelope ``y_i >= a_i^T x
        + b_i`` for every projected row and, on mixed-bound rows
        (``l_i < 0 < u_i``), the upper chord ``y_i <= E_ii (a_i^T x + b_i -
        l_i)`` with ``E_ii = u_i / (u_i - l_i)``. The ``y_i >= 0`` line is
        omitted because the y-leaf box bound plus the PSD constraint already
        imply it.
    rlt_all_pairs : bool, default=False
        If True, RLT *every* pair of variables (not just cone-mates) by
        registering a cheap ``{0, p, q}`` 3x3 PSD minor for each pair so the
        all-pairs McCormick cuts actually bind, then firing ``_build_rlt_cuts``
        on the resulting (now all-pairs) cone structure. This is the middle
        tier between cone-mate-only ``rlt_cuts`` and the dense
        ``full_psd_cone``: ``baseline >= rlt_cuts >= rlt_all_pairs >=
        full_psd_cone + rlt_cuts``. Pairs already inside a larger cone add no
        new minor (``add_psd_cone_indices`` drops subsumed cones), so it only
        couples genuinely-uncoupled pairs. Implies RLT cut emission on its own
        (no need to also set ``rlt_cuts``).
    full_psd_cone : bool, default=False
        If True, union all PSD cones into a single full moment-matrix block
        at solve time. Combined with ``rlt_cuts=True`` this yields the full
        Shor + RLT relaxation (RLT then fires on every variable pair, not
        just cone-mates). Tightest but most expensive configuration.
    cross_cut : bool, default=False
        If True, add the PEP "cross" interpolation cuts ``z_a r_b <= 0`` over
        ordered pairs of ReLU output components, where ``z_a >= 0`` is a ReLU
        output and ``r_b = v_b - z_b <= 0`` is another ReLU's residual (valid
        for every orthant projection). Couples *different* steps, which the
        per-step encoding does not. Each emitted cut self-registers the minimal
        PSD cone over the indices it touches so it binds without
        ``full_psd_cone`` (Clarabel handles many small cones efficiently; the
        ``add_psd_cone_indices`` antichain subsumes a cone already covered by a
        larger one). For saturated-linear (box projection) steps this flag also
        emits the box normal-cone cuts (per coordinate ``j``: ``r_j(z_j-l_j)>=0``,
        ``r_j(z_j-u_j)>=0`` within a step, and ``r_j^b(z_j^a-z_j^b)<=0`` across
        same-box steps), via `_build_box_normalcone_cuts`.
    subgrad_mono_cut : bool, default=False
        If True, add the PEP subgradient-monotonicity cuts
        ``<s^i/tau_i - s^j/tau_j, z^i> >= 0`` over ordered pairs of registered
        prox steps, with ``s = v - z`` and ``tau`` the prox threshold
        (``tau = 1`` for ReLU/orthant, ``tau = lambda`` for soft-threshold).
        Normalizing by ``tau`` places the subgradient in ``d||.||_1`` so steps
        of the same (homogeneous) function are comparable. On the orthant this
        reduces to the cross cut; on the soft-threshold (ell-1 prox) it is a
        genuinely distinct coupling. Self-registers the PSD cone over the leaves
        each cut couples (without needing ``full_psd_cone``); note that for a
        composite prox (e.g. soft-threshold ``z = relu_pos - relu_neg``) this
        cone spans several index ranges, so it is not as tight as the cross
        cut's truly-minimal ``{0, ya, yb} U x_b``. For saturated-linear (box
        projection) steps this flag also
        emits the box firm-nonexpansiveness/monotonicity cuts (per coordinate:
        ``(z_j^a-z_j^b)(r_j^a-r_j^b)>=0`` across same-box steps), via
        `_build_box_mono_cuts`.
    """
    super(SDP, self).__init__(solver_params, solver)
    self.solver_name = solver.lower()  # Store for dispatch

    # cuClarabel cannot yet apply custom solver options (its Julia settings
    # are hardcoded). Reject construction-time params eagerly here rather
    # than surprising the user with a failure only at solve time.
    if self.solver_name == "cuclarabel" and self.solver_params:
        raise ValueError(
            "cuClarabel does not support custom solver options yet; "
            f"remove params={self.solver_params} from the Verifier."
        )
    self.range_marker = 1
    self.var_idx_map = {}

    self.Aeq_matrices = []
    self.beq_consts = []
    self.Aineq_matrices = []
    self.bineq_upperbounds = []
    self.psd_cone_indices = []
    self.bound_constraint_indices = {}  # var -> (start_idx, end_idx)

    self.rlt_cuts = rlt_cuts
    self.triangle_cuts = triangle_cuts
    self.ineq_product_cuts = ineq_product_cuts
    self.ineq_box_cuts = ineq_box_cuts
    self.ineq_square_cuts = ineq_square_cuts
    self.relu_envelope_cuts = relu_envelope_cuts
    self.rlt_all_pairs = rlt_all_pairs
    self.full_psd_cone = full_psd_cone
    self.cross_cut = cross_cut
    self.subgrad_mono_cut = subgrad_mono_cut
    self._cuts_dirty = True
    self._cut_ineq_matrices = []
    self._cut_bineq_upperbounds = []
    self._relu_steps = []  # registry for triangle / envelope / cross cuts
    self._prox_steps = []  # registry for subgradient-monotonicity cuts
    self._box_prox_steps = []  # registry for satlin (box projection) PEP cuts
    self._ineq_constraints = []  # (constraint, add_product_cuts) for inequality cuts

    self.C_obj = None
    self._objective_spec = None
    self._solved_copy = None
    self._initialize_corner_constraint()

add_constraint

add_constraint(constraint: Constraint) -> None

Add a constraint to the SDP instance.

Dispatches to the appropriate implementation based on the constraint operator type (equality or inequality).

Parameters:

Name	Type	Description	Default
constraint	Constraint	Constraint object to add to the SDP instance.	required

Source code in fomtools/canonicalizers/sdp.py

def add_constraint(self, constraint: Constraint) -> None:
    """
    Add a constraint to the SDP instance.

    Dispatches to the appropriate implementation based on the constraint
    operator type (equality or inequality).

    Parameters
    ----------
    constraint : Constraint
        Constraint object to add to the SDP instance.
    """
    match constraint.operator:
        case "=":
            self._add_equality_constraint_impl(constraint)
        case "<=":
            self._add_inequality_constraint_impl(constraint)
        case "complementarity":
            self._add_complementarity_constraint_impl(constraint)
        case "sum_squares":
            self._add_sum_squares_constraint_impl(constraint)
        case _:
            raise ValueError(f"Unknown constraint operator: {constraint.operator}")

add_param_stack_var

add_param_stack_var(param_stack_var, param_list)

Stack parameter variables (not implemented for SDP).

Raises:

Type	Description
NotImplementedError	This method is not supported in the SDP canonicalizer.

Source code in fomtools/canonicalizers/sdp.py

def add_param_stack_var(self, param_stack_var, param_list):
    """
    Stack parameter variables (not implemented for SDP).

    Raises
    ------
    NotImplementedError
        This method is not supported in the SDP canonicalizer.
    """
    raise NotImplementedError

add_psd_cone_indices

add_psd_cone_indices(new_indices)

Register a new PSD cone constraint on a submatrix.

Adds the constraint that the submatrix of X indexed by new_indices (including the homogenization index 0) is positive semidefinite.

This method maintains an antichain: no stored set is a subset of another. If a matrix X is PSD, any principal submatrix is also PSD, so subset constraints are redundant.

Parameters:

Name	Type	Description	Default
new_indices	list[tuple[int, int]]	List of (start, end) index ranges defining the submatrix. Index 0 is automatically included for homogenization.	required

Source code in fomtools/canonicalizers/sdp.py

def add_psd_cone_indices(self, new_indices):
    """
    Register a new PSD cone constraint on a submatrix.

    Adds the constraint that the submatrix of X indexed by ``new_indices``
    (including the homogenization index 0) is positive semidefinite.

    This method maintains an antichain: no stored set is a subset of another.
    If a matrix X is PSD, any principal submatrix is also PSD, so subset
    constraints are redundant.

    Parameters
    ----------
    new_indices : list[tuple[int, int]]
        List of (start, end) index ranges defining the submatrix.
        Index 0 is automatically included for homogenization.
    """
    new_idx_set = range_to_indices(new_indices)
    new_idx_set.add(0)

    # Check if new set is contained in any existing set (redundant)
    for existing_set in self.psd_cone_indices:
        if new_idx_set <= existing_set:
            return

    # Filter out existing sets that are strict subsets of the new set
    self.psd_cone_indices = [s for s in self.psd_cone_indices if not s < new_idx_set]

    self.psd_cone_indices.append(new_idx_set)
    self._cuts_dirty = True

add_theory_cut

add_theory_cut(margin: float, target_expr: LinearExpression, lb: float | ndarray, ub: float | ndarray)

Add theory cut constraints (not yet implemented for SDP).

Parameters:

Name	Type	Description	Default
margin	float	Relaxation margin for the bounds.	required
target_expr	LinearExpression	The expression whose bounds are to be relaxed.	required
lb	float | ndarray	Lower bound(s) for target_expr.	required
ub	float | ndarray	Upper bound(s) for target_expr.	required

Raises:

Type	Description
NotImplementedError	This method is not yet implemented for the SDP canonicalizer.

Source code in fomtools/canonicalizers/sdp.py

def add_theory_cut(
    self,
    margin: float,
    target_expr: LinearExpression,
    lb: float | np.ndarray,
    ub: float | np.ndarray,
):
    """
    Add theory cut constraints (not yet implemented for SDP).

    Parameters
    ----------
    margin : float
        Relaxation margin for the bounds.
    target_expr : LinearExpression
        The expression whose bounds are to be relaxed.
    lb : float | np.ndarray
        Lower bound(s) for target_expr.
    ub : float | np.ndarray
        Upper bound(s) for target_expr.

    Raises
    ------
    NotImplementedError
        This method is not yet implemented for the SDP canonicalizer.
    """
    raise NotImplementedError(
        "add_theory_cut is not yet implemented for the SDP canonicalizer."
    )

add_var

add_var(var: LinearExpression, **kwargs) -> None

Add a variable to the SDP with bound constraints.

Embeds the variable in the SDP matrix diagonal and adds quadratic bound constraints of the form (x - l)(x - u) <= 0 for each component, which encode the box constraints l <= x <= u.

Parameters:

Name	Type	Description	Default
var	LinearExpression	The variable to add. Must have finite bounds in var.lb and var.ub.	required

Raises:

Type	Description
NotImplementedError	If any bound is infinite.

Source code in fomtools/canonicalizers/sdp.py

def add_var(self, var: LinearExpression, **kwargs) -> None:
    """
    Add a variable to the SDP with bound constraints.

    Embeds the variable in the SDP matrix diagonal and adds quadratic
    bound constraints of the form (x - l)(x - u) <= 0 for each
    component, which encode the box constraints l <= x <= u.

    Parameters
    ----------
    var : LinearExpression
        The variable to add. Must have finite bounds in ``var.lb``
        and ``var.ub``.

    Raises
    ------
    NotImplementedError
        If any bound is infinite.
    """
    lb, ub = var.lb, var.ub
    dim = var.ndim
    bounds = (self.range_marker, self.range_marker + dim)
    self.range_marker += dim
    self.var_idx_map[var] = bounds

    # whenever a new var is added, need to pad existing A matrices
    self._pad_existing_A_matrices()

    curr_prob_dim = self.get_sdp_var_dim()
    constraint_start = len(self.Aineq_matrices)

    for i, l, u in zip(range(bounds[0], bounds[1]), lb, ub):
        if np.isinf(l) or np.isinf(u):
            raise NotImplementedError("Infinite bounds not yet supported in SDP canonicalizer")
        new_A = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
        new_A[i, i] = 1
        new_A[i, 0] = -(l + u)

        new_A = new_A.tocsc()
        new_A = symmetrize(new_A)
        self.Aineq_matrices.append(new_A)
        self.bineq_upperbounds.append(-u * l)

    constraint_end = len(self.Aineq_matrices)
    self.bound_constraint_indices[var] = (constraint_start, constraint_end)
    self._cuts_dirty = True

canonicalize_affine

canonicalize_affine(step: Affine) -> None

Encode an Affine step as a linear equality constraint.

The Affine step asserts step.output_mat @ step.output == step.arg; this is the same equality MIP encodes (mip.py:404). On SDP it flows through add_constraint → _add_equality_constraint_impl, which emits the linear matrix, the quadratic strengthening, and the PSD cone membership for the involved leaves.

Parameters:

Name	Type	Description	Default
step	Affine	The affine step to canonicalize.	required

Source code in fomtools/canonicalizers/sdp.py

def canonicalize_affine(self, step: Affine) -> None:
    """
    Encode an Affine step as a linear equality constraint.

    The Affine step asserts ``step.output_mat @ step.output == step.arg``;
    this is the same equality MIP encodes (``mip.py:404``). On SDP it
    flows through ``add_constraint`` → ``_add_equality_constraint_impl``,
    which emits the linear matrix, the quadratic strengthening, and
    the PSD cone membership for the involved leaves.

    Parameters
    ----------
    step : Affine
        The affine step to canonicalize.
    """
    self.add_constraint(step.output_mat @ step.output == step.arg)

canonicalize_relu

canonicalize_relu(step: Relu) -> None

Encode a ReLU step as SDP constraints.

Converts the relationship y = ReLU(Ax + b) into equality constraints on the SDP matrix variable and registers the appropriate PSD cone constraint for the lifted formulation.

Operates uniformly over step.proj_indices and binds equalities to step.output_leaves[0]. For full ReLU, proj_indices covers every arg row and output_leaves[0] is the full output leaf — equivalent to a standard full-output encoding. For partial-projection ReLU, only the projected rows get lifted equalities and output_leaves[0] is the smaller fresh proj-leaf; the non-projected rows are structurally fused into shared arg leaves at composition time, so no equalities and no SDP rows are emitted for them.

Parameters:

Name	Type	Description	Default
step	Relu	The ReLU step containing input expression (arg) and output variable (output).	required

Source code in fomtools/canonicalizers/sdp.py

def canonicalize_relu(self, step: Relu) -> None:
    """
    Encode a ReLU step as SDP constraints.

    Converts the relationship y = ReLU(Ax + b) into equality constraints
    on the SDP matrix variable and registers the appropriate PSD cone
    constraint for the lifted formulation.

    Operates uniformly over ``step.proj_indices`` and binds equalities to
    ``step.output_leaves[0]``. For full ReLU, ``proj_indices`` covers every
    arg row and ``output_leaves[0]`` is the full output leaf — equivalent to a
    standard full-output encoding. For partial-projection ReLU, only the
    projected rows get lifted equalities and ``output_leaves[0]`` is the
    smaller fresh proj-leaf; the non-projected rows are structurally
    fused into shared arg leaves at composition time, so no equalities
    and no SDP rows are emitted for them.

    Parameters
    ----------
    step : Relu
        The ReLU step containing input expression (arg) and output
        variable (output).
    """
    arg = step.arg
    A_mat, x_idx_bounds = self.linexpr_to_Amat_and_bounds(arg)
    b_vec = arg.const

    y_leaf = step.output_leaves[0]
    proj_list = sorted(step.proj_indices)

    y_idx_bounds = [self.var_idx_map[y_leaf]]
    relu_psd_cone_indices = y_idx_bounds + x_idx_bounds

    y_idx_ranges_2d = process_2d_ranges(y_idx_bounds, y_idx_bounds)
    y_idx_ranges_1d = process_1d_ranges(y_idx_bounds)
    xy_idx_ranges = process_2d_ranges(x_idx_bounds, y_idx_bounds)
    I = np.eye(y_leaf.ndim)

    curr_prob_dim = self.get_sdp_var_dim()
    for j, i in enumerate(proj_list):
        # j indexes y_leaf (size = len(proj_list));
        # i indexes arg row in A_mat / b_vec (size = arg.ndim).
        Ij = I[j]
        new_A = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
        new_A[y_idx_ranges_2d] = np.outer(Ij, Ij)
        new_A[xy_idx_ranges] = -np.outer(A_mat[i], Ij)
        new_A[y_idx_ranges_1d] = -b_vec[i] * Ij

        self.Aeq_matrices.append(symmetrize(new_A.tocsc()))
        self.beq_consts.append(0)

    self.add_psd_cone_indices(relu_psd_cone_indices)
    self._relu_steps.append(step)

    # Register the ReLU as a prox step (orthant projection, tau=1) for the
    # subgradient-monotonicity cut — unless this ReLU is an internal piece of
    # a soft-threshold decomposition, in which case the composite prox is
    # registered by the decomposer instead (see SDPStepDecomposer). The ReLU
    # still joins ``_relu_steps`` above so the cross cut sees every node.
    if not getattr(step, "_skip_prox_registration", False):
        self.register_prox_step(z=step.output, v=step.arg, tau=1.0, func="nonneg_orthant")

    self._cuts_dirty = True

copy

copy() -> SDP

Create an independent copy of this SDP canonicalizer.

Returns a new SDP canonicalizer with deep copies of all mutable state (constraint matrices, cone indices, etc.). The copy can be modified without affecting the original.

Returns:

Type	Description
SDP	A new canonicalizer instance with copied state.

Source code in fomtools/canonicalizers/sdp.py

def copy(self) -> "SDP":
    """
    Create an independent copy of this SDP canonicalizer.

    Returns a new SDP canonicalizer with deep copies of all mutable
    state (constraint matrices, cone indices, etc.). The copy can be
    modified without affecting the original.

    Returns
    -------
    SDP
        A new canonicalizer instance with copied state.
    """
    new_canon = SDP.__new__(SDP)
    new_canon.solver_params = dict(self.solver_params) if self.solver_params else None
    new_canon.solver_name = self.solver_name
    new_canon.solver = None  # SDP solver is instantiated at solve time
    new_canon.range_marker = self.range_marker
    new_canon.var_idx_map = self.var_idx_map.copy()  # shallow copy ok (immutable values)

    # Deep copy mutable arrays/lists
    new_canon.Aeq_matrices = [m.copy() for m in self.Aeq_matrices]
    new_canon.beq_consts = self.beq_consts.copy()
    new_canon.Aineq_matrices = [m.copy() for m in self.Aineq_matrices]
    new_canon.bineq_upperbounds = self.bineq_upperbounds.copy()
    new_canon.psd_cone_indices = [s.copy() for s in self.psd_cone_indices]
    new_canon.bound_constraint_indices = self.bound_constraint_indices.copy()
    new_canon.rlt_cuts = self.rlt_cuts
    new_canon.triangle_cuts = self.triangle_cuts
    new_canon.ineq_product_cuts = self.ineq_product_cuts
    new_canon.ineq_box_cuts = self.ineq_box_cuts
    new_canon.ineq_square_cuts = self.ineq_square_cuts
    new_canon.relu_envelope_cuts = self.relu_envelope_cuts
    new_canon.rlt_all_pairs = self.rlt_all_pairs
    new_canon.full_psd_cone = self.full_psd_cone
    new_canon.cross_cut = self.cross_cut
    new_canon.subgrad_mono_cut = self.subgrad_mono_cut
    # Cuts get rebuilt from current state on the copy's first solve.
    new_canon._cuts_dirty = True
    new_canon._cut_ineq_matrices = []
    new_canon._cut_bineq_upperbounds = []
    new_canon._relu_steps = list(self._relu_steps)  # shallow: Relu objs are shared
    new_canon._prox_steps = list(self._prox_steps)  # shallow: exprs/scalars immutable
    new_canon._box_prox_steps = list(self._box_prox_steps)  # shallow: exprs/arrays immutable
    new_canon._ineq_constraints = list(self._ineq_constraints)  # shallow: (constr, flag) tuples
    new_canon.C_obj = self.C_obj.copy() if self.C_obj is not None else None
    new_canon.obbt_handler = self.obbt_handler
    new_canon._objective_spec = self._objective_spec
    new_canon._solved_copy = None

    return new_canon

get_sdp_var_dim

get_sdp_var_dim() -> int

Get the current dimension of the SDP matrix variable.

Returns:

Type	Description
int	The current size of the symmetric matrix variable X.

Source code in fomtools/canonicalizers/sdp.py

def get_sdp_var_dim(self) -> int:
    """
    Get the current dimension of the SDP matrix variable.

    Returns
    -------
    int
        The current size of the symmetric matrix variable X.
    """
    return self.range_marker

get_solution_value

get_solution_value(expression: LinearExpression) -> ndarray

Get the solution value for an expression (not yet implemented for SDP).

Parameters:

Name	Type	Description	Default
expression	LinearExpression	The expression to evaluate.	required

Returns:

Type	Description
ndarray	The solution values.

Raises:

Type	Description
NotImplementedError	This method is not yet implemented for the SDP canonicalizer.

Source code in fomtools/canonicalizers/sdp.py

def get_solution_value(self, expression: LinearExpression) -> np.ndarray:
    """
    Get the solution value for an expression (not yet implemented for SDP).

    Parameters
    ----------
    expression : LinearExpression
        The expression to evaluate.

    Returns
    -------
    np.ndarray
        The solution values.

    Raises
    ------
    NotImplementedError
        This method is not yet implemented for the SDP canonicalizer.
    """
    raise NotImplementedError(
        "get_solution_value is not yet implemented for the SDP canonicalizer. "
        "SDP solves a relaxation and does not provide primal variable values."
    )

linexpr_to_Amat_and_bounds

linexpr_to_Amat_and_bounds(expr: LinearExpression)

Extract the coefficient matrix and SDP indices from a LinearExpression.

Converts a LinearExpression of the form Ax + b into the coefficient matrix A and the index ranges where the corresponding variables are located in the SDP matrix.

Parameters:

Name	Type	Description	Default
expr	LinearExpression	The expression to decompose.	required

Returns:

Type	Description
tuple[ndarray, list[tuple[int, int]]]	A tuple containing: - The horizontally stacked coefficient matrix. - List of (start, end) index bounds for each variable block.

Source code in fomtools/canonicalizers/sdp.py

def linexpr_to_Amat_and_bounds(self, expr: LinearExpression):
    """
    Extract the coefficient matrix and SDP indices from a LinearExpression.

    Converts a LinearExpression of the form Ax + b into the coefficient
    matrix A and the index ranges where the corresponding variables are
    located in the SDP matrix.

    Parameters
    ----------
    expr : LinearExpression
        The expression to decompose.

    Returns
    -------
    tuple[np.ndarray, list[tuple[int, int]]]
        A tuple containing:
        - The horizontally stacked coefficient matrix.
        - List of (start, end) index bounds for each variable block.
    """
    all_A = []
    idx_bounds = []
    for leaf_var, mat in expr.decomposition_dict.items():
        all_A.append(mat.todense() if spa.issparse(mat) else mat)
        idx_bounds.append(self.var_idx_map[leaf_var])

    return np.hstack(all_A), idx_bounds

register_box_prox_step

register_box_prox_step(z: LinearExpression, v: LinearExpression, l, u) -> None

Register a saturated-linear (box projection) step for the PEP box cuts.

Records (z, v, l, u) where z = clip(v, l, u) is the composite projection output (relu(v - l) - relu(v - u) + l in the SDP lift), v is the input expression (step.arg; residual r = v - z), and l / u are the box bounds (constant arrays or LinearExpression if parametric). Consumed by _build_box_normalcone_cuts (under cross_cut) and _build_box_mono_cuts (under subgrad_mono_cut).

Parameters:

Name	Type	Description	Default
z	LinearExpression	Box-projection output (composite); leaves already in var_idx_map.	required
v	LinearExpression	Box-projection input (step.arg).	required
l	ndarray | LinearExpression	Lower/upper box bounds (per the operator's l / u).	required
u	ndarray | LinearExpression	Lower/upper box bounds (per the operator's l / u).	required

Source code in fomtools/canonicalizers/sdp.py

def register_box_prox_step(self, z: LinearExpression, v: LinearExpression, l, u) -> None:
    """
    Register a saturated-linear (box projection) step for the PEP box cuts.

    Records ``(z, v, l, u)`` where ``z = clip(v, l, u)`` is the composite
    projection output (``relu(v - l) - relu(v - u) + l`` in the SDP lift), ``v``
    is the input expression (``step.arg``; residual ``r = v - z``), and ``l`` /
    ``u`` are the box bounds (constant arrays or ``LinearExpression`` if
    parametric). Consumed by `_build_box_normalcone_cuts` (under
    ``cross_cut``) and `_build_box_mono_cuts` (under ``subgrad_mono_cut``).

    Parameters
    ----------
    z : LinearExpression
        Box-projection output (composite); leaves already in ``var_idx_map``.
    v : LinearExpression
        Box-projection input (``step.arg``).
    l, u : np.ndarray | LinearExpression
        Lower/upper box bounds (per the operator's ``l`` / ``u``).
    """
    self._box_prox_steps.append({"z": z, "v": v, "l": l, "u": u})
    self._cuts_dirty = True

register_prox_step

register_prox_step(z: LinearExpression, v: LinearExpression, tau: float, *, func: str) -> None

Register a prox step for the subgradient-monotonicity cut.

Records (z, v, tau, func) where z is the prox output expression, v is the prox input expression (so the subgradient is s = v - z), tau is the prox threshold (tau = 1 for orthant projection / ReLU, tau = lambda for soft-thresholding), and func names the sublinear function the step is the prox of. The subgradient cut builder (_build_subgrad_mono_cuts) consumes this registry and pairs only steps with the same func (the cut is invalid across functions).

Parameters:

Name	Type	Description	Default
z	LinearExpression	Prox output. May be composite (e.g. relu_pos - relu_neg for soft-thresholding); leaves must already be registered in var_idx_map at call time.	required
v	LinearExpression	Prox input (step.arg).	required
tau	float	Prox threshold used to normalize the subgradient.	required
func	str	Identifier of the sublinear function ("nonneg_orthant" for ReLU, "l1" for soft-thresholding).	required

Source code in fomtools/canonicalizers/sdp.py

def register_prox_step(
    self, z: LinearExpression, v: LinearExpression, tau: float, *, func: str
) -> None:
    """
    Register a prox step for the subgradient-monotonicity cut.

    Records ``(z, v, tau, func)`` where ``z`` is the prox output expression,
    ``v`` is the prox input expression (so the subgradient is ``s = v - z``),
    ``tau`` is the prox threshold (``tau = 1`` for orthant projection / ReLU,
    ``tau = lambda`` for soft-thresholding), and ``func`` names the sublinear
    function the step is the prox of. The subgradient cut builder
    (`_build_subgrad_mono_cuts`) consumes this registry and pairs only
    steps with the same ``func`` (the cut is invalid across functions).

    Parameters
    ----------
    z : LinearExpression
        Prox output. May be composite (e.g. ``relu_pos - relu_neg`` for
        soft-thresholding); leaves must already be registered in
        ``var_idx_map`` at call time.
    v : LinearExpression
        Prox input (``step.arg``).
    tau : float
        Prox threshold used to normalize the subgradient.
    func : str
        Identifier of the sublinear function (``"nonneg_orthant"`` for ReLU,
        ``"l1"`` for soft-thresholding).
    """
    self._prox_steps.append({"z": z, "v": v, "tau": float(tau), "func": func})
    self._cuts_dirty = True

set_infinity_norm_objective

set_infinity_norm_objective(expr_list, lb=None, ub=None) -> None

Store infinity-norm objective specification for deferred application.

Uses the identities: - abs(x) = relu(x) + relu(-x) - max(a, b) = a + relu(b - a)

Parameters:

Name	Type	Description	Default
expr_list	list[LinearExpression]	List of expressions whose infinity-norm maximum is the objective.	required
lb	optional	Unused (kept for interface compatibility with MIP).	None
ub	optional	Unused (kept for interface compatibility with MIP).	None

Source code in fomtools/canonicalizers/sdp.py

def set_infinity_norm_objective(self, expr_list, lb=None, ub=None) -> None:
    """
    Store infinity-norm objective specification for deferred application.

    Uses the identities:
    - abs(x) = relu(x) + relu(-x)
    - max(a, b) = a + relu(b - a)

    Parameters
    ----------
    expr_list : list[LinearExpression]
        List of expressions whose infinity-norm maximum is the objective.
    lb, ub : optional
        Unused (kept for interface compatibility with MIP).
    """
    self._objective_spec = ("infinity_norm", expr_list)

set_one_norm_objective

set_one_norm_objective(expr_list) -> None

Store one-norm objective specification for deferred application.

The objective will be applied to a copy of the canonicalizer at solve time using the identity: ||x||_1 = 1^T @ (relu(x) + relu(-x)).

Parameters:

Name	Type	Description	Default
expr_list	list[LinearExpression]	List of expressions whose one-norms are summed in the objective.	required

Source code in fomtools/canonicalizers/sdp.py

def set_one_norm_objective(self, expr_list) -> None:
    """
    Store one-norm objective specification for deferred application.

    The objective will be applied to a copy of the canonicalizer at
    solve time using the identity: ||x||_1 = 1^T @ (relu(x) + relu(-x)).

    Parameters
    ----------
    expr_list : list[LinearExpression]
        List of expressions whose one-norms are summed in the objective.
    """
    self._objective_spec = ("one_norm", expr_list)

set_two_norm_objective

set_two_norm_objective(expr_list, lb_list=None, ub_list=None) -> None

Store two-norm objective specification for deferred application.

The objective will be applied to a copy of the canonicalizer at solve time.

Parameters:

Name	Type	Description	Default
expr_list	list[LinearExpression]	List of expressions whose squared norms are summed in the objective.	required
lb_list	list[ndarray]	Lower bounds for each expression (unused in SDP, for interface consistency).	None
ub_list	list[ndarray]	Upper bounds for each expression (unused in SDP, for interface consistency).	None

Source code in fomtools/canonicalizers/sdp.py

def set_two_norm_objective(self, expr_list, lb_list=None, ub_list=None) -> None:
    """
    Store two-norm objective specification for deferred application.

    The objective will be applied to a copy of the canonicalizer at
    solve time.

    Parameters
    ----------
    expr_list : list[LinearExpression]
        List of expressions whose squared norms are summed in the
        objective.
    lb_list : list[np.ndarray], optional
        Lower bounds for each expression (unused in SDP, for interface consistency).
    ub_list : list[np.ndarray], optional
        Upper bounds for each expression (unused in SDP, for interface consistency).
    """
    self._objective_spec = ("two_norm", expr_list)

set_zero_objective

set_zero_objective()

Set objective to zero (not yet implemented for SDP).

Raises:

Type	Description
NotImplementedError	This method is not yet implemented for the SDP canonicalizer.

Source code in fomtools/canonicalizers/sdp.py

def set_zero_objective(self):
    """
    Set objective to zero (not yet implemented for SDP).

    Raises
    ------
    NotImplementedError
        This method is not yet implemented for the SDP canonicalizer.
    """
    raise NotImplementedError(
        "set_zero_objective is not yet implemented for the SDP canonicalizer."
    )

solve_model

solve_model(steps=None, convex_hull_cuts=False, **solver_kwargs)

Solve the SDP problem using the configured solver.

Creates a copy of the canonicalizer, applies the objective to the copy, solves it, and keeps a reference for future use. This prevents objective constraints from accumulating across multiple solve calls.

Parameters:

Name	Type	Description	Default
steps	list	Unused by SDP, accepted for API compatibility.	None
convex_hull_cuts	bool	Unused by SDP, accepted for API compatibility.	False
**solver_kwargs		Native solver options forwarded verbatim to the backend (e.g. max_iter=500, tol_gap_abs=1e-9 for Clarabel/cvxpy). They override construction-time params for this call. A ValueError is raised if the backend rejects an option.	{}

Returns:

Type	Description
float	The objective value.

Source code in fomtools/canonicalizers/sdp.py

def solve_model(self, steps=None, convex_hull_cuts=False, **solver_kwargs):
    """Solve the SDP problem using the configured solver.

    Creates a copy of the canonicalizer, applies the objective to the copy,
    solves it, and keeps a reference for future use. This prevents objective
    constraints from accumulating across multiple solve calls.

    Parameters
    ----------
    steps : list, optional
        Unused by SDP, accepted for API compatibility.
    convex_hull_cuts : bool, optional
        Unused by SDP, accepted for API compatibility.
    **solver_kwargs
        Native solver options forwarded verbatim to the backend
        (e.g. ``max_iter=500``, ``tol_gap_abs=1e-9`` for
        Clarabel/cvxpy). They override construction-time ``params``
        for this call. A ``ValueError`` is raised if the backend
        rejects an option.

    Returns
    -------
    float
        The objective value.
    """
    # Create copy to apply objective without modifying original
    self._solved_copy = self.copy()

    # Apply objective to the copy
    if self._objective_spec is not None:
        obj_type = self._objective_spec[0]
        match obj_type:
            case "two_norm":
                _, expr_list = self._objective_spec
                self._apply_two_norm_objective(self._solved_copy, expr_list)
            case "one_norm":
                _, expr_list = self._objective_spec
                self._apply_one_norm_objective(self._solved_copy, expr_list)
            case "infinity_norm":
                _, expr_list = self._objective_spec
                self._apply_infinity_norm_objective(self._solved_copy, expr_list)

    # Solve using the copy's data
    return self._solved_copy._solve_internal(**solver_kwargs)

solve_via_clarabel

solve_via_clarabel(**solver_kwargs)

Solve the SDP using the native Clarabel interface.

Directly invokes the Clarabel solver without going through CVXPY, which can be more efficient for large problems.

Parameters:

Name	Type	Description	Default
**solver_kwargs		Native Clarabel settings forwarded verbatim to the solver.	{}

Returns:

Type	Description
float	The optimal objective value.

Source code in fomtools/canonicalizers/sdp.py

def solve_via_clarabel(self, **solver_kwargs):
    """
    Solve the SDP using the native Clarabel interface.

    Directly invokes the Clarabel solver without going through CVXPY,
    which can be more efficient for large problems.

    Parameters
    ----------
    **solver_kwargs
        Native Clarabel settings forwarded verbatim to the solver.

    Returns
    -------
    float
        The optimal objective value.
    """
    self._ensure_objective_applied()
    return self._solve_via_clarabel_internal(**solver_kwargs)

solve_via_cuclarabel

solve_via_cuclarabel(**solver_kwargs)

Solve the SDP using GPU-accelerated Clarabel.

Uses the CUDA-enabled Clarabel solver for GPU acceleration. Requires CuPy and Julia with CUDA.jl and Clarabel.jl.

Parameters:

Name	Type	Description	Default
**solver_kwargs		Native Clarabel settings forwarded verbatim to the solver.	{}

Returns:

Type	Description
float	The optimal objective value.

Raises:

Type	Description
ValueError	If CUDA dependencies (cupy) are not available.

Source code in fomtools/canonicalizers/sdp.py

def solve_via_cuclarabel(self, **solver_kwargs):
    """
    Solve the SDP using GPU-accelerated Clarabel.

    Uses the CUDA-enabled Clarabel solver for GPU acceleration.
    Requires CuPy and Julia with CUDA.jl and Clarabel.jl.

    Parameters
    ----------
    **solver_kwargs
        Native Clarabel settings forwarded verbatim to the solver.

    Returns
    -------
    float
        The optimal objective value.

    Raises
    ------
    ValueError
        If CUDA dependencies (cupy) are not available.
    """
    self._ensure_objective_applied()
    return self._solve_via_cuclarabel_internal(**solver_kwargs)

solve_via_cvxpy

solve_via_cvxpy(**solver_kwargs)

Solve the SDP using the CVXPY modeling interface.

Constructs a CVXPY problem from the stored constraint matrices and solves it using the Clarabel backend via CVXPY.

Parameters:

Name	Type	Description	Default
**solver_kwargs		Native solver options forwarded verbatim to Problem.solve.	{}

Returns:

Type	Description
float	The optimal objective value.

Source code in fomtools/canonicalizers/sdp.py

def solve_via_cvxpy(self, **solver_kwargs):
    """
    Solve the SDP using the CVXPY modeling interface.

    Constructs a CVXPY problem from the stored constraint matrices
    and solves it using the Clarabel backend via CVXPY.

    Parameters
    ----------
    **solver_kwargs
        Native solver options forwarded verbatim to ``Problem.solve``.

    Returns
    -------
    float
        The optimal objective value.
    """
    self._ensure_objective_applied()
    return self._solve_via_cvxpy_internal(**solver_kwargs)

update_bounds

update_bounds(expression: LinearExpression, lb: float | ndarray, ub: float | ndarray) -> None

Update bounds for a variable by regenerating its constraint matrices.

Modifies the existing inequality constraint matrices for the given variable to reflect new lower and upper bounds. This is used during bound tightening procedures.

Parameters:

Name	Type	Description	Default
expression	LinearExpression	The variable whose bounds are being updated. Must already be registered via add_var.	required
lb	ndarray	New lower bounds. Must have length equal to expression.ndim.	required
ub	ndarray	New upper bounds. Must have length equal to expression.ndim.	required

Raises:

Type	Description
ValueError	If the expression is not found in the canonicalizer, or if the bounds do not have the expected length.
NotImplementedError	If any new bound is infinite.

Source code in fomtools/canonicalizers/sdp.py

def update_bounds(
    self, expression: LinearExpression, lb: float | np.ndarray, ub: float | np.ndarray
) -> None:
    """
    Update bounds for a variable by regenerating its constraint matrices.

    Modifies the existing inequality constraint matrices for the given
    variable to reflect new lower and upper bounds. This is used during
    bound tightening procedures.

    Parameters
    ----------
    expression : LinearExpression
        The variable whose bounds are being updated. Must already be
        registered via ``add_var``.
    lb : np.ndarray
        New lower bounds. Must have length equal to ``expression.ndim``.
    ub : np.ndarray
        New upper bounds. Must have length equal to ``expression.ndim``.

    Raises
    ------
    ValueError
        If the expression is not found in the canonicalizer, or if
        the bounds do not have the expected length.
    NotImplementedError
        If any new bound is infinite.
    """
    if expression not in self.bound_constraint_indices:
        raise ValueError("Expression not found in SDP canonicalizer")

    constraint_start, constraint_end = self.bound_constraint_indices[expression]
    sdp_start, sdp_end = self.var_idx_map[expression]
    curr_prob_dim = self.get_sdp_var_dim()

    lb = np.atleast_1d(lb)
    ub = np.atleast_1d(ub)

    expected_len = sdp_end - sdp_start
    if len(lb) != expected_len:
        raise ValueError(f"Expected lb of length {expected_len}, got {len(lb)}")
    if len(ub) != expected_len:
        raise ValueError(f"Expected ub of length {expected_len}, got {len(ub)}")

    for j, (i, l, u) in enumerate(zip(range(sdp_start, sdp_end), lb, ub)):
        if np.isinf(l) or np.isinf(u):
            raise NotImplementedError("Infinite bounds not yet supported in SDP canonicalizer")

        new_A = spa.lil_matrix((curr_prob_dim, curr_prob_dim))
        new_A[i, i] = 1
        new_A[i, 0] = -(l + u)

        constraint_idx = constraint_start + j
        self.Aineq_matrices[constraint_idx] = symmetrize(new_A.tocsc())
        self.bineq_upperbounds[constraint_idx] = -u * l

    self._cuts_dirty = True

range_to_indices

range_to_indices(idx_ranges)

Convert a list of (start, end) range tuples to a set of indices.

Parameters:

Name	Type	Description	Default
idx_ranges	list[tuple[int, int]]	List of (start, end) pairs defining half-open ranges.	required

Returns:

Type	Description
set[int]	The union of all indices in the specified ranges.

Source code in fomtools/canonicalizers/sdp.py

def range_to_indices(idx_ranges):
    """
    Convert a list of (start, end) range tuples to a set of indices.

    Parameters
    ----------
    idx_ranges : list[tuple[int, int]]
        List of (start, end) pairs defining half-open ranges.

    Returns
    -------
    set[int]
        The union of all indices in the specified ranges.
    """
    out_set = set()
    for bounds in idx_ranges:
        out_set.update(range(bounds[0], bounds[1]))
    return out_set

symmetrize

symmetrize(mat)

Symmetrize a sparse matrix.

Parameters:

Name	Type	Description	Default
mat	scipy.sparse matrix	The matrix to symmetrize.	required

Returns:

Type	Description
scipy.sparse matrix	The symmetric matrix (mat + mat.T) / 2.

Source code in fomtools/canonicalizers/sdp.py

def symmetrize(mat):
    """
    Symmetrize a sparse matrix.

    Parameters
    ----------
    mat : scipy.sparse matrix
        The matrix to symmetrize.

    Returns
    -------
    scipy.sparse matrix
        The symmetric matrix (mat + mat.T) / 2.
    """
    return (mat + mat.T) / 2