From 0339922998658291ddce555ffa40c36f111b9f9a Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 12:56:50 -0700 Subject: [PATCH 01/12] Define multifidelity QoI and error-estimate protocols --- romtools/workflows/models.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/romtools/workflows/models.py b/romtools/workflows/models.py index f4880629..748f878c 100644 --- a/romtools/workflows/models.py +++ b/romtools/workflows/models.py @@ -84,3 +84,31 @@ def compute_error_estimate(self, run_directory: str, parameter_sample: dict) -> AFTER run_model has been run ''' pass + + +class MultiFidelityQoiModel(QoiModel, Protocol): + """A QoI surrogate with predictions selectable by fidelity. + + Fidelity indices are ordered from lowest (0) to highest (-1). + Omitting fidelity preserves the original QoiModel call signature. + """ + + @property + def num_fidelities(self) -> int: + ... + + def compute_qoi( + self, run_directory: str, parameter_sample: dict, fidelity: int = -1, + ) -> np.ndarray: + ... + + +class MultiFidelityQoiModelWithErrorEstimate( + MultiFidelityQoiModel, QoiModelWithErrorEstimate, Protocol +): + """Fidelity-selectable QoIs and scalar error/uncertainty indicators.""" + + def compute_error_estimate( + self, run_directory: str, parameter_sample: dict, fidelity: int = -1, + ) -> float: + ... From cd504670ad2e25dfb08bc6f0bfb017f59f4bba1d Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 12:58:05 -0700 Subject: [PATCH 02/12] Implement joint N-level autoregressive GP and fidelity data contract --- romtools/rom/multifidelity.py | 304 ++++++++++++++++++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 romtools/rom/multifidelity.py diff --git a/romtools/rom/multifidelity.py b/romtools/rom/multifidelity.py new file mode 100644 index 00000000..388b7b55 --- /dev/null +++ b/romtools/rom/multifidelity.py @@ -0,0 +1,304 @@ +"""Jointly conditioned autoregressive multifidelity Gaussian-process QoI surrogate. + +Each fidelity has a different, potentially non-nested set of training inputs. +The public interface supports any number of ordered fidelities (two or more). +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional, Sequence + +import numpy as np +from scipy.linalg import cho_factor, cho_solve +from scipy.optimize import minimize + +from romtools.rom.qoi_surrogates import GaussianProcessKernel + + +@dataclass +class FidelityData: + """Accepted observations at one fidelity, in matching row order.""" + + parameters: np.ndarray + qois: np.ndarray + training_dirs: tuple[str, ...] = () + name: Optional[str] = None + + +@dataclass +class GPLevelConfig: + """Base kernel for level zero; discrepancy kernel and rho for later levels.""" + + kernel: GaussianProcessKernel + rho: float = 1.0 + noise_variance: float = 1e-8 + + +@dataclass +class AutoregressiveGPConfig: + """Configuration of a joint autoregressive GP hierarchy.""" + + levels: Sequence[GPLevelConfig] + tune_hyperparameters: bool = True + normalize_parameters: bool = True + optimizer_restarts: int = 4 + length_scale_bounds: tuple[float, float] = (1e-2, 1e2) + signal_variance_bounds: tuple[float, float] = (1e-6, 1e2) + rho_bounds: tuple[float, float] = (-5.0, 5.0) + + +class AutoregressiveGPQoiModel: + """Scalar-QoI Kennedy--O'Hagan autoregressive GP with joint conditioning. + + f_0 = g_0; f_l = rho_l * f_(l-1) + delta_l, where the base + process and all discrepancies are independent zero-mean GPs. + Hyperparameters are fitted by joint marginal likelihood; in particular, + the high-fidelity observations also inform the base process posterior. + """ + + def __init__( + self, + fidelity_data: Sequence[FidelityData], + config: AutoregressiveGPConfig, + parameter_names: Optional[Sequence[str]] = None, + ) -> None: + if len(fidelity_data) < 2: + raise ValueError("at least two fidelity levels are required") + if len(config.levels) != len(fidelity_data): + raise ValueError("one GPLevelConfig is required for each fidelity level") + if (not isinstance(config.optimizer_restarts, int) + or config.optimizer_restarts < 1): + raise ValueError("optimizer_restarts must be a positive integer") + for name, bounds in ( + ("length_scale_bounds", config.length_scale_bounds), + ("signal_variance_bounds", config.signal_variance_bounds), + ): + if (len(bounds) != 2 or not np.all(np.isfinite(bounds)) + or bounds[0] <= 0 or bounds[0] >= bounds[1]): + raise ValueError(f"{name} must have finite increasing positive bounds") + if (len(config.rho_bounds) != 2 + or not np.all(np.isfinite(config.rho_bounds)) + or config.rho_bounds[0] >= config.rho_bounds[1]): + raise ValueError("rho_bounds must be finite and increasing") + + xs, ys = [], [] + for index, data in enumerate(fidelity_data): + x = np.asarray(data.parameters, dtype=float) + y = np.asarray(data.qois, dtype=float) + if y.ndim == 2 and y.shape[1] == 1: + y = y[:, 0] + if (x.ndim != 2 or not len(x) or y.ndim != 1 + or len(y) != len(x) + or not np.all(np.isfinite(x)) or not np.all(np.isfinite(y))): + raise ValueError(f"invalid scalar QoI training data at fidelity {index}") + if xs and x.shape[1] != xs[0].shape[1]: + raise ValueError("parameter dimension must match across fidelity levels") + if data.training_dirs and len(data.training_dirs) != len(x): + raise ValueError("training_dirs must match the number of samples") + xs.append(x.copy()) + ys.append(y.copy()) + + self.parameter_names = ( + tuple(parameter_names) if parameter_names is not None else None + ) + if self.parameter_names is not None and len(self.parameter_names) != xs[0].shape[1]: + raise ValueError("parameter_names must match parameter dimension") + self.config = config + self.num_fidelities = len(xs) + self._param_min = np.min(np.vstack(xs), axis=0) + param_range = np.ptp(np.vstack(xs), axis=0) + self._param_scale = np.where(param_range > 0, param_range, 1.0) + if config.normalize_parameters: + self._xs = [(x - self._param_min) / self._param_scale for x in xs] + else: + self._xs = xs + self._ys = ys + self._y = np.concatenate(ys) + self._levels = np.concatenate([ + np.full(len(y), l, dtype=int) for l, y in enumerate(ys) + ]) + self._kernels = [ + GaussianProcessKernel( + length_scale=np.asarray(c.kernel.length_scale, dtype=float).copy(), + signal_variance=float(c.kernel.signal_variance), + ) for c in config.levels + ] + self._rhos = np.array([c.rho for c in config.levels[1:]], dtype=float) + self._noise = np.array([c.noise_variance for c in config.levels], dtype=float) + if not np.all(np.isfinite(self._rhos)): + raise ValueError("rho must be finite") + if not np.all(np.isfinite(self._noise)) or np.any(self._noise < 0): + raise ValueError("noise variances must be finite and nonnegative") + self._validate_kernels() + if config.tune_hyperparameters: + self._tune() + self._fit() + + def _validate_kernels(self) -> None: + d = self._xs[0].shape[1] + for kernel in self._kernels: + # Reuse existing kernel validation with correctly shaped inputs. + kernel(np.zeros((1, d)), np.zeros((1, d))) + + def _scale(self, x: np.ndarray) -> np.ndarray: + return ((x - self._param_min) / self._param_scale + if self.config.normalize_parameters else x) + + def _weights(self, level: int) -> np.ndarray: + weights = np.ones(level + 1) + for k in range(level): + weights[k] = np.prod(self._rhos[k:level]) + return weights + + def _cov(self, x_a: np.ndarray, a: int, x_b: np.ndarray, b: int) -> np.ndarray: + wa, wb = self._weights(a), self._weights(b) + result = np.zeros((len(x_a), len(x_b))) + for k in range(min(a, b) + 1): + result += wa[k] * wb[k] * self._kernels[k](x_a, x_b) + return result + + def _training_cov(self) -> np.ndarray: + sizes = [len(x) for x in self._xs] + offsets = np.cumsum([0] + sizes) + result = np.empty((len(self._y), len(self._y))) + for a in range(self.num_fidelities): + ia = slice(offsets[a], offsets[a + 1]) + for b in range(a + 1): + ib = slice(offsets[b], offsets[b + 1]) + block = self._cov(self._xs[a], a, self._xs[b], b) + result[ia, ib] = block + if a != b: + result[ib, ia] = block.T + result[ia, ia] += self._noise[a] * np.eye(sizes[a]) + return result + + def _factor(self): + matrix = self._training_cov() + # Fixed numerical jitter is not included in the observation noise. + matrix.flat[::len(matrix) + 1] += 1e-12 + return cho_factor(matrix, lower=True, check_finite=True) + + def _pack(self): + values = [] + for k in self._kernels: + scales = np.asarray(k.length_scale, dtype=float) + if scales.ndim == 0: + scales = np.full(self._xs[0].shape[1], float(scales)) + values.extend(np.log(scales)) + values.append(np.log(k.signal_variance)) + values.extend(self._rhos) + return np.asarray(values) + + def _unpack(self, theta): + d = self._xs[0].shape[1] + pos = 0 + for kernel in self._kernels: + kernel.length_scale = np.exp(theta[pos:pos + d]) + kernel.signal_variance = float(np.exp(theta[pos + d])) + pos += d + 1 + self._rhos = np.asarray(theta[pos:], dtype=float).copy() + + def _tune(self): + d = self._xs[0].shape[1] + bounds = [] + for _ in range(self.num_fidelities): + bounds.extend([tuple(np.log(self.config.length_scale_bounds))] * d) + bounds.append(tuple(np.log(self.config.signal_variance_bounds))) + bounds.extend([self.config.rho_bounds] * (self.num_fidelities - 1)) + initial = np.clip(self._pack(), [b[0] for b in bounds], [b[1] for b in bounds]) + rng = np.random.default_rng(0) + starts = [initial] + starts.extend(rng.uniform([b[0] for b in bounds], + [b[1] for b in bounds]) + for _ in range(self.config.optimizer_restarts - 1)) + + def objective(theta): + self._unpack(theta) + try: + chol = self._factor() + alpha = cho_solve(chol, self._y) + logdet_half = np.sum(np.log(np.diag(chol[0]))) + return (0.5 * self._y @ alpha + logdet_half + + 0.5 * len(self._y) * np.log(2 * np.pi)) + except (np.linalg.LinAlgError, ValueError, FloatingPointError): + return 1e100 + + results = [minimize(objective, start, method="L-BFGS-B", bounds=bounds) + for start in starts] + valid = [r for r in results if np.isfinite(r.fun) and r.fun < 1e99] + if not valid: + raise RuntimeError("joint GP hyperparameter optimization failed") + best = min(valid, key=lambda r: r.fun) + self._unpack(best.x) + self._validate_kernels() + + def _fit(self) -> None: + self._chol = self._factor() + self._alpha = cho_solve(self._chol, self._y) + + def _resolve_fidelity(self, fidelity: int) -> int: + if not isinstance(fidelity, (int, np.integer)): + raise TypeError("fidelity must be an integer") + index = fidelity if fidelity >= 0 else self.num_fidelities + fidelity + if index < 0 or index >= self.num_fidelities: + raise ValueError("fidelity index out of range") + return int(index) + + def predict(self, parameters, fidelity: int = -1, return_std: bool = False): + """Predict latent QoI mean and optionally posterior standard deviation.""" + level = self._resolve_fidelity(fidelity) + x = np.asarray(parameters, dtype=float) + if x.ndim == 1: + x = x[None, :] + if x.ndim != 2 or x.shape[1] != self._xs[0].shape[1]: + raise ValueError("prediction parameter dimension mismatch") + if not np.all(np.isfinite(x)): + raise ValueError("prediction parameters must be finite") + x = self._scale(x) + k_star = np.hstack([ + self._cov(x, level, xx, l) for l, xx in enumerate(self._xs) + ]) + mean = k_star @ self._alpha + if not return_std: + return mean + variance = np.diag(self._cov(x, level, x, level)) + variance -= np.sum(k_star * cho_solve(self._chol, k_star.T).T, axis=1) + return mean, np.sqrt(np.maximum(variance, 0.0)) + + def _parameter_row(self, parameter_sample) -> np.ndarray: + if isinstance(parameter_sample, dict): + if self.parameter_names is None: + raise ValueError("parameter_names required for dictionary inputs") + return np.array([parameter_sample[name] for name in self.parameter_names]) + return np.asarray(parameter_sample, dtype=float).ravel() + + def populate_run_directory(self, run_directory, parameter_sample) -> None: + pass + + def run_model(self, run_directory, parameter_sample) -> int: + return 0 + + def compute_qoi(self, run_directory, parameter_sample, fidelity: int = -1): + return self.predict(self._parameter_row(parameter_sample), fidelity=fidelity) + + def compute_error_estimate( + self, run_directory, parameter_sample, fidelity: int = -1, + ) -> float: + _, std = self.predict(self._parameter_row(parameter_sample), + fidelity=fidelity, return_std=True) + return float(std[0]) + + +class AutoregressiveGPBuilder: + """Build a jointly trained AR-GP from all accepted fidelity datasets.""" + + def __init__(self, config: AutoregressiveGPConfig, + parameter_names: Optional[Sequence[str]] = None): + self.config = config + self.parameter_names = parameter_names + + def build(self, offline_data_dir: str, training_data: Sequence[FidelityData]): + return AutoregressiveGPQoiModel( + fidelity_data=training_data, config=self.config, + parameter_names=self.parameter_names, + ) From 13f0a7e2aee8a250805b660798116882ab158793 Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 12:58:17 -0700 Subject: [PATCH 03/12] Add generic multifidelity builder protocol --- romtools/workflows/model_builders.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/romtools/workflows/model_builders.py b/romtools/workflows/model_builders.py index 31cd5b56..8cad23b5 100644 --- a/romtools/workflows/model_builders.py +++ b/romtools/workflows/model_builders.py @@ -65,3 +65,14 @@ def __init__(self): def build_from_training_dirs(self,offline_data_dir: str, training_data_dirs: List[str]) -> QoiModelWithErrorEstimate: return QoiModelWithErrorEstimate + + +class MultiFidelityQoiModelBuilder(Protocol): + """Construct a multifidelity surrogate from ordered per-level datasets. + + The builder must not execute fidelity models itself. The adaptive workflow + owns data acquisition and provides accepted observations at all levels. + """ + + def build(self, offline_data_dir: str, training_data) -> QoiModel: + ... From 271c376cb90be88ed49caa4ff878cbcb7e969b7b Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 12:58:48 -0700 Subject: [PATCH 04/12] Test multi-level joint conditioning, fidelity error estimates and training contract --- tests/romtools/rom/test_multifidelity_gp.py | 116 ++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 tests/romtools/rom/test_multifidelity_gp.py diff --git a/tests/romtools/rom/test_multifidelity_gp.py b/tests/romtools/rom/test_multifidelity_gp.py new file mode 100644 index 00000000..fc7b64c3 --- /dev/null +++ b/tests/romtools/rom/test_multifidelity_gp.py @@ -0,0 +1,116 @@ +"""Tests for jointly trained multifidelity GP and its QoI protocol.""" +import numpy as np +import pytest + +from romtools.rom.multifidelity import ( + FidelityData, GPLevelConfig, AutoregressiveGPConfig, + AutoregressiveGPBuilder, AutoregressiveGPQoiModel, +) +from romtools.rom.qoi_surrogates import GaussianProcessKernel +from romtools.workflows.models import ( + MultiFidelityQoiModel, MultiFidelityQoiModelWithErrorEstimate, +) + + +def make_config(n, tune=False): + return AutoregressiveGPConfig( + levels=[ + GPLevelConfig( + kernel=GaussianProcessKernel(length_scale=0.35, + signal_variance=1.0), + rho=0.75 if i else 1.0, + noise_variance=1e-9, + ) + for i in range(n) + ], + tune_hyperparameters=tune, + optimizer_restarts=2, + ) + + +def test_two_level_non_nested_gp_and_protocol(): + x0 = np.linspace(0.0, 1.0, 15)[:, None] + x1 = np.array([0.08, 0.29, 0.61, 0.93])[:, None] + f0 = np.sin(4 * x0[:, 0]) + f1 = 0.75 * np.sin(4 * x1[:, 0]) + 0.15 * x1[:, 0] + model = AutoregressiveGPQoiModel( + [FidelityData(x0, f0), FidelityData(x1, f1)], + make_config(2), parameter_names=["mu"], + ) + assert isinstance(model, MultiFidelityQoiModelWithErrorEstimate) + assert isinstance(model, MultiFidelityQoiModel) + assert model.num_fidelities == 2 + for fidelity, x, y in [(0, x0, f0), (1, x1, f1)]: + mean, std = model.predict(x, fidelity=fidelity, return_std=True) + np.testing.assert_allclose(mean, y, atol=5e-5) + assert np.all(std >= 0) + assert np.max(std) < 1e-3 + probe = {"mu": 0.35} + assert model.compute_qoi("", probe).shape == (1,) + assert model.compute_qoi("", probe, fidelity=0).shape == (1,) + assert model.compute_error_estimate("", probe, fidelity=0) >= 0 + assert model.compute_error_estimate("", probe, fidelity=1) >= 0 + np.testing.assert_allclose( + model.compute_qoi("", probe), + model.compute_qoi("", probe, fidelity=1), + ) + with pytest.raises(ValueError, match="out of range"): + model.predict(x1, fidelity=2) + + +def test_three_level_cross_fidelity_joint_conditioning(): + x0 = np.array([0., 0.3, 0.6, 1.])[:, None] + x1 = np.array([0.2, 0.5, 0.8])[:, None] + x2 = np.array([0.45, 0.9])[:, None] + data = [ + FidelityData(x0, np.sin(2 * x0[:, 0])), + FidelityData(x1, 0.7 * np.sin(2 * x1[:, 0]) + x1[:, 0]), + FidelityData(x2, 0.5 * (0.7 * np.sin(2 * x2[:, 0]) + x2[:, 0]) + + 0.2 * x2[:, 0]), + ] + model = AutoregressiveGPQoiModel(data, make_config(3)) + assert model.num_fidelities == 3 + for fidelity in range(3): + mean, std = model.predict(np.array([[0.4], [0.7]]), + fidelity=fidelity, return_std=True) + assert mean.shape == std.shape == (2,) + assert np.all(np.isfinite(mean)) + assert np.all(np.isfinite(std)) + assert np.all(std >= 0) + # Additional high fidelity observations must update the low-level + # posterior through the joint cross-fidelity covariance. + baseline = AutoregressiveGPQoiModel( + [data[0], data[1], FidelityData( + np.array([[0.9]]), data[2].qois[-1:])], make_config(3) + ) + assert abs(model.predict([0.45], fidelity=0)[0] + - baseline.predict([0.45], fidelity=0)[0]) > 1e-6 + + +def test_builder_and_joint_hyperparameter_tuning(): + x0 = np.linspace(0, 1, 10)[:, None] + x1 = np.array([0.12, 0.41, 0.77])[:, None] + data = [ + FidelityData(x0, np.sin(3 * x0[:, 0]), name="low"), + FidelityData(x1, 1.2 * np.sin(3 * x1[:, 0]), name="high"), + ] + model = AutoregressiveGPBuilder( + config=make_config(2, tune=True), parameter_names=["mu"], + ).build("/unused", data) + prediction, std = model.predict(np.array([[0.5]]), return_std=True) + assert prediction.shape == std.shape == (1,) + assert np.isfinite(prediction[0]) and np.isfinite(std[0]) + assert all(np.all(np.asarray(k.length_scale) > 0) + for k in model._kernels) + + +@pytest.mark.parametrize("bad_data", [ + [FidelityData(np.zeros((3, 1)), np.ones(3))], + [FidelityData(np.zeros((3, 1)), np.ones(3)), + FidelityData(np.zeros((2, 2)), np.ones(2))], + [FidelityData(np.zeros((3, 1)), np.ones((3, 2))), + FidelityData(np.zeros((2, 1)), np.ones(2))], +]) +def test_rejects_incompatible_fidelity_data(bad_data): + with pytest.raises(ValueError): + AutoregressiveGPQoiModel(bad_data, make_config(len(bad_data))) From b5b746bc1e0296ebc018951a927955c1fb225307 Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 12:58:59 -0700 Subject: [PATCH 05/12] Enable runtime structural protocol checks for multifidelity models --- romtools/workflows/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/romtools/workflows/models.py b/romtools/workflows/models.py index 748f878c..24875bed 100644 --- a/romtools/workflows/models.py +++ b/romtools/workflows/models.py @@ -86,6 +86,7 @@ def compute_error_estimate(self, run_directory: str, parameter_sample: dict) -> pass +@runtime_checkable class MultiFidelityQoiModel(QoiModel, Protocol): """A QoI surrogate with predictions selectable by fidelity. @@ -103,6 +104,7 @@ def compute_qoi( ... +@runtime_checkable class MultiFidelityQoiModelWithErrorEstimate( MultiFidelityQoiModel, QoiModelWithErrorEstimate, Protocol ): From 95717a8713115b0b2c23536771414d8f203aff5a Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 13:19:17 -0700 Subject: [PATCH 06/12] Introduce fixed-allocation random multifidelity adaptive workflow and HF K-fold validation --- romtools/workflows/multifidelity.py | 321 ++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 romtools/workflows/multifidelity.py diff --git a/romtools/workflows/multifidelity.py b/romtools/workflows/multifidelity.py new file mode 100644 index 00000000..a7fdb2d2 --- /dev/null +++ b/romtools/workflows/multifidelity.py @@ -0,0 +1,321 @@ +"""Fixed-allocation random adaptive training for ordered multifidelity QoI models. + +This first workflow intentionally keeps fidelity allocation fixed. All model +evaluations use the existing blocking/nonblocking execution abstraction. +""" +from __future__ import annotations + +from dataclasses import dataclass +import os +from typing import Callable, Optional, Sequence + +import numpy as np + +from romtools.rom.multifidelity import FidelityData +from romtools.workflows.models import QoiModel +from romtools.workflows.parameter_spaces import ParameterSpace +from romtools.workflows._adaptive.execution import _evaluate_batch, _process_state +from romtools.workflows._adaptive.sampling import ( + _parameter_names, _positive_integer, +) +from romtools.workflows._adaptive.validation import _error_value, relative_l2_error + + +@dataclass(frozen=True) +class FidelityLevel: + """Executable model and fixed nominal cost, ordered low to high.""" + + model: QoiModel + name: str + cost: float = 1.0 + + +@dataclass(frozen=True) +class MultiFidelityValidationContext: + """Inputs to a custom validation strategy; training data are copies.""" + + rom_model: QoiModel + rom_model_builder: object + training_data: tuple[FidelityData, ...] + parameter_names: tuple[str, ...] + iteration: int + validation_directory: str + random_seed: int + error_functor: Callable + + +@dataclass(frozen=True) +class MultiFidelityKFoldValidation: + """K-fold CV of the highest fidelity, retaining all lower-level data. + + Lower-fidelity data at held-out high-fidelity locations remain available: + CV assesses HF prediction when inexpensive evaluations are known. + The result is the maximum per-sample held-out relative L2 error, + consistent with ROMtools' existing KFoldValidation. + """ + + num_folds: int = 5 + + def __post_init__(self): + _positive_integer(self.num_folds, "num_folds") + if self.num_folds < 2: + raise ValueError("num_folds must be at least two") + + def __call__(self, context: MultiFidelityValidationContext) -> float: + hf = context.training_data[-1] + n = len(hf.parameters) + if n < self.num_folds: + raise ValueError("highest-fidelity samples must be >= num_folds") + rng = np.random.default_rng(context.random_seed) + folds = np.array_split(rng.permutation(n), self.num_folds) + max_error = 0.0 + for index, holdout in enumerate(folds): + keep = np.ones(n, dtype=bool) + keep[holdout] = False + train_hf = FidelityData( + parameters=hf.parameters[keep].copy(), + qois=hf.qois[keep].copy(), + training_dirs=tuple(np.asarray(hf.training_dirs)[keep]) + if hf.training_dirs else (), + name=hf.name, + ) + training = tuple(context.training_data[:-1]) + (train_hf,) + fold_dir = os.path.join(context.validation_directory, f"fold_{index}") + offline = os.path.join(fold_dir, "offline_data") + os.makedirs(offline, exist_ok=False) + with _process_state(int(rng.integers(0, 2**32))): + model = context.rom_model_builder.build(offline, training) + for held in holdout: + row = hf.parameters[int(held)] + parameter_sample = dict(zip(context.parameter_names, row)) + # The surrogate is a QoiModel: preserve the workflow's + # established populate/run/compute execution contract. + run_dir = os.path.join(fold_dir, f"run_{int(held)}") + os.makedirs(run_dir, exist_ok=False) + with _process_state(int(rng.integers(0, 2**32))): + model.populate_run_directory(run_dir, parameter_sample) + status = model.run_model(run_dir, parameter_sample) + if status != 0: + raise RuntimeError(f"fold {index} surrogate evaluation failed: {status}") + predicted = np.asarray(model.compute_qoi(run_dir, parameter_sample), + dtype=float).reshape(-1) + reference = np.asarray(hf.qois[int(held)], dtype=float).reshape(-1) + if predicted.shape != reference.shape: + raise ValueError("held-out QoI dimension mismatch") + error = _error_value(context.error_functor(predicted, reference), + "validation error") + max_error = max(max_error, error) + return max_error + + +@dataclass +class AdaptiveMultiFidelityResult: + rom_model: QoiModel + training_data: tuple[FidelityData, ...] + validation_errors: np.ndarray + sample_counts: np.ndarray + total_cost: float + converged: bool + termination_reason: str + + +def _counts(value, n, name): + if len(value) != n: + raise ValueError(f"{name} must have one count per fidelity") + result = tuple(value) + for i, count in enumerate(result): + _positive_integer(count, f"{name}[{i}]") + return result + + +def _copy_data(data): + """Do not let a builder or validation strategy change accumulated data.""" + copies = [] + for level in data: + parameters = level.parameters.copy() + qois = level.qois.copy() + parameters.setflags(write=False) + qois.setflags(write=False) + copies.append(FidelityData(parameters, qois, level.training_dirs, level.name)) + return tuple(copies) + + +def _random_batches(parameter_space, names, counts, paired, rng): + """Draw exactly the requested samples per fidelity, sharing higher points. + + When paired, samples for level l are a superset of level l+1. + No additional LF evaluation is counted outside the specified allocation. + """ + n = len(counts) + result = [None] * n + higher = np.empty((0, len(names))) + for level in reversed(range(n)): + shared = higher if paired else np.empty((0, len(names))) + additional = counts[level] - len(shared) + seed = int(rng.integers(0, 2**32)) + drawn = np.asarray(parameter_space.generate_samples(additional, seed=seed)) + if drawn.ndim != 2 or drawn.shape != (additional, len(names)): + raise ValueError("ParameterSpace returned samples with an unexpected shape") + if drawn.dtype.kind not in "biuf" or not np.all(np.isfinite(drawn)): + raise ValueError("multifidelity sampling requires finite real parameters") + result[level] = np.concatenate((shared, drawn)) if paired else drawn + higher = result[level] + return result + + +def run_adaptive_multifidelity( + fidelities: Sequence[FidelityLevel], + surrogate_builder, + parameter_space: ParameterSpace, + absolute_work_dir: str, + tolerance: float, + initial_samples: Sequence[int], + batch_sizes: Sequence[int], + *, + adaptive_sampling_strategy: str = "random", + pair_fidelity_samples: bool = True, + validation_strategy: Optional[Callable] = None, + error_functor: Callable = relative_l2_error, + max_iterations: int = 20, + max_total_cost: Optional[float] = None, + max_concurrent_fom_runs: int = 1, + random_seed: int = 1, +) -> AdaptiveMultiFidelityResult: + """Build a multifidelity surrogate after each randomized sampling round. + + An iteration always schedules the requested counts at *every* fidelity. + Iteration zero uses initial_samples; subsequent rounds use batch_sizes. + With pairing, every higher-fidelity location is sampled at all lower + levels and per-level counts must be nonincreasing. ParameterSpace controls + the underlying random sampling distribution. Costs are fixed user inputs; + incomplete rounds are not automatically retried or restarted. + """ + levels = tuple(fidelities) + if len(levels) < 2: + raise ValueError("at least two fidelity levels are required") + names = _parameter_names(parameter_space) + if not names: + raise ValueError("multifidelity sampling requires a nonempty parameter space") + if not isinstance(pair_fidelity_samples, bool): + raise TypeError("pair_fidelity_samples must be a bool") + if adaptive_sampling_strategy != "random": + raise ValueError("only adaptive_sampling_strategy='random' is supported") + if not callable(getattr(surrogate_builder, "build", None)): + raise TypeError("surrogate_builder must provide build()") + if not callable(error_functor): + raise TypeError("error_functor must be callable") + for name, value in (("max_iterations", max_iterations), + ("max_concurrent_fom_runs", max_concurrent_fom_runs)): + _positive_integer(value, name) + tolerance = _error_value(tolerance, "tolerance") + initial = _counts(initial_samples, len(levels), "initial_samples") + batch = _counts(batch_sizes, len(levels), "batch_sizes") + if pair_fidelity_samples: + if any(a < b for counts in (initial, batch) + for a, b in zip(counts[:-1], counts[1:])): + raise ValueError("paired sampling requires nonincreasing per-level counts") + if len(set(level.name for level in levels)) != len(levels): + raise ValueError("fidelity names must be unique") + for level in levels: + cost = _error_value(level.cost, "fidelity cost") + if cost <= 0: + raise ValueError("fidelity costs must be positive") + for method in ("populate_run_directory", "run_model", "compute_qoi"): + if not callable(getattr(level.model, method, None)): + raise TypeError(f"fidelity {level.name} lacks {method}") + if max_total_cost is not None: + max_total_cost = _error_value(max_total_cost, "max_total_cost") + strategy = MultiFidelityKFoldValidation() if validation_strategy is None else validation_strategy + if not callable(strategy): + raise TypeError("validation_strategy must be callable") + if isinstance(strategy, MultiFidelityKFoldValidation) and initial[-1] < strategy.num_folds: + raise ValueError("initial highest-fidelity count must be >= num_folds") + initial_cost = sum(level.cost * n for level, n in zip(levels, initial)) + if max_total_cost is not None and initial_cost > max_total_cost: + raise ValueError("initial sample allocation exceeds max_total_cost") + + directory = os.fspath(absolute_work_dir) + if not os.path.isabs(directory): + raise ValueError("absolute_work_dir must be absolute") + if os.path.exists(directory) and (not os.path.isdir(directory) or os.listdir(directory)): + raise ValueError("absolute_work_dir must be absent or empty") + os.makedirs(directory, exist_ok=True) + + sampling_rng, model_rng, validation_rng, build_rng = map( + np.random.default_rng, np.random.SeedSequence(random_seed).spawn(4) + ) + data = [FidelityData(np.empty((0, len(names))), np.empty((0, 1)), + name=level.name) for level in levels] + errors, count_history = [], [] + total_cost = 0.0 + surrogate = None + reason = "max_iterations" + with open(os.path.join(directory, "multifidelity_status.log"), "w", encoding="utf8") as log: + for iteration in range(max_iterations): + allocations = initial if iteration == 0 else batch + round_cost = sum(level.cost * n for level, n in zip(levels, allocations)) + if max_total_cost is not None and total_cost + round_cost > max_total_cost + 1e-12: + reason = "max_total_cost" + break + batches = _random_batches(parameter_space, names, allocations, + pair_fidelity_samples, sampling_rng) + for fidelity, (level, points) in enumerate(zip(levels, batches)): + old = data[fidelity] + start = len(old.parameters) + indices = list(range(start, start + len(points))) + run_dirs = [ + os.path.join(directory, f"fidelity_{fidelity}", f"run_{i}") + for i in indices + ] + seeds = [int(model_rng.integers(0, 2**32)) for _ in indices] + qois, _ = _evaluate_batch( + level.model, names, points, run_dirs, indices, seeds, + max_concurrent_fom_runs, + None if start == 0 else old.qois.shape[1], + ) + if start and qois.shape[1] != old.qois.shape[1]: + raise ValueError("fidelity QoI dimension changed across rounds") + data[fidelity] = FidelityData( + np.concatenate((old.parameters, points)), + np.concatenate((old.qois, qois)) if start else qois, + old.training_dirs + tuple(run_dirs), level.name, + ) + total_cost += round_cost + round_dir = os.path.join(directory, f"iteration_{iteration}") + offline = os.path.join(round_dir, "offline_data") + os.makedirs(offline, exist_ok=False) + training = _copy_data(data) + with _process_state(int(build_rng.integers(0, 2**32))): + surrogate = surrogate_builder.build(offline, training) + validation_dir = os.path.join(round_dir, "validation") + os.makedirs(validation_dir, exist_ok=False) + seed = int(validation_rng.integers(0, 2**32)) + context = MultiFidelityValidationContext( + surrogate, surrogate_builder, training, tuple(names), iteration, + validation_dir, seed, error_functor, + ) + with _process_state(seed): + error = _error_value(strategy(context), "validation result") + errors.append(error) + count_history.append([len(level_data.parameters) for level_data in data]) + log.write(f"Iteration {iteration}: counts={count_history[-1]}, " + f"cost={total_cost:.12g}, validation_error={error:.12g}\n") + log.flush() + if error <= tolerance: + reason = "tolerance" + break + + np.savez( + os.path.join(directory, "multifidelity_stats.npz"), + sample_counts=np.asarray(count_history, dtype=int), + validation_errors=np.asarray(errors), + total_cost=total_cost, + termination_reason=reason, + **{f"parameters_{l}": entry.parameters for l, entry in enumerate(data)}, + **{f"qois_{l}": entry.qois for l, entry in enumerate(data)}, + ) + return AdaptiveMultiFidelityResult( + surrogate, _copy_data(data), np.asarray(errors), + np.asarray(count_history, dtype=int).reshape((-1, len(levels))), + total_cost, reason == "tolerance", reason, + ) From f75513ab80e7c631f67fade4f372c3115b000da6 Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 13:19:36 -0700 Subject: [PATCH 07/12] Reuse established model evaluator in multifidelity cross-validation --- romtools/workflows/multifidelity.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/romtools/workflows/multifidelity.py b/romtools/workflows/multifidelity.py index a7fdb2d2..ff9f1030 100644 --- a/romtools/workflows/multifidelity.py +++ b/romtools/workflows/multifidelity.py @@ -14,7 +14,7 @@ from romtools.rom.multifidelity import FidelityData from romtools.workflows.models import QoiModel from romtools.workflows.parameter_spaces import ParameterSpace -from romtools.workflows._adaptive.execution import _evaluate_batch, _process_state +from romtools.workflows._adaptive.execution import _evaluate_batch, _evaluate_sample, _process_state from romtools.workflows._adaptive.sampling import ( _parameter_names, _positive_integer, ) @@ -88,17 +88,13 @@ def __call__(self, context: MultiFidelityValidationContext) -> float: for held in holdout: row = hf.parameters[int(held)] parameter_sample = dict(zip(context.parameter_names, row)) - # The surrogate is a QoiModel: preserve the workflow's - # established populate/run/compute execution contract. - run_dir = os.path.join(fold_dir, f"run_{int(held)}") - os.makedirs(run_dir, exist_ok=False) - with _process_state(int(rng.integers(0, 2**32))): - model.populate_run_directory(run_dir, parameter_sample) - status = model.run_model(run_dir, parameter_sample) - if status != 0: - raise RuntimeError(f"fold {index} surrogate evaluation failed: {status}") - predicted = np.asarray(model.compute_qoi(run_dir, parameter_sample), - dtype=float).reshape(-1) + # Reuse the same blocking/nonblocking model execution contract + # as the existing single-fidelity K-fold validator. + predicted, _ = _evaluate_sample( + model, context.parameter_names, row, + os.path.join(fold_dir, f"run_{int(held)}"), + int(held), int(rng.integers(0, 2**32)), + ) reference = np.asarray(hf.qois[int(held)], dtype=float).reshape(-1) if predicted.shape != reference.shape: raise ValueError("held-out QoI dimension mismatch") From 84f4d2f594dc2bb425023ca68dbb4c643d0a2492 Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 13:20:15 -0700 Subject: [PATCH 08/12] Test random multifidelity workflow pairing, nonnested data, K-fold CV and budget --- .../romtools/workflows/test_multifidelity.py | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 tests/romtools/workflows/test_multifidelity.py diff --git a/tests/romtools/workflows/test_multifidelity.py b/tests/romtools/workflows/test_multifidelity.py new file mode 100644 index 00000000..c185371c --- /dev/null +++ b/tests/romtools/workflows/test_multifidelity.py @@ -0,0 +1,132 @@ +"""Integration coverage for fixed-allocation multifidelity sampling and CV.""" +import numpy as np +import pytest + +from romtools.rom.multifidelity import ( + AutoregressiveGPBuilder, AutoregressiveGPConfig, GPLevelConfig, +) +from romtools.rom.qoi_surrogates import GaussianProcessKernel +from romtools.workflows.multifidelity import ( + FidelityLevel, MultiFidelityKFoldValidation, run_adaptive_multifidelity, +) +from romtools.workflows.parameter_spaces import UniformParameterSpace +from romtools.workflows.sampling_methods import MonteCarloSampler + + +class SyntheticQoiModel: + def __init__(self, factor=1.0): + self.factor = factor + + def populate_run_directory(self, run_directory, parameter_sample): + pass + + def run_model(self, run_directory, parameter_sample): + return 0 + + def compute_qoi(self, run_directory, parameter_sample): + x = parameter_sample["x"] + return np.array([self.factor * np.sin(2.0 * x) + 0.1 * x]) + + +def make_options(tmp_path, paired=True, budget=None, max_iterations=2): + return dict( + fidelities=[ + FidelityLevel(SyntheticQoiModel(1.0), name="low", cost=1.0), + FidelityLevel(SyntheticQoiModel(1.2), name="high", cost=10.0), + ], + surrogate_builder=AutoregressiveGPBuilder( + AutoregressiveGPConfig( + levels=[ + GPLevelConfig(GaussianProcessKernel(length_scale=0.3), + noise_variance=1e-8), + GPLevelConfig(GaussianProcessKernel(length_scale=0.3), + rho=1.0, noise_variance=1e-8), + ], + tune_hyperparameters=False, + ), + parameter_names=["x"], + ), + parameter_space=UniformParameterSpace( + ["x"], [0.0], [1.0], MonteCarloSampler, + ), + absolute_work_dir=str(tmp_path), + tolerance=0.0, + initial_samples=[6, 3], + batch_sizes=[2, 1], + adaptive_sampling_strategy="random", + pair_fidelity_samples=paired, + validation_strategy=MultiFidelityKFoldValidation(num_folds=3), + max_iterations=max_iterations, + max_total_cost=budget, + random_seed=31, + ) + + +def test_random_multifidelity_workflow_pairs_and_counts(tmp_path): + result = run_adaptive_multifidelity(**make_options(tmp_path)) + assert result.termination_reason == "max_iterations" + np.testing.assert_array_equal(result.sample_counts, [[6, 3], [8, 4]]) + assert result.total_cost == 48.0 + assert result.validation_errors.shape == (2,) + assert np.all(np.isfinite(result.validation_errors)) + low, high = result.training_data + assert low.qois.shape == (8, 1) + assert high.qois.shape == (4, 1) + assert set(high.parameters[:, 0]).issubset(set(low.parameters[:, 0])) + assert all((tmp_path / "fidelity_1" / f"run_{i}" / "passed.txt").is_file() + for i in range(4)) + assert (tmp_path / "multifidelity_stats.npz").is_file() + + +def test_random_multifidelity_without_pairing(tmp_path): + result = run_adaptive_multifidelity(**make_options(tmp_path, paired=False)) + low, high = result.training_data + assert not set(low.parameters[:, 0]).intersection(set(high.parameters[:, 0])) + np.testing.assert_array_equal(result.sample_counts[-1], [8, 4]) + + +def test_budget_stops_before_unaffordable_round(tmp_path): + result = run_adaptive_multifidelity( + **make_options(tmp_path, budget=36.0, max_iterations=4)) + assert result.termination_reason == "max_total_cost" + np.testing.assert_array_equal(result.sample_counts, [[6, 3]]) + assert result.total_cost == 36.0 + assert not (tmp_path / "iteration_1").exists() + + +def test_initial_budget_is_validated_before_execution(tmp_path): + with pytest.raises(ValueError, match="initial sample allocation"): + run_adaptive_multifidelity(**make_options(tmp_path, budget=35.0)) + assert not tmp_path.exists() + + +def test_paired_counts_must_be_nonincreasing(tmp_path): + options = make_options(tmp_path) + options["batch_sizes"] = [1, 2] + with pytest.raises(ValueError, match="nonincreasing"): + run_adaptive_multifidelity(**options) + + +def test_three_level_random_batch_pairing_and_cost(tmp_path): + options = make_options(tmp_path, max_iterations=1) + options["fidelities"] = [ + FidelityLevel(SyntheticQoiModel(1), "low", 1), + FidelityLevel(SyntheticQoiModel(1.1), "middle", 3), + FidelityLevel(SyntheticQoiModel(1.2), "high", 10), + ] + options["surrogate_builder"] = AutoregressiveGPBuilder( + AutoregressiveGPConfig( + levels=[GPLevelConfig(GaussianProcessKernel(length_scale=0.3), + noise_variance=1e-8) + for _ in range(3)], + tune_hyperparameters=False, + ), + parameter_names=["x"], + ) + options["initial_samples"] = [8, 5, 3] + options["batch_sizes"] = [4, 2, 1] + result = run_adaptive_multifidelity(**options) + np.testing.assert_array_equal(result.sample_counts, [[8, 5, 3]]) + assert result.total_cost == 53 + for lower, higher in zip(result.training_data[:-1], result.training_data[1:]): + assert set(higher.parameters[:, 0]).issubset(set(lower.parameters[:, 0])) From c1f47dc348766feb736418bd8f8ee38649a7b7b9 Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 13:29:50 -0700 Subject: [PATCH 09/12] Fix multifidelity CI regressions before CDR integration --- romtools/rom/multifidelity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/romtools/rom/multifidelity.py b/romtools/rom/multifidelity.py index 388b7b55..4fd6423e 100644 --- a/romtools/rom/multifidelity.py +++ b/romtools/rom/multifidelity.py @@ -261,7 +261,7 @@ def predict(self, parameters, fidelity: int = -1, return_std: bool = False): mean = k_star @ self._alpha if not return_std: return mean - variance = np.diag(self._cov(x, level, x, level)) + variance = np.diag(self._cov(x, level, x, level)).copy() variance -= np.sum(k_star * cho_solve(self._chol, k_star.T).T, axis=1) return mean, np.sqrt(np.maximum(variance, 0.0)) From 80c4f71ec67bca93cf1b7a4a1e4b94dc313673ba Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 13:29:52 -0700 Subject: [PATCH 10/12] Fix multifidelity CI regressions before CDR integration --- tests/romtools/workflows/test_multifidelity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/romtools/workflows/test_multifidelity.py b/tests/romtools/workflows/test_multifidelity.py index c185371c..fe96aafb 100644 --- a/tests/romtools/workflows/test_multifidelity.py +++ b/tests/romtools/workflows/test_multifidelity.py @@ -97,7 +97,7 @@ def test_budget_stops_before_unaffordable_round(tmp_path): def test_initial_budget_is_validated_before_execution(tmp_path): with pytest.raises(ValueError, match="initial sample allocation"): run_adaptive_multifidelity(**make_options(tmp_path, budget=35.0)) - assert not tmp_path.exists() + assert not (tmp_path / "multifidelity_status.log").exists() def test_paired_counts_must_be_nonincreasing(tmp_path): From ac5ecac1b377fb9c3509223fc9314403c6026f77 Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 13:30:38 -0700 Subject: [PATCH 11/12] Exercise random multifidelity GP workflow on coarse/fine steady CDR PDE --- .../workflows/test_multifidelity_cdr.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 tests/romtools/workflows/test_multifidelity_cdr.py diff --git a/tests/romtools/workflows/test_multifidelity_cdr.py b/tests/romtools/workflows/test_multifidelity_cdr.py new file mode 100644 index 00000000..d6a5322c --- /dev/null +++ b/tests/romtools/workflows/test_multifidelity_cdr.py @@ -0,0 +1,117 @@ +"""CDR PDE integration test for multifidelity random adaptive GP training. + +Reuses the established steady CDR model with two spatial mesh resolutions. +A grid-independent scalar QoI (domain integral of the state) is extracted from +the same solver output; the original vector-QoI wrapper is not modified. +""" +import sys +from pathlib import Path + +import numpy as np + +MODELS = Path(__file__).resolve().parents[3] / "examples" / "models" +if str(MODELS) not in sys.path: + sys.path.insert(0, str(MODELS)) + +from steady_cdr_model import SteadyCdrQoiModel + +from romtools.rom.multifidelity import ( + AutoregressiveGPBuilder, AutoregressiveGPConfig, GPLevelConfig, +) +from romtools.rom.qoi_surrogates import GaussianProcessKernel +from romtools.workflows.multifidelity import ( + FidelityLevel, MultiFidelityKFoldValidation, run_adaptive_multifidelity, +) +from romtools.workflows.parameter_spaces import UniformParameterSpace +from romtools.workflows.sampling_methods import MonteCarloSampler + + +class IntegralSteadyCdrQoiModel(SteadyCdrQoiModel): + """Scalar physical-domain integral of the steady CDR solution.""" + + def compute_qoi(self, run_directory, parameter_sample): + with np.load(Path(run_directory) / "solution.npz") as saved: + solution = saved["state"] + return np.array([solution.sum() * self.system.dx * self.system.dy]) + + +def test_adaptive_multifidelity_on_steady_cdr(tmp_path): + names = ["bmag", "theta", "nu", "sigma"] + parameter_space = UniformParameterSpace( + names, + [0.2, 0.3, 0.002, 0.4], + [0.8, 1.2, 0.010, 1.5], + MonteCarloSampler, + ) + low = IntegralSteadyCdrQoiModel(nx=9, ny=9) + high = IntegralSteadyCdrQoiModel(nx=21, ny=21) + + builder = AutoregressiveGPBuilder( + AutoregressiveGPConfig( + levels=[ + GPLevelConfig(GaussianProcessKernel(length_scale=0.5), + noise_variance=1e-8), + GPLevelConfig(GaussianProcessKernel(length_scale=0.5), + rho=1.0, noise_variance=1e-8), + ], + tune_hyperparameters=False, + ), + parameter_names=names, + ) + result = run_adaptive_multifidelity( + fidelities=[ + FidelityLevel(low, "coarse_9x9", cost=1), + FidelityLevel(high, "fine_21x21", cost=10), + ], + surrogate_builder=builder, + parameter_space=parameter_space, + absolute_work_dir=str(tmp_path / "cdr_multifidelity"), + tolerance=0.0, # force all rounds unless the holdout error is exactly zero + initial_samples=[9, 4], + batch_sizes=[3, 1], + adaptive_sampling_strategy="random", + pair_fidelity_samples=True, + validation_strategy=MultiFidelityKFoldValidation(num_folds=2), + max_iterations=2, + random_seed=2026, + ) + + np.testing.assert_array_equal(result.sample_counts, [[9, 4], [12, 5]]) + assert result.total_cost == 62 + assert result.validation_errors.shape == (2,) + assert np.all(np.isfinite(result.validation_errors)) + assert np.all(result.validation_errors >= 0) + assert result.termination_reason == "max_iterations" + + lf_data, hf_data = result.training_data + assert lf_data.qois.shape == (12, 1) + assert hf_data.qois.shape == (5, 1) + assert set(map(tuple, hf_data.parameters)).issubset( + set(map(tuple, lf_data.parameters)) + ) + # Discretization levels are correlated but not identical. + paired_lf = { + tuple(row): float(qoi[0]) + for row, qoi in zip(lf_data.parameters, lf_data.qois) + } + paired_errors = np.array([ + float(qoi[0]) - paired_lf[tuple(row)] + for row, qoi in zip(hf_data.parameters, hf_data.qois) + ]) + assert np.all(np.isfinite(paired_errors)) + assert np.any(np.abs(paired_errors) > 1e-8) + + # Check the final hierarchical surrogate against an *unseen* fine-grid solve. + sample = dict(zip(names, [0.47, 0.73, 0.006, 0.91])) + validation_dir = tmp_path / "independent_fine_grid" + validation_dir.mkdir() + high.populate_run_directory(str(validation_dir), sample) + assert high.run_model(str(validation_dir), sample) == 0 + truth = high.compute_qoi(str(validation_dir), sample) + prediction = result.rom_model.compute_qoi("", sample) + std = result.rom_model.compute_error_estimate("", sample) + assert truth.shape == prediction.shape == (1,) + assert np.all(np.isfinite(prediction)) + assert np.isfinite(std) and std >= 0 + assert np.isfinite(abs(float(prediction[0] - truth[0]))) + assert (tmp_path / "cdr_multifidelity" / "multifidelity_stats.npz").is_file() From 694fa1899294448e3dd660ba88cebe030f6dba90 Mon Sep 17 00:00:00 2001 From: eparish1 Date: Thu, 19 Nov 2026 13:31:23 -0700 Subject: [PATCH 12/12] Add reproducible CDR multifidelity benchmark with independent HF accuracy comparison --- examples/multifidelity_cdr.py | 144 ++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 examples/multifidelity_cdr.py diff --git a/examples/multifidelity_cdr.py b/examples/multifidelity_cdr.py new file mode 100644 index 00000000..7505649f --- /dev/null +++ b/examples/multifidelity_cdr.py @@ -0,0 +1,144 @@ +"""Run a two-fidelity autoregressive GP benchmark on the steady CDR PDE. + +Run from the repository root: + python examples/multifidelity_cdr.py --smoke + python examples/multifidelity_cdr.py --output-dir /tmp/cdr-mf + +The benchmark uses the same solver at two spatial resolutions and predicts +the scalar integral of the PDE state. A fixed independent HF test set is used +only for final reporting, never for adaptive sampling or cross-validation. +""" +import argparse +from pathlib import Path +import sys + +import numpy as np + +MODELS = Path(__file__).resolve().parent / "models" +sys.path.insert(0, str(MODELS)) +from steady_cdr_model import SteadyCdrQoiModel + +from romtools.rom.multifidelity import ( + AutoregressiveGPBuilder, AutoregressiveGPConfig, GPLevelConfig, +) +from romtools.rom.qoi_surrogates import GaussianProcessKernel, GaussianProcessQoiModel +from romtools.workflows.multifidelity import ( + FidelityLevel, MultiFidelityKFoldValidation, run_adaptive_multifidelity, +) +from romtools.workflows.parameter_spaces import UniformParameterSpace +from romtools.workflows.sampling_methods import MonteCarloSampler + + +class IntegralCdrModel(SteadyCdrQoiModel): + """QoI: integral of the steady CDR solution over the unit square.""" + + def compute_qoi(self, run_directory, parameter_sample): + with np.load(Path(run_directory) / "solution.npz") as data: + solution = data["state"] + return np.array([solution.sum() * self.system.dx * self.system.dy]) + + +def reference_qois(model, parameter_names, samples, directory): + directory.mkdir(parents=True, exist_ok=False) + outputs = [] + for i, values in enumerate(samples): + run_dir = directory / f"run_{i}" + run_dir.mkdir() + parameters = dict(zip(parameter_names, values)) + model.populate_run_directory(str(run_dir), parameters) + if model.run_model(str(run_dir), parameters) != 0: + raise RuntimeError(f"reference evaluation {i} failed") + outputs.append(float(model.compute_qoi(str(run_dir), parameters)[0])) + return np.array(outputs) + + +def run_benchmark(output_dir, smoke=False): + names = ["bmag", "theta", "nu", "sigma"] + parameter_space = UniformParameterSpace( + names, [0.2, 0.3, 0.002, 0.4], + [0.8, 1.2, 0.010, 1.5], MonteCarloSampler, + ) + low = IntegralCdrModel(9 if smoke else 15, 9 if smoke else 15) + high = IntegralCdrModel(21 if smoke else 39, 21 if smoke else 39) + config = AutoregressiveGPConfig( + levels=[ + GPLevelConfig(GaussianProcessKernel(length_scale=0.5), + noise_variance=1e-8), + GPLevelConfig(GaussianProcessKernel(length_scale=0.5), + rho=1.0, noise_variance=1e-8), + ], + tune_hyperparameters=not smoke, + optimizer_restarts=2, + ) + result = run_adaptive_multifidelity( + fidelities=[ + FidelityLevel(low, "coarse_CDR", cost=1), + FidelityLevel(high, "fine_CDR", cost=10), + ], + surrogate_builder=AutoregressiveGPBuilder(config, parameter_names=names), + parameter_space=parameter_space, + absolute_work_dir=str(output_dir / "adaptive"), + tolerance=0.0, + initial_samples=[9, 4] if smoke else [16, 6], + batch_sizes=[3, 1] if smoke else [8, 2], + adaptive_sampling_strategy="random", + pair_fidelity_samples=True, + validation_strategy=MultiFidelityKFoldValidation(num_folds=2 if smoke else 3), + max_iterations=2 if smoke else 3, + random_seed=2026, + ) + # This test set is independent of training and the K-fold partitions. + test_samples = parameter_space.generate_samples( + 5 if smoke else 20, seed=93145, + ) + truth = reference_qois(high, names, test_samples, output_dir / "reference") + low_prediction = reference_qois( + low, names, test_samples, output_dir / "reference_low" + ) + mf_prediction = np.asarray(result.rom_model.predict(test_samples)).ravel() + hf = result.training_data[-1] + hf_only_model = GaussianProcessQoiModel( + hf.parameters, hf.qois, + kernel=GaussianProcessKernel(length_scale=0.5), + noise_variance=1e-8, + normalize_parameters=True, + ) + hf_only_prediction = np.array([ + hf_only_model.compute_qoi("", dict(zip(names, x)))[0] + for x in test_samples + ]) + + denom = max(np.linalg.norm(truth), 1e-12) + metrics = { + "low_fidelity_direct_relative_l2": float(np.linalg.norm(low_prediction - truth) / denom), + "hf_only_gp_relative_l2": float(np.linalg.norm(hf_only_prediction - truth) / denom), + "multifidelity_gp_relative_l2": float(np.linalg.norm(mf_prediction - truth) / denom), + } + output_dir.mkdir(parents=True, exist_ok=True) + np.savez( + output_dir / "benchmark_results.npz", + test_parameters=test_samples, reference_qois=truth, + low_fidelity_qois=low_prediction, + hf_only_gp_predictions=hf_only_prediction, + multifidelity_gp_predictions=mf_prediction, + kfold_errors=result.validation_errors, + sample_counts=result.sample_counts, + nominal_cost=result.total_cost, + **metrics, + ) + print("CDR multifidelity benchmark") + print("sample counts (LF, HF):", result.sample_counts.tolist()) + print("highest-fidelity K-fold errors:", result.validation_errors.tolist()) + print("nominal total training cost:", result.total_cost) + for name, value in metrics.items(): + print(f"{name}: {value:.6e}") + print("Results:", output_dir / "benchmark_results.npz") + return metrics + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--smoke", action="store_true", help="small, fast comparison") + parser.add_argument("--output-dir", type=Path, default=Path("cdr_mf_results")) + args = parser.parse_args() + run_benchmark(args.output_dir.resolve(), smoke=args.smoke)