diff --git a/romtools/rom/qoi_surrogates.py b/romtools/rom/qoi_surrogates.py index a7c08a457..c8cdf1914 100644 --- a/romtools/rom/qoi_surrogates.py +++ b/romtools/rom/qoi_surrogates.py @@ -8,16 +8,28 @@ @dataclass class GaussianProcessKernel: - length_scale: float = 1.0 + length_scale: float | Sequence[float] = 1.0 signal_variance: float = 1.0 def __call__(self, x1: np.ndarray, x2: np.ndarray) -> np.ndarray: - x1 = np.atleast_2d(x1) - x2 = np.atleast_2d(x2) - x1_sq = np.sum(x1 * x1, axis=1)[:, None] - x2_sq = np.sum(x2 * x2, axis=1)[None, :] - sq_dist = x1_sq + x2_sq - 2.0 * x1 @ x2.T - return self.signal_variance * np.exp(-0.5 * sq_dist / (self.length_scale ** 2)) + x1 = np.atleast_2d(np.asarray(x1, dtype=float)) + x2 = np.atleast_2d(np.asarray(x2, dtype=float)) + if x1.shape[1] != x2.shape[1]: + raise ValueError("Input parameter dimensions must match.") + scales = np.asarray(self.length_scale, dtype=float) + if scales.ndim > 1 or (scales.ndim == 1 and scales.size != x1.shape[1]): + raise ValueError("length_scale must be a scalar or one value per parameter.") + if not np.all(np.isfinite(scales)) or np.any(scales <= 0): + raise ValueError("length_scale must contain positive finite values.") + if not np.isfinite(self.signal_variance) or self.signal_variance <= 0: + raise ValueError("signal_variance must be positive and finite.") + x1_scaled = x1 / scales + x2_scaled = x2 / scales + # Squared norms minus a dot product suffers catastrophic cancellation + # when coordinates are large relative to the length scales. + from scipy.spatial.distance import cdist + sq_dist = cdist(x1_scaled, x2_scaled, metric="sqeuclidean") + return self.signal_variance * np.exp(-0.5 * sq_dist) class GaussianProcessRegressorLite: @@ -106,7 +118,9 @@ def __init__(self, length_scale_grid: Optional[Sequence[float]] = None, signal_variance_grid: Optional[Sequence[float]] = None, normalize_parameters: bool = False, - normalize_targets: bool = False) -> None: + normalize_targets: bool = False, + anisotropic: bool = False, + optimizer_restarts: int = 4) -> None: self.parameter_names = list(parameter_names) if parameter_names is not None else None self.parameters = np.asarray(parameters, dtype=float) self.qois = np.asarray(qois, dtype=float) @@ -121,6 +135,10 @@ def __init__(self, self.signal_variance_grid = signal_variance_grid self.normalize_parameters = normalize_parameters self.normalize_targets = normalize_targets + self.anisotropic = anisotropic + self.optimizer_restarts = optimizer_restarts + if optimizer_restarts < 1: + raise ValueError("optimizer_restarts must be positive") self._mean_qoi: Optional[np.ndarray] = None self._pod_modes: Optional[np.ndarray] = None @@ -289,6 +307,42 @@ def _tuned_kernel(self, signal_variances = (self.signal_variance_grid if self.signal_variance_grid is not None else np.logspace(-6, 2, 9)) + if self.anisotropic: + from scipy.optimize import minimize + + dimension = parameters.shape[1] + scales = np.asarray(self.kernel.length_scale, dtype=float) + if scales.ndim == 0: + scales = np.full(dimension, float(scales)) + if scales.shape != (dimension,) or np.any(scales <= 0): + raise ValueError("Initial ARD length scales must match input dimension.") + length_bounds = (float(np.min(length_scales)), float(np.max(length_scales))) + variance_bounds = (float(np.min(signal_variances)), float(np.max(signal_variances))) + if length_bounds[0] <= 0 or variance_bounds[0] <= 0: + raise ValueError("GP hyperparameter bounds must be positive.") + bounds = [tuple(np.log(length_bounds))] * dimension + [tuple(np.log(variance_bounds))] + initial = np.log(np.r_[scales, self.kernel.signal_variance]) + initial = np.clip(initial, [b[0] for b in bounds], [b[1] for b in bounds]) + rng = np.random.default_rng(0) + starts = [initial] + starts.extend(rng.uniform([b[0] for b in bounds], + [b[1] for b in bounds]) + for _ in range(self.optimizer_restarts - 1)) + + def objective(log_params): + candidate = GaussianProcessKernel( + length_scale=np.exp(log_params[:-1]), + signal_variance=float(np.exp(log_params[-1]))) + lml = GaussianProcessRegressorLite.log_marginal_likelihood( + parameters, target, candidate, noise_variance) + return -lml if np.isfinite(lml) else 1e100 + + best = min((minimize(objective, start, method="L-BFGS-B", bounds=bounds) + for start in starts), key=lambda result: result.fun) + return GaussianProcessKernel( + length_scale=np.exp(best.x[:-1]), + signal_variance=float(np.exp(best.x[-1]))) + best_kernel = self.kernel best_lml = float("-inf") for length_scale in length_scales: diff --git a/romtools/workflows/inverse/eki_bound_transforms.py b/romtools/workflows/inverse/eki_bound_transforms.py index eff12e86b..debb64e7a 100644 --- a/romtools/workflows/inverse/eki_bound_transforms.py +++ b/romtools/workflows/inverse/eki_bound_transforms.py @@ -500,6 +500,8 @@ def _auto_rom_builder(parameter_space, rom_type, rom_args): signal_variance_grid=rom_args.get("signal_variance_grid"), normalize_parameters=rom_args.get("normalize_parameters", False), normalize_targets=rom_args.get("normalize_targets", False), + anisotropic=rom_args.get("anisotropic", False), + optimizer_restarts=rom_args.get("optimizer_restarts", 4), ) if rom_type in ("nn", "neural_network", "neural-network"): return _mf_eki_module.NeuralNetworkQoiModelBuilderWithTrainingData( diff --git a/romtools/workflows/inverse/mf_eki_drivers.py b/romtools/workflows/inverse/mf_eki_drivers.py index 332f4ff27..476c77f60 100644 --- a/romtools/workflows/inverse/mf_eki_drivers.py +++ b/romtools/workflows/inverse/mf_eki_drivers.py @@ -69,7 +69,9 @@ def __init__(self, length_scale_grid: Optional[list] = None, signal_variance_grid: Optional[list] = None, normalize_parameters: bool = False, - normalize_targets: bool = False) -> None: + normalize_targets: bool = False, + anisotropic: bool = False, + optimizer_restarts: int = 4) -> None: self.parameter_names = ( list(parameter_names) if parameter_names is not None else None ) @@ -84,6 +86,8 @@ def __init__(self, self.signal_variance_grid = signal_variance_grid self.normalize_parameters = normalize_parameters self.normalize_targets = normalize_targets + self.anisotropic = anisotropic + self.optimizer_restarts = optimizer_restarts def build_from_training_dirs(self, offline_data_dir: str, @@ -105,6 +109,8 @@ def build_from_training_dirs(self, signal_variance_grid=self.signal_variance_grid, normalize_parameters=self.normalize_parameters, normalize_targets=self.normalize_targets, + anisotropic=self.anisotropic, + optimizer_restarts=self.optimizer_restarts, ) @@ -1317,6 +1323,8 @@ def mf_eki_with_auto_rom(model: QoiModel, signal_variance_grid=rom_args.get("signal_variance_grid"), normalize_parameters=rom_args.get("normalize_parameters", False), normalize_targets=rom_args.get("normalize_targets", False), + anisotropic=rom_args.get("anisotropic", False), + optimizer_restarts=rom_args.get("optimizer_restarts", 4), ) elif rom_type_normalized in ("nn", "neural_network", "neural-network"): rom_model_builder = NeuralNetworkQoiModelBuilderWithTrainingData( diff --git a/romtools/workflows/inverse/vi/api.py b/romtools/workflows/inverse/vi/api.py index df8bea942..975739d24 100644 --- a/romtools/workflows/inverse/vi/api.py +++ b/romtools/workflows/inverse/vi/api.py @@ -317,6 +317,8 @@ def mf_vi_with_auto_rom(model: QoiModel, signal_variance_grid=rom_args.get("signal_variance_grid"), normalize_parameters=rom_args.get("normalize_parameters", False), normalize_targets=rom_args.get("normalize_targets", False), + anisotropic=rom_args.get("anisotropic", False), + optimizer_restarts=rom_args.get("optimizer_restarts", 4), ) else: raise ValueError(f"Unsupported rom_type '{rom_type}'.") diff --git a/tests/romtools/rom/test_gp.py b/tests/romtools/rom/test_gp.py index 4cef4ab40..0862b5a40 100644 --- a/tests/romtools/rom/test_gp.py +++ b/tests/romtools/rom/test_gp.py @@ -138,3 +138,57 @@ def test_predict_mean_and_std_raises_before_fit(): test_predict_mean_and_std_vector_query_matches_reference() test_predict_mean_and_std_mean_matches_predict() test_predict_mean_and_std_raises_before_fit() + + +def test_ard_kernel_matches_manual_covariance(): + kernel = GaussianProcessKernel(length_scale=[0.2, 2.0], signal_variance=1.5) + x = np.array([[0.0, 0.0], [0.2, 2.0]]) + expected = 1.5 * np.exp(-1.0) + np.testing.assert_allclose(kernel(x, x)[0, 1], expected) + + +def test_scalar_and_repeated_vector_length_scales_agree(): + x = np.array([[0.0, 0.5], [1.0, 2.0], [0.3, 1.0]]) + np.testing.assert_allclose( + GaussianProcessKernel(length_scale=0.7)(x, x), + GaussianProcessKernel(length_scale=[0.7, 0.7])(x, x)) + + +@pytest.mark.parametrize("length_scale", [[1.0], [1.0, -2.0], [1.0, np.nan]]) +def test_ard_rejects_invalid_length_scales(length_scale): + with pytest.raises(ValueError, match="length_scale"): + GaussianProcessKernel(length_scale=length_scale)( + np.array([[0.0, 0.0]]), np.array([[1.0, 1.0]])) + + +def test_ard_hyperparameter_tuning_fits_anisotropic_function(): + rng = np.random.default_rng(123) + x = rng.uniform(0, 1, (28, 2)) + y = np.sin(12 * x[:, 0]) + 0.05 * x[:, 1] + model = GaussianProcessQoiModel( + x, y, tune_hyperparameters=True, anisotropic=True, + normalize_parameters=True, normalize_targets=True, + optimizer_restarts=3) + scales = np.asarray(model._gps[0].kernel.length_scale) + assert scales.shape == (2,) + assert np.all(scales > 0) + assert scales[0] < scales[1] + + +def test_ard_kernel_preserves_small_separations_at_large_offsets(): + # Norm-square/dot-product distance formulas incorrectly round this + # squared separation to zero after division by a small length scale. + x = np.array([[1e5, 1e5], [1e5 + 1e-3, 1e5]]) + kernel = GaussianProcessKernel(length_scale=1e-3) + covariance = kernel(x, x) + expected = np.exp(-0.5) + np.testing.assert_allclose(covariance[0, 1], expected, rtol=1e-7) + np.testing.assert_allclose(covariance[1, 0], expected, rtol=1e-7) + np.testing.assert_allclose(np.diag(covariance), 1.0, rtol=0, atol=1e-14) + + +def test_ard_kernel_large_offset_with_anisotropic_scales_and_cross_covariance(): + x = np.array([[1e5, -2e5], [1e5 + 1e-3, -2e5 + 2e-3]]) + kernel = GaussianProcessKernel(length_scale=[1e-3, 2e-3], signal_variance=1.5) + expected = 1.5 * np.exp(-1.0) + np.testing.assert_allclose(kernel(x[:1], x[1:])[0, 0], expected, rtol=1e-7)