check_search_multimetric_scoring

yohou.testing.search.check_search_multimetric_scoring(search_cv, y, X_actual=None, forecasting_horizon=3, X_future=None, X_forecast=None)

Check multi-metric scoring with dict scorer works correctly.

Parameters

Name	Type	Description	Default
search_cv	BaseSearchCV	Unfitted search CV instance with dict scoring	required
y	DataFrame	Training target data	required
X_actual	DataFrame	Training features	None
forecasting_horizon	int	Number of steps ahead to forecast	3

Raises

Type	Description
AssertionError	If multi-metric scoring doesn't work correctly

Source Code

View on GitHub

Show/Hide source

def check_search_multimetric_scoring(
    search_cv,
    y: pl.DataFrame,
    X_actual: pl.DataFrame | None = None,
    forecasting_horizon: int = 3,
    X_future: pl.DataFrame | None = None,
    X_forecast: pl.DataFrame | None = None,
) -> None:
    """Check multi-metric scoring with dict scorer works correctly.

    Parameters
    ----------
    search_cv : BaseSearchCV
        Unfitted search CV instance with dict scoring
    y : pl.DataFrame
        Training target data
    X_actual : pl.DataFrame, optional
        Training features
    forecasting_horizon : int, default=3
        Number of steps ahead to forecast

    Raises
    ------
    AssertionError
        If multi-metric scoring doesn't work correctly

    """
    search_cv_clone = clone(search_cv)

    # Check that scoring is a dict
    if not isinstance(search_cv_clone.scoring, dict):
        raise ValueError("This check requires search_cv.scoring to be a dict")

    search_cv_clone.fit(y, X_actual, forecasting_horizon=forecasting_horizon, X_future=X_future, X_forecast=X_forecast)

    # Check multimetric_ flag
    assert search_cv_clone.multimetric_, "multimetric_ should be True when scoring is dict"

    # Check that all metrics are in cv_results_
    for metric_name in search_cv_clone.scoring:
        mean_key = f"mean_test_{metric_name}"
        rank_key = f"rank_test_{metric_name}"

        assert mean_key in search_cv_clone.cv_results_, (
            f"cv_results_ must have '{mean_key}' key for multi-metric scoring"
        )
        assert rank_key in search_cv_clone.cv_results_, (
            f"cv_results_ must have '{rank_key}' key for multi-metric scoring"
        )

    # Check scorer_ is dict
    assert isinstance(search_cv_clone.scorer_, dict), (
        f"scorer_ should be dict for multi-metric, got {type(search_cv_clone.scorer_)}"
    )

    # Check that best_score_ matches the refit metric
    if isinstance(search_cv_clone.refit, str):
        refit_metric = search_cv_clone.refit
        expected_best_score = search_cv_clone.cv_results_[f"mean_test_{refit_metric}"][search_cv_clone.best_index_]
        assert np.isclose(search_cv_clone.best_score_, expected_best_score), (
            f"best_score_ should match mean_test_{refit_metric} at best_index_, "
            f"got {search_cv_clone.best_score_} vs {expected_best_score}"
        )