{ "tool_id": "art-489-model-test-battery", "note": "golden_hash filled by golden-parity.test.mjs --update; output_payload filled by fill-fixture-payloads.mjs. Vector 1: full battery, all five tests pass against policy thresholds. Vector 2: PSI/CSI breach + a missing calibration threshold (skipped, not defaulted). Vector 3: no policy thresholds supplied at all and no discrimination/stability/backtest data -- every test skipped (finite gate, none defaulted).", "vectors": [ { "name": "full_battery_all_pass", "description": "Six scored records (3 pos/3 neg, perfectly separated -> Gini=KS=1), stable population/characteristic snapshots, and two calibration bins all within their policy thresholds -> five pass results.", "policy_parameters": { "as_of_date": "2026-01-15", "threshold_version": "policy-2026-Q1", "thresholds": { "gini_min": 0.4, "ks_min": 0.3, "psi_max": 0.25, "csi_max": 0.25, "calibration_max_diff": 0.05 }, "discrimination": { "scored_outcomes": [ { "id": "a", "score": 0.9, "outcome": 1 }, { "id": "b", "score": 0.8, "outcome": 1 }, { "id": "c", "score": 0.3, "outcome": 0 }, { "id": "d", "score": 0.2, "outcome": 0 }, { "id": "e", "score": 0.6, "outcome": 1 }, { "id": "f", "score": 0.1, "outcome": 0 } ] }, "stability": { "population_bins": [ { "bin": "low", "expected_count": 100, "actual_count": 90 }, { "bin": "high", "expected_count": 100, "actual_count": 110 } ], "characteristic_bins": [ { "characteristic": "ltv", "bin": "low", "expected_count": 50, "actual_count": 40 }, { "characteristic": "ltv", "bin": "high", "expected_count": 50, "actual_count": 60 } ] }, "backtest": { "bins": [ { "bin": "1", "predicted_rate": 0.05, "actual_rate": 0.07, "n": 100 }, { "bin": "2", "predicted_rate": 0.1, "actual_rate": 0.09, "n": 80 } ] } }, "output_payload": { "as_of_date": "2026-01-15", "threshold_version": "policy-2026-Q1", "thresholds_applied": { "gini_min": 0.4, "ks_min": 0.3, "psi_max": 0.25, "csi_max": 0.25, "calibration_max_diff": 0.05 }, "tests": [ { "name": "gini_coefficient", "standard_name": "Gini coefficient", "threshold_field": "gini_min", "threshold": 0.4, "comparison": ">=", "metric_value": 1, "detail": { "auc": 1, "n_scored": 6, "n_positive": 3, "n_negative": 3 }, "result": "pass" }, { "name": "ks_statistic", "standard_name": "Kolmogorov-Smirnov statistic", "threshold_field": "ks_min", "threshold": 0.3, "comparison": ">=", "metric_value": 1, "detail": { "n_scored": 6, "n_positive": 3, "n_negative": 3 }, "result": "pass" }, { "name": "population_stability_index", "standard_name": "Population Stability Index (PSI)", "threshold_field": "psi_max", "threshold": 0.25, "comparison": "<=", "metric_value": 0.0100335348, "detail": [ { "bin": "low", "expected_pct": 0.5, "actual_pct": 0.45, "contribution": 0.0052680258 }, { "bin": "high", "expected_pct": 0.5, "actual_pct": 0.55, "contribution": 0.004765509 } ], "result": "pass" }, { "name": "characteristic_stability_index", "standard_name": "Characteristic Stability Index (CSI, max across characteristics)", "threshold_field": "csi_max", "threshold": 0.25, "comparison": "<=", "metric_value": 0.0405465108, "detail": [ { "characteristic": "ltv", "index": 0.0405465108, "detail": [ { "bin": "low", "expected_pct": 0.5, "actual_pct": 0.4, "contribution": 0.0223143551 }, { "bin": "high", "expected_pct": 0.5, "actual_pct": 0.6, "contribution": 0.0182321557 } ] } ], "result": "pass" }, { "name": "calibration", "standard_name": "Calibration (predicted vs actual rate per bin, max abs diff)", "threshold_field": "calibration_max_diff", "threshold": 0.05, "comparison": "<=", "metric_value": 0.02, "detail": [ { "bin": "1", "predicted_rate": 0.05, "actual_rate": 0.07, "n": 100, "abs_diff": 0.02 }, { "bin": "2", "predicted_rate": 0.1, "actual_rate": 0.09, "n": 80, "abs_diff": 0.01 } ], "result": "pass" } ], "back_test": { "per_bin": [ { "bin": "1", "predicted_rate": 0.05, "actual_rate": 0.07, "n": 100, "abs_diff": 0.02 }, { "bin": "2", "predicted_rate": 0.1, "actual_rate": 0.09, "n": 80, "abs_diff": 0.01 } ] }, "tests_run": 5, "tests_passed": 5, "tests_breached": 0, "tests_skipped_no_threshold": 0, "tests_skipped_insufficient_data": 0 }, "golden_hash": "d36a8080cced37d1819c447cff8583aa26fdb9e52c9143952af47ccce9c76e2a" }, { "name": "psi_csi_breach_calibration_no_threshold", "description": "Population snapshot has drifted heavily (PSI and CSI breach a tight 0.05 threshold) and the caller did not supply calibration_max_diff -- calibration is computed but its result is 'skipped_no_threshold', never defaulted.", "policy_parameters": { "as_of_date": "2026-02-01", "threshold_version": "policy-2026-Q1", "thresholds": { "gini_min": 0.4, "ks_min": 0.3, "psi_max": 0.05, "csi_max": 0.05 }, "discrimination": { "scored_outcomes": [ { "id": "a", "score": 0.9, "outcome": 1 }, { "id": "b", "score": 0.4, "outcome": 1 }, { "id": "c", "score": 0.3, "outcome": 0 }, { "id": "d", "score": 0.2, "outcome": 0 } ] }, "stability": { "population_bins": [ { "bin": "low", "expected_count": 200, "actual_count": 50 }, { "bin": "high", "expected_count": 100, "actual_count": 250 } ], "characteristic_bins": [ { "characteristic": "ltv", "bin": "low", "expected_count": 200, "actual_count": 50 }, { "characteristic": "ltv", "bin": "high", "expected_count": 100, "actual_count": 250 } ] }, "backtest": { "bins": [ { "bin": "1", "predicted_rate": 0.05, "actual_rate": 0.08, "n": 100 } ] } }, "output_payload": { "as_of_date": "2026-02-01", "threshold_version": "policy-2026-Q1", "thresholds_applied": { "gini_min": 0.4, "ks_min": 0.3, "psi_max": 0.05, "csi_max": 0.05, "calibration_max_diff": null }, "tests": [ { "name": "gini_coefficient", "standard_name": "Gini coefficient", "threshold_field": "gini_min", "threshold": 0.4, "comparison": ">=", "metric_value": 1, "detail": { "auc": 1, "n_scored": 4, "n_positive": 2, "n_negative": 2 }, "result": "pass" }, { "name": "ks_statistic", "standard_name": "Kolmogorov-Smirnov statistic", "threshold_field": "ks_min", "threshold": 0.3, "comparison": ">=", "metric_value": 1, "detail": { "n_scored": 4, "n_positive": 2, "n_negative": 2 }, "result": "pass" }, { "name": "population_stability_index", "standard_name": "Population Stability Index (PSI)", "threshold_field": "psi_max", "threshold": 0.05, "comparison": "<=", "metric_value": 1.1512925465, "detail": [ { "bin": "low", "expected_pct": 0.66666667, "actual_pct": 0.16666667, "contribution": 0.6931471806 }, { "bin": "high", "expected_pct": 0.33333333, "actual_pct": 0.83333333, "contribution": 0.4581453659 } ], "result": "breach" }, { "name": "characteristic_stability_index", "standard_name": "Characteristic Stability Index (CSI, max across characteristics)", "threshold_field": "csi_max", "threshold": 0.05, "comparison": "<=", "metric_value": 1.1512925465, "detail": [ { "characteristic": "ltv", "index": 1.1512925465, "detail": [ { "bin": "low", "expected_pct": 0.66666667, "actual_pct": 0.16666667, "contribution": 0.6931471806 }, { "bin": "high", "expected_pct": 0.33333333, "actual_pct": 0.83333333, "contribution": 0.4581453659 } ] } ], "result": "breach" }, { "name": "calibration", "standard_name": "Calibration (predicted vs actual rate per bin, max abs diff)", "threshold_field": "calibration_max_diff", "threshold": null, "comparison": "<=", "metric_value": 0.03, "detail": [ { "bin": "1", "predicted_rate": 0.05, "actual_rate": 0.08, "n": 100, "abs_diff": 0.03 } ], "result": "skipped_no_threshold", "reason": "threshold not supplied in policy input" } ], "back_test": { "per_bin": [ { "bin": "1", "predicted_rate": 0.05, "actual_rate": 0.08, "n": 100, "abs_diff": 0.03 } ] }, "tests_run": 4, "tests_passed": 2, "tests_breached": 2, "tests_skipped_no_threshold": 1, "tests_skipped_insufficient_data": 0 }, "golden_hash": "256759db0a1f3740c8735f0725b848a7d6922af1af9df8fa1e318360e97f259b" }, { "name": "no_data_no_thresholds_edge", "description": "No thresholds and no discrimination/stability/backtest data supplied at all -- every test resolves 'skipped_insufficient_data', tests_run is 0 (finite gate, nothing defaulted or fabricated).", "policy_parameters": { "as_of_date": "2026-03-01", "threshold_version": "policy-2026-Q1", "thresholds": {} }, "output_payload": { "as_of_date": "2026-03-01", "threshold_version": "policy-2026-Q1", "thresholds_applied": { "gini_min": null, "ks_min": null, "psi_max": null, "csi_max": null, "calibration_max_diff": null }, "tests": [ { "name": "gini_coefficient", "standard_name": "Gini coefficient", "threshold_field": "gini_min", "threshold": null, "comparison": ">=", "metric_value": null, "detail": null, "result": "skipped_insufficient_data", "reason": "discrimination.scored_outcomes missing, empty, or single-class (need both outcome=0 and outcome=1)" }, { "name": "ks_statistic", "standard_name": "Kolmogorov-Smirnov statistic", "threshold_field": "ks_min", "threshold": null, "comparison": ">=", "metric_value": null, "detail": null, "result": "skipped_insufficient_data", "reason": "discrimination.scored_outcomes missing, empty, or single-class (need both outcome=0 and outcome=1)" }, { "name": "population_stability_index", "standard_name": "Population Stability Index (PSI)", "threshold_field": "psi_max", "threshold": null, "comparison": "<=", "metric_value": null, "detail": null, "result": "skipped_insufficient_data", "reason": "stability.population_bins missing, empty, or both snapshot totals are zero" }, { "name": "characteristic_stability_index", "standard_name": "Characteristic Stability Index (CSI, max across characteristics)", "threshold_field": "csi_max", "threshold": null, "comparison": "<=", "metric_value": null, "detail": null, "result": "skipped_insufficient_data", "reason": "stability.characteristic_bins missing, empty, or both snapshot totals are zero for every characteristic" }, { "name": "calibration", "standard_name": "Calibration (predicted vs actual rate per bin, max abs diff)", "threshold_field": "calibration_max_diff", "threshold": null, "comparison": "<=", "metric_value": null, "detail": null, "result": "skipped_insufficient_data", "reason": "backtest.bins missing, empty, or no bin had both predicted_rate and actual_rate" } ], "back_test": null, "tests_run": 0, "tests_passed": 0, "tests_breached": 0, "tests_skipped_no_threshold": 0, "tests_skipped_insufficient_data": 5 }, "golden_hash": "354d3ce903384a0b1742cd260183a4b6937212875f3567f9928a8dc674d34e95" } ] }