#!/usr/bin/env -S bun --no-env-file --config=/dev/null

import * as fs from 'node:fs';

import * as path from 'node:path';

import { isPartialEval, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, parseTrialOutcomes, sanitizeTrialError,

TRIAL_OUTCOME_SCHEMA, type EvalCaseKind, type EvalResult, type TrialOutcomeRecord } from '../test/helpers/eval-store';

import { flakeLedgerPath, type FlakeLedgerEntry } from './test-free-shards';

import { E2E_KINDS, E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';

import { CASE_QUARANTINE, EVAL_POLICY } from '../test/helpers/periodic-exclude-data';

import { CASE_TEST_NAMES } from './test-paid-shards';

import { resolveStateRoot } from '../lib/state-root';

import { GH_JOBS, PASS_RATES_USAGE, criticalPath, formatCriticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs,

redLedger, triageRun, CENSUS_RED_GUIDE, type CriticalPath } from './lib/eval-history';

import { downloadRunArtifacts, GH_HISTORY, isPooledTrialRun, isWeeklyHistoryRun, listWeeklyRuns, parseFlakeLedger, repoSlug, TRIAL_OUTCOMES_MAX_BYTES, type WeeklyRun } from './lib/ci-history';

interface TestSeries {

export function aggregate(evalFiles: string[]): Map<string, TestSeries> {

export function collectEvalFiles(dir: string, sinceDays = 60): string[] {

function readFreeLedger(): FlakeLedgerEntry[] {

export type TrialRecord = TrialOutcomeRecord & { series_identity?: string; series_fingerprint?: string; harness_version?: number };

export function readTrialOutcomeDir(dir: string): { records: TrialRecord[]; errors: string[] } {

export interface Registry {

export const LIVE_REGISTRY: Registry = {

kinds: E2E_KINDS, tiers: E2E_TIERS, touchfiles: E2E_TOUCHFILES, judgeTouchfiles: LLM_JUDGE_TOUCHFILES,

globals: GLOBAL_TOUCHFILES, testNames: CASE_TEST_NAMES,

export function caseTier(id: string, registry: Registry = LIVE_REGISTRY): string {

return registry.tiers[id] ?? (id in registry.judgeTouchfiles ? 'judge' : 'unknown');

export function attributeLegacyRecord(name: string, shard: string | undefined, registry: Registry = LIVE_REGISTRY): string | null {

export function backfillEvalFiles(files: string[], run?: { run_id: string; sha?: string; timestamp?: string },

export function wilsonInterval(k: number, n: number, z = 1.96): { lo: number; hi: number } {

function logChoose(n: number, k: number): number {

export function fisherOneSidedLower(curPass: number, curN: number, prevPass: number, prevN: number): number {

export function holmRejections(pValues: number[], alpha: number): Set<number> {

export type PassRateLabel = 'INCONCLUSIVE' | 'BROKEN' | 'FLAKY' | 'FAILING' | 'PASSING';

export type AlarmKind = 'drift' | 'rule-as-behavior' | 'regression' | 'quarantine-exit' | 'quarantine-expired'

| 'quarantine-cap' | 'quarantine-invalid' | 'headroom';

export interface PassRatePolicy {

export type QuarantineEntry = (typeof CASE_QUARANTINE)[string];

export const BLOCKING_TIERS: readonly string[] = ['gate', 'periodic'];

const QUARANTINE_FAILURE_CLASSES: readonly string[] = ['detector', 'harness', 'model-latency'];

export interface SeriesStats {

export interface CasePassRate {

export interface Alarm { kind: AlarmKind; case: string; message: string }

export interface PassRateReport {

export interface AnalyzeOptions {

const at = (record: TrialRecord) => record.recorded_at ?? '';

const runOf = (record: TrialRecord) => `${record.run_id ?? record.sha ?? 'local'}#${record.attempt}`;

function seriesStats(key: string, records: TrialRecord[]): SeriesStats {

export function quarantineRunsSince(enteredAt: string, weeklyRuns: string[] | undefined, now: number): number {

export function quarantinePolicyProblems(quarantine: Record<string, QuarantineEntry>,

export function analyzePassRates(records: TrialRecord[], options: AnalyzeOptions = {}): PassRateReport {

function pct(value: number): string { return `${Math.round(value * 1000) / 10}%`; }

function formatStats(stats: SeriesStats | null): string {

if (!stats) return '-';

return `${stats.passes}/${stats.trials} [${pct(stats.interval.lo)}–${pct(stats.interval.hi)}]${stats.infra ? ` (${stats.infra} infra)` : ''}`;

export function formatPassRates(report: PassRateReport, options: { caseFilter?: string } = {}): string {