{{ message }}
Repository navigation
Expand file tree
/
Copy pathreportv2.go
More file actions
1261 lines (1189 loc) · 57.4 KB
/
Copy pathreportv2.go
File metadata and controls
1261 lines (1189 loc) · 57.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package bench
// Report v2 types for the discovery-effectiveness profiler (spec 083). Every
// struct mirrors specs/083-discovery-profiler/contracts/report-v2.schema.json
// — that schema file is the contract; these types are its Go projection, and
// bench/reportv2_test.go validates a sample against it.
//
// Determinism (FR-010): marshaling is canonical — struct field order is fixed,
// map keys are sorted by encoding/json, and no writer here injects wall-clock
// time (GeneratedAt is caller-supplied data, set once per run).
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
)
// ReportVersion2 is the report_version const of the v2 envelope.
const ReportVersion2 = 2
// Provenance labels for headline metrics (SC-005): a number is either measured
// (observed over the real protocol), computed (derived arithmetic over
// measured inputs), or estimated (model with documented assumptions).
const (
ProvenanceMeasured = "measured"
ProvenanceComputed = "computed"
ProvenanceEstimated = "estimated"
)
// TokenizerInfo names the token estimator and carries the mandatory accuracy
// caveat rendered wherever absolute numbers appear (research D11, SC-005).
type TokenizerInfo struct {
Name string `json:"name"`
Caveat string `json:"caveat"`
}
// ProxyInfo records the measured proxy's identity and discovery configuration
// (FR-021): interpreting response cost requires knowing tools_limit and
// routing_mode; tool_count vs expected_tool_count surfaces corpus drift.
type ProxyInfo struct {
Version string `json:"version,omitempty"`
ToolCount int `json:"tool_count,omitempty"`
ExpectedToolCount int `json:"expected_tool_count,omitempty"`
ToolsLimit int `json:"tools_limit,omitempty"`
RoutingMode string `json:"routing_mode,omitempty"`
}
// DegenerateDescriptions counts corpus tools whose descriptions trip the
// FR-020 quality rules, with the rule list echoed for reproducibility.
type DegenerateDescriptions struct {
Count int `json:"count"`
Rules []string `json:"rules,omitempty"`
}
// CorpusDescriptor identifies one measured corpus with license/attribution
// provenance (FR-011/012/013).
type CorpusDescriptor struct {
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
ToolCount int `json:"tool_count"`
License string `json:"license"`
Attribution string `json:"attribution,omitempty"`
Committed bool `json:"committed"`
DegenerateDescriptions *DegenerateDescriptions `json:"degenerate_descriptions,omitempty"`
}
// SkipExample is one (tool, error) pair from an arm's skipped tools (FR-009).
type SkipExample struct {
ToolID string `json:"tool_id"`
Error string `json:"error"`
}
// ToolTokenEntry is one heaviest-tools row (FR-020).
type ToolTokenEntry struct {
ToolID string `json:"tool_id"`
Tokens int `json:"tokens"`
}
// RetrievalScore is the flat retrieval-quality DTO of the v2 contract. It is
// mapped from the existing nested RetrievalMetrics by MapRetrievalMetrics —
// the single conversion point — so the report schema stays flat and stable.
type RetrievalScore struct {
RecallAt1 float64 `json:"recall_at_1"`
RecallAt3 float64 `json:"recall_at_3"`
RecallAt5 float64 `json:"recall_at_5"`
RecallAt10 float64 `json:"recall_at_10"`
MRR float64 `json:"mrr"`
NDCGAt10 float64 `json:"ndcg_at_10"`
MAP float64 `json:"map"`
// MetricNote documents the gain formula (FR-012), or — for an
// index-altering arm scored on a corpus without relevance labels —
// explains the absence of numbers.
MetricNote string `json:"metric_note,omitempty"`
}
// MapRetrievalMetrics converts the existing nested bench.RetrievalMetrics
// (recall_at as a map) into the flat report DTO. nil in, nil out — a nil score
// marks a quality-neutral arm (FR-008).
func MapRetrievalMetrics(m *RetrievalMetrics) *RetrievalScore {
if m == nil {
return nil
}
return &RetrievalScore{
RecallAt1: m.Metrics.RecallAt[1],
RecallAt3: m.Metrics.RecallAt[3],
RecallAt5: m.Metrics.RecallAt[5],
RecallAt10: m.Metrics.RecallAt[10],
MRR: m.Metrics.MRR,
NDCGAt10: m.Metrics.NDCGAt10,
MAP: m.Metrics.MAP,
}
}
// ArmResult is one encoding arm measured on one corpus (FR-005/006/007/009).
//
// Contract conditionals: a skipped row requires SkipReason; a non-skipped row
// requires the token stats; a results-class row requires FixtureID and the
// tabular/non-tabular split (pointers so an explicit 0 still serializes); a
// non-skipped index-altering row requires the quality key (nullable only when
// the corpus has no relevance labels, with MetricNote explaining the absence).
type ArmResult struct {
Arm string `json:"arm"`
CorpusID string `json:"corpus_id"`
Skipped bool `json:"skipped"`
SkipReason string `json:"skip_reason,omitempty"`
LowerBound bool `json:"lower_bound,omitempty"`
IndexAltering bool `json:"index_altering"`
PayloadClass string `json:"payload_class,omitempty"` // "listing" | "results"
FixtureID string `json:"fixture_id,omitempty"`
TabularCount *int `json:"tabular_count,omitempty"`
NonTabularCount *int `json:"non_tabular_count,omitempty"`
TotalTokens int `json:"total_tokens"`
MeanTokens float64 `json:"mean_tokens"`
P95Tokens int `json:"p95_tokens"`
SavingsVsBaselinePct float64 `json:"savings_vs_baseline_pct"`
SkippedTools int `json:"skipped_tools"`
SkipExamples []SkipExample `json:"skip_examples,omitempty"`
HeaviestTools []ToolTokenEntry `json:"heaviest_tools,omitempty"`
Quality *RetrievalScore `json:"quality"`
}
// DiscoveryResponseMeasurement is one golden query's retrieve_tools response
// cost with its span-attributed component breakdown (FR-001/002). Invariant:
// the component values sum EXACTLY to TotalTokens — enforced by construction
// in the span attributor (bench/respcost.go), never by re-tokenizing fields.
type DiscoveryResponseMeasurement struct {
QueryID string `json:"query_id"`
TotalTokens int `json:"total_tokens"`
ResultCount int `json:"result_count,omitempty"`
LatencyMs float64 `json:"latency_ms,omitempty"`
Components map[string]int `json:"components"`
}
// ResponseCostSummary aggregates per-query response measurements (FR-001).
type ResponseCostSummary struct {
P50 int `json:"p50"`
P95 int `json:"p95"`
Max int `json:"max"`
Mean float64 `json:"mean"`
PerQuery []DiscoveryResponseMeasurement `json:"per_query,omitempty"`
}
// BreakEvenAnalysis is the FR-003 break-even computation with its inputs
// echoed (FR-004): break_even_calls = (naive − proxy_menu) / mean_response.
type BreakEvenAnalysis struct {
NaiveFullMenuTokens int `json:"naive_full_menu_tokens"`
ProxyMenuTokens int `json:"proxy_menu_tokens"`
MeanResponseTokens float64 `json:"mean_response_tokens"`
BreakEvenCalls float64 `json:"break_even_calls"`
NoBreakEven bool `json:"no_break_even,omitempty"`
}
// SessionCostEstimate is one row of the FR-019 session estimator (provenance
// is always "estimated"; retry-rate defaults documented in research D8).
type SessionCostEstimate struct {
Arm string `json:"arm"`
CallsPerSession int `json:"calls_per_session"`
RetryRate float64 `json:"retry_rate"`
EstimatedTokens int `json:"estimated_tokens"`
}
// LatencyAggregate is one nearest-rank percentile summary of client-measured
// latencies (FR-023) for a single measured surface.
type LatencyAggregate struct {
P50Ms float64 `json:"p50_ms"`
P95Ms float64 `json:"p95_ms"`
P99Ms float64 `json:"p99_ms"`
MaxMs float64 `json:"max_ms"`
}
// LatencyV2 is the client-measured latency block (FR-023). Two DIFFERENT
// surfaces are measured and must never be conflated: the REST
// /api/v1/index/search calls used for retrieval scoring, and the MCP
// retrieve_tools calls the discovery-response rows time. The flat fields
// summarize the REST search calls (kept additive for existing consumers and
// mirrored in RESTSearch); MCPDiscovery, when present, is the retrieve_tools
// aggregate over the real MCP protocol.
type LatencyV2 struct {
P50Ms float64 `json:"p50_ms"`
P95Ms float64 `json:"p95_ms"`
P99Ms float64 `json:"p99_ms"`
MaxMs float64 `json:"max_ms"`
// RESTSearch labels the flat fields' surface explicitly:
// GET /api/v1/index/search round-trips.
RESTSearch *LatencyAggregate `json:"rest_search,omitempty"`
// MCPDiscovery aggregates the MCP retrieve_tools call latencies from the
// per-query DiscoveryResponseMeasurement rows.
MCPDiscovery *LatencyAggregate `json:"mcp_discovery,omitempty"`
}
// LapVerdict is the pinned independent LAP run (FR-015/016, SC-006).
type LapVerdict struct {
Executed bool `json:"executed"`
SkipReason string `json:"skip_reason,omitempty"`
Version string `json:"version,omitempty"`
MenuTokens int `json:"menu_tokens,omitempty"`
InHouseMenuTokens int `json:"in_house_menu_tokens,omitempty"`
DivergencePct float64 `json:"divergence_pct,omitempty"`
Grade string `json:"grade,omitempty"`
ArtifactPath string `json:"artifact_path,omitempty"`
}
// SubsetInfo records the seeded query subset of a public-corpus run (FR-014):
// same revision + seed + size ⇒ same subset.
type SubsetInfo struct {
Seed int `json:"seed"`
Size int `json:"size"`
}
// --- Spec 103 additive blocks (contracts/report-v2-additions.md) ---
//
// The replay, agent-loop and payload-decomposition blocks are ADDITIVE
// optional members of this same v2 envelope: ReportVersion2 stays 2 and no
// existing field changes meaning. That rule is what forces the two shapes
// below to exist rather than reusing what is already here.
//
// Why a per-BLOCK accounting source. The envelope carries ONE tokenizer
// identity (TokenizerInfo) for the whole document. Once provider-reported
// usage enters the same document, that field would silently claim the entire
// report was tokenizer-counted. Narrowing it to "the deterministic sections
// only" is a meaning change, and a meaning change requires a version bump —
// which this feature must not do. So the document-level field keeps its
// meaning intact (it names the deterministic estimator, still true of every
// section that has one) and each new block names its own source explicitly. A
// reader never has to infer scope from a document-level field.
//
// Why a per-ROW provenance. ReportV2.Provenance is section-level: one badge
// per section key. FR-013 lets measured and estimated figures coexist INSIDE
// one block, which a single badge cannot express. The concrete failure it
// prevents: RetryRateForArm returns 0.0 for an unknown arm, indistinguishable
// from a measured 0.0, so one table can mix a measured rate with a defaulted
// one under a single badge. Every new row type therefore carries its own
// provenance, constrained to the same three values as the section map.
// Accounting-source kinds. Figures from different kinds are NEVER summed: a
// cross-source aggregate is withheld with a stated reason (data-model.md
// invariant 2), reusing the harness's withhold-rather-than-compute pattern.
const (
// AccountingKindTokenizer marks figures counted by the pinned
// deterministic estimator named in the document-level TokenizerInfo.
AccountingKindTokenizer = "deterministic-tokenizer"
// AccountingKindProvider marks figures taken from provider-reported
// usage, which requires a pinned model to be interpretable at all.
AccountingKindProvider = "provider-usage"
)
// Exclusion reason codes for replay sessions (FR-003). Every supplied session
// that does not reach a figure is counted under exactly one of these — silence
// is never success (data-model.md invariant 3).
const (
ReplayExclusionTruncated = "truncated"
ReplayExclusionBodiesMissed = "bodies_missing"
ReplayExclusionSensitive = "sensitive"
ReplayExclusionUnreplayable = "unreplayable"
ReplayExclusionUnattributed = "unattributed"
)
// Spec-102 verdicts (FR-025). The verdict is stated explicitly per fleet
// shape; an absent verdict is not "confirmed by default".
const (
Spec102Confirmed = "confirmed"
Spec102Corrected = "corrected"
)
// AccountingSource names where one block's numbers came from. It is
// deliberately a value, not a string: a provider-sourced block is only
// interpretable with the model pinned alongside the provider, and squeezing
// both into one enum string would make the "never sum across sources"
// comparison a string-parsing exercise.
type AccountingSource struct {
// Kind is AccountingKindTokenizer or AccountingKindProvider.
Kind string `json:"kind"`
// Identity names the estimator ("cl100k_base") or the provider.
Identity string `json:"identity"`
// Model pins the model for provider-sourced figures. Mandatory for
// AccountingKindProvider, empty otherwise — usage numbers from an
// unpinned model are not comparable to any later run.
Model string `json:"model,omitempty"`
}
// IsZero reports whether the source was never populated. A block with a zero
// source cannot enter any aggregate (data-model.md: accounting_source is
// mandatory), so emission-time validation rejects it rather than defaulting.
func (a AccountingSource) IsZero() bool {
return a.Kind == "" && a.Identity == "" && a.Model == ""
}
// FleetShape is the tool count and definition-size distribution a figure holds
// for. Every published percentage carries one (IC-004): a saving quoted
// without its fleet shape is a saving quoted at an unstated fleet size, which
// is how spec 102 over-projected from a single corpus.
type FleetShape struct {
ID string `json:"id"`
ToolCount int `json:"tool_count"`
MeanDefinitionTokens float64 `json:"mean_definition_tokens,omitempty"`
P95DefinitionTokens int `json:"p95_definition_tokens,omitempty"`
// SchemalessTools counts tools in this fleet that carried no priceable
// input schema.
//
// A fleet is refused outright only when NO tool has a real schema. A
// PARTIAL regression — 3 of 45, or 20 of 45 — passes that guard and prices
// the stubbed ones at nothing, quietly shrinking the baseline and
// inflating the headline. Without this count the report carries no trace
// of it, so the reader cannot tell a clean fleet from a partly stubbed
// one, and a drift from 3 to 20 between runs is invisible.
SchemalessTools int `json:"schemaless_tools,omitempty"`
}
// ReplayExclusion counts the supplied sessions dropped for one reason (FR-003,
// SC-008). Reasons are the Replay* constants above.
type ReplayExclusion struct {
Reason string `json:"reason"`
Sessions int `json:"sessions"`
}
// ReplayCellCost is one mode cell's recomputed cost over the recorded
// workload. Bodies-off it carries a MEASURED menu cost and, at best, an
// ESTIMATED response cost derived from pre-truncation byte LENGTHS — which is
// exactly the mix that per-row provenance exists to express.
// ReplayLoaderReason is one reason code and its count, as the loader recorded
// it. The reason strings are the loader's own vocabulary rather than the
// report's closed enum: they name causes the report-level enum has no term for
// (a truncated internal response that would overstate, a byte-gap record), and
// flattening them into the closed set would destroy the very distinction that
// makes them actionable.
type ReplayLoaderReason struct {
Reason string `json:"reason"`
Count int `json:"count"`
}
// ReplayLoaderAccounting is the loader-level companion to ReplayBlock.Exclusions.
//
// The three buckets are deliberately separate rather than one total. A dropped
// record never entered a unit of work; a withheld cost was suppressed on a
// record that DID contribute; a flagged record contributed and was counted but
// carries a usability caveat. "12 exclusions" spanning all three is
// uninterpretable, and an uninterpretable count satisfies the letter of
// "everything is counted" while destroying its purpose.
type ReplayLoaderAccounting struct {
RecordsDropped int `json:"records_dropped"`
Dropped []ReplayLoaderReason `json:"dropped,omitempty"`
CostsWithheld int `json:"costs_withheld"`
Withheld []ReplayLoaderReason `json:"withheld,omitempty"`
RecordsFlagged int `json:"records_flagged"`
Flagged []ReplayLoaderReason `json:"flagged,omitempty"`
// OrphanedSubCalls counts sub-calls whose parent fell outside the exported
// window. They ARE counted in the workload, at top level; what was lost is
// their attribution to a parent. Reported because a total that hides it
// cannot be audited.
OrphanedSubCalls int `json:"orphaned_sub_calls,omitempty"`
}
type ReplayCellCost struct {
CellID string `json:"cell_id"`
// Provenance is measured/computed/estimated for THIS row.
Provenance string `json:"provenance"`
Calls int `json:"calls"`
// MenuTokens is the tool-surface cost of what the agent was shown under
// this cell, computed from the supplied fleet input.
MenuTokens int `json:"menu_tokens"`
// ResponseTokens is present only when a response figure could be
// produced at all; bodies-off it is an estimate from byte lengths, never
// a measurement, and a pointer so a genuine 0 is distinguishable from
// "not computed".
ResponseTokens *int `json:"response_tokens,omitempty"`
// AbsoluteWorkloadWithheld records that no absolute complete-workload
// figure is available for this cell. Bodies-off that is true of EVERY
// cell, because complete workload includes every consumed response and
// that text is absent. It is reported as unavailable, never as zero.
AbsoluteWorkloadWithheld bool `json:"absolute_workload_withheld,omitempty"`
WithheldReason string `json:"withheld_reason,omitempty"`
}
// ReplayDirectDelta is the honest bodies-off headline: the cross-mode delta
// between the two direct cells, whose identical call responses cancel out of
// the comparison. It is a row in its own right and carries its own provenance
// (computed — it is arithmetic over two measured menu costs).
type ReplayDirectDelta struct {
FromCellID string `json:"from_cell_id"`
ToCellID string `json:"to_cell_id"`
Provenance string `json:"provenance"`
DeltaTokens int `json:"delta_tokens"`
DeltaPct float64 `json:"delta_pct"`
}
// ReplayBlock is the deterministic recomputation over a real recorded workload
// (US1). It is a COUNTERFACTUAL: recorded call shape scored against the
// SUPPLIED fleet, not the fleet as it stood when the session was recorded, and
// never observed agent behaviour (FR-004). No recorded content reaches this
// struct — only counts, sizes and derived measurements (FR-006).
type ReplayBlock struct {
AccountingSource AccountingSource `json:"accounting_source"`
// Counterfactual is the mandatory FR-004 label. It is a required field
// rather than a rendering concern so a figure cannot be emitted without
// it and then lose the caveat downstream.
Counterfactual string `json:"counterfactual"`
// BodiesIncluded records whether the export carried response bodies.
// False is the default and the safe mode; true is an explicit opt-in
// over raw unmasked traffic (the export path does not mask).
BodiesIncluded bool `json:"bodies_included"`
FleetShape FleetShape `json:"fleet_shape"`
// SessionsSupplied/SessionsUsed plus Exclusions are the FR-003 accounting:
// supplied minus used must be explained by the exclusion counts.
SessionsSupplied int `json:"sessions_supplied"`
SessionsUsed int `json:"sessions_used"`
Exclusions []ReplayExclusion `json:"exclusions,omitempty"`
// LoaderAccounting is the loader's own exclusion detail: what was dropped
// before joining a unit of work, what had its cost withheld INSIDE an
// admitted unit, what was admitted but flagged, and how many sub-calls
// lost their parent. The session rows above cannot express any of that,
// and a withheld cost that collapses a response total with no row saying
// why is precisely the silent accounting SC-008 forbids.
LoaderAccounting *ReplayLoaderAccounting `json:"loader_accounting,omitempty"`
Cells []ReplayCellCost `json:"cells,omitempty"`
// DirectDelta is absent when only one direct cell was measured.
DirectDelta *ReplayDirectDelta `json:"direct_delta,omitempty"`
// SensitiveFlagBestEffort restates, in the report itself, that the
// sensitive-data flag is derived from detection metadata written
// asynchronously AFTER persistence: its absence does not prove a record
// is clean. It is a reducer, never a guarantee (Principle IV).
SensitiveFlagBestEffort bool `json:"sensitive_flag_best_effort,omitempty"`
Inputs *InputProvenance `json:"inputs,omitempty"`
Records *RawRecordsRef `json:"records,omitempty"`
}
// AgentLoopCell is one mode cell measured over a live agent loop (US2). Its
// numbers come from provider-reported usage, so they may never be summed with
// the deterministic blocks' figures.
type AgentLoopCell struct {
CellID string `json:"cell_id"`
// Provenance is measured for a cell backed by real runs; a row still
// carrying an assumed default stays estimated and must be rendered as
// such beside its measured neighbours (FR-013).
Provenance string `json:"provenance"`
// Runs and SpreadPct are the FR-021 consistency requirement: a
// model-dependent figure needs at least four runs plus a spread before
// it may be a headline.
Runs int `json:"runs"`
// SpreadPct is a POINTER: with fewer than two per-run figures the spread is
// UNDEFINED, not zero. Serialising 0 there renders "±0.0%" — a claim of
// perfect consistency from a single run, which is the opposite of what
// FR-021 asks the field to convey.
SpreadPct *float64 `json:"spread_pct"`
// PartialRuns counts runs that did not finish. They are excluded from
// the figures above and reported, never silently absorbed (FR-032).
PartialRuns int `json:"partial_runs,omitempty"`
// TokensPerCompletedTask is the FR-018 headline, and CompletionRatePct
// travels with it at equal prominence — a cheap mode that finishes
// fewer tasks is not a saving.
TokensPerCompletedTask float64 `json:"tokens_per_completed_task"`
CompletionRatePct float64 `json:"completion_rate_pct"`
// FirstAttemptSuccessPct is a POINTER for the same reason: with no
// determinate intents there is no rate, and a serialised 0 reads as "the
// agent never got a call right first time" rather than "nothing was
// measured".
FirstAttemptSuccessPct *float64 `json:"first_attempt_success_pct"`
// Corrective and infrastructure retries are counted separately: only the
// first kind says anything about how well the mode serves the agent.
RetriesCorrective int `json:"retries_corrective"`
RetriesInfrastructure int `json:"retries_infrastructure"`
// Input/output/cache-read consumption stays split (FR-014); collapsing
// them hides the cache behaviour that dominates a long agent loop.
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheReadTokens int `json:"cache_read_tokens"`
// Headline is false whenever the FR-021 bar is unmet; the row is still
// published, just not as a headline.
Headline bool `json:"headline"`
// Regression marks a cell whose completion rate falls more than the
// stated threshold below the baseline's. Such a cell must never be
// described as a saving regardless of its token cost (FR-019).
Regression bool `json:"regression,omitempty"`
// Withheld carries a figure that cannot be computed honestly — a
// cross-source aggregate above all — with the reason stated.
Withheld bool `json:"withheld,omitempty"`
WithheldReason string `json:"withheld_reason,omitempty"`
}
// AgentLoopBlock is the live measured block (US2). Its AccountingSource is
// provider-usage plus the pinned model, which is why the block-level field
// exists: this block and the deterministic ones sit in one document and must
// stay separable.
type AgentLoopBlock struct {
AccountingSource AccountingSource `json:"accounting_source"`
// CompletionRegressionThresholdPct is the stated FR-019 threshold this
// block was evaluated against. Serialised so a reader knows which number
// separated "saving" from "regression", and so a verdict recomputed from a
// loaded report uses the SAME threshold the run applied rather than
// whatever the current default happens to be.
CompletionRegressionThresholdPct float64 `json:"completion_regression_threshold_pct,omitempty"`
// Suite and SuiteVersion pin the task suite so a later run is comparable
// to an earlier one (FR-028).
Suite string `json:"suite,omitempty"`
SuiteVersion string `json:"suite_version,omitempty"`
FleetShape FleetShape `json:"fleet_shape"`
Cells []AgentLoopCell `json:"cells,omitempty"`
Inputs *InputProvenance `json:"inputs,omitempty"`
Records *RawRecordsRef `json:"records,omitempty"`
}
// PayloadDecompositionRow attributes tool-definition cost for ONE fleet shape
// (FR-024/025).
type PayloadDecompositionRow struct {
FleetShape FleetShape `json:"fleet_shape"`
Provenance string `json:"provenance"`
// The four attributions, as percentages of the definition payload.
ShareNamesPct float64 `json:"share_names_pct"`
ShareDescriptionsPct float64 `json:"share_descriptions_pct"`
// ShareAnnotationsPct is a POINTER because null and zero mean different
// things here. The frozen corpora carry no annotations field, so a
// corpus-based decomposition cannot measure this share at all — and a 0
// would read as "measured, and annotations cost nothing", which is the
// silent-zero failure the rest of this report works to avoid. nil marshals
// to null: not measured.
ShareAnnotationsPct *float64 `json:"share_annotations_pct"`
ShareSchemasPct float64 `json:"share_schemas_pct"`
// AchievableCeilingPct is recomputed for THIS shape. Carrying a
// previously published ceiling forward is precisely the error spec 102
// made when it projected from a single corpus.
AchievableCeilingPct float64 `json:"achievable_ceiling_pct"`
// Spec102Verdict is an explicit confirmed/corrected call on spec 102's
// conclusion, with the delta when corrected.
Spec102Verdict string `json:"spec102_verdict"`
Spec102DeltaPct *float64 `json:"spec102_delta_pct,omitempty"`
}
// PayloadDecompositionBlock holds one row per fleet shape (at least two, per
// FR-024) so the reader can see how the attribution moves with fleet size
// rather than trusting a single-corpus projection.
type PayloadDecompositionBlock struct {
AccountingSource AccountingSource `json:"accounting_source"`
Shapes []PayloadDecompositionRow `json:"shapes,omitempty"`
Inputs *InputProvenance `json:"inputs,omitempty"`
}
// IsValidProvenance reports whether s is one of the three closed-enum values
// shared by the section-level map and every per-row provenance field.
func IsValidProvenance(s string) bool {
return s == ProvenanceMeasured || s == ProvenanceComputed || s == ProvenanceEstimated
}
// ValidateAdditiveBlocks enforces, at emission time, the two properties the
// JSON schema cannot: that every present block names a populated accounting
// source, and that every row of every block carries a provenance from the
// closed enum.
//
// The schema declares these blocks but has no additionalProperties:false and
// cannot require a field to be non-empty, so a block whose accounting source
// was never set would validate silently — which is the exact failure the
// contract calls out ("a field that exists but is never set does not satisfy
// the contract"). Callers that build a report should run this before writing.
func (r *ReportV2) ValidateAdditiveBlocks() error {
checkSource := func(block string, src AccountingSource) error {
if src.IsZero() {
return fmt.Errorf("%s block: accounting_source is unset", block)
}
switch src.Kind {
case AccountingKindTokenizer, AccountingKindProvider:
default:
return fmt.Errorf("%s block: accounting_source.kind %q not in {%s,%s}",
block, src.Kind, AccountingKindTokenizer, AccountingKindProvider)
}
if src.Identity == "" {
return fmt.Errorf("%s block: accounting_source.identity is empty", block)
}
if src.Kind == AccountingKindProvider && src.Model == "" {
return fmt.Errorf("%s block: provider-sourced figures need a pinned model", block)
}
return nil
}
checkRow := func(block, row string, provenance string) error {
if provenance == "" {
return fmt.Errorf("%s block: %s has no provenance", block, row)
}
if !IsValidProvenance(provenance) {
return fmt.Errorf("%s block: %s provenance %q not in {%s,%s,%s}",
block, row, provenance, ProvenanceMeasured, ProvenanceComputed, ProvenanceEstimated)
}
return nil
}
// The US3 marks are optional (a report that sets none of them validates
// exactly as before, which is what keeps this feature additive), but a
// mark that IS set and is internally inconsistent is an error here rather
// than a silently-valid document downstream.
if err := r.RunStatus.Validate(); err != nil {
return err
}
if b := r.Replay; b != nil {
if err := checkSource("replay", b.AccountingSource); err != nil {
return err
}
if err := b.Inputs.Validate("replay"); err != nil {
return err
}
if err := b.Records.Validate("replay"); err != nil {
return err
}
for i := range b.Cells {
if err := checkRow("replay", fmt.Sprintf("cells[%d] (%s)", i, b.Cells[i].CellID), b.Cells[i].Provenance); err != nil {
return err
}
}
if b.DirectDelta != nil {
if err := checkRow("replay", "direct_delta", b.DirectDelta.Provenance); err != nil {
return err
}
}
}
if b := r.AgentLoop; b != nil {
if err := checkSource("agent_loop", b.AccountingSource); err != nil {
return err
}
if err := b.Inputs.Validate("agent_loop"); err != nil {
return err
}
if err := b.Records.Validate("agent_loop"); err != nil {
return err
}
for i := range b.Cells {
if err := checkRow("agent_loop", fmt.Sprintf("cells[%d] (%s)", i, b.Cells[i].CellID), b.Cells[i].Provenance); err != nil {
return err
}
}
}
if b := r.PayloadDecomposition; b != nil {
if err := checkSource("payload_decomposition", b.AccountingSource); err != nil {
return err
}
if err := b.Inputs.Validate("payload_decomposition"); err != nil {
return err
}
for i := range b.Shapes {
if err := checkRow("payload_decomposition", fmt.Sprintf("shapes[%d] (%s)", i, b.Shapes[i].FleetShape.ID), b.Shapes[i].Provenance); err != nil {
return err
}
}
}
return nil
}
// ReportV2 is the versioned report envelope (research D12). Additive over the
// v1 report: existing consumers are unaffected (reports are never committed,
// Spec 065 CN-003).
type ReportV2 struct {
ReportVersion int `json:"report_version"`
GeneratedAt string `json:"generated_at"`
Tokenizer TokenizerInfo `json:"tokenizer"`
Proxy *ProxyInfo `json:"proxy,omitempty"`
Corpora []CorpusDescriptor `json:"corpora"`
Arms []ArmResult `json:"arms"`
ResponseCost *ResponseCostSummary `json:"response_cost,omitempty"`
BreakEven *BreakEvenAnalysis `json:"break_even,omitempty"`
// SessionEstimates carries SessionCostRow, not the bare estimate: FR-013
// requires a measured retry rate and a defaulted one to be
// distinguishable INSIDE one table, and RetryRateForArm returns 0.0 for an
// unknown arm — indistinguishable from a measured 0.0 without the per-row
// provenance the row type adds.
SessionEstimates []SessionCostRow `json:"session_estimates,omitempty"`
Latency *LatencyV2 `json:"latency,omitempty"`
Lap *LapVerdict `json:"lap,omitempty"`
Subset *SubsetInfo `json:"subset,omitempty"`
// Spec 103 blocks. Additive and optional: omitted they do not appear at
// all, so existing offline reports and their consumers are byte-
// unaffected and no report_version bump is needed. Each names its own
// accounting source rather than borrowing the document-level Tokenizer.
RunStatus *RunStatus `json:"run_status,omitempty"`
Replay *ReplayBlock `json:"replay,omitempty"`
AgentLoop *AgentLoopBlock `json:"agent_loop,omitempty"`
PayloadDecomposition *PayloadDecompositionBlock `json:"payload_decomposition,omitempty"`
Provenance map[string]string `json:"provenance"`
}
// WriteJSON writes the v2 report as indented JSON into dir/report.json.
//
// Before marshaling it resolves every raw-record reference against dir, the
// run-local root those references are relative to: a reference whose artifact
// is no longer there degrades to "records not retained" instead of being
// written out as a path that resolves to nothing (T057/FR-029). This is the
// only moment both halves are in hand, and the report lands in the same
// gitignored tree the records do — so a reference that was accurate when the
// block was built can already be stale by the time it is serialized.
//
// It stays deterministic (FR-010/SC-002): the same run directory produces the
// same bytes, and no wall-clock time is injected.
func (r *ReportV2) WriteJSON(dir string) (string, error) {
if err := os.MkdirAll(dir, 0o755); err != nil {
return "", fmt.Errorf("mkdir %q: %w", dir, err)
}
if r.Replay != nil {
r.Replay.Records.resolve(dir)
}
if r.AgentLoop != nil {
r.AgentLoop.Records.resolve(dir)
}
path := filepath.Join(dir, "report.json")
data, err := json.MarshalIndent(r, "", " ")
if err != nil {
return "", fmt.Errorf("marshal report v2: %w", err)
}
if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil {
return "", fmt.Errorf("write %q: %w", path, err)
}
return path, nil
}
// ---------------------------------------------------------------------------
// US3 — an outsider reproduces the numbers (FR-029/030/032).
//
// The three structures below all answer the same question from different
// angles: what would a reader have to take on trust? Each one exists because
// its absence reads as a guarantee. An unlabelled figure derived from a
// private recording reads as reproducible. A run that stopped after four of
// six cells reads as a complete comparison. A report that references no raw
// records reads as if the records were never needed. None of those readings
// is corrected by a README, because a report travels without one.
//
// All three are ADDITIVE and OPTIONAL: a report that sets none of them
// marshals exactly as before (the contract forbids a report_version bump for
// this feature), and ValidateAdditiveBlocks only checks what is present. The
// strictness lives in PublicationCheck instead — the one moment where "not
// declared" must not be allowed to pass as "fine".
// ---------------------------------------------------------------------------
// Input availability (FR-030). Every input is either in the repository,
// obtainable by a documented pinned procedure, or a private recording that an
// outsider simply cannot get.
//
// The third value is not a lesser version of the first two. Repository and
// pinned-procedure inputs differ only in where the bytes live; a private
// recording differs in kind, because FR-006 forbids publishing the recording
// and there is no procedure that would hand it over. That is a permanent
// limitation of any figure built on it, and the report says so rather than
// leaving a reader to assume the usual.
const (
// InputAvailabilityRepository marks inputs committed in this repository —
// the frozen corpora and fixtures. Anyone with a clone has them.
InputAvailabilityRepository = "repository"
// InputAvailabilityPinnedProcedure marks inputs an outsider fetches by
// following a documented, version-pinned procedure (a pinned dataset
// revision, a pinned task suite). Reproducible, just not in-tree.
InputAvailabilityPinnedProcedure = "pinned-procedure"
// InputAvailabilityPrivateRecording marks inputs that are one operator's
// own recorded traffic. Not publishable, therefore not obtainable,
// therefore not independently reproducible.
InputAvailabilityPrivateRecording = "private-recording"
)
// InputProvenance is one block's FR-030 mark: where its inputs come from and,
// consequently, whether an outsider can reproduce its figures.
//
// IndependentlyReproducible is deliberately redundant with Availability — it
// is exactly `Availability != private-recording`, and Validate enforces that.
// The redundancy is the point: a renderer, a dashboard or an outside reader
// consuming the JSON should not have to know which enum values imply
// reproducibility. The one-word answer is in the document, and the enum
// explains it. A mismatch between the two is a hard error, so the redundancy
// can never drift into a contradiction.
type InputProvenance struct {
Availability string `json:"availability"`
IndependentlyReproducible bool `json:"independently_reproducible"`
// Limitation states, in the report itself, WHY an outsider cannot
// reproduce this block. Mandatory whenever the block is not
// independently reproducible: an unexplained "false" is only marginally
// better than no mark, because the reader cannot tell whether the input
// is private, expired, or merely undocumented.
Limitation string `json:"limitation,omitempty"`
// Procedure names the documented, pinned procedure that obtains the
// inputs. Mandatory for pinned-procedure availability — a procedure that
// is not named is not documented, and FR-030 asks for a documented one.
Procedure string `json:"procedure,omitempty"`
}
// PrivateRecordingInputs marks a block as built on one operator's own recorded
// sessions (T056/FR-030). It is a constructor rather than a struct literal so
// the not-reproducible flag cannot be forgotten at a call site: the whole
// value of this mark is that it is never omitted on the one block that needs
// it most.
func PrivateRecordingInputs(limitation string) *InputProvenance {
return &InputProvenance{
Availability: InputAvailabilityPrivateRecording,
IndependentlyReproducible: false,
Limitation: limitation,
}
}
// ReproducibleInputs marks a block whose inputs an outsider can obtain, either
// from the repository or by following the named pinned procedure.
func ReproducibleInputs(availability, procedure string) *InputProvenance {
return &InputProvenance{
Availability: availability,
IndependentlyReproducible: true,
Procedure: procedure,
}
}
// Validate reports whether the mark is internally consistent. block names the
// report block for the error message.
func (p *InputProvenance) Validate(block string) error {
if p == nil {
return nil
}
switch p.Availability {
case InputAvailabilityRepository, InputAvailabilityPinnedProcedure, InputAvailabilityPrivateRecording:
default:
return fmt.Errorf("%s block: inputs.availability %q not in {%s,%s,%s}",
block, p.Availability, InputAvailabilityRepository,
InputAvailabilityPinnedProcedure, InputAvailabilityPrivateRecording)
}
wantReproducible := p.Availability != InputAvailabilityPrivateRecording
if p.IndependentlyReproducible != wantReproducible {
return fmt.Errorf("%s block: inputs.availability %q implies independently_reproducible=%t, got %t",
block, p.Availability, wantReproducible, p.IndependentlyReproducible)
}
if !p.IndependentlyReproducible && p.Limitation == "" {
return fmt.Errorf("%s block: inputs are not independently reproducible but state no limitation", block)
}
if p.Availability == InputAvailabilityPinnedProcedure && p.Procedure == "" {
return fmt.Errorf("%s block: pinned-procedure inputs name no procedure", block)
}
return nil
}
// Run completeness (FR-032). A run is one harness invocation producing one
// report; it is complete only when every planned cell either produced a figure
// or was skipped for a stated reason.
const (
RunCompletenessComplete = "complete"
RunCompletenessPartial = "partial"
)
// RunStatus is the FR-032 mark (T058): whether this report covers the whole
// comparison it set out to make.
//
// Why the three counts rather than one boolean. A cell can be absent for two
// very different reasons, and collapsing them would make the mark useless. A
// DELIBERATE skip — code_exec skipped because code execution is disabled — is
// accounted for, expected, and does not make the comparison partial; the
// harness already reports such cells as skip rows naming what they collapse
// onto. An UNEXPLAINED gap is a cell the run was supposed to measure and did
// not, which is exactly the subset-presented-as-the-whole failure FR-032
// exists to catch. Planned − measured − skipped is that gap, and MissingCells
// names it, because "5 of 6" tells a reader nothing about which comparison
// they are missing.
//
// Interrupted is separate again: a run killed during the fourth repetition of
// the last cell may have touched every cell and still be partial, since the
// per-cell figures are averages over a repetition count the operator chose.
type RunStatus struct {
Completeness string `json:"completeness"`
// Reason is mandatory on a partial run. A partial with no reason is the
// same silence as no mark at all — the reader learns something is
// missing but not whether it matters.
Reason string `json:"reason,omitempty"`
Interrupted bool `json:"interrupted,omitempty"`
CellsPlanned int `json:"cells_planned"`
CellsMeasured int `json:"cells_measured"`
CellsSkipped int `json:"cells_skipped,omitempty"`
MissingCells []string `json:"missing_cells,omitempty"`
}
// DeriveRunStatus computes the completeness label from what the run actually
// did, rather than accepting one the caller asserts.
//
// This direction matters. A producer that stamps its own "complete" stamps it
// from the same optimism that lost the cells; a label derived from the planned
// set and the measured set cannot disagree with them. Validate then re-checks
// the same arithmetic on the way out, so a status that was hand-edited (or
// deserialized from somewhere else) is caught at emission time too.
//
// planned, measured and skipped are cell ids. reason is optional: when the run
// turns out to be partial and the caller supplied none, a factual one is
// synthesized, because a partial run must never reach the report unexplained.
func DeriveRunStatus(planned, measured, skipped []string, interrupted bool, reason string) *RunStatus {
accounted := make(map[string]bool, len(measured)+len(skipped))
for _, id := range measured {
accounted[id] = true
}
for _, id := range skipped {
accounted[id] = true
}
// Iterate over planned, not over the maps: the missing list must be
// deterministic (FR-010/SC-002) and in the operator's own cell order.
missing := make([]string, 0, len(planned))
for _, id := range planned {
if !accounted[id] {
missing = append(missing, id)
}
}
status := &RunStatus{
Completeness: RunCompletenessComplete,
Interrupted: interrupted,
CellsPlanned: len(planned),
CellsMeasured: len(measured),
CellsSkipped: len(skipped),
Reason: reason,
}
if len(missing) > 0 {
status.MissingCells = missing
}
if len(missing) > 0 || interrupted {
status.Completeness = RunCompletenessPartial
if status.Reason == "" {
switch {
case len(missing) > 0 && interrupted:
status.Reason = fmt.Sprintf("run was interrupted; %d planned cell(s) unmeasured and unaccounted for: %s",
len(missing), strings.Join(missing, ", "))
case len(missing) > 0:
status.Reason = fmt.Sprintf("%d planned cell(s) unmeasured and unaccounted for: %s",
len(missing), strings.Join(missing, ", "))
default:
status.Reason = "run was interrupted before it finished"
}
}
}
return status
}
// IsPartial reports whether this run may not be published as a complete
// comparison. An unset status is NOT treated as complete here — callers that
// need that distinction check for nil themselves, and PublicationCheck does.
func (s *RunStatus) IsPartial() bool {
return s != nil && s.Completeness == RunCompletenessPartial
}
// Validate re-derives the completeness label from the counts and refuses a
// status that claims more than the counts support. The interesting case is a
// hand-stamped "complete" over an unexplained gap: that is the exact document
// FR-032 forbids, and it is well-formed JSON, so nothing else would catch it.
func (s *RunStatus) Validate() error {
if s == nil {
return nil
}
switch s.Completeness {
case RunCompletenessComplete, RunCompletenessPartial:
default:
return fmt.Errorf("run_status: completeness %q not in {%s,%s}",
s.Completeness, RunCompletenessComplete, RunCompletenessPartial)
}
if s.CellsPlanned <= 0 {
return fmt.Errorf("run_status: cells_planned is %d — a run that planned nothing compares nothing", s.CellsPlanned)
}
if s.CellsMeasured < 0 || s.CellsSkipped < 0 {
return fmt.Errorf("run_status: negative cell counts (measured=%d skipped=%d)", s.CellsMeasured, s.CellsSkipped)
}
gap := s.CellsPlanned - s.CellsMeasured - s.CellsSkipped
if gap < 0 {
return fmt.Errorf("run_status: measured(%d)+skipped(%d) exceeds planned(%d)",
s.CellsMeasured, s.CellsSkipped, s.CellsPlanned)
}
if gap > 0 && len(s.MissingCells) == 0 {
return fmt.Errorf("run_status: %d planned cell(s) unaccounted for but missing_cells is empty — name them", gap)
}
mustBePartial := gap > 0 || s.Interrupted
if mustBePartial && s.Completeness != RunCompletenessPartial {
return fmt.Errorf("run_status: claims %q while %d cell(s) are unaccounted for (interrupted=%t) — that is a subset presented as the whole",
s.Completeness, gap, s.Interrupted)
}
if s.Completeness == RunCompletenessComplete && len(s.MissingCells) > 0 {
return fmt.Errorf("run_status: claims %q but lists %d missing cell(s)", s.Completeness, len(s.MissingCells))
}
if s.Completeness == RunCompletenessPartial && s.Reason == "" {
return fmt.Errorf("run_status: partial run states no reason")
}
return nil
}
// Raw per-run record retention (FR-029).
const (
// RecordsRetained means the artifact exists in this run's output
// directory, at RunLocalPath, right now.
RecordsRetained = "retained"
// RecordsNotRetained means there is no artifact to point at. It is the
// honest terminal state of a reference whose file is gone, and it is
You can’t perform that action at this time.