FilesExpand file tree

main

/

run_mincut_bench.sh

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

executable file

·

176 lines (155 loc) · 6.99 KB

main

/

run_mincut_bench.sh

Copy path

Top

File metadata and controls

• Code
• Blame

executable file

·

176 lines (155 loc) · 6.99 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

#!/usr/bin/env bash

# run_mincut_bench.sh -- 1k-sample grid runner for min-cut gating vs softmax

#

# Usage:

# ./scripts/run_mincut_bench.sh [--samples N] [--output-dir DIR]

#

# Runs a grid search over lambda and tau parameters, collecting:

# - Coherence delta metrics

# - Memory pressure profiles

# - Power/energy measurements

# - Latency distributions (p50/p95/p99)

# - Witness chain (JSONL + RVF bundle)

set -euo pipefail

# ---------------------------------------------------------------------------

# Defaults

# ---------------------------------------------------------------------------

SAMPLES=1000

SHORT_SAMPLES=500

LONG_SAMPLES=500

SHORT_MAX_LEN=128

LONG_MIN_LEN=256

LONG_MAX_LEN=1024

OUTPUT_DIR="results/mincut-bench"

LAMBDA_GRID="0.3 0.5 0.7"

TAU_GRID="0 2"

EPS=0.01

SEED=42

# ---------------------------------------------------------------------------

# Parse arguments

# ---------------------------------------------------------------------------

while [[ $# -gt 0 ]]; do

case $1 in

--samples) SAMPLES="$2"; shift 2 ;;

--output-dir) OUTPUT_DIR="$2"; shift 2 ;;

--lambda) LAMBDA_GRID="$2"; shift 2 ;;

--tau) TAU_GRID="$2"; shift 2 ;;

--seed) SEED="$2"; shift 2 ;;

*) echo "Unknown option: $1"; exit 1 ;;

esac

done

SHORT_SAMPLES=$((SAMPLES / 2))

LONG_SAMPLES=$((SAMPLES - SHORT_SAMPLES))

# ---------------------------------------------------------------------------

# Setup

# ---------------------------------------------------------------------------

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"

PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"

mkdir -p "$OUTPUT_DIR"/{csv,witness,figs}

echo "============================================="

echo "Min-Cut Gating Benchmark"

echo "============================================="

echo "Samples: $SAMPLES ($SHORT_SAMPLES short + $LONG_SAMPLES long)"

echo "Lambda grid: $LAMBDA_GRID"

echo "Tau grid: $TAU_GRID"

echo "Epsilon: $EPS"

echo "Seed: $SEED"

echo "Output: $OUTPUT_DIR"

echo "============================================="

# ---------------------------------------------------------------------------

# Build (release mode for accurate benchmarks)

# ---------------------------------------------------------------------------

echo "[1/5] Building in release mode..."

cargo build --release \

-p ruvector-attn-mincut \

-p ruvector-coherence \

-p ruvector-profiler \

2>&1 | tail -5

# ---------------------------------------------------------------------------

# Run baseline (softmax)

# ---------------------------------------------------------------------------

echo "[2/5] Running baseline (softmax) on $SAMPLES samples..."

BASELINE_CSV="$OUTPUT_DIR/csv/baseline.csv"

echo "sample_id,seq_len,wall_time_us,peak_mem_bytes,energy_j" > "$BASELINE_CSV"

# Placeholder: in a real run, this would invoke the benchmark binary

# cargo run --release -p ruvector-bench-runner -- \

# --mode softmax \

# --short-samples $SHORT_SAMPLES --short-max-len $SHORT_MAX_LEN \

# --long-samples $LONG_SAMPLES --long-min-len $LONG_MIN_LEN --long-max-len $LONG_MAX_LEN \

# --seed $SEED \

# --output "$BASELINE_CSV"

echo " (baseline runner placeholder -- implement with bench binary)"

# ---------------------------------------------------------------------------

# Run grid search (min-cut gating)

# ---------------------------------------------------------------------------

echo "[3/5] Running min-cut gating grid search..."

RESULTS_CSV="$OUTPUT_DIR/csv/results.csv"

echo "setting,lambda,tau,coherence_delta,kv_cache_reduction,peak_mem_reduction,energy_reduction,p95_latency_us,accuracy" > "$RESULTS_CSV"

for lambda in $LAMBDA_GRID; do

for tau in $TAU_GRID; do

SETTING="mincut_l${lambda}_t${tau}"

echo " Running $SETTING..."

RUN_CSV="$OUTPUT_DIR/csv/${SETTING}.csv"

WITNESS_FILE="$OUTPUT_DIR/witness/${SETTING}.jsonl"

# Placeholder: invoke bench binary with min-cut params

# cargo run --release -p ruvector-bench-runner -- \

# --mode mincut \

# --lambda $lambda --tau $tau --eps $EPS \

# --short-samples $SHORT_SAMPLES --short-max-len $SHORT_MAX_LEN \

# --long-samples $LONG_SAMPLES --long-min-len $LONG_MIN_LEN --long-max-len $LONG_MAX_LEN \

# --seed $SEED \

# --output "$RUN_CSV" \

# --witness "$WITNESS_FILE"

echo " (grid runner placeholder -- implement with bench binary)"

done

done

# ---------------------------------------------------------------------------

# Compute aggregate metrics

# ---------------------------------------------------------------------------

echo "[4/5] Computing aggregate metrics..."

# Placeholder: post-processing script would:

# 1. Read all CSV files

# 2. Compute mean +/- 95% CI for coherence delta

# 3. Compare memory, energy, latency vs baseline

# 4. Write summary to results.csv

echo " (aggregation placeholder -- implement with post-processor)"

# ---------------------------------------------------------------------------

# Pack witness bundle (RVF)

# ---------------------------------------------------------------------------

echo "[5/5] Packing witness bundle..."

WITNESS_BUNDLE="$OUTPUT_DIR/witness/witness.rvf"

# Placeholder: concatenate witness JSONL files into RVF bundle

# The RVF format includes:

# - Header: config hash, model commit, weights hash

# - Body: per-sample witness entries with hash chain

# - Footer: aggregate stats, signature

echo " (RVF packer placeholder -- implement with witness tool)"

# ---------------------------------------------------------------------------

# Summary

# ---------------------------------------------------------------------------

echo ""

echo "============================================="

echo "Benchmark complete"

echo "============================================="

echo "Results: $OUTPUT_DIR/csv/results.csv"

echo "Witness: $OUTPUT_DIR/witness/"

echo "Figures: $OUTPUT_DIR/figs/ (generate with plot script)"

echo ""

echo "Expected results table:"

echo ""

echo "Setting | dCoherence | KV-Cache | Peak Mem | Energy/sample | p95 Latency"

echo "---------------------|------------|----------|----------|---------------|------------"

echo "Softmax (baseline) | -- | -- | -- | -- | --"

echo "Min-cut l=0.3, t=0 | +??% | -??% | -??% | -??% | ??us"

echo "Min-cut l=0.3, t=2 | +??% | -??% | -??% | -??% | ??us"

echo "Min-cut l=0.5, t=0 | +??% | -??% | -??% | -??% | ??us"

echo "Min-cut l=0.5, t=2 | +??% | -??% | -??% | -??% | ??us"

echo "Min-cut l=0.7, t=0 | +??% | -??% | -??% | -??% | ??us"

echo "Min-cut l=0.7, t=2 | +??% | -??% | -??% | -??% | ??us"

echo ""

echo "Success criteria:"

echo " >= 5% coherence delta with <= 1% accuracy loss"

echo " >= 15% KV-cache reduction"

echo " >= 10% energy/sample drop"

echo " p95 latency within +/-10% of baseline"

echo " Deterministic witness reproducible on second machine"