github · lpcox · Jun 12, 2026 · Jun 12, 2026 · Jun 12, 2026 · Jun 12, 2026
diff --git a/AGENTS.md b/AGENTS.md
@@ -417,6 +417,7 @@ DEBUG_COLORS=0 DEBUG=* ./awmg --config config.toml
 - `MCP_GATEWAY_HMAC_SECRET` - Shared HMAC-SHA256 secret for request signing and replay protection; when set, requests to MCP handlers must carry valid `X-MCP-Timestamp`, `X-MCP-Nonce`, and `X-MCP-Signature` headers (sets default for `--hmac-secret`)
 - `OTEL_EXPORTER_OTLP_ENDPOINT` - OTLP HTTP endpoint for trace export; sets default for `--otlp-endpoint`
 - `OTEL_EXPORTER_OTLP_HEADERS` - Comma-separated `key=value` OTLP export headers (W3C Baggage format); used as fallback when `gateway.opentelemetry.headers` / `gateway.tracing.headers` is not set in config
+- `GH_AW_OTLP_ENDPOINTS` - Comma-separated list of OTLP HTTP endpoint URLs for multi-backend fan-out; when set, spans are exported to ALL listed endpoints (partial-failure tolerant). Takes precedence over `OTEL_EXPORTER_OTLP_ENDPOINT`. Shared headers from `OTEL_EXPORTER_OTLP_HEADERS` / config apply to every endpoint.
 - `OTEL_SERVICE_NAME` - Service name in traces; sets default for `--otlp-service-name`
 - `TAVILY_API_KEY` - Tavily API key; when unset, integration tests that use it are automatically skipped
 - `AWMG_BINARY_PATH` - Override binary path for integration tests

diff --git a/internal/cmd/root.go b/internal/cmd/root.go
@@ -333,9 +333,16 @@ func run(cmd *cobra.Command, args []string) error {
 			sampleRate = tracingCfg.GetSampleRate()
 			serviceName = tracingCfg.ServiceName
 		}
-		if endpoint != "" {
+		if tracingProvider.IsEnabled() {
+			// When GH_AW_OTLP_ENDPOINTS is set, InitProvider uses fan-out mode and
+			// it takes precedence over the primary endpoint; use the env var as the
+			// display value to accurately reflect what is actually receiving spans.
+			displayEndpoint := os.Getenv("GH_AW_OTLP_ENDPOINTS")
+			if displayEndpoint == "" {
+				displayEndpoint = endpoint
+			}
 			logger.StartupInfo("OpenTelemetry tracing enabled: endpoint=%s, service=%s, sampleRate=%.2f",
-				endpoint, serviceName, sampleRate)
+				displayEndpoint, serviceName, sampleRate)
 		} else {
 			logger.StartupInfo("OpenTelemetry tracing disabled (no OTLP endpoint configured)")
 		}

diff --git a/internal/tracing/config_resolver.go b/internal/tracing/config_resolver.go
@@ -140,6 +140,41 @@ func resolveHeaders(cfg *config.TracingConfig) map[string]string {
 	return parseOTLPHeaders(raw)
 }
 
+// resolveExtraEndpoints returns the additional OTLP endpoints from the
+// GH_AW_OTLP_ENDPOINTS environment variable as a slice of fully-qualified URL
+// strings (with the signal path appended). Each URL is normalised using the
+// same signal-path rules as resolveEndpoint. Empty and whitespace-only entries
+// are skipped. Returns nil when the variable is unset or yields no valid URLs.
+func resolveExtraEndpoints(cfg *config.TracingConfig) []string {
+	raw := os.Getenv("GH_AW_OTLP_ENDPOINTS")
+	if raw == "" {
+		return nil
+	}
+	signalPath := ""
+	if cfg != nil {
+		signalPath = cfg.SignalPath
+	}
+	var endpoints []string
+	for _, ep := range strings.Split(raw, ",") {
+		ep = strings.TrimSpace(ep)
+		if ep == "" {
+			continue
+		}
+		normalized := resolveEndpoint(&config.TracingConfig{
+			Endpoint:   ep,
+			SignalPath: signalPath,
+		})
+		if normalized != "" {
+			endpoints = append(endpoints, normalized)
+		}
+	}
+	if len(endpoints) == 0 {
+		return nil
+	}
+	logTracing.Printf("GH_AW_OTLP_ENDPOINTS: resolved %d extra endpoint(s)", len(endpoints))
+	return endpoints
+}
+
 // resolveParentContext builds a context carrying the W3C remote parent span context
 // from the configured traceId and spanId (spec §4.1.3.6).
 // If traceId is absent, or either ID is malformed, the original context is returned unchanged.

diff --git a/internal/tracing/fanout.go b/internal/tracing/fanout.go
@@ -0,0 +1,74 @@
+package tracing
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+// fanoutExporter is a SpanExporter that fans out span export to multiple
+// underlying exporters. All exporters are attempted even when earlier ones
+// fail (partial-failure tolerance), and collected errors are joined before
+// returning.
+type fanoutExporter struct {
+	exporters []sdktrace.SpanExporter
+}
+
+// newFanoutExporter returns a SpanExporter that forwards to all given exporters.
+// When only one exporter is provided it is returned directly to avoid overhead.
+func newFanoutExporter(exporters []sdktrace.SpanExporter) sdktrace.SpanExporter {
+	if len(exporters) == 1 {
+		return exporters[0]
+	}
+	return &fanoutExporter{exporters: exporters}
+}
+
+// ExportSpans exports spans to each underlying exporter concurrently. All
+// exporters are invoked in parallel so that a slow or hung backend cannot
+// delay delivery to the others. Errors from all exporters are collected and
+// joined before returning.
+func (f *fanoutExporter) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error {
+	var (
+		wg   sync.WaitGroup
+		mu   sync.Mutex
+		errs []error
+	)
+	for _, exp := range f.exporters {
+		wg.Add(1)
+		go func(e sdktrace.SpanExporter) {
+			defer wg.Done()
+			if err := e.ExportSpans(ctx, spans); err != nil {
+				mu.Lock()
+				errs = append(errs, err)
+				mu.Unlock()
+			}
+		}(exp)
+	}
+	wg.Wait()
+	return errors.Join(errs...)
+}
+
+// Shutdown shuts down each underlying exporter concurrently, collecting any
+// errors. All errors are joined and returned.
+func (f *fanoutExporter) Shutdown(ctx context.Context) error {
+	var (
+		wg   sync.WaitGroup
+		mu   sync.Mutex
+		errs []error
+	)
+	for _, exp := range f.exporters {
+		wg.Add(1)
+		go func(e sdktrace.SpanExporter) {
+			defer wg.Done()
+			if err := e.Shutdown(ctx); err != nil {
+				mu.Lock()
+				errs = append(errs, err)
+				mu.Unlock()
+			}
+		}(exp)
+	}
+	wg.Wait()
+	return errors.Join(errs...)
+}
diff --git a/internal/tracing/fanout_test.go b/internal/tracing/fanout_test.go
@@ -0,0 +1,143 @@
+package tracing
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
+// stubExporter is a minimal SpanExporter that records calls for test assertions.
+type stubExporter struct {
+	exported    [][]sdktrace.ReadOnlySpan
+	shutdowns   int
+	exportErr   error
+	shutdownErr error
+}
+
+func (s *stubExporter) ExportSpans(_ context.Context, spans []sdktrace.ReadOnlySpan) error {
+	s.exported = append(s.exported, spans)
+	return s.exportErr
+}
+
+func (s *stubExporter) Shutdown(_ context.Context) error {
+	s.shutdowns++
+	return s.shutdownErr
+}
+
+// TestNewFanoutExporter_SingleExporter returns the exporter directly.
+func TestNewFanoutExporter_SingleExporter(t *testing.T) {
+	exp := &stubExporter{}
+	result := newFanoutExporter([]sdktrace.SpanExporter{exp})
+	assert.Same(t, exp, result)
+}
+
+// TestNewFanoutExporter_MultipleExporters wraps in fanoutExporter.
+func TestNewFanoutExporter_MultipleExporters(t *testing.T) {
+	a, b := &stubExporter{}, &stubExporter{}
+	result := newFanoutExporter([]sdktrace.SpanExporter{a, b})
+	_, ok := result.(*fanoutExporter)
+	assert.True(t, ok, "expected *fanoutExporter when more than one exporter is given")
+}
+
+// TestFanoutExporter_ExportSpans_AllReceiveSpans verifies that all exporters
+// receive the same spans on each ExportSpans call.
+func TestFanoutExporter_ExportSpans_AllReceiveSpans(t *testing.T) {
+	ctx := context.Background()
+	a, b := &stubExporter{}, &stubExporter{}
+	exp := &fanoutExporter{exporters: []sdktrace.SpanExporter{a, b}}
+
+	sr := tracetest.NewSpanRecorder()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+	_, span := tp.Tracer("test").Start(ctx, "op")
+	span.End()
+	require.NoError(t, tp.Shutdown(ctx))
+
+	spans := sr.Ended()
+	require.NotEmpty(t, spans)
+
+	err := exp.ExportSpans(ctx, spans)
+	require.NoError(t, err)
+
+	assert.Len(t, a.exported, 1, "exporter A should have received one batch")
+	assert.Len(t, b.exported, 1, "exporter B should have received one batch")
+	assert.Equal(t, spans, a.exported[0])
+	assert.Equal(t, spans, b.exported[0])
+}
+
+// TestFanoutExporter_ExportSpans_ContinuesOnError verifies partial-failure
+// tolerance: a failing exporter does not prevent delivery to later exporters.
+func TestFanoutExporter_ExportSpans_ContinuesOnError(t *testing.T) {
+	ctx := context.Background()
+	failing := &stubExporter{exportErr: errors.New("backend unavailable")}
+	ok := &stubExporter{}
+
+	exp := &fanoutExporter{exporters: []sdktrace.SpanExporter{failing, ok}}
+
+	sr := tracetest.NewSpanRecorder()
+	tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+	_, span := tp.Tracer("test").Start(ctx, "op")
+	span.End()
+	require.NoError(t, tp.Shutdown(ctx))
+
+	spans := sr.Ended()
+	err := exp.ExportSpans(ctx, spans)
+
+	assert.Error(t, err, "error from failing exporter should be returned")
+	assert.Contains(t, err.Error(), "backend unavailable")
+	assert.Len(t, ok.exported, 1, "healthy exporter should still receive spans")
+}
+
+// TestFanoutExporter_ExportSpans_BothErrors returns all errors joined.
+func TestFanoutExporter_ExportSpans_BothErrors(t *testing.T) {
+	ctx := context.Background()
+	e1 := &stubExporter{exportErr: errors.New("error one")}
+	e2 := &stubExporter{exportErr: errors.New("error two")}
+	exp := &fanoutExporter{exporters: []sdktrace.SpanExporter{e1, e2}}
+
+	err := exp.ExportSpans(ctx, nil)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "error one")
+	assert.Contains(t, err.Error(), "error two")
+}
+
+// TestFanoutExporter_Shutdown_CallsAll verifies Shutdown is forwarded to every exporter.
+func TestFanoutExporter_Shutdown_CallsAll(t *testing.T) {
+	ctx := context.Background()
+	a, b, c := &stubExporter{}, &stubExporter{}, &stubExporter{}
+	exp := &fanoutExporter{exporters: []sdktrace.SpanExporter{a, b, c}}
+
+	require.NoError(t, exp.Shutdown(ctx))
+	assert.Equal(t, 1, a.shutdowns)
+	assert.Equal(t, 1, b.shutdowns)
+	assert.Equal(t, 1, c.shutdowns)
+}
+
+// TestFanoutExporter_Shutdown_ContinuesOnError verifies that a Shutdown error
+// does not prevent subsequent exporters from being shut down.
+func TestFanoutExporter_Shutdown_ContinuesOnError(t *testing.T) {
+	ctx := context.Background()
+	failing := &stubExporter{shutdownErr: errors.New("shutdown error")}
+	ok := &stubExporter{}
+	exp := &fanoutExporter{exporters: []sdktrace.SpanExporter{failing, ok}}
+
+	err := exp.Shutdown(ctx)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "shutdown error")
+	assert.Equal(t, 1, ok.shutdowns, "healthy exporter should still be shut down")
+}
+
+// TestFanoutExporter_ExportSpans_Empty verifies that ExportSpans with a nil/empty
+// span slice is forwarded to the underlying exporter (which may be a no-op internally).
+func TestFanoutExporter_ExportSpans_Empty(t *testing.T) {
+	ctx := context.Background()
+	a := &stubExporter{}
+	exp := &fanoutExporter{exporters: []sdktrace.SpanExporter{a}}
+
+	require.NoError(t, exp.ExportSpans(ctx, nil))
+	assert.Len(t, a.exported, 1)
+}