Add chasm standby task verification by yux0 · Pull Request #12618 · temporalio/temporal · GitHub
Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions chasm/lib/tests/tasks.go
93 changes: 93 additions & 0 deletions chasm/lib/tests/tasks_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package tests_test

import (
"testing"
"time"

"github.com/stretchr/testify/require"
"go.temporal.io/api/serviceerror"
"go.temporal.io/server/chasm"
"go.temporal.io/server/chasm/chasmtest"
"go.temporal.io/server/chasm/lib/tests"
"go.temporal.io/server/chasm/lib/tests/gen/testspb/v1"
"go.temporal.io/server/common/clock"
"go.temporal.io/server/common/log"
"go.temporal.io/server/common/payload"
"go.temporal.io/server/common/testing/protorequire"
)

var _ chasm.StandbyTaskValidator[*testspb.TestPayloadTTLSideEffectTask] = (*tests.PayloadTTLSideEffectTaskHandler)(nil)

func TestPayloadTTLSideEffectTaskValidateStandby(t *testing.T) {
t.Parallel()

for _, tc := range []struct {
name string
ttl time.Duration
removePayload bool
missingStore bool
valid bool
}{
{name: "expiration matches task - retry", ttl: time.Hour, valid: true},
{name: "expiration before task - retry", ttl: 30 * time.Minute, valid: true},
{name: "expiration after task - discard", ttl: 2 * time.Hour},
{name: "payload removed - discard", ttl: time.Hour, removePayload: true},
{name: "payload has no TTL - discard"},
{name: "store missing - propagate read error", missingStore: true},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()

registry := chasm.NewRegistry(log.NewNoopLogger())
require.NoError(t, registry.Register(&chasm.CoreLibrary{}))
require.NoError(t, registry.Register(tests.Library))
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
engine := chasmtest.NewEngine(t, registry, chasmtest.WithTimeSource(clock.NewEventTimeSource().Update(now)))
ctx := chasm.NewEngineContext(t.Context(), engine)
key := chasm.ExecutionKey{NamespaceID: "test-namespace", BusinessID: "payload-store"}
if !tc.missingStore {
result, err := chasm.StartExecution(ctx, key,
func(ctx chasm.MutableContext, _ any) (*tests.PayloadStore, error) {
store, err := tests.NewPayloadStore(ctx)
if err != nil {
return nil, err
}
_, err = store.AddPayload(ctx, tests.AddPayloadRequest{
PayloadKey: "payload", Payload: payload.EncodeString("value"), TTL: tc.ttl,
})
return store, err
}, nil)
require.NoError(t, err)
key = result.ExecutionKey
}
ref := chasm.NewComponentRef[*tests.PayloadStore](key)
if tc.removePayload {
_, _, err := chasm.UpdateComponent(ctx, ref, (*tests.PayloadStore).RemovePayload, "payload")
require.NoError(t, err)
}

var before tests.DescribePayloadStoreResponse
if !tc.missingStore {
var err error
before, err = chasm.ReadComponent(ctx, ref, (*tests.PayloadStore).Describe, tests.DescribePayloadStoreRequest{})
require.NoError(t, err)
}

handler := &tests.PayloadTTLSideEffectTaskHandler{}
valid, err := handler.ValidateStandby(ctx, ref, chasm.TaskInvocation{
TaskAttributes: chasm.TaskAttributes{ScheduledTime: now.Add(time.Hour)}, Attempt: 3,
}, &testspb.TestPayloadTTLSideEffectTask{PayloadKey: "payload"})
require.Equal(t, tc.valid, valid)
if tc.missingStore {
var notFound *serviceerror.NotFound
require.ErrorAs(t, err, &notFound)
return
}
require.NoError(t, err)

after, err := chasm.ReadComponent(ctx, ref, (*tests.PayloadStore).Describe, tests.DescribePayloadStoreRequest{})
require.NoError(t, err)
protorequire.ProtoEqual(t, before.State, after.State)
})
}
}
16 changes: 14 additions & 2 deletions chasm/registrable_task.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ type (
goType reflect.Type
componentGoType reflect.Type // It is not clear how this one is used.
validateFn validateFn
validateStandbyFn func(context.Context, ComponentRef, TaskInvocation, any) (bool, error)
pureTaskExecuteFn pureTaskExecuteFn
sideEffectTaskExecuteFn sideEffectTaskExecuteFn
sideEffectTaskDiscardFn sideEffectTaskDiscardFn
Expand All @@ -45,13 +46,13 @@ type (
)

// NewRegistrableSideEffectTask creates a new registrable side-effect task. NOTE: C is not Component but any.
// The handler's Discard method is called on standby clusters when a task has been pending past the discard delay.
// The handler's Discard method is called when standby validation returns false.
func NewRegistrableSideEffectTask[C any, T any](
taskType string,
handler SideEffectTaskHandler[C, T],
opts ...RegistrableTaskOption,
) *RegistrableTask {
return newRegistrableTask(
rt := newRegistrableTask(
taskType,
reflect.TypeFor[T](),
reflect.TypeFor[C](),
Expand Down Expand Up @@ -84,6 +85,17 @@ func NewRegistrableSideEffectTask[C any, T any](
},
opts...,
)
rt.validateStandbyFn = func(ctx context.Context, ref ComponentRef, invocation TaskInvocation, task any) (bool, error) {
return ReadComponent(ctx, ref, func(component C, ctx Context, _ struct{}) (bool, error) {
return handler.Validate(ctx, component, invocation, task.(T))
}, struct{}{})
}
if validator, ok := handler.(StandbyTaskValidator[T]); ok {
rt.validateStandbyFn = func(ctx context.Context, ref ComponentRef, invocation TaskInvocation, task any) (bool, error) {
return validator.ValidateStandby(ctx, ref, invocation, task.(T))
}
}
return rt
}

func NewRegistrablePureTask[C any, T any](
Expand Down
18 changes: 15 additions & 3 deletions chasm/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
)

// ErrTaskDiscarded is the error returned by the default [SideEffectTaskHandlerBase] Discard implementation,
// indicating that a side-effect task on a standby cluster has been pending past the discard delay.
// indicating that standby validation rejected a task without a custom discard handler.
var ErrTaskDiscarded = errors.New("standby task pending for too long")

type (
Expand All @@ -25,7 +25,7 @@ type (
Destination string
}

// TaskInvocation is passed to a task's Validate callback. It carries the task's [TaskAttributes]
// TaskInvocation is passed to a task's Validate and ValidateStandby callbacks. It carries the task's [TaskAttributes]
// together with framework-supplied state for the current processing attempt.
TaskInvocation struct {
TaskAttributes
Expand All @@ -44,7 +44,7 @@ type (
TaskValidator[C, T]
Execute(context.Context, ComponentRef, TaskAttributes, T) error
// Discard implements custom discard behavior on standby clusters. When a side-effect task has been
// pending on standby past the discard delay, the framework calls Discard instead of silently dropping
// rejected by standby validation, the framework calls Discard instead of silently dropping
// the task. For example, the activity dispatch handler implements this to spill tasks to matching.
// The ctx carries engine access, but implementations must avoid mutating component state on standby
// clusters.
Expand All @@ -60,6 +60,16 @@ type (
pureTaskHandler()
}

// StandbyTaskValidator optionally overrides standby validation for a side-effect task.
// The callback runs without holding the execution lock and may perform I/O. Component
// state must be accessed through ReadComponent; it must not be mutated on standby.
// Handlers without this interface fall back to Validate through a locked component read.
StandbyTaskValidator[T any] interface {
// ValidateStandby returns true to retry the task, false to invoke Discard, or an error
// to fail the processing attempt. It receives an engine-enabled context.
ValidateStandby(context.Context, ComponentRef, TaskInvocation, T) (bool, error)
}

// TaskValidator is implemented by both [SideEffectTaskHandler] and [PureTaskHandler] to gate
// whether a task should proceed with execution.
TaskValidator[C any, T any] interface {
Expand All @@ -78,6 +88,8 @@ type (
// The framework automatically calls Validate at key points, such as after closing transactions, to check all
// generated tasks before they execute.
//
// Side-effect handlers may implement [StandbyTaskValidator] to override validation on standby.
//
// Returns:
// - (true, nil) if the task is valid and should be executed
// - (false, nil) if the task should be silently dropped (it's no longer relevant)
Expand Down
39 changes: 39 additions & 0 deletions chasm/task_mock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

66 changes: 59 additions & 7 deletions chasm/tree.go
Loading
Loading