FilesExpand file tree

main

/

agent_validation_model_test.go

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

151 lines (133 loc) · 4.34 KB

main

/

agent_validation_model_test.go

Copy path

Top

File metadata and controls

• Code
• Blame

151 lines (133 loc) · 4.34 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

//go:build !integration

package workflow

import (

"os"

"path/filepath"

"testing"

"github.com/github/gh-aw/pkg/testutil"

"github.com/stretchr/testify/assert"

"github.com/stretchr/testify/require"

)

func TestValidateUniversalLLMConsumerModel(t *testing.T) {

compiler := NewCompiler()

opencodeEngine, err := NewBehaviorDefinedEngine(&EngineDefinition{

ID: "opencode",

DisplayName: "OpenCode",

Behaviors: &EngineBehaviorDefinition{

SecretStrategy: behaviorSecretStrategyUniversalLLMConsumer,

},

})

require.NoError(t, err)

t.Run("non universal engine skips validation", func(t *testing.T) {

err := compiler.validateUniversalLLMConsumerModel(

map[string]any{

"engine": map[string]any{

"id": "copilot",

},

},

NewCopilotEngine(),

)

assert.NoError(t, err, "Non-universal engines should skip model validation")

})

t.Run("opencode requires model", func(t *testing.T) {

err := compiler.validateUniversalLLMConsumerModel(

map[string]any{

"engine": map[string]any{

"id": "opencode",

},

},

opencodeEngine,

)

require.Error(t, err, "Missing model should fail for opencode")

require.ErrorContains(t, err, "engine.model is required for engine 'opencode'")

})

t.Run("opencode requires provider/model format", func(t *testing.T) {

err := compiler.validateUniversalLLMConsumerModel(

map[string]any{

"engine": map[string]any{

"id": "opencode",

"model": "gpt-4.1",

},

},

opencodeEngine,

)

require.Error(t, err, "Unqualified model should fail for opencode")

require.ErrorContains(t, err, "provider/model format")

})

t.Run("unsupported provider fails", func(t *testing.T) {

err := compiler.validateUniversalLLMConsumerModel(

map[string]any{

"engine": map[string]any{

"id": "opencode",

"model": "groq/llama-4",

},

},

opencodeEngine,

)

require.Error(t, err, "Unsupported provider should fail")

require.ErrorContains(t, err, "unsupported provider")

})

t.Run("supported provider passes", func(t *testing.T) {

err := compiler.validateUniversalLLMConsumerModel(

map[string]any{

"engine": map[string]any{

"id": "opencode",

"model": "anthropic/claude-sonnet-4",

},

},

opencodeEngine,

)

assert.NoError(t, err, "Supported provider/model should pass")

})

}

func TestValidateContextWindowSupport(t *testing.T) {

t.Run("missing context-window does not warn", func(t *testing.T) {

compiler := NewCompiler()

compiler.validateContextWindowSupport(&EngineConfig{}, NewCodexEngine())

assert.Zero(t, compiler.GetWarningCount())

})

t.Run("nil engine config does not warn", func(t *testing.T) {

compiler := NewCompiler()

compiler.validateContextWindowSupport(nil, NewCodexEngine())

assert.Zero(t, compiler.GetWarningCount())

})

engineConfig := &EngineConfig{ContextWindow: 1000000}

t.Run("Pi supports context-window", func(t *testing.T) {

compiler := NewCompiler()

compiler.validateContextWindowSupport(engineConfig, NewPiEngine())

assert.Zero(t, compiler.GetWarningCount())

})

for _, engine := range []CodingAgentEngine{

NewClaudeEngine(),

NewCodexEngine(),

NewCopilotEngine(),

NewGeminiEngine(),

} {

t.Run(engine.GetID()+" warns when context-window is unsupported", func(t *testing.T) {

compiler := NewCompiler()

compiler.validateContextWindowSupport(engineConfig, engine)

assert.Equal(t, 1, compiler.GetWarningCount())

})

}

}

func TestCompileWorkflowWarnsForImportedUnsupportedContextWindow(t *testing.T) {

compileWithSharedEngine := func(t *testing.T, sharedEngine string) int {

t.Helper()

tmpDir := testutil.TempDir(t, "imported-context-window")

require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "shared.md"), []byte("---\nengine:\n"+sharedEngine+"---\n\nShared engine configuration.\n"), 0o644))

workflowPath := filepath.Join(tmpDir, "workflow.md")

require.NoError(t, os.WriteFile(workflowPath, []byte(`---

on: workflow_dispatch

imports:

- shared.md

---

Run the workflow.

`), 0o644))

compiler := NewCompiler(WithVersion("dev"))

require.NoError(t, compiler.CompileWorkflow(workflowPath))

return compiler.GetWarningCount()

}

baseline := compileWithSharedEngine(t, " id: codex\n")

withContextWindow := compileWithSharedEngine(t, " id: codex\n context-window: 1000000\n")

assert.Equal(t, baseline+1, withContextWindow, "Imported context-window on an unsupported engine should warn")

}