|
| 1 | +"""Exercise the runner CLI in disposable repositories without loading the SDK.""" |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +import shutil |
| 6 | +import subprocess |
| 7 | +import sys |
| 8 | +import tempfile |
| 9 | +import unittest |
| 10 | +from pathlib import Path |
| 11 | + |
| 12 | +from run_repo_skill_tests import test_environment |
| 13 | + |
| 14 | + |
| 15 | +class RepoSkillRunnerTests(unittest.TestCase): |
| 16 | + def setUp(self) -> None: |
| 17 | + directory = tempfile.TemporaryDirectory() |
| 18 | + self.addCleanup(directory.cleanup) |
| 19 | + self.repo = Path(directory.name) |
| 20 | + self.runner = self.repo / ".github/scripts/run_repo_skill_tests.py" |
| 21 | + self.runner.parent.mkdir(parents=True) |
| 22 | + shutil.copyfile(Path(__file__).with_name(self.runner.name), self.runner) |
| 23 | + |
| 24 | + def write_suite(self, skill: str, source: str) -> Path: |
| 25 | + path = self.repo / ".agents/skills" / skill / "scripts/test_fixture.py" |
| 26 | + path.parent.mkdir(parents=True) |
| 27 | + path.write_text(source, encoding="utf-8") |
| 28 | + return path |
| 29 | + |
| 30 | + def run_cli( |
| 31 | + self, *args: str, env: dict[str, str] | None = None |
| 32 | + ) -> subprocess.CompletedProcess[str]: |
| 33 | + return subprocess.run( |
| 34 | + [sys.executable, str(self.runner), *args], |
| 35 | + cwd=self.repo.parent, |
| 36 | + env=test_environment() if env is None else env, |
| 37 | + capture_output=True, |
| 38 | + text=True, |
| 39 | + timeout=30, |
| 40 | + check=False, |
| 41 | + ) |
| 42 | + |
| 43 | + def test_inventory_is_sorted_and_listing_does_not_execute_tests(self) -> None: |
| 44 | + for name in ("z-last", "a-first"): |
| 45 | + self.write_suite(name, "raise RuntimeError('must not execute during listing')\n") |
| 46 | + result = self.run_cli("--list") |
| 47 | + self.assertEqual(result.returncode, 0, result.stderr) |
| 48 | + self.assertEqual( |
| 49 | + result.stdout.splitlines(), |
| 50 | + [ |
| 51 | + "Discovered 2 repository skill test modules:", |
| 52 | + " .agents/skills/a-first/scripts/test_fixture.py", |
| 53 | + " .agents/skills/z-last/scripts/test_fixture.py", |
| 54 | + ], |
| 55 | + ) |
| 56 | + |
| 57 | + def test_isolated_modules_keep_sibling_imports_and_exclude_sdk_tests(self) -> None: |
| 58 | + for name in ("first", "second"): |
| 59 | + suite = self.write_suite( |
| 60 | + name, |
| 61 | + "import unittest\nfrom helper import VALUE\n" |
| 62 | + "class Fixture(unittest.TestCase):\n" |
| 63 | + f" def test_value(self): self.assertEqual(VALUE, {name!r})\n", |
| 64 | + ) |
| 65 | + suite.with_name("helper.py").write_text(f"VALUE = {name!r}\n", encoding="utf-8") |
| 66 | + sdk_suite = self.repo / "tests/test_sdk.py" |
| 67 | + sdk_suite.parent.mkdir() |
| 68 | + sdk_suite.write_text("raise RuntimeError('SDK suite must not be collected')\n") |
| 69 | + result = self.run_cli() |
| 70 | + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) |
| 71 | + self.assertIn("Completed 2 modules; 0 failed.", result.stdout) |
| 72 | + self.assertEqual(result.stderr.count("Ran 1 test"), 2) |
| 73 | + |
| 74 | + def test_failure_propagates_and_later_modules_still_execute(self) -> None: |
| 75 | + self.write_suite( |
| 76 | + "a-failing", |
| 77 | + "import unittest\nclass Fixture(unittest.TestCase):\n" |
| 78 | + " def test_failure(self): self.fail('deliberate fixture failure')\n", |
| 79 | + ) |
| 80 | + self.write_suite( |
| 81 | + "z-passing", |
| 82 | + "import unittest\nfrom pathlib import Path\nclass Fixture(unittest.TestCase):\n" |
| 83 | + " def test_later(self): Path('executed.txt').write_text('ran')\n", |
| 84 | + ) |
| 85 | + result = self.run_cli() |
| 86 | + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) |
| 87 | + self.assertIn("deliberate fixture failure", result.stderr) |
| 88 | + self.assertIn("Completed 2 modules; 1 failed.", result.stdout) |
| 89 | + self.assertTrue((self.repo / ".agents/skills/z-passing/scripts/executed.txt").is_file()) |
| 90 | + |
| 91 | + def test_import_failure_propagates(self) -> None: |
| 92 | + self.write_suite("broken", "raise ImportError('broken helper import')\n") |
| 93 | + result = self.run_cli() |
| 94 | + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) |
| 95 | + self.assertIn("broken helper import", result.stderr) |
| 96 | + |
| 97 | + def test_empty_skill_inventory_fails(self) -> None: |
| 98 | + result = self.run_cli() |
| 99 | + self.assertNotEqual(result.returncode, 0) |
| 100 | + self.assertIn("No repository skill test modules found", result.stderr) |
| 101 | + |
| 102 | + def test_children_drop_credentials_and_git_only_allows_local_remotes(self) -> None: |
| 103 | + self.write_suite( |
| 104 | + "environment", |
| 105 | + """import os |
| 106 | +import subprocess |
| 107 | +import sys |
| 108 | +import unittest |
| 109 | +
|
| 110 | +class Fixture(unittest.TestCase): |
| 111 | + def test_environment(self): |
| 112 | + for name in ('OPENAI_API_KEY', 'OPENAI_API_KEY_SOURCE', 'GH_TOKEN', 'GITHUB_TOKEN', |
| 113 | + 'AZURE_OPENAI_API_KEY', 'AWS_SECRET_ACCESS_KEY', 'PYTHONPATH'): |
| 114 | + self.assertNotIn(name, os.environ) |
| 115 | + subprocess.run([sys.executable, '-c', |
| 116 | + "import os; assert 'OPENAI_API_KEY' not in os.environ"], check=True) |
| 117 | + subprocess.run(['git', 'init', '--bare', 'origin.git'], check=True, capture_output=True) |
| 118 | + local = subprocess.run(['git', 'ls-remote', 'origin.git'], capture_output=True) |
| 119 | + self.assertEqual(local.returncode, 0, local.stderr) |
| 120 | + remote = subprocess.run(['git', 'ls-remote', 'https://example.invalid/repo.git'], |
| 121 | + capture_output=True, text=True) |
| 122 | + self.assertNotEqual(remote.returncode, 0) |
| 123 | + self.assertIn("transport 'https' not allowed", remote.stderr) |
| 124 | +""", |
| 125 | + ) |
| 126 | + env = test_environment() |
| 127 | + # Only synthetic values enter this fixture; inherited credentials are never forwarded. |
| 128 | + env.update( |
| 129 | + OPENAI_API_KEY="fixture-only", |
| 130 | + OPENAI_API_KEY_SOURCE="fixture-only", |
| 131 | + GH_TOKEN="fixture-only", |
| 132 | + GITHUB_TOKEN="fixture-only", |
| 133 | + AZURE_OPENAI_API_KEY="fixture-only", |
| 134 | + AWS_SECRET_ACCESS_KEY="fixture-only", |
| 135 | + PYTHONPATH=str(self.repo / "unused"), |
| 136 | + ) |
| 137 | + result = self.run_cli(env=env) |
| 138 | + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) |
| 139 | + |
| 140 | + |
| 141 | +if __name__ == "__main__": |
| 142 | + unittest.main() |
0 commit comments