{{ message }}
-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Expand file tree
/
Copy pathcmd_quick.py
More file actions
478 lines (399 loc) · 14.7 KB
/
Copy pathcmd_quick.py
File metadata and controls
478 lines (399 loc) · 14.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
#!/usr/bin/env python
"""
Quick update for test files from CPython.
Usage:
# Library + test: copy lib, then patch + auto-mark test + commit
python scripts/update_lib quick cpython/Lib/dataclasses.py
# Shortcut: just the module name
python scripts/update_lib quick dataclasses
# Test file: patch + auto-mark
python scripts/update_lib quick cpython/Lib/test/test_foo.py
# Test file: migrate only
python scripts/update_lib quick cpython/Lib/test/test_foo.py --no-auto-mark
# Test file: auto-mark only (Lib/ path implies --no-migrate)
python scripts/update_lib quick Lib/test/test_foo.py
# Directory: patch all + auto-mark all
python scripts/update_lib quick cpython/Lib/test/test_dataclasses/
# Skip git commit
python scripts/update_lib quick dataclasses --no-commit
"""
import argparse
import pathlib
import sys
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
from update_lib.deps import DEPENDENCIES, get_test_paths
from update_lib.file_utils import (
construct_lib_path,
get_cpython_dir,
get_module_name,
get_test_files,
is_lib_path,
is_test_path,
lib_to_test_path,
parse_lib_path,
resolve_module_path,
safe_read_text,
)
def collect_original_methods(
lib_path: pathlib.Path,
) -> set[tuple[str, str]] | dict[pathlib.Path, set[tuple[str, str]]] | None:
"""
Collect original test methods from lib path before patching.
Returns:
- For file: set of (class_name, method_name) or None if file doesn't exist
- For directory: dict mapping file path to set of methods, or None if dir doesn't exist
"""
from update_lib.cmd_auto_mark import extract_test_methods
if not lib_path.exists():
return None
if lib_path.is_file():
content = safe_read_text(lib_path)
return extract_test_methods(content) if content else set()
else:
result = {}
for lib_file in get_test_files(lib_path):
content = safe_read_text(lib_file)
if content:
result[lib_file.resolve()] = extract_test_methods(content)
return result
def quick(
src_path: pathlib.Path,
no_migrate: bool = False,
no_auto_mark: bool = False,
mark_failure: bool = False,
verbose: bool = True,
skip_build: bool = False,
) -> list[pathlib.Path]:
"""
Process a file or directory: migrate + auto-mark.
Args:
src_path: Source path (file or directory)
no_migrate: Skip migration step
no_auto_mark: Skip auto-mark step
mark_failure: Add @expectedFailure to ALL failing tests
verbose: Print progress messages
skip_build: Skip cargo build, use pre-built binary
Returns:
List of extra paths (data dirs, hard deps) that were copied/migrated.
"""
from update_lib.cmd_auto_mark import auto_mark_directory, auto_mark_file
from update_lib.cmd_migrate import patch_directory, patch_file
extra_paths: list[pathlib.Path] = []
# Determine lib_path and whether to migrate
if is_lib_path(src_path):
no_migrate = True
lib_path = src_path
else:
lib_path = parse_lib_path(src_path)
is_dir = src_path.is_dir()
# Capture original test methods before migration (for smart auto-mark)
original_methods = collect_original_methods(lib_path)
# Step 1: Migrate
if not no_migrate:
if is_dir:
patch_directory(src_path, lib_path, verbose=verbose)
else:
patch_file(src_path, lib_path, verbose=verbose)
# Step 1.5: Handle test dependencies
from update_lib.deps import get_test_dependencies
test_deps = get_test_dependencies(src_path)
# Migrate dependency files
for dep_src in test_deps["hard_deps"]:
dep_lib = parse_lib_path(dep_src)
if verbose:
print(f"Migrating dependency: {dep_src.name}")
if dep_src.is_dir():
patch_directory(dep_src, dep_lib, verbose=False)
else:
patch_file(dep_src, dep_lib, verbose=False)
extra_paths.append(dep_lib)
# Copy data directories (no migration)
import shutil
for data_src in test_deps["data"]:
data_lib = parse_lib_path(data_src)
if verbose:
print(f"Copying data: {data_src.name}")
if data_lib.exists():
if data_lib.is_dir():
shutil.rmtree(data_lib)
else:
data_lib.unlink()
if data_src.is_dir():
shutil.copytree(data_src, data_lib)
else:
data_lib.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(data_src, data_lib)
extra_paths.append(data_lib)
# Step 2: Auto-mark
if not no_auto_mark:
if not lib_path.exists():
raise FileNotFoundError(f"Path not found: {lib_path}")
if is_dir:
num_added, num_removed, _ = auto_mark_directory(
lib_path,
mark_failure=mark_failure,
verbose=verbose,
original_methods_per_file=original_methods,
skip_build=skip_build,
)
else:
num_added, num_removed, _ = auto_mark_file(
lib_path,
mark_failure=mark_failure,
verbose=verbose,
original_methods=original_methods,
skip_build=skip_build,
)
if verbose:
if num_added:
print(f"Added expectedFailure to {num_added} tests")
print(f"Removed expectedFailure from {num_removed} tests")
return extra_paths
def get_cpython_version(cpython_dir: pathlib.Path) -> str:
"""Get CPython version from git tag."""
import subprocess
result = subprocess.run(
["git", "describe", "--tags"],
cwd=cpython_dir,
capture_output=True,
text=True,
check=True,
)
return result.stdout.strip()
def git_commit(
name: str,
lib_path: pathlib.Path | None,
test_paths: list[pathlib.Path] | pathlib.Path | None,
cpython_dir: pathlib.Path,
hard_deps: list[pathlib.Path] | None = None,
verbose: bool = True,
) -> bool:
"""Commit changes with CPython author.
Args:
name: Module name (e.g., "dataclasses")
lib_path: Path to library file/directory (or None)
test_paths: Path(s) to test file/directory (or None)
cpython_dir: Path to cpython directory
hard_deps: Path(s) to hard dependency files (or None)
verbose: Print progress messages
Returns:
True if commit was created, False otherwise
"""
import subprocess
# Normalize test_paths to list
if test_paths is None:
test_paths = []
elif isinstance(test_paths, pathlib.Path):
test_paths = [test_paths]
# Normalize hard_deps to list
if hard_deps is None:
hard_deps = []
# Stage changes
paths_to_add = []
if lib_path and lib_path.exists():
paths_to_add.append(str(lib_path))
for test_path in test_paths:
if test_path and test_path.exists():
paths_to_add.append(str(test_path))
for dep_path in hard_deps:
if dep_path and dep_path.exists():
paths_to_add.append(str(dep_path))
if not paths_to_add:
return False
version = get_cpython_version(cpython_dir)
subprocess.run(["git", "add"] + paths_to_add, check=True)
# Check if there are staged changes
result = subprocess.run(
["git", "diff", "--cached", "--quiet"],
capture_output=True,
)
if result.returncode == 0:
if verbose:
print("No changes to commit")
return False
# Commit with CPython author
message = f"Update {name} from {version}"
subprocess.run(
[
"git",
"commit",
"--author",
"CPython Developers <>",
"-m",
message,
],
check=True,
)
if verbose:
print(f"Committed: {message}")
return True
def _expand_shortcut(path: pathlib.Path) -> pathlib.Path:
"""Expand simple name to cpython/Lib path if it exists.
Examples:
dataclasses -> cpython/Lib/dataclasses.py (if exists)
json -> cpython/Lib/json/ (if exists)
test_types -> cpython/Lib/test/test_types.py (if exists)
regrtest -> cpython/Lib/test/libregrtest (from DEPENDENCIES)
"""
# Only expand if it's a simple name (no path separators) and doesn't exist
if "/" in str(path) or path.exists():
return path
name = str(path)
# Check DEPENDENCIES table for path overrides (e.g., regrtest)
from update_lib.deps import DEPENDENCIES
if name in DEPENDENCIES and "lib" in DEPENDENCIES[name]:
lib_paths = DEPENDENCIES[name]["lib"]
if lib_paths:
override_path = construct_lib_path("cpython", lib_paths[0])
if override_path.exists():
return override_path
# Test shortcut: test_foo -> cpython/Lib/test/test_foo
if name.startswith("test_"):
resolved = resolve_module_path(f"test/{name}", "cpython", prefer="dir")
if resolved.exists():
return resolved
# Library shortcut: foo -> cpython/Lib/foo
resolved = resolve_module_path(name, "cpython", prefer="file")
if resolved.exists():
return resolved
# Extension module shortcut: winreg -> cpython/Lib/test/test_winreg
# For C/Rust extension modules that have no Python source but have tests
resolved = resolve_module_path(f"test/test_{name}", "cpython", prefer="dir")
if resolved.exists():
return resolved
# Return original (will likely fail later with a clear error)
return path
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"path",
type=pathlib.Path,
help="Source path (file or directory)",
)
parser.add_argument(
"--copy",
action=argparse.BooleanOptionalAction,
default=True,
help="Copy library file (default: enabled, implied disabled if test path)",
)
parser.add_argument(
"--migrate",
action=argparse.BooleanOptionalAction,
default=True,
help="Migrate test file (default: enabled, implied disabled if Lib/ path)",
)
parser.add_argument(
"--auto-mark",
action=argparse.BooleanOptionalAction,
default=True,
help="Auto-mark test failures (default: enabled)",
)
parser.add_argument(
"--mark-failure",
action="store_true",
help="Add @expectedFailure to failing tests",
)
parser.add_argument(
"--commit",
action=argparse.BooleanOptionalAction,
default=True,
help="Create git commit (default: enabled)",
)
parser.add_argument(
"--build",
action=argparse.BooleanOptionalAction,
default=True,
help="Build with cargo (default: enabled)",
)
args = parser.parse_args(argv)
try:
src_path = args.path
# Shortcut: expand simple name to cpython/Lib path
src_path = _expand_shortcut(src_path)
original_src = src_path # Keep for commit
# Track library path for commit
lib_file_path = None
test_path = None
hard_deps_for_commit = []
# If it's a library path (not test path), do copy_lib first
if not is_test_path(src_path):
# Get library destination path for commit
lib_file_path = parse_lib_path(src_path)
if args.copy:
from update_lib.cmd_copy_lib import copy_lib
copy_lib(src_path)
# Get all test paths from DEPENDENCIES (or fall back to default)
module_name = get_module_name(original_src)
cpython_dir = get_cpython_dir(original_src)
test_src_paths = get_test_paths(module_name, str(cpython_dir))
# Fall back to default test path if DEPENDENCIES has no entry
if not test_src_paths:
default_test = lib_to_test_path(original_src)
if default_test.exists():
test_src_paths = (default_test,)
# Collect hard dependencies for commit
lib_deps = DEPENDENCIES.get(module_name, {})
for dep_name in lib_deps.get("hard_deps", []):
dep_lib_path = pathlib.Path("Lib") / dep_name
if dep_lib_path.exists():
hard_deps_for_commit.append(dep_lib_path)
# Process all test paths
test_paths_for_commit = []
for test_src in test_src_paths:
if not test_src.exists():
print(f"Warning: Test path does not exist: {test_src}")
continue
test_lib_path = parse_lib_path(test_src)
test_paths_for_commit.append(test_lib_path)
extra = quick(
test_src,
no_migrate=not args.migrate,
no_auto_mark=not args.auto_mark,
mark_failure=args.mark_failure,
skip_build=not args.build,
)
hard_deps_for_commit.extend(extra)
test_paths = test_paths_for_commit
else:
# It's a test path - process single test
test_path = (
parse_lib_path(src_path) if not is_lib_path(src_path) else src_path
)
extra = quick(
src_path,
no_migrate=not args.migrate,
no_auto_mark=not args.auto_mark,
mark_failure=args.mark_failure,
skip_build=not args.build,
)
hard_deps_for_commit.extend(extra)
test_paths = [test_path]
# Step 3: Git commit
if args.commit:
cpython_dir = get_cpython_dir(original_src)
git_commit(
get_module_name(original_src),
lib_file_path,
test_paths,
cpython_dir,
hard_deps=hard_deps_for_commit,
)
return 0
except ValueError as e:
print(f"Error: {e}", file=sys.stderr)
return 1
except FileNotFoundError as e:
print(f"Error: {e}", file=sys.stderr)
return 1
except Exception as e:
# Handle TestRunError with a clean message
from update_lib.cmd_auto_mark import TestRunError
if isinstance(e, TestRunError):
print(f"Error: {e}", file=sys.stderr)
return 1
raise
if __name__ == "__main__":
sys.exit(main())
You can’t perform that action at this time.