Bytecode parity (#7507) · RustPython/RustPython@3a8fb76 · GitHub
Skip to content

Commit 3a8fb76

Browse files
youknowoneyouknowone
authored
Bytecode parity (#7507)
* Bytecode parity phase 3 Compiler changes: - Emit TO_BOOL in and/or short-circuit evaluation (COPY+TO_BOOL+JUMP) - Add module-level __conditional_annotations__ cell (PEP 649) - Only set conditional annotations for AnnAssign, not function params - Skip __classdict__ cell when future annotations are active - Convert list literals to tuples in for-loop iterables - Fix cell variable ordering: parameters first, then alphabetical - Fix RESUME DEPTH1 flag for yield-from/await - Don't propagate __classdict__/__conditional_annotations__ freevar through regular functions — only annotation/type-param scopes - Inline string compilation path * Skip test_thread_safety in _test_multiprocessing SIGSEGV in _finalizer_registry dict access under aggressive GC and thread switching. Root cause is dict thread-safety in VM. * Skip list→tuple optimization for async for; propagate future_annotations to nested scopes
1 parent a91127c commit 3a8fb76

6 files changed

Lines changed: 127 additions & 39 deletions

File tree

‎Lib/test/_test_multiprocessing.py‎

Lines changed: 2 additions & 0 deletions

‎Lib/test/test_peepholer.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -645,7 +645,6 @@ def containtest():
645645
self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0)
646646
self.check_lnotab(containtest)
647647

648-
@unittest.expectedFailure # TODO: RUSTPYTHON; no BUILD_LIST to BUILD_TUPLE optimization
649648
def test_iterate_literal_list(self):
650649
def forloop():
651650
for x in [a, b]:

‎crates/codegen/src/compile.rs‎

Lines changed: 50 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1074,12 +1074,12 @@ impl Compiler {
10741074
.filter(|(_, s)| {
10751075
s.scope == SymbolScope::Cell || s.flags.contains(SymbolFlags::COMP_CELL)
10761076
})
1077-
.map(|(name, _)| name.clone())
1077+
.map(|(name, sym)| (name.clone(), sym.flags))
10781078
.collect();
10791079
let mut param_cells = Vec::new();
10801080
let mut nonparam_cells = Vec::new();
1081-
for name in cell_symbols {
1082-
if varname_cache.contains(&name) {
1081+
for (name, flags) in cell_symbols {
1082+
if flags.contains(SymbolFlags::PARAMETER) {
10831083
param_cells.push(name);
10841084
} else {
10851085
nonparam_cells.push(name);
@@ -1110,8 +1110,9 @@ impl Compiler {
11101110
}
11111111

11121112
// Handle implicit __conditional_annotations__ cell if needed
1113-
// Only for class scope - module scope uses NAME operations, not DEREF
1114-
if ste.has_conditional_annotations && scope_type == CompilerScope::Class {
1113+
if ste.has_conditional_annotations
1114+
&& matches!(scope_type, CompilerScope::Class | CompilerScope::Module)
1115+
{
11151116
cellvar_cache.insert("__conditional_annotations__".to_string());
11161117
}
11171118

@@ -1794,8 +1795,27 @@ impl Compiler {
17941795
let size_before = self.code_stack.len();
17951796
// Set future_annotations from symbol table (detected during symbol table scan)
17961797
self.future_annotations = symbol_table.future_annotations;
1798+
1799+
// Module-level __conditional_annotations__ cell
1800+
let has_module_cond_ann = symbol_table.has_conditional_annotations;
1801+
if has_module_cond_ann {
1802+
self.current_code_info()
1803+
.metadata
1804+
.cellvars
1805+
.insert("__conditional_annotations__".to_string());
1806+
}
1807+
17971808
self.symbol_table_stack.push(symbol_table);
17981809

1810+
// Emit MAKE_CELL for module-level cells (before RESUME)
1811+
if has_module_cond_ann {
1812+
let ncells = self.code_stack.last().unwrap().metadata.cellvars.len();
1813+
for i in 0..ncells {
1814+
let i_varnum: oparg::VarNum = u32::try_from(i).expect("too many cellvars").into();
1815+
emit!(self, Instruction::MakeCell { i: i_varnum });
1816+
}
1817+
}
1818+
17991819
self.emit_resume_for_scope(CompilerScope::Module, 1);
18001820

18011821
let (doc, statements) = split_doc(&body.body, &self.opts);
@@ -5437,7 +5457,25 @@ impl Compiler {
54375457
let mut end_async_for_target = BlockIdx::NULL;
54385458

54395459
// The thing iterated:
5440-
self.compile_expression(iter)?;
5460+
// Optimize: `for x in [a, b, c]` → use tuple instead of list
5461+
// (list creation is wasteful for iteration)
5462+
// Skip optimization if any element is starred (e.g., `[a, *b, c]`)
5463+
if !is_async
5464+
&& let ast::Expr::List(ast::ExprList { elts, .. }) = iter
5465+
&& !elts.iter().any(|e| matches!(e, ast::Expr::Starred(_)))
5466+
{
5467+
for elt in elts {
5468+
self.compile_expression(elt)?;
5469+
}
5470+
emit!(
5471+
self,
5472+
Instruction::BuildTuple {
5473+
count: u32::try_from(elts.len()).expect("too many elements"),
5474+
}
5475+
);
5476+
} else {
5477+
self.compile_expression(iter)?;
5478+
}
54415479

54425480
if is_async {
54435481
if self.ctx.func != FunctionContext::AsyncFunction {
@@ -7033,6 +7071,7 @@ impl Compiler {
70337071
/// For `And`, emits `PopJumpIfFalse`; for `Or`, emits `PopJumpIfTrue`.
70347072
fn emit_short_circuit_test(&mut self, op: &ast::BoolOp, target: BlockIdx) {
70357073
emit!(self, Instruction::Copy { i: 1 });
7074+
emit!(self, Instruction::ToBool);
70367075
match op {
70377076
ast::BoolOp::And => {
70387077
emit!(self, Instruction::PopJumpIfFalse { delta: target });
@@ -8554,11 +8593,11 @@ impl Compiler {
85548593

85558594
// fn block_done()
85568595

8557-
/// Convert a string literal AST node to Wtf8Buf, handling surrogates correctly.
8596+
/// Convert a string literal AST node to Wtf8Buf, handling surrogate literals correctly.
85588597
fn compile_string_value(&self, string: &ast::ExprStringLiteral) -> Wtf8Buf {
85598598
let value = string.value.to_str();
85608599
if value.contains(char::REPLACEMENT_CHARACTER) {
8561-
// Might have a surrogate literal; reparse from source to preserve them
8600+
// Might have a surrogate literal; reparse from source to preserve them.
85628601
string
85638602
.value
85648603
.iter()
@@ -8601,8 +8640,9 @@ impl Compiler {
86018640
}
86028641
},
86038642
ast::Expr::StringLiteral(s) => {
8604-
let value = self.compile_string_value(s);
8605-
constants.push(ConstantData::Str { value });
8643+
constants.push(ConstantData::Str {
8644+
value: self.compile_string_value(s),
8645+
});
86068646
}
86078647
ast::Expr::BytesLiteral(b) => {
86088648
constants.push(ConstantData::Bytes {

‎crates/codegen/src/ir.rs‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2172,11 +2172,19 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) {
21722172
blocks[bi].instructions[i].except_handler = handler_info;
21732173

21742174
// Track YIELD_VALUE except stack depth
2175-
if matches!(
2176-
blocks[bi].instructions[i].instr.real(),
2177-
Some(Instruction::YieldValue { .. })
2178-
) {
2179-
last_yield_except_depth = stack.len() as i32;
2175+
// Only count for direct yield (arg=0), not yield-from/await (arg=1)
2176+
// The yield-from's internal SETUP_FINALLY is not an external except depth
2177+
if let Some(Instruction::YieldValue { .. }) =
2178+
blocks[bi].instructions[i].instr.real()
2179+
{
2180+
let yield_arg = u32::from(blocks[bi].instructions[i].arg);
2181+
if yield_arg == 0 {
2182+
// Direct yield: count actual except depth
2183+
last_yield_except_depth = stack.len() as i32;
2184+
} else {
2185+
// yield-from/await: subtract 1 for the internal SETUP_FINALLY
2186+
last_yield_except_depth = (stack.len() as i32) - 1;
2187+
}
21802188
}
21812189

21822190
// Set RESUME DEPTH1 flag based on last yield's except depth

‎crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_bool_op.snap‎

Lines changed: 23 additions & 14 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/codegen/src/symboltable.rs‎

Lines changed: 39 additions & 9 deletions

0 commit comments

Comments
 (0)