Preserve lone surrogates across WASM conversions (#8508) · RustPython/RustPython@d0baa1c · GitHub
Skip to content

Commit d0baa1c

Browse files
HyoJongParkHyoJongPark
authored
Preserve lone surrogates across WASM conversions (#8508)
Assisted-by: Codex:GPT-5
1 parent 2690c16 commit d0baa1c

3 files changed

Lines changed: 86 additions & 28 deletions

File tree

‎crates/vm/src/py_serde.rs‎

Lines changed: 4 additions & 1 deletion

‎crates/wasm/src/convert.rs‎

Lines changed: 73 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,40 @@
22

33
use crate::js_module;
44
use crate::vm_class::{WASMVirtualMachine, stored_vm_from_wasm};
5-
use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array};
5+
use js_sys::{
6+
Array, ArrayBuffer, JsString, Map, Object, Promise, Reflect, SyntaxError, Uint8Array,
7+
};
8+
use rustpython_common::wtf8::{Wtf8, Wtf8Buf};
69
use rustpython_vm::{
710
AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine,
8-
builtins::{PyBaseException, PyBaseExceptionRef},
11+
builtins::{PyBaseException, PyBaseExceptionRef, PyDict, PyList, PyStr, PyTuple},
912
compiler::{CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType},
1013
exceptions,
1114
function::{ArgBytesLike, FuncArgs},
1215
py_serde,
1316
};
1417
use wasm_bindgen::{JsCast, closure::Closure, prelude::*};
1518

19+
pub(crate) fn js_string_to_wtf8(value: &JsString) -> Wtf8Buf {
20+
Wtf8Buf::from_wide(&value.iter().collect::<Vec<_>>())
21+
}
22+
23+
fn wtf8_to_js_string(value: &Wtf8) -> JsString {
24+
const CHUNK_SIZE: usize = 8192;
25+
26+
if let Ok(value) = value.as_str() {
27+
return value.into();
28+
}
29+
30+
value
31+
.encode_wide()
32+
.collect::<Vec<_>>()
33+
.chunks(CHUNK_SIZE)
34+
.map(JsString::from_char_code)
35+
.collect::<Array>()
36+
.join("")
37+
}
38+
1639
#[wasm_bindgen(inline_js = r"
1740
export class PyError extends Error {
1841
constructor(info) {
@@ -119,12 +142,9 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue {
119142
if let Some(ref kwargs) = kwargs {
120143
for pair in object_entries(kwargs) {
121144
let (key, val) = pair?;
122-
py_func_args.kwargs.insert(
123-
// JS strings coming in are UTF-16; go through Rust `String`
124-
// (kwargs keys are now WTF-8, so convert String -> Wtf8Buf).
125-
String::from(js_sys::JsString::from(key)).into(),
126-
js_to_py(vm, val),
127-
);
145+
py_func_args
146+
.kwargs
147+
.insert(js_string_to_wtf8(&key.into()), js_to_py(vm, val));
128148
}
129149
}
130150
let result = py_obj.call(py_func_args, vm);
@@ -151,17 +171,44 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue {
151171
}
152172

153173
if let Ok(bytes) = ArgBytesLike::try_from_borrowed_object(vm, &py_obj) {
154-
bytes.with_ref(|bytes| unsafe {
174+
return bytes.with_ref(|bytes| unsafe {
155175
// `Uint8Array::view` is an `unsafe fn` because it provides
156176
// a direct view into the WASM linear memory; if you were to allocate
157177
// something with Rust that view would probably become invalid. It's safe
158178
// because we then copy the array using `Uint8Array::slice`.
159179
let view = Uint8Array::view(bytes);
160180
view.slice(0, bytes.len() as u32).into()
161-
})
181+
});
182+
}
183+
py_serde_to_js(vm, &py_obj).unwrap_or(JsValue::UNDEFINED)
184+
}
185+
186+
fn py_serde_to_js(
187+
vm: &VirtualMachine,
188+
py_obj: &PyObjectRef,
189+
) -> Result<JsValue, serde_wasm_bindgen::Error> {
190+
if let Some(value) = py_obj.downcast_ref::<PyStr>() {
191+
Ok(wtf8_to_js_string(value.as_wtf8()).into())
192+
} else if let Some(value) = py_obj.downcast_ref::<PyList>() {
193+
let array = Array::new();
194+
for item in value.borrow_vec().iter() {
195+
array.push(&py_serde_to_js(vm, item)?);
196+
}
197+
Ok(array.into())
198+
} else if let Some(value) = py_obj.downcast_ref::<PyTuple>() {
199+
let array = Array::new();
200+
for item in value {
201+
array.push(&py_serde_to_js(vm, item)?);
202+
}
203+
Ok(array.into())
204+
} else if let Some(value) = py_obj.downcast_ref::<PyDict>() {
205+
let map = Map::new();
206+
for (key, value) in value {
207+
map.set(&py_serde_to_js(vm, &key)?, &py_serde_to_js(vm, &value)?);
208+
}
209+
Ok(map.into())
162210
} else {
163-
py_serde::serialize(vm, &py_obj, &serde_wasm_bindgen::Serializer::new())
164-
.unwrap_or(JsValue::UNDEFINED)
211+
py_serde::serialize(vm, py_obj, &serde_wasm_bindgen::Serializer::new())
165212
}
166213
}
167214

@@ -199,6 +246,15 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef {
199246
.map(|val| js_to_py(vm, val.expect("Iteration over array failed")))
200247
.collect();
201248
vm.ctx.new_list(elems).into()
249+
} else if let Some(map) = js_val.dyn_ref::<Map>() {
250+
let dict = vm.ctx.new_dict();
251+
for entry in map.entries() {
252+
let entry = Array::from(&entry.expect("Iteration over map failed"));
253+
let key = js_to_py(vm, entry.get(0));
254+
dict.set_item(&*key, js_to_py(vm, entry.get(1)), vm)
255+
.unwrap();
256+
}
257+
dict.into()
202258
} else if ArrayBuffer::is_view(&js_val) || js_val.is_instance_of::<ArrayBuffer>() {
203259
// unchecked_ref because if it's not an ArrayBuffer it could either be a TypedArray
204260
// or a DataView, but they all have a `buffer` property
@@ -216,12 +272,8 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef {
216272
for pair in object_entries(&Object::from(js_val)) {
217273
let (key, val) = pair.expect("iteration over object to not fail");
218274
let py_val = js_to_py(vm, val);
219-
dict.set_item(
220-
String::from(js_sys::JsString::from(key)).as_str(),
221-
py_val,
222-
vm,
223-
)
224-
.unwrap();
275+
dict.set_item(&*js_string_to_wtf8(&key.into()), py_val, vm)
276+
.unwrap();
225277
}
226278
dict.into()
227279
}
@@ -232,9 +284,7 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef {
232284
move |args: FuncArgs, vm: &VirtualMachine| -> PyResult {
233285
let this = Object::new();
234286
for (k, v) in args.kwargs {
235-
// WTF-8 -> JS string: lone surrogates in the key become U+FFFD
236-
// (wasm-bindgen only accepts Rust `String`); acceptable at this boundary.
237-
Reflect::set(&this, &k.to_string().into(), &py_to_js(vm, v))
287+
Reflect::set(&this, &wtf8_to_js_string(&k).into(), &py_to_js(vm, v))
238288
.expect("property to be settable");
239289
}
240290
let js_args = args
@@ -253,6 +303,8 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef {
253303
} else if js_val.is_undefined() {
254304
// Because `JSON.stringify(undefined)` returns undefined
255305
vm.ctx.none()
306+
} else if js_val.is_string() {
307+
vm.ctx.new_str(js_string_to_wtf8(&js_val.into())).into()
256308
} else {
257309
py_serde::deserialize(vm, serde_wasm_bindgen::Deserializer::from(js_val))
258310
.unwrap_or_else(|_| vm.ctx.none())

‎crates/wasm/src/vm_class.rs‎

Lines changed: 9 additions & 6 deletions

0 commit comments

Comments
 (0)