From dac74197acc0da496f537aa657e57935774b7018 Mon Sep 17 00:00:00 2001 From: zzarbttoo Date: Sun, 9 Aug 2026 16:29:20 +0900 Subject: [PATCH 1/3] Accept Unicode decimal digits in int(), Decimal() and complex() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CPython runs a string argument through _PyUnicode_TransformDecimalAndSpaceToASCII before parsing it, so decimal digits from any script are accepted: int('١٢٣') # 123 int('0x١f', 16) # 31 Decimal('١٢٣') # Decimal('123') complex('１+２j') # (1+2j) RustPython only did this for float(), which had the transform inlined. int() handed the raw UTF-8 bytes to bytes_to_int(), whose digit check is is_ascii_alphanumeric(), so every non-ASCII digit was rejected — even though float() accepted the same string. Lift the inlined transform out of float_from_string() into common::str::transform_decimal_and_space_to_ascii() and apply it to the str paths of int() and complex() too. The result is always ASCII: as in CPython, a character that is neither ASCII, whitespace nor a decimal digit becomes '?' and truncates the string, which no parser accepts at any base, leaving the caller to raise the error from the original string. Bytes-like input keeps going straight to the parser, matching CPython's split between PyLong_FromUnicodeObject and PyLong_FromString. This unmarks two expectedFailure tests: test_int.test_unicode and test_decimal.test_unicode_digits. --- Lib/test/test_decimal.py | 1 - Lib/test/test_int.py | 1 - crates/common/src/str.rs | 54 +++++++++++++++++++++++++++++++ crates/vm/src/builtins/complex.rs | 3 +- crates/vm/src/builtins/float.rs | 15 ++------- crates/vm/src/builtins/int.rs | 16 ++++++++- crates/vm/src/protocol/number.rs | 2 +- 7 files changed, 74 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 47ab0a79702..c0b5671421e 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -854,7 +854,6 @@ class CExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): class PyExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): decimal = P - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode_digits(self): return super().test_unicode_digits() diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index a18683098e1..e281763a6c9 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -247,7 +247,6 @@ def test_invalid_signs(self): with self.assertRaises(ValueError): int(' + 1 ') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode(self): self.assertEqual(int("१२३४५६७८९०1234567890"), 12345678901234567890) self.assertEqual(int('١٢٣٤٥٦٧٨٩٠'), 1234567890) diff --git a/crates/common/src/str.rs b/crates/common/src/str.rs index c006a5f4db4..5937a9b1b62 100644 --- a/crates/common/src/str.rs +++ b/crates/common/src/str.rs @@ -2,6 +2,7 @@ use crate::atomic::{PyAtomic, Radium}; use crate::format::CharLen; use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; +use alloc::borrow::Cow; use ascii::{AsciiChar, AsciiStr, AsciiString}; use core::fmt; use core::ops::{Bound, RangeBounds}; @@ -658,10 +659,63 @@ pub fn char_to_decimal(ch: char) -> Option { .map(|i| (i % 10) as u8) } +/// Replace Unicode decimal digits with their ASCII equivalents and any Unicode +/// whitespace with a plain space, so the byte-oriented numeric parsers can read +/// them. Mirrors CPython's `_PyUnicode_TransformDecimalAndSpaceToASCII`. +/// +/// The result is always ASCII. Any other non-ASCII character cannot appear in a +/// numeric literal, so it becomes a `?` and the rest of the string is dropped: +/// `?` is rejected by every parser at every base, which leaves the caller — the +/// one that knows the base and owns the original string — to raise the error. +#[must_use] +pub fn transform_decimal_and_space_to_ascii(s: &str) -> Cow<'_, str> { + if s.is_ascii() { + return Cow::Borrowed(s); + } + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if (c as u32) < 127 { + out.push(c); + } else if c.is_whitespace() { + out.push(' '); + } else if let Some(n) = char_to_decimal(c) { + out.push(char::from_digit(n.into(), 10).unwrap()); + } else { + out.push('?'); + break; + } + } + debug_assert!(out.is_ascii()); + Cow::Owned(out) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn transform_decimal_and_space() { + // ASCII input is passed through untouched, without allocating. + assert!(matches!( + transform_decimal_and_space_to_ascii("123"), + Cow::Borrowed("123") + )); + // Decimal digits from any script fold to ASCII. + assert_eq!(transform_decimal_and_space_to_ascii("١٢٣"), "123"); + assert_eq!(transform_decimal_and_space_to_ascii("１２३"), "123"); + assert_eq!(transform_decimal_and_space_to_ascii("1٢3"), "123"); + // Unicode whitespace folds to a plain space. + assert_eq!(transform_decimal_and_space_to_ascii("\u{3000}٣"), " 3"); + // ASCII characters ride through untouched, whatever they are. + assert_eq!(transform_decimal_and_space_to_ascii("0x١f"), "0x1f"); + assert_eq!(transform_decimal_and_space_to_ascii("-١_٢"), "-1_2"); + // Anything else poisons the literal and truncates it, so the result stays + // ASCII and the caller's parser is guaranteed to reject it. + assert_eq!(transform_decimal_and_space_to_ascii("½가"), "?"); + assert_eq!(transform_decimal_and_space_to_ascii("١٢가٣"), "12?"); + assert_eq!(transform_decimal_and_space_to_ascii("١\u{7f}"), "1?"); + } + #[test] fn get_chars_basic() { let s = "0123456789"; diff --git a/crates/vm/src/builtins/complex.rs b/crates/vm/src/builtins/complex.rs index c54b3bc1731..1ede962aeec 100644 --- a/crates/vm/src/builtins/complex.rs +++ b/crates/vm/src/builtins/complex.rs @@ -222,7 +222,8 @@ impl Constructor for PyComplex { } let (re, im) = s .to_str() - .and_then(rustpython_literal::complex::parse_str) + .map(crate::common::str::transform_decimal_and_space_to_ascii) + .and_then(|s| rustpython_literal::complex::parse_str(&s)) .ok_or_else(|| vm.new_value_error("complex() arg is a malformed string"))?; return Ok(Self::from(Complex64 { re, im })); } else { diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index 1c861b14fc6..c9747052e5c 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -212,19 +212,8 @@ pub fn float_from_string(val: PyObjectRef, vm: &VirtualMachine) -> PyResult match s.as_str_kind() { PyKindStr::Ascii(s) => s.trim().as_bytes(), PyKindStr::Utf8(s) => { - mapped_string = s - .trim() - .chars() - .map(|c| { - if let Some(n) = rustpython_common::str::char_to_decimal(c) { - char::from_digit(n.into(), 10).unwrap() - } else if c.is_whitespace() { - ' ' - } else { - c - } - }) - .collect::(); + mapped_string = + rustpython_common::str::transform_decimal_and_space_to_ascii(s.trim()); mapped_string.as_bytes() } // if there are surrogates, it's not gonna parse anyway, diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index 278a9cecbb1..85b8154adbd 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -9,6 +9,7 @@ use crate::{ format::FormatSpec, hash, int::{bigint_to_finite_float, bytes_to_int, true_div}, + str::{PyKindStr, transform_decimal_and_space_to_ascii}, wtf8::Wtf8Buf, }, convert::{IntoPyException, ToPyObject, ToPyResult}, @@ -19,6 +20,7 @@ use crate::{ protocol::{PyNumberMethods, handle_bytes_to_int_err}, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; +use alloc::borrow::Cow; use alloc::fmt; use core::cell::Cell; use core::ops::{Neg, Not}; @@ -801,10 +803,22 @@ struct IntToByteArgs { signed: OptionalArg, } +/// Normalize a `str` for the byte-oriented int parser: Unicode decimal digits and +/// whitespace fold to their ASCII equivalents, the way CPython's +/// `PyLong_FromUnicodeObject` does. A string holding surrogates can never be a +/// valid literal, so it folds to an empty — and therefore invalid — one. +pub(crate) fn int_literal_from_str(s: &PyStr) -> Cow<'_, str> { + match s.as_str_kind() { + PyKindStr::Ascii(s) => Cow::Borrowed(s.trim().as_str()), + PyKindStr::Utf8(s) => transform_decimal_and_space_to_ascii(s.trim()), + PyKindStr::Wtf8(_) => Cow::Borrowed(""), + } +} + fn try_int_radix(obj: &PyObject, base: u32, vm: &VirtualMachine) -> PyResult { match_class!(match obj.to_owned() { string @ PyStr => { - let s = string.as_wtf8().trim(); + let s = int_literal_from_str(&string); bytes_to_int(s.as_bytes(), base, vm.state.int_max_str_digits.load()) .map_err(|e| handle_bytes_to_int_err(e, obj, vm)) } diff --git a/crates/vm/src/protocol/number.rs b/crates/vm/src/protocol/number.rs index 301499aa115..8352af9db44 100644 --- a/crates/vm/src/protocol/number.rs +++ b/crates/vm/src/protocol/number.rs @@ -59,7 +59,7 @@ impl PyObject { } else if let Some(i) = self.number().int(vm).or_else(|| self.try_index_opt(vm)) { i } else if let Some(s) = self.downcast_ref::() { - try_convert(self, s.as_wtf8().trim().as_bytes(), vm) + try_convert(self, int::int_literal_from_str(s).as_bytes(), vm) } else if let Some(bytes) = self.downcast_ref::() { try_convert(self, bytes, vm) } else if let Some(bytearray) = self.downcast_ref::() { From d6a7ccc19c0b93aabb24f3e404494a6d5e3d9346 Mon Sep 17 00:00:00 2001 From: zzarbttoo Date: Sun, 9 Aug 2026 16:31:07 +0900 Subject: [PATCH 2/3] Share one PyStr-to-numeric-literal step across int, float and complex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three constructors need the same thing from a str argument: trim it, fold Unicode decimal digits and whitespace to ASCII, and give up on a string holding surrogates. Each expressed that last part differently — float matched PyKindStr and returned b"", complex leaned on to_str() returning None, int returned an empty Cow — so the rule lived in three places at once. Move it into protocol::numeric_literal_from_str() and have all three call it. CPython repeats this per type because its wrapper is three lines over a single PyUnicode representation; ours has to match over Ascii/Utf8/Wtf8, which is worth writing once. Only the shared step moves. int keeps its base handling, int and float keep accepting bytes-like input, complex keeps rejecting it, and each keeps raising its own error, because none of that is shared. No behavior change: the CPython differential suite is byte-identical before and after. --- crates/vm/src/builtins/complex.rs | 9 ++++----- crates/vm/src/builtins/float.rs | 14 ++------------ crates/vm/src/builtins/int.rs | 18 ++---------------- crates/vm/src/protocol/mod.rs | 1 + crates/vm/src/protocol/number.rs | 27 +++++++++++++++++++++++++-- 5 files changed, 34 insertions(+), 35 deletions(-) diff --git a/crates/vm/src/builtins/complex.rs b/crates/vm/src/builtins/complex.rs index 1ede962aeec..7dcbedf7e17 100644 --- a/crates/vm/src/builtins/complex.rs +++ b/crates/vm/src/builtins/complex.rs @@ -220,11 +220,10 @@ impl Constructor for PyComplex { "complex() can't take second arg if first is a string", )); } - let (re, im) = s - .to_str() - .map(crate::common::str::transform_decimal_and_space_to_ascii) - .and_then(|s| rustpython_literal::complex::parse_str(&s)) - .ok_or_else(|| vm.new_value_error("complex() arg is a malformed string"))?; + let (re, im) = rustpython_literal::complex::parse_str( + &crate::protocol::numeric_literal_from_str(s), + ) + .ok_or_else(|| vm.new_value_error("complex() arg is a malformed string"))?; return Ok(Self::from(Complex64 { re, im })); } else { return Err(vm.new_type_error(format!( diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index c9747052e5c..8b1881272fe 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -208,18 +208,8 @@ impl Constructor for PyFloat { pub fn float_from_string(val: PyObjectRef, vm: &VirtualMachine) -> PyResult { let (bytearray, buffer, buffer_lock, mapped_string); let b = if let Some(s) = val.downcast_ref::() { - use crate::common::str::PyKindStr; - match s.as_str_kind() { - PyKindStr::Ascii(s) => s.trim().as_bytes(), - PyKindStr::Utf8(s) => { - mapped_string = - rustpython_common::str::transform_decimal_and_space_to_ascii(s.trim()); - mapped_string.as_bytes() - } - // if there are surrogates, it's not gonna parse anyway, - // so we can just choose a known bad value - PyKindStr::Wtf8(_) => b"", - } + mapped_string = crate::protocol::numeric_literal_from_str(s); + mapped_string.as_bytes() } else if let Some(bytes) = val.downcast_ref::() { bytes.as_bytes() } else if let Some(buf) = val.downcast_ref::() { diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index 85b8154adbd..4e07ae80920 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -9,7 +9,6 @@ use crate::{ format::FormatSpec, hash, int::{bigint_to_finite_float, bytes_to_int, true_div}, - str::{PyKindStr, transform_decimal_and_space_to_ascii}, wtf8::Wtf8Buf, }, convert::{IntoPyException, ToPyObject, ToPyResult}, @@ -17,10 +16,9 @@ use crate::{ ArgByteOrder, ArgIntoBool, FuncArgs, OptionalArg, OptionalOption, PyArithmeticValue, PyComparisonValue, }, - protocol::{PyNumberMethods, handle_bytes_to_int_err}, + protocol::{PyNumberMethods, handle_bytes_to_int_err, numeric_literal_from_str}, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; -use alloc::borrow::Cow; use alloc::fmt; use core::cell::Cell; use core::ops::{Neg, Not}; @@ -803,22 +801,10 @@ struct IntToByteArgs { signed: OptionalArg, } -/// Normalize a `str` for the byte-oriented int parser: Unicode decimal digits and -/// whitespace fold to their ASCII equivalents, the way CPython's -/// `PyLong_FromUnicodeObject` does. A string holding surrogates can never be a -/// valid literal, so it folds to an empty — and therefore invalid — one. -pub(crate) fn int_literal_from_str(s: &PyStr) -> Cow<'_, str> { - match s.as_str_kind() { - PyKindStr::Ascii(s) => Cow::Borrowed(s.trim().as_str()), - PyKindStr::Utf8(s) => transform_decimal_and_space_to_ascii(s.trim()), - PyKindStr::Wtf8(_) => Cow::Borrowed(""), - } -} - fn try_int_radix(obj: &PyObject, base: u32, vm: &VirtualMachine) -> PyResult { match_class!(match obj.to_owned() { string @ PyStr => { - let s = int_literal_from_str(&string); + let s = numeric_literal_from_str(&string); bytes_to_int(s.as_bytes(), base, vm.state.int_max_str_digits.load()) .map_err(|e| handle_bytes_to_int_err(e, obj, vm)) } diff --git a/crates/vm/src/protocol/mod.rs b/crates/vm/src/protocol/mod.rs index 411aa4dfad3..e5ca3f85362 100644 --- a/crates/vm/src/protocol/mod.rs +++ b/crates/vm/src/protocol/mod.rs @@ -14,5 +14,6 @@ pub use mapping::{PyMapping, PyMappingMethods, PyMappingSlots}; pub use number::{ PyNumber, PyNumberBinaryFunc, PyNumberBinaryOp, PyNumberMethods, PyNumberSlots, PyNumberTernaryFunc, PyNumberTernaryOp, PyNumberUnaryFunc, handle_bytes_to_int_err, + numeric_literal_from_str, }; pub use sequence::{PySequence, PySequenceMethods, PySequenceSlots}; diff --git a/crates/vm/src/protocol/number.rs b/crates/vm/src/protocol/number.rs index 8352af9db44..6f566431da7 100644 --- a/crates/vm/src/protocol/number.rs +++ b/crates/vm/src/protocol/number.rs @@ -8,11 +8,34 @@ use crate::{ builtins::{ PyBaseExceptionRef, PyByteArray, PyBytes, PyComplex, PyFloat, PyInt, PyIntRef, PyStr, int, }, - common::int::{BytesToIntError, bytes_to_int}, + common::{ + int::{BytesToIntError, bytes_to_int}, + str::{PyKindStr, transform_decimal_and_space_to_ascii}, + }, function::ArgBytesLike, object::{Traverse, TraverseFn}, stdlib::_warnings, }; +use alloc::borrow::Cow; + +/// Normalize a `str` for the byte-oriented numeric parsers: Unicode decimal digits +/// and whitespace fold to their ASCII equivalents, the way CPython runs every +/// numeric constructor's string argument through +/// `_PyUnicode_TransformDecimalAndSpaceToASCII` first. +/// +/// `int`, `float` and `complex` share this step and nothing else — only `int` takes +/// a base, and only `int` and `float` accept bytes-like input, so each keeps its own +/// entry point around this one. +/// +/// A string holding surrogates can never be a valid literal, so it folds to an +/// empty — and therefore invalid — one. +pub fn numeric_literal_from_str(s: &PyStr) -> Cow<'_, str> { + match s.as_str_kind() { + PyKindStr::Ascii(s) => Cow::Borrowed(s.trim().as_str()), + PyKindStr::Utf8(s) => transform_decimal_and_space_to_ascii(s.trim()), + PyKindStr::Wtf8(_) => Cow::Borrowed(""), + } +} pub type PyNumberUnaryFunc = fn(PyNumber<'_>, &VirtualMachine) -> PyResult; pub type PyNumberBinaryFunc = fn(&PyObject, &PyObject, &VirtualMachine) -> PyResult; @@ -59,7 +82,7 @@ impl PyObject { } else if let Some(i) = self.number().int(vm).or_else(|| self.try_index_opt(vm)) { i } else if let Some(s) = self.downcast_ref::() { - try_convert(self, int::int_literal_from_str(s).as_bytes(), vm) + try_convert(self, numeric_literal_from_str(s).as_bytes(), vm) } else if let Some(bytes) = self.downcast_ref::() { try_convert(self, bytes, vm) } else if let Some(bytearray) = self.downcast_ref::() { From 0d81b087aea6a9b3835c9cc09f26d42f17369e91 Mon Sep 17 00:00:00 2001 From: zzarbttoo Date: Fri, 14 Aug 2026 16:46:58 +0900 Subject: [PATCH 3/3] Drop the now-empty test_unicode_digits override in test_decimal Co-Authored-By: Claude Opus 5 --- Lib/test/test_decimal.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index c0b5671421e..c621b7ac08c 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -854,9 +854,6 @@ class CExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): class PyExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): decimal = P - def test_unicode_digits(self): - return super().test_unicode_digits() - class ImplicitConstructionTest: '''Unit tests for Implicit Construction cases of Decimal.'''