diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 7ae7943e..344eedd1 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.x' cache: "pip" diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 1ad0a1e6..6e9c8c14 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-slim steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: ruff check run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67c5e52e..bc7efe06 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,10 +26,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.py }} allow-prereleases: true diff --git a/.github/workflows/test_debug.yml b/.github/workflows/test_debug.yml index 0cfb2006..5fbf185e 100644 --- a/.github/workflows/test_debug.yml +++ b/.github/workflows/test_debug.yml @@ -11,9 +11,9 @@ jobs: PYTHON_VERSION: 3.14.6 steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: /opt/python-debug key: python-debug-${{ runner.os }}-${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 8234ab4e..6250cfd0 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -26,8 +26,8 @@ jobs: name: Build wheels on ${{ matrix.os }}${{ matrix.name_suffix || '' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" cache: "pip" @@ -39,12 +39,12 @@ jobs: - name: Set up QEMU for emulation if: matrix.cibw_archs == 'riscv64' - uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0 + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 with: platforms: ${{ matrix.cibw_archs }} - name: Build - uses: pypa/cibuildwheel@294735312765b09d24a2fbec22660ce817587d55 # v4.1.0 + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -99,6 +99,6 @@ jobs: path: dist merge-multiple: true - - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 #with: # To test: repository-url: https://test.pypi.org/legacy/ diff --git a/docs/conf.py b/docs/conf.py index 28116cd6..d7215a94 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -269,7 +269,7 @@ # The format is a list of tuples containing the path and title. # epub_pre_files = [] -# HTML files shat should be inserted after the pages created by sphinx. +# HTML files that should be inserted after the pages created by sphinx. # The format is a list of tuples containing the path and title. # epub_post_files = [] diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 277239d8..e816c814 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -110,7 +110,7 @@ cdef class Packer: cdef bint autoreset cdef bint datetime - def __cinit__(self, buf_size=256*1024, **_kwargs): + def __cinit__(self, size_t buf_size=256*1024, **_kwargs): self.pk.buf = PyMem_Malloc(buf_size) if self.pk.buf == NULL: raise MemoryError("Unable to allocate internal buffer.") diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index f758c0c4..7e406622 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -46,7 +46,7 @@ cdef extern from "unpack.h": Py_ssize_t count ctypedef int (*execute_fn)(unpack_context* ctx, const char* data, - Py_ssize_t len, Py_ssize_t* off) except? -1 + Py_ssize_t len, Py_ssize_t* off) except -1 execute_fn unpack_construct execute_fn unpack_skip execute_fn read_array_header @@ -190,15 +190,16 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, use_list, raw, timestamp, strict_map_key, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) ret = unpack_construct(&ctx, buf, buf_len, &off) + if ret == 1: + obj = unpack_data(&ctx) + if off < buf_len: + # buf may point into a temporary contiguous copy owned by view, + # so the extra data must be copied out before releasing view. + raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) + return obj finally: PyBuffer_Release(&view); - if ret == 1: - obj = unpack_data(&ctx) - if off < buf_len: - raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) - return obj - unpack_clear(&ctx) if ret == 0: raise ValueError("Unpack failed: incomplete input") @@ -206,8 +207,6 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, raise FormatError elif ret == -3: raise StackError - elif PyErr_Occurred(): - raise else: raise ValueError("Unpack failed: error = %d" % (ret,)) @@ -502,8 +501,6 @@ cdef class Unpacker: raise FormatError elif ret == -3: raise StackError - elif PyErr_Occurred(): - raise else: raise ValueError("Unpack failed: error = %d" % (ret,)) finally: diff --git a/msgpack/ext.py b/msgpack/ext.py index 92ea4530..abaaad90 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -2,6 +2,8 @@ import struct from collections import namedtuple +_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + class ExtType(namedtuple("ExtType", "code data")): """ExtType represents ext type in msgpack.""" @@ -156,8 +158,7 @@ def to_datetime(self): :rtype: `datetime.datetime` """ - utc = datetime.timezone.utc - return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta( + return _EPOCH + datetime.timedelta( seconds=self.seconds, microseconds=self.nanoseconds // 1000 ) @@ -167,4 +168,17 @@ def from_datetime(dt): :rtype: Timestamp """ - return Timestamp(seconds=int(dt.timestamp() // 1), nanoseconds=dt.microsecond * 1000) + # Use integer timedelta arithmetic (like the Cython packer) rather than + # ``int(dt.timestamp() // 1)``. ``datetime.timestamp()`` returns a float + # that cannot hold microsecond precision for datetimes far from the epoch, + # so it may round the whole-second part up while the exact ``microsecond`` + # is still used for the nanoseconds -- producing a Timestamp one second in + # the future (and OverflowError near datetime.max). + if dt.tzinfo is None: + # Match datetime.timestamp(): a naive datetime is treated as local time. + dt = dt.astimezone() + delta = dt - _EPOCH + return Timestamp( + seconds=delta.days * 86400 + delta.seconds, + nanoseconds=delta.microseconds * 1000, + ) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 824f59d5..e219786e 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -582,7 +582,10 @@ def __next__(self): next = __next__ def skip(self): - self._unpack(EX_SKIP) + try: + self._unpack(EX_SKIP) + except RecursionError: + raise StackError self._consume() def unpack(self): diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 55cfdab9..eb4330ac 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -40,11 +40,6 @@ struct unpack_context; typedef struct unpack_context unpack_context; typedef int (*execute_fn)(unpack_context *ctx, const char* data, Py_ssize_t len, Py_ssize_t* off); -static inline msgpack_unpack_object unpack_callback_root(unpack_user* u) -{ - return NULL; -} - static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) { PyObject *p = PyLong_FromLong((long)d); diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index ab5887ae..797a2f91 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -35,11 +35,6 @@ struct unpack_context { unsigned int cs; unsigned int trail; unsigned int top; - /* - unpack_stack* stack; - unsigned int stack_size; - unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE]; - */ unpack_stack stack[MSGPACK_EMBED_STACK_SIZE]; }; @@ -49,22 +44,9 @@ static inline void unpack_init(unpack_context* ctx) ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; - /* - ctx->stack = ctx->embed_stack; - ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; - */ - ctx->stack[0].obj = unpack_callback_root(&ctx->user); + ctx->stack[0].obj = NULL; } -/* -static inline void unpack_destroy(unpack_context* ctx) -{ - if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { - free(ctx->stack); - } -} -*/ - static inline PyObject* unpack_data(unpack_context* ctx) { return (ctx)->stack[0].obj; @@ -94,9 +76,6 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char unsigned int cs = ctx->cs; unsigned int top = ctx->top; unpack_stack* stack = ctx->stack; - /* - unsigned int stack_size = ctx->stack_size; - */ unpack_user* user = &ctx->user; PyObject* obj = NULL; @@ -319,6 +298,7 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); default: + PyErr_Format(PyExc_RuntimeError, "Invalid state: %d", cs); goto _failed; } } @@ -355,6 +335,7 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char goto _header_again; default: + PyErr_Format(PyExc_RuntimeError, "Invalid container type: %u", c->ct); goto _failed; } diff --git a/test/test_except.py b/test/test_except.py index a3bf4675..1e9c0045 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -97,6 +97,11 @@ def test_invalidvalue(): with raises(StackError): unpackb(b"\x91" * 3000) # nested fixarray(len=1) + with raises(StackError): + unpacker = Unpacker() + unpacker.feed(b"\x91" * 3000) + unpacker.skip() + def test_no_memory_leak_on_nested_invalid_tag() -> None: """Regression test: unpacking nested arrays containing an invalid tag must not leak objects.""" diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 3f6a39d4..fd124dce 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -2,7 +2,9 @@ from array import array -from msgpack import packb, unpackb +from pytest import raises + +from msgpack import ExtraData, packb, unpackb def make_array(f, data): @@ -109,3 +111,20 @@ def test_unpack_noncontiguous_memoryview(): noncont = memoryview(bytes(padded))[::2] assert not noncont.c_contiguous assert unpackb(noncont) == 2**32 + + +def test_unpack_noncontiguous_memoryview_extra_data(): + # See https://github.com/msgpack/msgpack-python/issues/720 + # ExtraData.extra must be copied out of the temporary contiguous copy + # before that copy is released. + packed = packb(0) + b"extra" + padded = bytearray() + for byte in packed: + padded.append(byte) + padded.append(0) + noncont = memoryview(bytes(padded))[::2] + assert not noncont.c_contiguous + with raises(ExtraData) as excinfo: + unpackb(noncont) + assert excinfo.value.unpacked == 0 + assert excinfo.value.extra == b"extra" diff --git a/test/test_pack.py b/test/test_pack.py index 374d1549..f44bd557 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -179,3 +179,24 @@ def test_get_buffer(): expected = packb([1, 2], use_bin_type=True) assert written == expected + + +@pytest.mark.skipif( + Packer.__module__ == "msgpack.fallback", + reason="buf_size only allocates in the C extension", +) +def test_buf_size_is_converted_once(): + # Asking twice let the allocation and the recorded capacity disagree, + # so the packer overflowed a buffer smaller than the size it recorded. + class Counting: + count = 0 + + def __int__(self): + self.count += 1 + return 600 + + __index__ = __int__ + + buf_size = Counting() + Packer(buf_size=buf_size) + assert buf_size.count == 1 diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 7c8e3e83..2083356a 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -111,6 +111,40 @@ def test_timestamp_datetime(): assert ts_pre_epoch.to_datetime() == pre_epoch +def test_from_datetime_far_future_precision(): + # Regression: Timestamp.from_datetime used the float datetime.timestamp(), + # which cannot hold microsecond precision far from the epoch. It rounded the + # whole-second part up by one while still taking the exact microsecond for the + # nanoseconds, yielding a Timestamp one second in the future -- and raised + # OverflowError near datetime.max. It must instead match the integer + # arithmetic used by the Cython packer. + utc = datetime.timezone.utc + epoch = datetime.datetime(1970, 1, 1, tzinfo=utc) + + for dt in [ + datetime.datetime(2515, 1, 1, 0, 0, 0, 999999, tzinfo=utc), + datetime.datetime(3000, 1, 1, 0, 0, 0, 999999, tzinfo=utc), + datetime.datetime(5000, 6, 15, 12, 30, 45, 123456, tzinfo=utc), + # Near datetime.max: previously raised OverflowError. + datetime.datetime(9999, 12, 31, 23, 59, 59, 999999, tzinfo=utc), + ]: + ts = Timestamp.from_datetime(dt) + # Round-trips exactly (was one second in the future). + assert ts.to_datetime() == dt + # Whole-second part is exact, computed independently of the implementation. + assert ts.seconds == (dt - epoch) // datetime.timedelta(seconds=1) + assert ts.nanoseconds == dt.microsecond * 1000 + # The pure-Python path agrees with packing the datetime directly (the + # reference path), i.e. no off-by-one-second divergence. + assert msgpack.packb(ts) == msgpack.packb(dt, datetime=True) + + # Explicit value: 3000-01-01T00:00:00.999999Z is 32503680000 s after the + # epoch; the float path produced 32503680001 (one second in the future). + ts = Timestamp.from_datetime(datetime.datetime(3000, 1, 1, 0, 0, 0, 999999, tzinfo=utc)) + assert ts.seconds == 32503680000 + assert ts.nanoseconds == 999999000 + + def test_unpack_datetime(): t = Timestamp(42, 14) utc = datetime.timezone.utc diff --git a/test/test_unpack.py b/test/test_unpack.py index 81bc976b..5811c27f 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -5,7 +5,15 @@ from pytest import mark, raises -from msgpack import ExtType, OutOfData, Unpacker, packb +from msgpack import ( + ExtraData, + ExtType, + OutOfData, + Unpacker, + packb, + unpack, + unpackb, +) def test_unpack_array_header_from_file(): @@ -82,10 +90,10 @@ def test_unpacker_tell_read_bytes(): objects = 1, "abc", "ghi" packed = b"\x01\x02\xa3abc\xa3def\xa3ghi" raw_data = b"\x02", b"\xa3def", b"" - lenghts = 1, 4, 999 + lengths = 1, 4, 999 positions = 1, 6, 14 unpacker = Unpacker(BytesIO(packed)) - for obj, unp, pos, n, raw in zip(objects, unpacker, positions, lenghts, raw_data): + for obj, unp, pos, n, raw in zip(objects, unpacker, positions, lengths, raw_data): assert obj == unp assert pos == unpacker.tell() assert unpacker.read_bytes(n) == raw @@ -142,3 +150,20 @@ def ext_hook(code, data): up.feed(b"\xdc" + struct.pack(">H", 11) + b"\xd4\x05A" + b"\x2a" * 10) with raises(RuntimeError): up.unpack() + + +def test_unpackb_raises_extra_data_with_trailing_bytes(): + packed = packb(42) + packb("trailing") + with raises(ExtraData) as exc_info: + unpackb(packed) + err = exc_info.value + assert err.unpacked == 42 + assert err.extra == packb("trailing") + + +def test_unpack_raises_extra_data_on_stream_with_trailing_bytes(): + stream = BytesIO(packb(100) + packb(200)) + with raises(ExtraData) as exc_info: + unpack(stream) + assert exc_info.value.unpacked == 100 + assert exc_info.value.extra == packb(200)