From 63a9350c973816b9fcbfc9671926d488e229acb8 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Tue, 30 Jan 2018 16:37:31 -0600 Subject: [PATCH 1/8] Add compresslevel support for LZMA --- Lib/test/test_zipfile.py | 15 +++++++++++ Lib/zipfile.py | 54 ++++++++++++++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 61c3e349a69ef4..8f20eb589bf368 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2,6 +2,7 @@ import io import os import importlib.util +import lzma import pathlib import posixpath import time @@ -576,6 +577,20 @@ class LzmaTestsWithSourceFile(AbstractTestsWithSourceFile, unittest.TestCase): compression = zipfile.ZIP_LZMA + def test_write_preset(self): + # Initialize an LZMA compressor with a preset and get the generated + # property header. + props_header = zipfile.LZMACompressor(preset=1).compress(b'') + + # Write a ZIP archive with that LZMA compression preset and ensure + # that the property header above is written to that archive. + with io.BytesIO() as fp: + kwargs = {'compression': self.compression, 'compresslevel': 1} + with zipfile.ZipFile(fp, 'w', **kwargs) as zipfp: + zipfp.writestr('dummy_file', b'dummy_contents') + written_bytes = fp.getvalue() + self.assertIn(props_header, written_bytes) + class AbstractTestZip64InSmallFiles: # These tests test the ZIP64 functionality without using large files, diff --git a/Lib/zipfile.py b/Lib/zipfile.py index b90b60f72e2bcd..9f09c0d01587bc 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -3,6 +3,7 @@ XXX references to utf-8 need further investigation. """ +import collections import io import os import importlib.util @@ -567,15 +568,53 @@ def decrypter(data): class LZMACompressor: - - def __init__(self): + # LZMA uses "filter chains" to control compression settings (e.g., how + # much memory and CPU to use) in details. liblzma defines certain filter + # chain configurations as "preset levels," akin to the compression levels + # in zlib or bzip2. + _PresetOptions = collections.namedtuple( + '_PresetOptions', + ('lc', 'lp', 'pb', 'dict_size', 'mode', 'mf', 'nice_len', 'depth') + ) + _PRESET_OPTIONS_MAP = { + # Levels 0 through 9, without the "extreme" setting + 0x00000000: _PresetOptions(3, 0, 2, 1 << 18, 1, 3, 128, 4), + 0x00000001: _PresetOptions(3, 0, 2, 1 << 20, 1, 4, 128, 8), + 0x00000002: _PresetOptions(3, 0, 2, 1 << 21, 1, 4, 273, 24), + 0x00000003: _PresetOptions(3, 0, 2, 1 << 22, 1, 4, 273, 48), + 0x00000004: _PresetOptions(3, 0, 2, 1 << 22, 2, 20, 16, 0), + 0x00000005: _PresetOptions(3, 0, 2, 1 << 23, 2, 20, 32, 0), + 0x00000006: _PresetOptions(3, 0, 2, 1 << 23, 2, 20, 64, 0), + 0x00000007: _PresetOptions(3, 0, 2, 1 << 24, 2, 20, 64, 0), + 0x00000008: _PresetOptions(3, 0, 2, 1 << 25, 2, 20, 64, 0), + 0x00000009: _PresetOptions(3, 0, 2, 1 << 26, 2, 20, 64, 0), + # Levels 0 through 9, OR-ed with the "extreme" setting (PRESET_EXTREME) + 0x80000000: _PresetOptions(3, 0, 2, 1 << 18, 2, 20, 273, 512), + 0x80000001: _PresetOptions(3, 0, 2, 1 << 20, 2, 20, 273, 512), + 0x80000002: _PresetOptions(3, 0, 2, 1 << 21, 2, 20, 273, 512), + 0x80000003: _PresetOptions(3, 0, 2, 1 << 22, 2, 20, 192, 0), + 0x80000004: _PresetOptions(3, 0, 2, 1 << 22, 2, 20, 273, 512), + 0x80000005: _PresetOptions(3, 0, 2, 1 << 23, 2, 20, 192, 0), + 0x80000006: _PresetOptions(3, 0, 2, 1 << 23, 2, 20, 273, 512), + 0x80000007: _PresetOptions(3, 0, 2, 1 << 24, 2, 20, 273, 512), + 0x80000008: _PresetOptions(3, 0, 2, 1 << 25, 2, 20, 273, 512), + 0x80000009: _PresetOptions(3, 0, 2, 1 << 26, 2, 20, 273, 512), + } + + def __init__(self, preset=None): self._comp = None + if (preset is not None) and (preset not in self._PRESET_OPTIONS_MAP): + raise ValueError(f'invalid preset {preset}') + self._preset = preset def _init(self): - props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1}) - self._comp = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=[ - lzma._decode_filter_properties(lzma.FILTER_LZMA1, props) - ]) + # Translate the encoder settings into the value to be written to + # the LZMA Properties Header in the ZIP file. + opts = {'id': lzma.FILTER_LZMA1} + if self._preset is not None: + opts.update(self._PRESET_OPTIONS_MAP[self._preset]._asdict()) + props = lzma._encode_filter_properties(opts) + self._comp = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=[opts]) return struct.pack(' Date: Sun, 4 Feb 2018 13:18:23 -0600 Subject: [PATCH 2/8] Add NEWS entry --- .../next/Library/2018-02-04-13-18-14.bpo-32728.2u0pLO.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-02-04-13-18-14.bpo-32728.2u0pLO.rst diff --git a/Misc/NEWS.d/next/Library/2018-02-04-13-18-14.bpo-32728.2u0pLO.rst b/Misc/NEWS.d/next/Library/2018-02-04-13-18-14.bpo-32728.2u0pLO.rst new file mode 100644 index 00000000000000..61aec60ae730b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-04-13-18-14.bpo-32728.2u0pLO.rst @@ -0,0 +1,2 @@ +Archives created by the ``zipfile`` module can now use LZMA compression +presets. From 47d7dbc8b1b936796d4c23798dff3670f4414cd0 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sun, 4 Feb 2018 13:33:28 -0600 Subject: [PATCH 3/8] Add docs entry for lzma compresslevel --- Doc/library/zipfile.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index c0f2a89a3a17fa..58513cd74ff908 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -163,11 +163,14 @@ ZipFile Objects The *compresslevel* parameter controls the compression level to use when writing files to the archive. - When using :const:`ZIP_STORED` or :const:`ZIP_LZMA` it has no effect. + When using :const:`ZIP_STORED` it has no effect. When using :const:`ZIP_DEFLATED` integers ``0`` through ``9`` are accepted (see :class:`zlib ` for more information). When using :const:`ZIP_BZIP2` integers ``1`` through ``9`` are accepted (see :class:`bz2 ` for more information). + When using :const:`ZIP_LZMA` integers ``0`` through ``9`` are accepted, + as are the values ``0 | lzma.PRESET_EXTREME`` through ``9 | lzma.PRESET_EXTREME``. + (see :class:`lzma ` for more information) If the file is created with mode ``'w'``, ``'x'`` or ``'a'`` and then :meth:`closed ` without adding any files to the archive, the appropriate @@ -203,6 +206,9 @@ ZipFile Objects .. versionchanged:: 3.7 Add the *compresslevel* parameter. + .. versionchanged:: 3.8 + Add support for using LZMA presets with the *compresslevel* parameter. + .. method:: ZipFile.close() From 37b8945188930f8e825e6bcfb4a07e4d14c92202 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Mon, 5 Aug 2019 18:21:50 -0500 Subject: [PATCH 4/8] Update Doc/library/zipfile.rst Co-Authored-By: Zackery Spytz --- Doc/library/zipfile.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 8145e2fba80f20..a387f8a8919a5d 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -225,7 +225,7 @@ ZipFile Objects The *strict_timestamps* keyword-only argument .. versionchanged:: 3.8 - Add support for using LZMA presets with the *compresslevel* parameter. + Add support for using LZMA presets with the *compresslevel* parameter. .. method:: ZipFile.close() From 3ed3f2495d6386f30cd3101059c51b0c997ae734 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Mon, 5 Aug 2019 18:22:01 -0500 Subject: [PATCH 5/8] Update Misc/NEWS.d/next/Library/2018-02-04-13-18-14.bpo-32728.2u0pLO.rst Co-Authored-By: Zackery Spytz --- .../next/Library/2018-02-04-13-18-14.bpo-32728.2u0pLO.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2018-02-04-13-18-14.bpo-32728.2u0pLO.rst b/Misc/NEWS.d/next/Library/2018-02-04-13-18-14.bpo-32728.2u0pLO.rst index 61aec60ae730b0..2992c9f00182ee 100644 --- a/Misc/NEWS.d/next/Library/2018-02-04-13-18-14.bpo-32728.2u0pLO.rst +++ b/Misc/NEWS.d/next/Library/2018-02-04-13-18-14.bpo-32728.2u0pLO.rst @@ -1,2 +1,2 @@ -Archives created by the ``zipfile`` module can now use LZMA compression +Archives created by the :mod:`zipfile` module can now use LZMA compression presets. From b7499b06da2dab7c83edbf1b2174df93c4348a5f Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Mon, 5 Aug 2019 18:25:58 -0500 Subject: [PATCH 6/8] Adjust punctuation --- Doc/library/zipfile.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index a387f8a8919a5d..dc99d98cc83b7f 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -178,8 +178,8 @@ ZipFile Objects When using :const:`ZIP_BZIP2` integers ``1`` through ``9`` are accepted (see :class:`bz2 ` for more information). When using :const:`ZIP_LZMA` integers ``0`` through ``9`` are accepted, - as are the values ``0 | lzma.PRESET_EXTREME`` through ``9 | lzma.PRESET_EXTREME``. - (see :class:`lzma ` for more information) + as are the values ``0 | lzma.PRESET_EXTREME`` through ``9 | lzma.PRESET_EXTREME`` + (see :class:`lzma ` for more information). The *strict_timestamps* argument, when set to ``False``, allows to zip files older than 1980-01-01 at the cost of setting the From 978bf3feca99e5d54453236b41a4ac0baae36715 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Mon, 5 Aug 2019 19:15:24 -0500 Subject: [PATCH 7/8] Remove collections import --- Lib/zipfile.py | 67 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 170f2870392741..dd9f3a8952e44f 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -3,7 +3,6 @@ XXX references to utf-8 need further investigation. """ -import collections import binascii import functools import importlib.util @@ -601,33 +600,49 @@ class LZMACompressor: # much memory and CPU to use) in details. liblzma defines certain filter # chain configurations as "preset levels," akin to the compression levels # in zlib or bzip2. - _PresetOptions = collections.namedtuple( - '_PresetOptions', - ('lc', 'lp', 'pb', 'dict_size', 'mode', 'mf', 'nice_len', 'depth') - ) _PRESET_OPTIONS_MAP = { # Levels 0 through 9, without the "extreme" setting - 0x00000000: _PresetOptions(3, 0, 2, 1 << 18, 1, 3, 128, 4), - 0x00000001: _PresetOptions(3, 0, 2, 1 << 20, 1, 4, 128, 8), - 0x00000002: _PresetOptions(3, 0, 2, 1 << 21, 1, 4, 273, 24), - 0x00000003: _PresetOptions(3, 0, 2, 1 << 22, 1, 4, 273, 48), - 0x00000004: _PresetOptions(3, 0, 2, 1 << 22, 2, 20, 16, 0), - 0x00000005: _PresetOptions(3, 0, 2, 1 << 23, 2, 20, 32, 0), - 0x00000006: _PresetOptions(3, 0, 2, 1 << 23, 2, 20, 64, 0), - 0x00000007: _PresetOptions(3, 0, 2, 1 << 24, 2, 20, 64, 0), - 0x00000008: _PresetOptions(3, 0, 2, 1 << 25, 2, 20, 64, 0), - 0x00000009: _PresetOptions(3, 0, 2, 1 << 26, 2, 20, 64, 0), + 0x00000000: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 18, + 'mode': 1, 'mf': 3, 'nice_len': 128, 'depth': 4}, + 0x00000001: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 20, + 'mode': 1, 'mf': 4, 'nice_len': 128, 'depth': 8}, + 0x00000002: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 21, + 'mode': 1, 'mf': 4, 'nice_len': 273, 'depth': 24}, + 0x00000003: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 22, + 'mode': 1, 'mf': 4, 'nice_len': 273, 'depth': 48}, + 0x00000004: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 22, + 'mode': 2, 'mf': 20, 'nice_len': 16, 'depth': 0}, + 0x00000005: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 23, + 'mode': 2, 'mf': 20, 'nice_len': 32, 'depth': 0}, + 0x00000006: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 23, + 'mode': 2, 'mf': 20, 'nice_len': 64, 'depth': 0}, + 0x00000007: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 24, + 'mode': 2, 'mf': 20, 'nice_len': 64, 'depth': 0}, + 0x00000008: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 25, + 'mode': 2, 'mf': 20, 'nice_len': 64, 'depth': 0}, + 0x00000009: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 26, + 'mode': 2, 'mf': 20, 'nice_len': 64, 'depth': 0}, # Levels 0 through 9, OR-ed with the "extreme" setting (PRESET_EXTREME) - 0x80000000: _PresetOptions(3, 0, 2, 1 << 18, 2, 20, 273, 512), - 0x80000001: _PresetOptions(3, 0, 2, 1 << 20, 2, 20, 273, 512), - 0x80000002: _PresetOptions(3, 0, 2, 1 << 21, 2, 20, 273, 512), - 0x80000003: _PresetOptions(3, 0, 2, 1 << 22, 2, 20, 192, 0), - 0x80000004: _PresetOptions(3, 0, 2, 1 << 22, 2, 20, 273, 512), - 0x80000005: _PresetOptions(3, 0, 2, 1 << 23, 2, 20, 192, 0), - 0x80000006: _PresetOptions(3, 0, 2, 1 << 23, 2, 20, 273, 512), - 0x80000007: _PresetOptions(3, 0, 2, 1 << 24, 2, 20, 273, 512), - 0x80000008: _PresetOptions(3, 0, 2, 1 << 25, 2, 20, 273, 512), - 0x80000009: _PresetOptions(3, 0, 2, 1 << 26, 2, 20, 273, 512), + 0x80000000: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 262144, + 'mode': 2, 'mf': 20, 'nice_len': 273, 'depth': 512}, + 0x80000001: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1048576, + 'mode': 2, 'mf': 20, 'nice_len': 273, 'depth': 512}, + 0x80000002: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 2097152, + 'mode': 2, 'mf': 20, 'nice_len': 273, 'depth': 512}, + 0x80000003: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 22, + 'mode': 2, 'mf': 20, 'nice_len': 192, 'depth': 0}, + 0x80000004: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 22, + 'mode': 2, 'mf': 20, 'nice_len': 273, 'depth': 512}, + 0x80000005: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 23, + 'mode': 2, 'mf': 20, 'nice_len': 192, 'depth': 0}, + 0x80000006: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 23, + 'mode': 2, 'mf': 20, 'nice_len': 273, 'depth': 512}, + 0x80000007: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 24, + 'mode': 2, 'mf': 20, 'nice_len': 273, 'depth': 512}, + 0x80000008: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 25, + 'mode': 2, 'mf': 20, 'nice_len': 273, 'depth': 512}, + 0x80000009: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 26, + 'mode': 2, 'mf': 20, 'nice_len': 273, 'depth': 512}, } def __init__(self, preset=None): @@ -641,7 +656,7 @@ def _init(self): # the LZMA Properties Header in the ZIP file. opts = {'id': lzma.FILTER_LZMA1} if self._preset is not None: - opts.update(self._PRESET_OPTIONS_MAP[self._preset]._asdict()) + opts.update(self._PRESET_OPTIONS_MAP[self._preset]) props = lzma._encode_filter_properties(opts) self._comp = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=[opts]) return struct.pack(' Date: Mon, 5 Aug 2019 19:26:00 -0500 Subject: [PATCH 8/8] Prettify dict_size --- Lib/zipfile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index dd9f3a8952e44f..6fbf101c060ce5 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -623,11 +623,11 @@ class LZMACompressor: 0x00000009: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 26, 'mode': 2, 'mf': 20, 'nice_len': 64, 'depth': 0}, # Levels 0 through 9, OR-ed with the "extreme" setting (PRESET_EXTREME) - 0x80000000: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 262144, + 0x80000000: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 18, 'mode': 2, 'mf': 20, 'nice_len': 273, 'depth': 512}, - 0x80000001: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1048576, + 0x80000001: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 20, 'mode': 2, 'mf': 20, 'nice_len': 273, 'depth': 512}, - 0x80000002: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 2097152, + 0x80000002: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 21, 'mode': 2, 'mf': 20, 'nice_len': 273, 'depth': 512}, 0x80000003: {'lc': 3, 'lp': 0, 'pb': 2, 'dict_size': 1 << 22, 'mode': 2, 'mf': 20, 'nice_len': 192, 'depth': 0},