From 6c9efd1f2ada5d57dcb897167d252fdcdcbb40de Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Sep 2017 11:22:40 +0200 Subject: [PATCH 1/3] bpo-31170: Write unit tests for Expat UTF-8 bug Non-regression tests for the Expat 2.2.3 UTF-8 decoder bug. --- Lib/test/test_xml_etree.py | 36 ++++++++++++++++++++++ Lib/test/xmltestdata/expat224_utf8_bug.xml | 2 ++ 2 files changed, 38 insertions(+) create mode 100644 Lib/test/xmltestdata/expat224_utf8_bug.xml diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index baa4e1f53427f6d..bfb1f36378306fc 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -34,6 +34,7 @@ except UnicodeEncodeError: raise unittest.SkipTest("filename is not encodable to utf8") SIMPLE_NS_XMLFILE = findfile("simple-ns.xml", subdir="xmltestdata") +UTF8_BUG_XMLFILE = findfile("expat224_utf8_bug.xml", subdir="xmltestdata") SAMPLE_XML = """\ @@ -1739,6 +1740,41 @@ def __eq__(self, other): self.assertIsInstance(e[0].tag, str) self.assertEqual(e[0].tag, 'changed') + def check_expat224_utf8_bug(self, text): + xml = b'' % text + root = ET.XML(xml) + self.assertEqual(root.get('b'), text.decode('utf-8')) + + def test_expat224_utf8_bug(self): + # bpo-31170: Expat 2.2.3 had a bug in its UTF-8 decoder. + # Check that Expat 2.2.4 fixed the bug. + + # 1 KB buffer + text = b'x' * 1023 + b'\xc3\xa0' + self.check_expat224_utf8_bug(text) + + # 2 KB buffer + text = b'\xc3\xa0' * 1024 + self.check_expat224_utf8_bug(text) + + # 2 KB + 1 B buffer + text = b'x' + b'\xc3\xa0' * 1024 + self.check_expat224_utf8_bug(text) + + def test_expat224_utf8_bug_file(self): + with open(UTF8_BUG_XMLFILE, 'rb') as fp: + raw = fp.read() + root = ET.fromstring(raw) + xmlattr = root.get('b') + + # "Parse" manually the XML file to extract the value of the 'b' + # attribute of the XML element + text = raw.decode('utf-8').strip() + text = text.replace('\r\n', ' ') + text = text[6:-4] + self.assertEqual(root.get('b'), text) + + # -------------------------------------------------------------------- diff --git a/Lib/test/xmltestdata/expat224_utf8_bug.xml b/Lib/test/xmltestdata/expat224_utf8_bug.xml new file mode 100644 index 000000000000000..d66a8e6b50f93b7 --- /dev/null +++ b/Lib/test/xmltestdata/expat224_utf8_bug.xml @@ -0,0 +1,2 @@ + From b9788ab3ca54bee9207001cdce5c399dde5585ef Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 16 Sep 2017 01:59:23 +0200 Subject: [PATCH 2/3] Remove redundant test, add comment, KB => KiB --- Lib/test/test_xml_etree.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index bfb1f36378306fc..ae02ccf99f17a20 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1748,16 +1748,14 @@ def check_expat224_utf8_bug(self, text): def test_expat224_utf8_bug(self): # bpo-31170: Expat 2.2.3 had a bug in its UTF-8 decoder. # Check that Expat 2.2.4 fixed the bug. + # + # Test buffer bounds at odd and even positions. - # 1 KB buffer - text = b'x' * 1023 + b'\xc3\xa0' - self.check_expat224_utf8_bug(text) - - # 2 KB buffer + # 2 KiB buffer (even) text = b'\xc3\xa0' * 1024 self.check_expat224_utf8_bug(text) - # 2 KB + 1 B buffer + # 2 KiB + 1 B buffer (odd) text = b'x' + b'\xc3\xa0' * 1024 self.check_expat224_utf8_bug(text) From f62b331f515dfeb830d1975097fd70fbfb1209b3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 21 Sep 2017 18:48:48 +0200 Subject: [PATCH 3/3] Remove wrong comments --- Lib/test/test_xml_etree.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index ae02ccf99f17a20..661ad8b9d4dfeff 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1751,11 +1751,9 @@ def test_expat224_utf8_bug(self): # # Test buffer bounds at odd and even positions. - # 2 KiB buffer (even) text = b'\xc3\xa0' * 1024 self.check_expat224_utf8_bug(text) - # 2 KiB + 1 B buffer (odd) text = b'x' + b'\xc3\xa0' * 1024 self.check_expat224_utf8_bug(text)