From f1e70111c87e8b2d7f43af70d1762862900dfde1 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 19 Dec 2018 14:13:29 -0800 Subject: [PATCH 01/21] Use _getfinalpathname to implement realpath --- Lib/ntpath.py | 14 +++++++++++--- Lib/test/test_ntpath.py | 20 +++++++++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 11bb297e16bf4e0..579f402b6757aac 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -527,8 +527,6 @@ def abspath(path): except (OSError, ValueError): return _abspath_fallback(path) -# realpath is a no-op on systems without islink support -realpath = abspath # Win9x family and earlier have no Unicode filename support. supports_unicode_filenames = (hasattr(sys, "getwindowsversion") and sys.getwindowsversion()[3] >= 2) @@ -647,6 +645,15 @@ def commonpath(paths): # Windows XP and non-Windows OS'es will mock _getfinalpathname. if sys.getwindowsversion()[:2] >= (6, 0): from nt import _getfinalpathname + + def realpath(p): + try: + p = _getfinalpathname(p) + # GetFinalPathNameByHandle returns path in \\?\ syntax + # remove leading \\?\ prefix + return p[4:] + except FileNotFoundError: + return abspath(p) else: raise ImportError except (AttributeError, ImportError): @@ -656,7 +663,8 @@ def commonpath(paths): # approximation. def _getfinalpathname(f): return normcase(abspath(f)) - + # realpath is a no-op on systems without islink support + realpath = abspath try: # The genericpath.isdir implementation uses os.stat and checks the mode diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 223e50f12c6d568..74eac58d0e8688d 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -5,7 +5,7 @@ import warnings from test.support import TestFailed, FakePath from test import support, test_genericpath -from tempfile import TemporaryFile +from tempfile import TemporaryFile, TemporaryDirectory try: import nt @@ -14,6 +14,12 @@ # but for those that require it we import here. nt = None +try: + import _winapi +except ImportError: + # realpath tests require _winapi + _winapi = None + def tester(fn, wantResult): fn = fn.replace("\\", "\\\\") gotResult = eval(fn) @@ -277,6 +283,18 @@ def test_expanduser(self): tester('ntpath.expanduser("~/foo/bar")', 'C:\\idle\\eric/foo/bar') + @unittest.skipUnless(nt and _winapi, "realpath requires 'nt' and '_winapi' modules") + def test_realpath(self): + with TemporaryDirectory() as d: + f = ntpath.join(d, "f") + os.mkdir(f) + f2 = ntpath.join(d, "g") + _winapi.CreateJunction(f, f2) + p1 = ntpath.realpath(f) + p2 = ntpath.realpath(f2) + os.unlink(f2) + self.assertEqualCI(p1, p2) + @unittest.skipUnless(nt, "abspath requires 'nt' module") def test_abspath(self): tester('ntpath.abspath("C:\\")', "C:\\") From 62028089498a4f06a1d8b225d52b6ba2b43cbf90 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 19 Dec 2018 15:27:27 -0800 Subject: [PATCH 02/21] added NEWS --- Lib/ntpath.py | 2 +- .../next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 579f402b6757aac..8c4229dd0bed2c3 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -645,7 +645,7 @@ def commonpath(paths): # Windows XP and non-Windows OS'es will mock _getfinalpathname. if sys.getwindowsversion()[:2] >= (6, 0): from nt import _getfinalpathname - + def realpath(p): try: p = _getfinalpathname(p) diff --git a/Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst b/Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst new file mode 100644 index 000000000000000..df5e124e647dab5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst @@ -0,0 +1 @@ +use _getfinalpathname to implement realpath From 58518f4192208fe094909cab2b39a777506125b8 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 19 Dec 2018 15:57:33 -0800 Subject: [PATCH 03/21] Handle OSError --- Lib/ntpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 8c4229dd0bed2c3..558f27f404ca789 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -652,7 +652,7 @@ def realpath(p): # GetFinalPathNameByHandle returns path in \\?\ syntax # remove leading \\?\ prefix return p[4:] - except FileNotFoundError: + except (FileNotFoundError, OSError): return abspath(p) else: raise ImportError From eb1127b1798a420320f1555c3cff8c5aa859d6f4 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 19 Dec 2018 21:50:24 -0800 Subject: [PATCH 04/21] Keep only OSError --- Lib/ntpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 558f27f404ca789..fb15d272dd34837 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -652,7 +652,7 @@ def realpath(p): # GetFinalPathNameByHandle returns path in \\?\ syntax # remove leading \\?\ prefix return p[4:] - except (FileNotFoundError, OSError): + except OSError: return abspath(p) else: raise ImportError From bb1c124050dd60aae3a3c359625389e4207ad1d0 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 26 Dec 2018 13:14:21 -0800 Subject: [PATCH 05/21] implement nt.realpath similarly to posix.realpath --- Lib/ntpath.py | 75 ++++++++++++++++++++++++++++------------- Lib/test/test_ntpath.py | 45 ++++++++++++++++++++++--- 2 files changed, 92 insertions(+), 28 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index fb15d272dd34837..9bbd425726472b6 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -22,6 +22,7 @@ import stat import genericpath from genericpath import * +from nt import _getfinalpathname __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", @@ -640,31 +641,57 @@ def commonpath(paths): # determine if two files are in fact the same file -try: - # GetFinalPathNameByHandle is available starting with Windows 6.0. - # Windows XP and non-Windows OS'es will mock _getfinalpathname. - if sys.getwindowsversion()[:2] >= (6, 0): - from nt import _getfinalpathname - - def realpath(p): - try: - p = _getfinalpathname(p) - # GetFinalPathNameByHandle returns path in \\?\ syntax - # remove leading \\?\ prefix - return p[4:] - except OSError: - return abspath(p) +def realpath(filename): + filename = os.fspath(filename) + extended_length_prefix = '\\\\?\\' if isinstance(filename, str) else b'\\\\?\\' + is_extended_path = filename.startswith(extended_length_prefix) + unresolved = filename if is_extended_path else abspath(filename) + resolved_parts = [] + while True: + try: + resolved_parts.append(_getfinalpathname(unresolved)) + break + except OSError: + unresolved, tail = split(unresolved) + if not tail: + resolved_parts.append(unresolved) + break + resolved_parts.append(tail) + resolved = join(*reversed(resolved_parts)) + # try to convert extended path to normal if + # initial path did not use \\?\ prefix and result uses it + if not is_extended_path and resolved.startswith(extended_length_prefix): + resolved = _extended_to_normal(resolved) + return resolved + +def _extended_to_normal(path): + drive, rest = splitdrive(path) + if not rest: + return path + if isinstance(path, str): + sep = '\\' + colon = ':' + letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + extended_unc_prefix = '\\\\?\\UNC' + else: + sep = b'\\' + colon = b':' + letters = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + extended_unc_prefix = b'\\\\?\\UNC' + drive = drive.upper() + if drive == extended_unc_prefix: + # UNC path with \\?\ prefix - drop prefix + # 7 is len('\\?\UNC') + normal_path = sep + path[7:] else: - raise ImportError -except (AttributeError, ImportError): - # On Windows XP and earlier, two files are the same if their absolute - # pathnames are the same. - # Non-Windows operating systems fake this method with an XP - # approximation. - def _getfinalpathname(f): - return normcase(abspath(f)) - # realpath is a no-op on systems without islink support - realpath = abspath + assert len(drive) == 6 and drive[4:5] in letters and drive[5:6] == colon + # extended path with \\?\ prefix + # 4 is len('\\?\') + normal_path = path[4:] + + if 0 < len(normal_path) < 260 and normal_path == abspath(normal_path): + return normal_path + return path try: # The genericpath.isdir implementation uses os.stat and checks the mode diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 74eac58d0e8688d..0a4831bb1ba2d2a 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -285,15 +285,52 @@ def test_expanduser(self): @unittest.skipUnless(nt and _winapi, "realpath requires 'nt' and '_winapi' modules") def test_realpath(self): + def to_unc(p): + drive, rest = ntpath.splitdrive(file2) + return ntpath.join(f"\\\\localhost\\{drive[0]}$", rest) + def s2b(s): + return bytes(s, "utf-8") + with TemporaryDirectory() as d: + f4 = None f = ntpath.join(d, "f") os.mkdir(f) f2 = ntpath.join(d, "g") _winapi.CreateJunction(f, f2) - p1 = ntpath.realpath(f) - p2 = ntpath.realpath(f2) - os.unlink(f2) - self.assertEqualCI(p1, p2) + try: + # realpath for original path and junction is the same + self.assertEqualCI(ntpath.realpath(f), ntpath.realpath(f2)) + self.assertEqualCI(ntpath.realpath(s2b(f)), ntpath.realpath(s2b(f2))) + + # realpath for UNC path is the same + file = ntpath.join(f, "file1") + open(file, "w+").close() + file2 = ntpath.join(f2, "file1") + unc1 = to_unc(file) + unc2 = to_unc(file2) + self.assertEqualCI(unc1, ntpath.realpath(unc2)) + self.assertEqualCI(s2b(unc1), ntpath.realpath(s2b(unc2))) + + # realpath for non-existent file F in symlinked folder + # is original folder + F + file = ntpath.join(f, "missing") + file2 = ntpath.join(f2, "missing") + self.assertEqualCI(file, ntpath.realpath(file2)) + self.assertEqualCI(s2b(file), ntpath.realpath(s2b(file2))) + + # realpath for long path is in extended form + # even though initially it was not + f3 = ntpath.join(d, "f" * 255) + os.mkdir("\\\\?\\" + f3) + f4 = ntpath.join(d, "short") + _winapi.CreateJunction(f3, f4) + self.assertEqualCI("\\\\?\\" + f3, ntpath.realpath(f4)) + self.assertEqualCI(s2b("\\\\?\\" + f3), ntpath.realpath(s2b(f4))) + finally: + os.unlink(f2) + if f4: + os.unlink(f4) + @unittest.skipUnless(nt, "abspath requires 'nt' module") def test_abspath(self): From 6769908defc0c66bf5d35c76036af8478024fadd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 26 Dec 2018 13:18:03 -0800 Subject: [PATCH 06/21] Update Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst Co-Authored-By: vladima --- .../next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst b/Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst index df5e124e647dab5..1fa2ac72dc38920 100644 --- a/Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst +++ b/Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst @@ -1 +1 @@ -use _getfinalpathname to implement realpath +ntpath.realpath() now uses :c:func:`GetFinalPathNameByHandle()`. From 6842cde7afe6f6053124077a389fc19cbaa95c93 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 26 Dec 2018 15:13:33 -0800 Subject: [PATCH 07/21] handle missing _getfinalpathname --- Lib/ntpath.py | 108 ++++++++++++++++++++++++++------------------------ 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 9bbd425726472b6..f16c0599addf431 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -22,7 +22,6 @@ import stat import genericpath from genericpath import * -from nt import _getfinalpathname __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", @@ -640,58 +639,65 @@ def commonpath(paths): raise -# determine if two files are in fact the same file -def realpath(filename): - filename = os.fspath(filename) - extended_length_prefix = '\\\\?\\' if isinstance(filename, str) else b'\\\\?\\' - is_extended_path = filename.startswith(extended_length_prefix) - unresolved = filename if is_extended_path else abspath(filename) - resolved_parts = [] - while True: - try: - resolved_parts.append(_getfinalpathname(unresolved)) - break - except OSError: - unresolved, tail = split(unresolved) - if not tail: - resolved_parts.append(unresolved) +try: + from nt import _getfinalpathname + + # determine if two files are in fact the same file + def realpath(filename): + filename = os.fspath(filename) + extended_length_prefix = '\\\\?\\' if isinstance(filename, str) else b'\\\\?\\' + is_extended_path = filename.startswith(extended_length_prefix) + unresolved = filename if is_extended_path else abspath(filename) + resolved_parts = [] + while True: + try: + resolved_parts.append(_getfinalpathname(unresolved)) break - resolved_parts.append(tail) - resolved = join(*reversed(resolved_parts)) - # try to convert extended path to normal if - # initial path did not use \\?\ prefix and result uses it - if not is_extended_path and resolved.startswith(extended_length_prefix): - resolved = _extended_to_normal(resolved) - return resolved - -def _extended_to_normal(path): - drive, rest = splitdrive(path) - if not rest: + except OSError: + unresolved, tail = split(unresolved) + if not tail: + resolved_parts.append(unresolved) + break + resolved_parts.append(tail) + resolved = join(*reversed(resolved_parts)) + # try to convert extended path to normal if + # initial path did not use \\?\ prefix and result uses it + if not is_extended_path and resolved.startswith(extended_length_prefix): + resolved = _extended_to_normal(resolved) + return resolved + + def _extended_to_normal(path): + drive, rest = splitdrive(path) + if not rest: + return path + if isinstance(path, str): + sep = '\\' + colon = ':' + letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + extended_unc_prefix = '\\\\?\\UNC' + else: + sep = b'\\' + colon = b':' + letters = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + extended_unc_prefix = b'\\\\?\\UNC' + drive = drive.upper() + if drive == extended_unc_prefix: + # UNC path with \\?\ prefix - drop prefix + # 7 is len('\\?\UNC') + normal_path = sep + path[7:] + else: + assert len(drive) == 6 and drive[4:5] in letters and drive[5:6] == colon + # extended path with \\?\ prefix + # 4 is len('\\?\') + normal_path = path[4:] + + if 0 < len(normal_path) < 260 and normal_path == abspath(normal_path): + return normal_path return path - if isinstance(path, str): - sep = '\\' - colon = ':' - letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - extended_unc_prefix = '\\\\?\\UNC' - else: - sep = b'\\' - colon = b':' - letters = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - extended_unc_prefix = b'\\\\?\\UNC' - drive = drive.upper() - if drive == extended_unc_prefix: - # UNC path with \\?\ prefix - drop prefix - # 7 is len('\\?\UNC') - normal_path = sep + path[7:] - else: - assert len(drive) == 6 and drive[4:5] in letters and drive[5:6] == colon - # extended path with \\?\ prefix - # 4 is len('\\?\') - normal_path = path[4:] - - if 0 < len(normal_path) < 260 and normal_path == abspath(normal_path): - return normal_path - return path +except ImportError: + realpath = abspath + def _getfinalpathname(path): + return normcase(abspath(path)) try: # The genericpath.isdir implementation uses os.stat and checks the mode From 58ef5f0e194170ccc2721b0fbc91a5dea43f14d7 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 26 Dec 2018 15:29:40 -0800 Subject: [PATCH 08/21] drop flaky test --- Lib/test/test_ntpath.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 0a4831bb1ba2d2a..b16558db2abf2f9 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -317,15 +317,6 @@ def s2b(s): file2 = ntpath.join(f2, "missing") self.assertEqualCI(file, ntpath.realpath(file2)) self.assertEqualCI(s2b(file), ntpath.realpath(s2b(file2))) - - # realpath for long path is in extended form - # even though initially it was not - f3 = ntpath.join(d, "f" * 255) - os.mkdir("\\\\?\\" + f3) - f4 = ntpath.join(d, "short") - _winapi.CreateJunction(f3, f4) - self.assertEqualCI("\\\\?\\" + f3, ntpath.realpath(f4)) - self.assertEqualCI(s2b("\\\\?\\" + f3), ntpath.realpath(s2b(f4))) finally: os.unlink(f2) if f4: From cb2c24c03996ac18e58c28c61608036ad867ac84 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 26 Dec 2018 16:10:58 -0800 Subject: [PATCH 09/21] add missing realpath call --- Lib/test/test_ntpath.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index b16558db2abf2f9..8ad82a2ad059fc9 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -308,15 +308,15 @@ def s2b(s): file2 = ntpath.join(f2, "file1") unc1 = to_unc(file) unc2 = to_unc(file2) - self.assertEqualCI(unc1, ntpath.realpath(unc2)) - self.assertEqualCI(s2b(unc1), ntpath.realpath(s2b(unc2))) + self.assertEqualCI(ntpath.realpath(unc1), ntpath.realpath(unc2)) + self.assertEqualCI(ntpath.realpath(s2b(unc1)), ntpath.realpath(s2b(unc2))) # realpath for non-existent file F in symlinked folder # is original folder + F file = ntpath.join(f, "missing") file2 = ntpath.join(f2, "missing") - self.assertEqualCI(file, ntpath.realpath(file2)) - self.assertEqualCI(s2b(file), ntpath.realpath(s2b(file2))) + self.assertEqualCI(ntpath.realpath(file), ntpath.realpath(file2)) + self.assertEqualCI(ntpath.realpath(s2b(file)), ntpath.realpath(s2b(file2))) finally: os.unlink(f2) if f4: From 9f5c9c06325da2685767d354b785433a67403db0 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 26 Dec 2018 21:58:36 -0800 Subject: [PATCH 10/21] added test with non 8.3 directory names --- Lib/test/test_ntpath.py | 12 +++++++++--- Lib/test/test_shutil.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 8ad82a2ad059fc9..efeb0bdf6cf0658 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -292,7 +292,6 @@ def s2b(s): return bytes(s, "utf-8") with TemporaryDirectory() as d: - f4 = None f = ntpath.join(d, "f") os.mkdir(f) f2 = ntpath.join(d, "g") @@ -317,10 +316,17 @@ def s2b(s): file2 = ntpath.join(f2, "missing") self.assertEqualCI(ntpath.realpath(file), ntpath.realpath(file2)) self.assertEqualCI(ntpath.realpath(s2b(file)), ntpath.realpath(s2b(file2))) + + # realpath for short names used for non 8.3 directory names + dir = ntpath.join(f, "somelongname") + os.mkdir(dir) + file = ntpath.join(dir, "f") + open(file, "w+").close() + file_in_f2 = ntpath.join(f2, "somelo~1", "f") + self.assertEqualCI(ntpath.realpath(file), ntpath.realpath(file_in_f2)) + self.assertEqualCI(ntpath.realpath(s2b(file)), ntpath.realpath(s2b(file_in_f2))) finally: os.unlink(f2) - if f4: - os.unlink(f4) @unittest.skipUnless(nt, "abspath requires 'nt' module") diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index ec8fcc3eef01e25..ec1b11168818bb8 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1731,7 +1731,7 @@ def test_move_dangling_symlink(self): self.assertTrue(os.path.islink(dst_link)) # On Windows, os.path.realpath does not follow symlinks (issue #9949) if os.name == 'nt': - self.assertEqual(os.path.realpath(src), os.readlink(dst_link)) + self.assertEqual(os.path.realpath(src), os.path.realpath(os.readlink(dst_link))) else: self.assertEqual(os.path.realpath(src), os.path.realpath(dst_link)) From 04502bf25925d28994ebe384338548186d8341eb Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Sun, 30 Dec 2018 22:25:19 -0800 Subject: [PATCH 11/21] address PR feedback: rename locals, put drive-letter case to elif --- Lib/ntpath.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index f16c0599addf431..e0382587a9e7170 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -645,8 +645,8 @@ def commonpath(paths): # determine if two files are in fact the same file def realpath(filename): filename = os.fspath(filename) - extended_length_prefix = '\\\\?\\' if isinstance(filename, str) else b'\\\\?\\' - is_extended_path = filename.startswith(extended_length_prefix) + extended_path_prefix = '\\\\?\\' if isinstance(filename, str) else b'\\\\?\\' + is_extended_path = filename.startswith(extended_path_prefix) unresolved = filename if is_extended_path else abspath(filename) resolved_parts = [] while True: @@ -662,7 +662,7 @@ def realpath(filename): resolved = join(*reversed(resolved_parts)) # try to convert extended path to normal if # initial path did not use \\?\ prefix and result uses it - if not is_extended_path and resolved.startswith(extended_length_prefix): + if not is_extended_path and resolved.startswith(extended_path_prefix): resolved = _extended_to_normal(resolved) return resolved @@ -674,30 +674,38 @@ def _extended_to_normal(path): sep = '\\' colon = ':' letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - extended_unc_prefix = '\\\\?\\UNC' + unc_dev = '\\\\?\\UNC' else: sep = b'\\' colon = b':' letters = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - extended_unc_prefix = b'\\\\?\\UNC' + unc_dev = b'\\\\?\\UNC' drive = drive.upper() - if drive == extended_unc_prefix: + if drive == unc_dev: # UNC path with \\?\ prefix - drop prefix # 7 is len('\\?\UNC') normal_path = sep + path[7:] - else: - assert len(drive) == 6 and drive[4:5] in letters and drive[5:6] == colon + elif len(drive) == 6 and drive[4:5] in letters and drive[5:6] == colon: # extended path with \\?\ prefix # 4 is len('\\?\') normal_path = path[4:] + else: + # not a UNC or drive-letter path + # return path as-is + return path if 0 < len(normal_path) < 260 and normal_path == abspath(normal_path): return normal_path return path except ImportError: - realpath = abspath + def realpath(filename): + filename = os.fspath(filename) + extended_path_prefix = '\\\\?\\' if isinstance(filename, str) else b'\\\\?\\' + is_extended_path = filename.startswith(extended_path_prefix) + return filename if is_extended_path else abspath(filename) + def _getfinalpathname(path): - return normcase(abspath(path)) + return normcase(realpath(path)) try: # The genericpath.isdir implementation uses os.stat and checks the mode From 13f167695edadbc4b88965baa3a8c21a7d67e616 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Mon, 31 Dec 2018 16:38:38 -0800 Subject: [PATCH 12/21] remove redundant path length check --- Lib/ntpath.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index e0382587a9e7170..37de766177b3c78 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -638,6 +638,7 @@ def commonpath(paths): genericpath._check_arg_types('commonpath', *paths) raise +MAX_PATH = 260 try: from nt import _getfinalpathname @@ -694,7 +695,7 @@ def _extended_to_normal(path): # return path as-is return path - if 0 < len(normal_path) < 260 and normal_path == abspath(normal_path): + if len(normal_path) < MAX_PATH and normal_path == abspath(normal_path): return normal_path return path except ImportError: From b5683d31b0f709574198a3891a52ff53cb9dc642 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 9 Jan 2019 16:43:56 -0800 Subject: [PATCH 13/21] combine str and bytes codepaths by fsdecoding filename at the beginning and fsencoding at the end --- Lib/ntpath.py | 23 +++++++------------ .../2018-12-19-15-26-35.bpo-14094.8Hotek.rst | 2 +- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 37de766177b3c78..ed75b41ad1b5e80 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -646,7 +646,9 @@ def commonpath(paths): # determine if two files are in fact the same file def realpath(filename): filename = os.fspath(filename) - extended_path_prefix = '\\\\?\\' if isinstance(filename, str) else b'\\\\?\\' + is_str = isinstance(filename, str) + filename = os.fsdecode(filename) + extended_path_prefix = '\\\\?\\' is_extended_path = filename.startswith(extended_path_prefix) unresolved = filename if is_extended_path else abspath(filename) resolved_parts = [] @@ -665,28 +667,19 @@ def realpath(filename): # initial path did not use \\?\ prefix and result uses it if not is_extended_path and resolved.startswith(extended_path_prefix): resolved = _extended_to_normal(resolved) - return resolved + return resolved if is_str else os.fsencode(resolved) def _extended_to_normal(path): drive, rest = splitdrive(path) if not rest: return path - if isinstance(path, str): - sep = '\\' - colon = ':' - letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - unc_dev = '\\\\?\\UNC' - else: - sep = b'\\' - colon = b':' - letters = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - unc_dev = b'\\\\?\\UNC' + letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' drive = drive.upper() - if drive == unc_dev: + if drive == '\\\\?\\UNC': # UNC path with \\?\ prefix - drop prefix # 7 is len('\\?\UNC') - normal_path = sep + path[7:] - elif len(drive) == 6 and drive[4:5] in letters and drive[5:6] == colon: + normal_path = '\\' + path[7:] + elif len(drive) == 6 and drive[4] in letters and drive[5] == ':': # extended path with \\?\ prefix # 4 is len('\\?\') normal_path = path[4:] diff --git a/Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst b/Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst index 1fa2ac72dc38920..85cfe82fae62728 100644 --- a/Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst +++ b/Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst @@ -1 +1 @@ -ntpath.realpath() now uses :c:func:`GetFinalPathNameByHandle()`. +ntpath.realpath() now uses ``GetFinalPathNameByHandle()``. From 1cacf6ac9ae6025f36cb472f92bd4d5d941a7592 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 9 Jan 2019 19:29:48 -0800 Subject: [PATCH 14/21] fsdecode only if filename is bytes --- Lib/ntpath.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index ed75b41ad1b5e80..173e2bce8311e18 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -647,7 +647,8 @@ def commonpath(paths): def realpath(filename): filename = os.fspath(filename) is_str = isinstance(filename, str) - filename = os.fsdecode(filename) + if not is_str: + filename = os.fsdecode(filename) extended_path_prefix = '\\\\?\\' is_extended_path = filename.startswith(extended_path_prefix) unresolved = filename if is_extended_path else abspath(filename) From 145a978f6aa85bb3a5357607be6ba82205895dce Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 9 Jan 2019 23:23:44 -0800 Subject: [PATCH 15/21] replace splitdrive with manual check of the prefix --- Lib/ntpath.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 173e2bce8311e18..f64380bcef33096 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -671,19 +671,18 @@ def realpath(filename): return resolved if is_str else os.fsencode(resolved) def _extended_to_normal(path): - drive, rest = splitdrive(path) - if not rest: - return path letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - drive = drive.upper() - if drive == '\\\\?\\UNC': - # UNC path with \\?\ prefix - drop prefix - # 7 is len('\\?\UNC') - normal_path = '\\' + path[7:] - elif len(drive) == 6 and drive[4] in letters and drive[5] == ':': + if (len(path) > 7 and + path[4].upper() in letters and + path[5] == ':' and + path[6] == '\\'): # extended path with \\?\ prefix # 4 is len('\\?\') normal_path = path[4:] + elif len(path) > 8 and path[:7].upper() == '\\\\?\\UNC\\': + # UNC path with \\?\ prefix - drop prefix + # 7 is len('\\?\UNC') + normal_path = '\\' + path[7:] else: # not a UNC or drive-letter path # return path as-is From 5da9b3650cc3a112c3df6240316500562f912224 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 9 Jan 2019 23:31:14 -0800 Subject: [PATCH 16/21] typo in prefix length --- Lib/ntpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index f64380bcef33096..5f96966f0fad2e3 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -679,7 +679,7 @@ def _extended_to_normal(path): # extended path with \\?\ prefix # 4 is len('\\?\') normal_path = path[4:] - elif len(path) > 8 and path[:7].upper() == '\\\\?\\UNC\\': + elif len(path) > 8 and path[:8].upper() == '\\\\?\\UNC\\': # UNC path with \\?\ prefix - drop prefix # 7 is len('\\?\UNC') normal_path = '\\' + path[7:] From ccb4e1d808c3697eb1e7b5eb67251424441f0670 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Thu, 10 Jan 2019 15:55:37 -0800 Subject: [PATCH 17/21] PR feedback --- Lib/ntpath.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 5f96966f0fad2e3..325eb229182329c 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -672,24 +672,24 @@ def realpath(filename): def _extended_to_normal(path): letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - if (len(path) > 7 and + if (len(path) > 6 and path[4].upper() in letters and path[5] == ':' and path[6] == '\\'): # extended path with \\?\ prefix # 4 is len('\\?\') - normal_path = path[4:] - elif len(path) > 8 and path[:8].upper() == '\\\\?\\UNC\\': + normal_path = normpath(path[4:]) + elif len(path) > 7 and path[:8].upper() == '\\\\?\\UNC\\': # UNC path with \\?\ prefix - drop prefix # 7 is len('\\?\UNC') - normal_path = '\\' + path[7:] + normal_path = normpath('\\' + path[7:]) else: # not a UNC or drive-letter path # return path as-is return path - if len(normal_path) < MAX_PATH and normal_path == abspath(normal_path): - return normal_path + if len(normal_path) < MAX_PATH and normal_path == _getfullpathname(normal_path): + return normpath(normal_path) return path except ImportError: def realpath(filename): From 497aa2034b01815e7810da855f460440db4946f7 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Thu, 10 Jan 2019 20:31:59 -0800 Subject: [PATCH 18/21] drop _getfinalpathname fallback --- Lib/ntpath.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 325eb229182329c..b6048b892289bda 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -698,9 +698,6 @@ def realpath(filename): is_extended_path = filename.startswith(extended_path_prefix) return filename if is_extended_path else abspath(filename) - def _getfinalpathname(path): - return normcase(realpath(path)) - try: # The genericpath.isdir implementation uses os.stat and checks the mode # attribute to tell whether or not the path is a directory. From 41705e3c8aa730c0fa0f2035108ee2ca9086a5e1 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Thu, 10 Jan 2019 21:11:03 -0800 Subject: [PATCH 19/21] drop redundant normpath call --- Lib/ntpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index b6048b892289bda..3f642537a39f143 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -689,7 +689,7 @@ def _extended_to_normal(path): return path if len(normal_path) < MAX_PATH and normal_path == _getfullpathname(normal_path): - return normpath(normal_path) + return normal_path return path except ImportError: def realpath(filename): From 01b426ee08ce13c33c663466439c1931b8b55228 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Thu, 10 Jan 2019 21:42:22 -0800 Subject: [PATCH 20/21] drop len(path) check in favor of slices --- Lib/ntpath.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 3f642537a39f143..bd66d6f3a2470ad 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -672,14 +672,11 @@ def realpath(filename): def _extended_to_normal(path): letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - if (len(path) > 6 and - path[4].upper() in letters and - path[5] == ':' and - path[6] == '\\'): + if path[5:7] == ":\\" and path[4].upper() in letters: # extended path with \\?\ prefix # 4 is len('\\?\') normal_path = normpath(path[4:]) - elif len(path) > 7 and path[:8].upper() == '\\\\?\\UNC\\': + elif path[:8].upper() == '\\\\?\\UNC\\': # UNC path with \\?\ prefix - drop prefix # 7 is len('\\?\UNC') normal_path = normpath('\\' + path[7:]) From fc2385a2d68b925f31b217209a51b75c102186ef Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Thu, 10 Jan 2019 21:51:01 -0800 Subject: [PATCH 21/21] PR feedback: formatting --- Lib/ntpath.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index bd66d6f3a2470ad..08f9dae30d223a2 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -685,13 +685,16 @@ def _extended_to_normal(path): # return path as-is return path - if len(normal_path) < MAX_PATH and normal_path == _getfullpathname(normal_path): + if (len(normal_path) < MAX_PATH and + normal_path == _getfullpathname(normal_path)): return normal_path return path except ImportError: def realpath(filename): filename = os.fspath(filename) - extended_path_prefix = '\\\\?\\' if isinstance(filename, str) else b'\\\\?\\' + extended_path_prefix = ( + '\\\\?\\' if isinstance(filename, str) else b'\\\\?\\' + ) is_extended_path = filename.startswith(extended_path_prefix) return filename if is_extended_path else abspath(filename)