Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
f1e7011
Use _getfinalpathname to implement realpath
vladima Dec 19, 2018
6202808
added NEWS
vladima Dec 19, 2018
58518f4
Handle OSError
vladima Dec 19, 2018
eb1127b
Keep only OSError
vladima Dec 20, 2018
613115c
Merge branch 'master' into realpath
vladima Dec 26, 2018
bb1c124
implement nt.realpath similarly to posix.realpath
vladima Dec 26, 2018
6769908
Update Misc/NEWS.d/next/Library/2018-12-19-15-26-35.bpo-14094.8Hotek.rst
vstinner Dec 26, 2018
6842cde
handle missing _getfinalpathname
vladima Dec 26, 2018
58ef5f0
drop flaky test
vladima Dec 26, 2018
cb2c24c
add missing realpath call
vladima Dec 27, 2018
9f5c9c0
added test with non 8.3 directory names
vladima Dec 27, 2018
04502bf
address PR feedback: rename locals, put drive-letter case to elif
vladima Dec 31, 2018
13f1676
remove redundant path length check
vladima Jan 1, 2019
6253ab3
Merge remote-tracking branch 'upstream/master' into realpath
vladima Jan 10, 2019
b5683d3
combine str and bytes codepaths by fsdecoding filename at the beginni…
vladima Jan 10, 2019
1cacf6a
fsdecode only if filename is bytes
vladima Jan 10, 2019
145a978
replace splitdrive with manual check of the prefix
vladima Jan 10, 2019
5da9b36
typo in prefix length
vladima Jan 10, 2019
ccb4e1d
PR feedback
vladima Jan 10, 2019
497aa20
drop _getfinalpathname fallback
vladima Jan 11, 2019
41705e3
drop redundant normpath call
vladima Jan 11, 2019
01b426e
drop len(path) check in favor of slices
vladima Jan 11, 2019
fc2385a
PR feedback: formatting
vladima Jan 11, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 56 additions & 16 deletions Lib/ntpath.py
Original file line number Diff line number Diff line change
Expand Up @@ -527,8 +527,6 @@ def abspath(path):
except (OSError, ValueError):
return _abspath_fallback(path)

# realpath is a no-op on systems without islink support
realpath = abspath
# Win9x family and earlier have no Unicode filename support.
supports_unicode_filenames = (hasattr(sys, "getwindowsversion") and
sys.getwindowsversion()[3] >= 2)
Expand Down Expand Up @@ -640,23 +638,65 @@ def commonpath(paths):
genericpath._check_arg_types('commonpath', *paths)
raise

MAX_PATH = 260

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

_INTERNAL_CONSTANT or else document it (as a legacy value that can largely be ignored)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My later suggestion may remove the need for this constant entirely.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's useful to document as a public constant, but maybe change it to MAX_PATH_DOS or MAX_PATH_LEGACY? Python still has an inter-operation concern even if our own process supports long paths (in addition to our manifest setting, this requires enabling at the OS level , which last I checked isn't the default setting). Along the same lines, CreateProcess doesn't allow the current directory (explicit or inherited) to be a long path.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We've never had it in the past, and it's becoming less relevant. I'd rather not add it now, and continue the work we do to handle it as transparently as possible.


# determine if two files are in fact the same file
try:
# GetFinalPathNameByHandle is available starting with Windows 6.0.
# Windows XP and non-Windows OS'es will mock _getfinalpathname.
if sys.getwindowsversion()[:2] >= (6, 0):
from nt import _getfinalpathname
else:
raise ImportError
except (AttributeError, ImportError):
# On Windows XP and earlier, two files are the same if their absolute
# pathnames are the same.
# Non-Windows operating systems fake this method with an XP
# approximation.
def _getfinalpathname(f):
return normcase(abspath(f))
from nt import _getfinalpathname

# determine if two files are in fact the same file
def realpath(filename):
filename = os.fspath(filename)
is_str = isinstance(filename, str)
if not is_str:
filename = os.fsdecode(filename)
extended_path_prefix = '\\\\?\\'

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Prefer module-level _PRIVATE_CONSTANTS

is_extended_path = filename.startswith(extended_path_prefix)
unresolved = filename if is_extended_path else abspath(filename)
resolved_parts = []
while True:
try:
resolved_parts.append(_getfinalpathname(unresolved))
break
except OSError:
unresolved, tail = split(unresolved)
if not tail:
resolved_parts.append(unresolved)
break
resolved_parts.append(tail)
resolved = join(*reversed(resolved_parts))
# try to convert extended path to normal if
# initial path did not use \\?\ prefix and result uses it
if not is_extended_path and resolved.startswith(extended_path_prefix):
resolved = _extended_to_normal(resolved)
return resolved if is_str else os.fsencode(resolved)

def _extended_to_normal(path):
letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
if path[5:7] == ":\\" and path[4].upper() in letters:
# extended path with \\?\ prefix
# 4 is len('\\?\')
normal_path = normpath(path[4:])
elif path[:8].upper() == '\\\\?\\UNC\\':
# UNC path with \\?\ prefix - drop prefix
# 7 is len('\\?\UNC')
normal_path = normpath('\\' + path[7:])
else:
# not a UNC or drive-letter path
# return path as-is
return path

if (len(normal_path) < MAX_PATH and

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd rather not check the length here and just handle the OSError that _getfullpathname raises if the path is too long (which may be 32k instead of 260 on more recent systems)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That would be convenient, but GetFullPathNameW only has the NT limit for a null-terminated string, UNICODE_STRING_MAX_CHARS - 1 (32766). Someone made a couple of mistakes in the docs. Windows 10 long-path support has no bearing on GetFullPathNameW, and we've never had to use the "\\?\" prefix to support up to 32,766 characters with this function. The only change I recall is that prior to Windows 7 the behavior wasn't well-defined if we passed it a path that exceeded 32,766 characters. I think it would type cast the length in bytes to USHORT. Since Windows 7, it calls the internal function RtlGetFullPathName_UEx, which returns an NTSTATUS code. If the input path length exceeds 32,766, RtlGetFullPathName_UEx fails with STATUS_NAME_TOO_LONG (i.e. ERROR_FILENAME_EXCED_RANGE). If the result would exceed 32,766 characters, it fails with STATUS_OBJECT_NAME_INVALID (i.e. ERROR_INVALID_NAME).

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That would be convenient, but …

I don't understand why you say "but"? It sounds like (and I confirmed from source) all our supported platforms can handle normal and excessively long paths here just fine, so there's no need for the check, correct?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're probably talking past each other here. IMO, we shouldn't remove the "\\?\" prefix for long paths prior to Windows 10, and in Windows 10 we shouldn't remove it if long-path support isn't enabled by the "LongPathsEnabled" value in "HKLM\SYSTEM\CurrentControlSet\Control\FileSystem". Maybe you mean that we can remove the prefix because none of our code will crash on long paths. My basis for suggesting this check was that realpath should return a usable result if possible, not a result that requires converting back to an extended path (including the special case for UNC paths) if long paths aren't enabled.

We can skip the MAX_PATH check if we have a way to check whether long paths are enabled. The only way to directly check this is to call the undocumented function RtlAreLongPathsEnabled, if it's defined in ntdll.dll. We could also check the Windows version number and "LongPathsEnabled" registry value at startup, but it's slightly race-prone since the value may have changed after the system read and cached it at startup. I suppose we could check it indirectly by trying to open a random 255-character name in the temp folder at startup, if that's at least MAX_PATH characters. If it fails with ERROR_FILE_NOT_FOUND then set a flag that long paths are enabled. If long paths aren't enabled I expect the error to be ERROR_PATH_NOT_FOUND.

I was also concerned about sharing paths via IPC and configuration settings with programs that don't support long paths. But now that I think about it more, I don't think extended paths help this case much in general since many programs don't support them. In this case we can use some type of junction, which could be a drive-letter junction via win32file.DefineDosDevice (or equivalently subst.exe) or a filesystem junction via _winapi.CreateJunction (or equivalently CMD's mklink /j). These extend the limit to about 4k characters. It's not the full 32k, but I have no idea who would really need paths that long.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see what you're thinking now.

I'd rather move the path length condition out of this function then, so that:

  • _extended_to_normal always returns a normal path, regardless of length
  • realpath returns a normal path if < 260 chars or an extended path if > 260 chars

The first point is the main part that was bothering me.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And _extended_to_normal should also have the prefix check, being a no-op if the path doesn't have it.

normal_path == _getfullpathname(normal_path)):
return normal_path
Comment thread
vladima marked this conversation as resolved.
return path
except ImportError:
def realpath(filename):
filename = os.fspath(filename)
extended_path_prefix = (
'\\\\?\\' if isinstance(filename, str) else b'\\\\?\\'
)
is_extended_path = filename.startswith(extended_path_prefix)
return filename if is_extended_path else abspath(filename)

try:
# The genericpath.isdir implementation uses os.stat and checks the mode
Expand Down
54 changes: 53 additions & 1 deletion Lib/test/test_ntpath.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import warnings
from test.support import TestFailed, FakePath
from test import support, test_genericpath
from tempfile import TemporaryFile
from tempfile import TemporaryFile, TemporaryDirectory

try:
import nt
Expand All @@ -14,6 +14,12 @@
# but for those that require it we import here.
nt = None

try:
import _winapi
except ImportError:
# realpath tests require _winapi
_winapi = None

def tester(fn, wantResult):
fn = fn.replace("\\", "\\\\")
gotResult = eval(fn)
Expand Down Expand Up @@ -277,6 +283,52 @@ def test_expanduser(self):
tester('ntpath.expanduser("~/foo/bar")',
'C:\\idle\\eric/foo/bar')

@unittest.skipUnless(nt and _winapi, "realpath requires 'nt' and '_winapi' modules")
def test_realpath(self):
def to_unc(p):
drive, rest = ntpath.splitdrive(file2)
return ntpath.join(f"\\\\localhost\\{drive[0]}$", rest)
def s2b(s):
return bytes(s, "utf-8")

with TemporaryDirectory() as d:
f = ntpath.join(d, "f")
os.mkdir(f)
f2 = ntpath.join(d, "g")
_winapi.CreateJunction(f, f2)
try:
# realpath for original path and junction is the same
self.assertEqualCI(ntpath.realpath(f), ntpath.realpath(f2))
self.assertEqualCI(ntpath.realpath(s2b(f)), ntpath.realpath(s2b(f2)))

# realpath for UNC path is the same
file = ntpath.join(f, "file1")
open(file, "w+").close()
file2 = ntpath.join(f2, "file1")
unc1 = to_unc(file)
unc2 = to_unc(file2)
self.assertEqualCI(ntpath.realpath(unc1), ntpath.realpath(unc2))
self.assertEqualCI(ntpath.realpath(s2b(unc1)), ntpath.realpath(s2b(unc2)))

# realpath for non-existent file F in symlinked folder
# is original folder + F
file = ntpath.join(f, "missing")
file2 = ntpath.join(f2, "missing")
self.assertEqualCI(ntpath.realpath(file), ntpath.realpath(file2))
self.assertEqualCI(ntpath.realpath(s2b(file)), ntpath.realpath(s2b(file2)))

# realpath for short names used for non 8.3 directory names
dir = ntpath.join(f, "somelongname")
os.mkdir(dir)
file = ntpath.join(dir, "f")
open(file, "w+").close()
file_in_f2 = ntpath.join(f2, "somelo~1", "f")
self.assertEqualCI(ntpath.realpath(file), ntpath.realpath(file_in_f2))
self.assertEqualCI(ntpath.realpath(s2b(file)), ntpath.realpath(s2b(file_in_f2)))
finally:
os.unlink(f2)


@unittest.skipUnless(nt, "abspath requires 'nt' module")
def test_abspath(self):
tester('ntpath.abspath("C:\\")', "C:\\")
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_shutil.py
Original file line number Diff line number Diff line change
Expand Up @@ -1756,7 +1756,7 @@ def test_move_dangling_symlink(self):
self.assertTrue(os.path.islink(dst_link))
# On Windows, os.path.realpath does not follow symlinks (issue #9949)
if os.name == 'nt':
self.assertEqual(os.path.realpath(src), os.readlink(dst_link))
self.assertEqual(os.path.realpath(src), os.path.realpath(os.readlink(dst_link)))
else:
self.assertEqual(os.path.realpath(src), os.path.realpath(dst_link))

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ntpath.realpath() now uses ``GetFinalPathNameByHandle()``.