diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 1478b34bc95514..77cfcc08d7c7a1 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -25,7 +25,8 @@ Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, ``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``mailto``, ``mms``, ``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtspu``, ``sftp``, ``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, ``telnet``, -``wais``, ``ws``, ``wss``. +``wais``, ``ws``, ``wss``. The behavior of other schemes may be controlled with +a ``UrlFlag`` passed to dependent functions. The :mod:`urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in @@ -37,30 +38,35 @@ URL Parsing The URL parsing functions focus on splitting a URL string into its components, or on combining URL components into a URL string. -.. function:: urlparse(urlstring, scheme='', allow_fragments=True) +.. function:: urlparse(urlstring, scheme='', allow_fragments=True, *, flags=None) - Parse a URL into six components, returning a 6-item :term:`named tuple`. This - corresponds to the general structure of a URL: - ``scheme://netloc/path;parameters?query#fragment``. - Each tuple item is a string, possibly empty. The components are not broken up - into smaller parts (for example, the network location is a single string), and % + Parse a URL into six components with respect to given scheme classes, + returning a 6-item :term:`named tuple`. This corresponds to the general + structure of a URL: ``scheme://netloc/path;parameters?query#fragment``. Each + tuple item is a string, possibly empty. The components are not broken up into + smaller parts (for example, the network location is a single string), and % escapes are not expanded. The delimiters as shown above are not part of the - result, except for a leading slash in the *path* component, which is retained if - present. For example: + result, except for a leading slash in the *path* component, which is retained + if present. + + The scheme of the URL determines whether or not parameters are parsed as + distinct from the path. To override the scheme and parse parameters anyway, + pass the corresponding SchemeFlag. + + For example: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse + >>> from urllib.parse import urlparse, PARAMS >>> urlparse("scheme://netloc/path;parameters?query#fragment") - ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='', - query='query', fragment='fragment') + ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='', query='query', fragment='fragment') + >>> urlparse("scheme://netloc/path;parameters?query#fragment", flags=PARAMS) + ParseResult(scheme='scheme', netloc='netloc', path='/path', params='parameters', query='query', fragment='fragment') >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?" ... "highlight=params#url-parsing") >>> o - ParseResult(scheme='http', netloc='docs.python.org:80', - path='/3/library/urllib.parse.html', params='', - query='highlight=params', fragment='url-parsing') + ParseResult(scheme='http', netloc='docs.python.org:80', path='/3/library/urllib.parse.html', params='', query='highlight=params', fragment='url-parsing') >>> o.scheme 'http' >>> o.netloc @@ -82,14 +88,11 @@ or on combining URL components into a URL string. >>> from urllib.parse import urlparse >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', - params='', query='', fragment='') + ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', params='', query='', fragment='') >>> urlparse('help/Python.html') - ParseResult(scheme='', netloc='', path='help/Python.html', params='', - query='', fragment='') + ParseResult(scheme='', netloc='', path='help/Python.html', params='', query='', fragment='') The *scheme* argument gives the default addressing scheme, to be used only if the URL does not specify one. It should be the same type @@ -152,11 +155,9 @@ or on combining URL components into a URL string. >>> from urllib.parse import urlparse >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') >>> u - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') >>> u._replace(scheme='http') - ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') .. versionchanged:: 3.2 @@ -348,19 +349,22 @@ or on combining URL components into a URL string. with an empty query; the RFC states that these are equivalent). -.. function:: urljoin(base, url, allow_fragments=True) +.. function:: urljoin(base, url, allow_fragments=True, *, flags=None) - Construct a full ("absolute") URL by combining a "base URL" (*base*) with - another URL (*url*). Informally, this uses components of the base URL, in - particular the addressing scheme, the network location and (part of) the - path, to provide missing components in the relative URL. For example: + Construct a full ("absolute") URL by combining a "base URL" + (*base*) with another URL (*url*), and with behavior given by a + ``SchemeFlag`` flag. Informally, this uses components of the base + URL, in particular the addressing scheme, the network location and + (part of) the path, to provide missing components in the relative + URL. For example: >>> from urllib.parse import urljoin >>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', 'FAQ.html') 'http://www.cwi.nl/%7Eguido/FAQ.html' - The *allow_fragments* argument has the same meaning and default as for - :func:`urlparse`. + The *allow_fragments* argument has the same meaning and default as + for :func:`urlparse`. As in :func:`urlparse`, a ``SchemeFlag`` may + be given to override behavior inferred by the scheme. .. note:: @@ -543,6 +547,58 @@ operating on :class:`bytes` or :class:`bytearray` objects: .. versionadded:: 3.2 +Special URL Behaviors and Scheme Flags +-------------------------------------- + +:mod:`urllib.parse` recognizes three special properties of URLs, namely relative +addressing (used in, for instance, the ``ftp``, ``http``, or ``gopher`` +protocols), netloc-sensitive resolution (used in the ``ftp``, ``http``, or +``git`` protocols), and URLs that may contain parameters (for instance, ``ftp`` +or ``telnet``). + +Relative addressing allows resolution of relative URLs, and netloc-sensitive +addressing allows resolution with respect to the netloc (domain name) of a URL. +As HTTP URLs have both behaviors by default, this is demonstrated in the +following example: + + >>> from urllib.parse import urljoin + >>> urljoin('http://example.org/post/x/', '../y/') + 'http://example.org/post/y/' + +Additionally, if it is not indicated that a URL is sensitive to parameters +(those specified after a semicolon in the path), then they'll be treated as part +of the path rather than as a distinct component. + +Without specifying optional parameters or modifying global variables, Python +will guess what parameters to apply based on the scheme. Schemes associated with +each are specified by three lists in :mod:`urllib.parse`: + +* ``urllib.parse.uses_relative`` +* ``urllib.parse.uses_netloc`` +* ``urllib.parse.uses_params`` + +In addition, any function that takes a ``flags`` parameter (for +instance, :func:`urlparse` and :func:`urljoin`) may override the +behavior of the ``uses`` lists, for instance, parsing a custom or +widely unused scheme with the same behavior as that of HTTP: + + >>> from urllib.parse import urljoin, NETLOC, RELATIVE + >>> urljoin( + ... 'my-protocol://example.org/post/x/', '../y/', + ... flags=(NETLOC | RELATIVE)) + 'my-protocol://example.org/post/y/' + +Also provided is the ``UNIVERSAL`` flag, which will use all +recognizable elements of a URL (``RELATIVE``, ``NETLOC``, and +``PARAMS``). It is exactly equivalent to the logical or of all other +flags. + +For reference, the following scheme classes are present: + +* ``urllib.parse.RELATIVE`` +* ``urllib.parse.NETLOC`` +* ``urllib.parse.PARAMS`` +* ``urllib.parse.UNIVERSAL`` URL Quoting ----------- diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 2f629c72ae784e..e1c40e5919f958 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -213,12 +213,12 @@ def _encode(t): split = (scheme,) + split self.checkRoundtrips(url, parsed, split) - def checkJoin(self, base, relurl, expected): + def checkJoin(self, base, relurl, expected, flags=urllib.parse.SchemeFlag(0)): str_components = (base, relurl, expected) - self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + self.assertEqual(urllib.parse.urljoin(base, relurl, flags=flags), expected) bytes_components = baseb, relurlb, expectedb = [ x.encode('ascii') for x in str_components] - self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + self.assertEqual(urllib.parse.urljoin(baseb, relurlb, flags=flags), expectedb) def test_unparse_parse(self): str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] @@ -417,6 +417,11 @@ def test_urljoins(self): self.checkJoin('svn+ssh://pathtorepo/dir1', 'dir2', 'svn+ssh://pathtorepo/dir2') self.checkJoin('ws://a/b','g','ws://a/g') self.checkJoin('wss://a/b','g','wss://a/g') + self.checkJoin( + 'nonsensebase://net.loc/url/', '..', + 'nonsensebase://net.loc/', + flags=(urllib.parse.SchemeFlag.RELATIVE | urllib.parse.SchemeFlag.NETLOC), + ) # XXX: The following tests are no longer compatible with RFC3986 # self.checkJoin(SIMPLE_BASE, '../../../g','http://a/../g') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 67ba308c409a2f..b65a067d93a73b 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -28,6 +28,7 @@ """ from collections import namedtuple +import enum import functools import re import sys @@ -39,12 +40,35 @@ "parse_qsl", "quote", "quote_plus", "quote_from_bytes", "unquote", "unquote_plus", "unquote_to_bytes", "DefragResult", "ParseResult", "SplitResult", + "SchemeFlag", "RELATIVE", "NETLOC", "PARAMS", "UNIVERSAL", "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] # A classification of schemes. # The empty string classifies URLs with no scheme specified, # being the default value returned by “urlsplit” and “urlparse”. +class SchemeFlag(enum.Flag): + """SchemeFlag is an enum with the members RELATIVE, NETLOC, and + PARAMS. These describe methods for URL resolution, usually by + scheme. These resolution classes determine, namely, whether a + scheme supports, respectively, relative addressing, preserving the + netloc (domain name), and preserving the parameters. + """ + RELATIVE = enum.auto() + NETLOC = enum.auto() + PARAMS = enum.auto() + UNIVERSAL = RELATIVE | NETLOC | PARAMS + + def __repr__(self): + return f'{self.__module__}.{self._name_}' + + __str__ = __repr__ + +RELATIVE, NETLOC, PARAMS = SchemeFlag +# UNIVERSAL must be assigned separately as it's a combination of other variants. +UNIVERSAL = SchemeFlag.UNIVERSAL + + uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', 'prospero', 'rtsp', 'rtspu', 'sftp', @@ -60,6 +84,30 @@ 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', 'mms', 'sftp', 'tel'] + +def _scheme_classes(scheme, overrides=None): + """Find out what scheme classes a given scheme fits in. + + This consults the variables uses_relative, uses_netloc, and + uses_params. It returns a set of all the classes that apply, with + at least the unique classes specified by the optional overrides + parameter. + """ + if overrides is None: + overrides = SchemeFlag(0) + + if scheme in uses_relative: + overrides |= RELATIVE + + if scheme in uses_netloc: + overrides |= NETLOC + + if scheme in uses_params: + overrides |= PARAMS + + return overrides + + # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) @@ -363,7 +411,7 @@ def _fix_result_transcoding(): _fix_result_transcoding() del _fix_result_transcoding -def urlparse(url, scheme='', allow_fragments=True): +def urlparse(url, scheme='', allow_fragments=True, *, flags=None): """Parse a URL into 6 components: :///;?# @@ -386,7 +434,8 @@ def urlparse(url, scheme='', allow_fragments=True): url, scheme, _coerce_result = _coerce_args(url, scheme) splitresult = urlsplit(url, scheme, allow_fragments) scheme, netloc, url, query, fragment = splitresult - if scheme in uses_params and ';' in url: + scheme_classes = _scheme_classes(scheme, overrides=flags) + if PARAMS in scheme_classes and ';' in url: url, params = _splitparams(url) else: params = '' @@ -500,7 +549,9 @@ def urlunsplit(components): empty query; the RFC states that these are equivalent).""" scheme, netloc, url, query, fragment, _coerce_result = ( _coerce_args(*components)) - if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'): + + scheme_classes = _scheme_classes(scheme) + if netloc or (scheme and (NETLOC in scheme_classes) and url[:2] != '//'): if url and url[:1] != '/': url = '/' + url url = '//' + (netloc or '') + url if scheme: @@ -511,9 +562,10 @@ def urlunsplit(components): url = url + '#' + fragment return _coerce_result(url) -def urljoin(base, url, allow_fragments=True): +def urljoin(base, url, allow_fragments=True, *, flags=None): """Join a base URL and a possibly relative URL to form an absolute - interpretation of the latter.""" + interpretation of the latter. Some logic may be enabled by setting + the flags variable.""" if not base: return url if not url: @@ -521,13 +573,15 @@ def urljoin(base, url, allow_fragments=True): base, url, _coerce_result = _coerce_args(base, url) bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ - urlparse(base, '', allow_fragments) + urlparse(base, '', allow_fragments, flags=flags) scheme, netloc, path, params, query, fragment = \ - urlparse(url, bscheme, allow_fragments) + urlparse(url, bscheme, allow_fragments, flags=flags) - if scheme != bscheme or scheme not in uses_relative: + scheme_classes = _scheme_classes(scheme, overrides=flags) + + if scheme != bscheme or RELATIVE not in scheme_classes: return _coerce_result(url) - if scheme in uses_netloc: + if NETLOC in scheme_classes: if netloc: return _coerce_result(urlunparse((scheme, netloc, path, params, query, fragment))) diff --git a/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst b/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst new file mode 100644 index 00000000000000..5af5eccc0aa969 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst @@ -0,0 +1 @@ +Expose scheme-specific URL options in ``urllib.parse.urljoin`` and ``urllib.parse.urlparse`` via the enum ``urllib.parse.SchemeFlag``.