From ee890ed6b4cfaea266e383a6cec8463477750ea0 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Mon, 10 Jan 2022 13:20:37 -0700 Subject: [PATCH 01/22] bpo-46337: Enumerate URL types for urllib's scheme-based behavior Some features in urllib are dependent on schemes, (i.e., preserving the netloc in url joining). Prior to this patch, this was governed by the uses_* lists (uses_relative, uses_netloc, uses_params) which hard code these attributes for certain schemes. Providing an enum interface and a 'constructor' that allows overrides makes this mechanism a bit more flexible for future modifications. --- Lib/urllib/parse.py | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 67ba308c409a2f9..a151e5fc99f8798 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -28,6 +28,7 @@ """ from collections import namedtuple +from enum import Enum import functools import re import sys @@ -38,13 +39,19 @@ "urlsplit", "urlunsplit", "urlencode", "parse_qs", "parse_qsl", "quote", "quote_plus", "quote_from_bytes", "unquote", "unquote_plus", "unquote_to_bytes", - "DefragResult", "ParseResult", "SplitResult", + "DefragResult", "ParseResult", "SplitResult", "SchemeClass", "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] # A classification of schemes. # The empty string classifies URLs with no scheme specified, # being the default value returned by “urlsplit” and “urlparse”. +"""SchemeClass is an enum with members. RELATIVE, NETLOC, and PARAMS. These +describe methods for URL resolution, usually by scheme. These resolution classes +determine, namely, whether a scheme supports, respectively, relative addressing, +preserving the netloc (domain name), and preserving the parameters.""" +SchemeClass = Enum('SchemeClass', 'RELATIVE NETLOC PARAMS') + uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', 'prospero', 'rtsp', 'rtspu', 'sftp', @@ -60,6 +67,24 @@ 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', 'mms', 'sftp', 'tel'] +def _scheme_classes(scheme, overrides=set()): + """Find out what scheme classes a given scheme fits in. This consults the + variables uses_relative, uses_netloc, and uses_params. It returns a set of + all the classes that apply, with at least the unique classes specified by + the optional overrides parameter.""" + scheme_classes = set(overrides) + + if scheme in uses_relative: + scheme_classes.add(SchemeClass.RELATIVE) + + if scheme in uses_netloc: + scheme_classes.add(SchemeClass.NETLOC) + + if scheme in uses_params: + scheme_classes.add(SchemeClass.PARAMS) + + return scheme_classes + # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) @@ -386,7 +411,8 @@ def urlparse(url, scheme='', allow_fragments=True): url, scheme, _coerce_result = _coerce_args(url, scheme) splitresult = urlsplit(url, scheme, allow_fragments) scheme, netloc, url, query, fragment = splitresult - if scheme in uses_params and ';' in url: + scheme_classes = _scheme_classes(scheme) + if SchemeClass.PARAMS in scheme_classes and ';' in url: url, params = _splitparams(url) else: params = '' @@ -500,7 +526,9 @@ def urlunsplit(components): empty query; the RFC states that these are equivalent).""" scheme, netloc, url, query, fragment, _coerce_result = ( _coerce_args(*components)) - if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'): + + scheme_classes = _scheme_classes(scheme) + if netloc or (scheme and SchemeClass.NETLOC in scheme_classes and url[:2] != '//'): if url and url[:1] != '/': url = '/' + url url = '//' + (netloc or '') + url if scheme: @@ -525,9 +553,11 @@ def urljoin(base, url, allow_fragments=True): scheme, netloc, path, params, query, fragment = \ urlparse(url, bscheme, allow_fragments) - if scheme != bscheme or scheme not in uses_relative: + scheme_classes = _scheme_classes(scheme) + + if scheme != bscheme or SchemeClass.RELATIVE not in scheme_classes: return _coerce_result(url) - if scheme in uses_netloc: + if SchemeClass.NETLOC in scheme_classes: if netloc: return _coerce_result(urlunparse((scheme, netloc, path, params, query, fragment))) From 53c6ccc1ae149c392f4ae39c013ddb6bc8207a45 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Mon, 10 Jan 2022 14:08:26 -0700 Subject: [PATCH 02/22] bpo-46337: Allow caller modification of url classes This allows the callers of urljoin and urlparse to add guaranteed scheme classes to the url regardless of the actual scheme, which may not be in the default uses_* lists of schemes. This call-time behavior is done through an optional parameter that preserves backwards compatibility. A test case is added for this, and requires the change present in test_urlparse.checkJoin. --- Lib/test/test_urlparse.py | 11 ++++++++--- Lib/urllib/parse.py | 15 ++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 2f629c72ae784e6..38accdda5dc92c9 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -213,12 +213,12 @@ def _encode(t): split = (scheme,) + split self.checkRoundtrips(url, parsed, split) - def checkJoin(self, base, relurl, expected): + def checkJoin(self, base, relurl, expected, classes=[]): str_components = (base, relurl, expected) - self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + self.assertEqual(urllib.parse.urljoin(base, relurl, classes=classes), expected) bytes_components = baseb, relurlb, expectedb = [ x.encode('ascii') for x in str_components] - self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + self.assertEqual(urllib.parse.urljoin(baseb, relurlb, classes=classes), expectedb) def test_unparse_parse(self): str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] @@ -417,6 +417,11 @@ def test_urljoins(self): self.checkJoin('svn+ssh://pathtorepo/dir1', 'dir2', 'svn+ssh://pathtorepo/dir2') self.checkJoin('ws://a/b','g','ws://a/g') self.checkJoin('wss://a/b','g','wss://a/g') + self.checkJoin( + 'nonsensebase://net.loc/url/', '..', + 'nonsensebase://net.loc/', + classes=[urllib.parse.SchemeClass.RELATIVE, urllib.parse.SchemeClass.NETLOC], + ) # XXX: The following tests are no longer compatible with RFC3986 # self.checkJoin(SIMPLE_BASE, '../../../g','http://a/../g') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index a151e5fc99f8798..3481664d0d0db48 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -388,7 +388,7 @@ def _fix_result_transcoding(): _fix_result_transcoding() del _fix_result_transcoding -def urlparse(url, scheme='', allow_fragments=True): +def urlparse(url, scheme='', allow_fragments=True, classes=set()): """Parse a URL into 6 components: :///;?# @@ -411,7 +411,7 @@ def urlparse(url, scheme='', allow_fragments=True): url, scheme, _coerce_result = _coerce_args(url, scheme) splitresult = urlsplit(url, scheme, allow_fragments) scheme, netloc, url, query, fragment = splitresult - scheme_classes = _scheme_classes(scheme) + scheme_classes = _scheme_classes(scheme, overrides=classes) if SchemeClass.PARAMS in scheme_classes and ';' in url: url, params = _splitparams(url) else: @@ -539,9 +539,10 @@ def urlunsplit(components): url = url + '#' + fragment return _coerce_result(url) -def urljoin(base, url, allow_fragments=True): +def urljoin(base, url, allow_fragments=True, classes=set()): """Join a base URL and a possibly relative URL to form an absolute - interpretation of the latter.""" + interpretation of the latter. Some logic may be enabled by setting + the classes variable.""" if not base: return url if not url: @@ -549,11 +550,11 @@ def urljoin(base, url, allow_fragments=True): base, url, _coerce_result = _coerce_args(base, url) bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ - urlparse(base, '', allow_fragments) + urlparse(base, '', allow_fragments, classes=classes) scheme, netloc, path, params, query, fragment = \ - urlparse(url, bscheme, allow_fragments) + urlparse(url, bscheme, allow_fragments, classes=classes) - scheme_classes = _scheme_classes(scheme) + scheme_classes = _scheme_classes(scheme, overrides=classes) if scheme != bscheme or SchemeClass.RELATIVE not in scheme_classes: return _coerce_result(url) From 41d3b58eda7dfe8d9fe5e932a49441ec8fddbac3 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 22:03:03 +0000 Subject: [PATCH 03/22] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst diff --git a/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst b/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst new file mode 100644 index 000000000000000..8580290c62f8dd9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst @@ -0,0 +1 @@ +Expose scheme-specific URL options in ``urllib.parse.urljoin`` and ``urllib.parse.urlparse`` via the enum ``urllib.parse.SchemeClass``. \ No newline at end of file From eee880c997dcfe90112fa762ab250c9ffc4e6684 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Sat, 26 Feb 2022 10:50:35 -0700 Subject: [PATCH 04/22] bpo-46337: Fix grammar of doc comment. --- Lib/urllib/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 3481664d0d0db48..17cacf613a823a6 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -46,7 +46,7 @@ # The empty string classifies URLs with no scheme specified, # being the default value returned by “urlsplit” and “urlparse”. -"""SchemeClass is an enum with members. RELATIVE, NETLOC, and PARAMS. These +"""SchemeClass is an enum with the members RELATIVE, NETLOC, and PARAMS. These describe methods for URL resolution, usually by scheme. These resolution classes determine, namely, whether a scheme supports, respectively, relative addressing, preserving the netloc (domain name), and preserving the parameters.""" From f9b59dd02d602df77703b0cf9297082c4eb0b73d Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Sat, 26 Feb 2022 11:48:50 -0700 Subject: [PATCH 05/22] bpo-46337: document SchemeClass behavior This functionality was exposed in 53c6ccc. --- Doc/library/urllib.parse.rst | 89 ++++++++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 15 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 1478b34bc95514a..750ee4cb389b500 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -25,7 +25,8 @@ Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, ``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``mailto``, ``mms``, ``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtspu``, ``sftp``, ``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, ``telnet``, -``wais``, ``ws``, ``wss``. +``wais``, ``ws``, ``wss``. The behavior of other schemes may be controlled with +a collection of ``UrlClass`` enums passed to dependent functions. The :mod:`urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in @@ -37,24 +38,33 @@ URL Parsing The URL parsing functions focus on splitting a URL string into its components, or on combining URL components into a URL string. -.. function:: urlparse(urlstring, scheme='', allow_fragments=True) +.. function:: urlparse(urlstring, scheme='', allow_fragments=True, classes=set()) - Parse a URL into six components, returning a 6-item :term:`named tuple`. This - corresponds to the general structure of a URL: - ``scheme://netloc/path;parameters?query#fragment``. - Each tuple item is a string, possibly empty. The components are not broken up - into smaller parts (for example, the network location is a single string), and % + Parse a URL into six components with respect to given scheme classes, + returning a 6-item :term:`named tuple`. This corresponds to the general + structure of a URL: ``scheme://netloc/path;parameters?query#fragment``. Each + tuple item is a string, possibly empty. The components are not broken up into + smaller parts (for example, the network location is a single string), and % escapes are not expanded. The delimiters as shown above are not part of the - result, except for a leading slash in the *path* component, which is retained if - present. For example: + result, except for a leading slash in the *path* component, which is retained + if present. + + The scheme of the URL determines whether or not parameters are parsed as + distinct from the path. To override the scheme and parse parameters anyway, + pass a set containing ``SchemeClass.PARAMS``. + + For example: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse + >>> from urllib.parse import urlparse, SchemeClass >>> urlparse("scheme://netloc/path;parameters?query#fragment") ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='', query='query', fragment='fragment') + >>> urlparse("scheme://netloc/path;parameters?query#fragment", classes=[SchemeClass.PARAMS]) + ParseResult(scheme='scheme', netloc='netloc', path='/path', + params=';parameters', query='query', fragment='fragment') >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?" ... "highlight=params#url-parsing") >>> o @@ -348,19 +358,21 @@ or on combining URL components into a URL string. with an empty query; the RFC states that these are equivalent). -.. function:: urljoin(base, url, allow_fragments=True) +.. function:: urljoin(base, url, allow_fragments=True, classes=set()) Construct a full ("absolute") URL by combining a "base URL" (*base*) with - another URL (*url*). Informally, this uses components of the base URL, in - particular the addressing scheme, the network location and (part of) the - path, to provide missing components in the relative URL. For example: + another URL (*url*), and with behavior given by a set of ``SchemeClass`` + enums. Informally, this uses components of the base URL, in particular the + addressing scheme, the network location and (part of) the path, to provide + missing components in the relative URL. For example: >>> from urllib.parse import urljoin >>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', 'FAQ.html') 'http://www.cwi.nl/%7Eguido/FAQ.html' The *allow_fragments* argument has the same meaning and default as for - :func:`urlparse`. + :func:`urlparse`. As in :func:`urlparse`, a ``SchemeClass`` set may be given + to override behavior inferred by the scheme. .. note:: @@ -543,6 +555,53 @@ operating on :class:`bytes` or :class:`bytearray` objects: .. versionadded:: 3.2 +Special URL Behaviors and Scheme Classes +---------------------------------------- + +:mod:`urllib.parse` recognizes three special properties of URLs, namely relative +addressing (used in, for instance, the ``ftp``, ``http``, or ``gopher`` +protocols), netloc-sensitive resolution (used in the ``ftp``, ``http``, or +``git`` protocols), and URLs that may contain parameters (for instance, ``ftp`` +or ``telnet``). + +Relative addressing allows resolution of relative URLs, and netloc-sensitive +addressing allows resolution with respect to the netloc (domain name) of a URL. +As HTTP URLs have both behaviors by default, this is demonstrated in the +following example: + + >>> from urllib.parse import urljoin + >>> urljoin('http://example.org/post/x', '../y') + 'http://example.org/post/y' + +Additionally, if it is not indicated that a URL is sensitive to parameters +(those specified after a semicolon in the path), then they'll be treated as part +of the path rather than as a distinct component. + +Without specifying optional parameters or modifying global variables, Python +will guess what parameters to apply based on the scheme. Schemes associated with +each are specified by three lists in :mod:`urllib.parse`: + +* ``urllib.uses_relative`` +* ``urllib.uses_netloc`` +* ``urllib.uses_params`` + +In addition, any function that takes a ``classes`` parameter (for instance, +:func:`urlparse` and :func:`urljoin`) may override the behavior of the uses +lists, for instance, parsing a custom or widely unused scheme with the same +behavior as that of HTTP: + + >>> from urllib.parse import urljoin, SchemeClass + >>> urljoin( + 'my-protocol://example.org/post/x', '../y', + classes=[SchemeClass.NETLOC, SchemeClass.RELATIVE]) + 'http://example.org/post/y' + +For reference, the following three scheme classes are present (exactly +corresponding to the uses lists): + +* ``urllib.SchemeClass.RELATIVE`` +* ``urllib.SchemeClass.NETLOC`` +* ``urllib.SchemeClass.PARAMS`` URL Quoting ----------- From 1691a1ed5a8256a05c9843e908c62f598e0b1424 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Sun, 27 Feb 2022 20:36:55 -0700 Subject: [PATCH 06/22] add newline to end of news file It looks like CI expects this when building documentation. --- .../next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst b/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst index 8580290c62f8dd9..c2d311527ddb174 100644 --- a/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst +++ b/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst @@ -1 +1 @@ -Expose scheme-specific URL options in ``urllib.parse.urljoin`` and ``urllib.parse.urlparse`` via the enum ``urllib.parse.SchemeClass``. \ No newline at end of file +Expose scheme-specific URL options in ``urllib.parse.urljoin`` and ``urllib.parse.urlparse`` via the enum ``urllib.parse.SchemeClass``. From c7ae936cade6168bd42a294d2f2cf43233bd7d03 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Fri, 11 Mar 2022 22:07:19 -0700 Subject: [PATCH 07/22] bpo-46337: fix doctest formatting --- Doc/library/urllib.parse.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 750ee4cb389b500..6f5ffe6fe5a7a1f 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -592,8 +592,8 @@ behavior as that of HTTP: >>> from urllib.parse import urljoin, SchemeClass >>> urljoin( - 'my-protocol://example.org/post/x', '../y', - classes=[SchemeClass.NETLOC, SchemeClass.RELATIVE]) + ... 'my-protocol://example.org/post/x', '../y', + ... classes=[SchemeClass.NETLOC, SchemeClass.RELATIVE]) 'http://example.org/post/y' For reference, the following three scheme classes are present (exactly From 4fc9059c6792f75c3a0ead651146a63a4ca72bde Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Sat, 12 Mar 2022 11:31:19 -0700 Subject: [PATCH 08/22] doc: fix some doctests urljoin will not treat `..` as moving up one directory rather than moving up one file, thus causing the doctests to fail due to a missing trailing slash. Both changes are of the form: http://example.org/post/x -> http://example.org/post/x/ Additionally, the my-protocol example's expected output had the wrong scheme. --- Doc/library/urllib.parse.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 6f5ffe6fe5a7a1f..ccd0f218c5564d3 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -570,8 +570,8 @@ As HTTP URLs have both behaviors by default, this is demonstrated in the following example: >>> from urllib.parse import urljoin - >>> urljoin('http://example.org/post/x', '../y') - 'http://example.org/post/y' + >>> urljoin('http://example.org/post/x/', '../y/') + 'http://example.org/post/y/' Additionally, if it is not indicated that a URL is sensitive to parameters (those specified after a semicolon in the path), then they'll be treated as part @@ -592,9 +592,9 @@ behavior as that of HTTP: >>> from urllib.parse import urljoin, SchemeClass >>> urljoin( - ... 'my-protocol://example.org/post/x', '../y', + ... 'my-protocol://example.org/post/x/', '../y/', ... classes=[SchemeClass.NETLOC, SchemeClass.RELATIVE]) - 'http://example.org/post/y' + 'my-protocol://example.org/post/y/' For reference, the following three scheme classes are present (exactly corresponding to the uses lists): From c07600cf249e6870bf432726de9aa9b720babf0b Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Sun, 13 Mar 2022 01:07:10 -0700 Subject: [PATCH 09/22] bpo-43677: fixup PEP 8 style Minor style things: + _scheme_classes' docstring's summary was made explicit. + _scheme_classes was prepended with and followed by two newlines. --- Lib/urllib/parse.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 17cacf613a823a6..9cc778e2939e8db 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -67,11 +67,16 @@ 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', 'mms', 'sftp', 'tel'] + def _scheme_classes(scheme, overrides=set()): - """Find out what scheme classes a given scheme fits in. This consults the - variables uses_relative, uses_netloc, and uses_params. It returns a set of - all the classes that apply, with at least the unique classes specified by - the optional overrides parameter.""" + """Find out what scheme classes a given scheme fits in. + + This consults the variables uses_relative, uses_netloc, and + uses_params. It returns a set of all the classes that apply, with + at least the unique classes specified by the optional overrides + parameter. + + """ scheme_classes = set(overrides) if scheme in uses_relative: @@ -85,6 +90,7 @@ def _scheme_classes(scheme, overrides=set()): return scheme_classes + # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) From 2c4aa3a6636107c368edce23bc6a43c107d1cc14 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Tue, 29 Mar 2022 19:01:08 -0600 Subject: [PATCH 10/22] urllib: use an enum.FLAG for SchemeClass's. This commit also contains some miscellaneous doctest fixes. --- Doc/library/urllib.parse.rst | 50 +++++++++++++++--------------------- Lib/test/test_urlparse.py | 4 +-- Lib/urllib/parse.py | 41 +++++++++++++++-------------- 3 files changed, 45 insertions(+), 50 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index ccd0f218c5564d3..6620b0412788d1a 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -51,7 +51,7 @@ or on combining URL components into a URL string. The scheme of the URL determines whether or not parameters are parsed as distinct from the path. To override the scheme and parse parameters anyway, - pass a set containing ``SchemeClass.PARAMS``. + pass a SchemeClass containing instances of ``SchemeClass.PARAMS``. For example: @@ -60,17 +60,13 @@ or on combining URL components into a URL string. >>> from urllib.parse import urlparse, SchemeClass >>> urlparse("scheme://netloc/path;parameters?query#fragment") - ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='', - query='query', fragment='fragment') - >>> urlparse("scheme://netloc/path;parameters?query#fragment", classes=[SchemeClass.PARAMS]) - ParseResult(scheme='scheme', netloc='netloc', path='/path', - params=';parameters', query='query', fragment='fragment') + ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='', query='query', fragment='fragment') + >>> urlparse("scheme://netloc/path;parameters?query#fragment", classes=SchemeClass.PARAMS) + ParseResult(scheme='scheme', netloc='netloc', path='/path', params='parameters', query='query', fragment='fragment') >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?" ... "highlight=params#url-parsing") >>> o - ParseResult(scheme='http', netloc='docs.python.org:80', - path='/3/library/urllib.parse.html', params='', - query='highlight=params', fragment='url-parsing') + ParseResult(scheme='http', netloc='docs.python.org:80', path='/3/library/urllib.parse.html', params='', query='highlight=params', fragment='url-parsing') >>> o.scheme 'http' >>> o.netloc @@ -92,14 +88,11 @@ or on combining URL components into a URL string. >>> from urllib.parse import urlparse >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', - params='', query='', fragment='') + ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', params='', query='', fragment='') >>> urlparse('help/Python.html') - ParseResult(scheme='', netloc='', path='help/Python.html', params='', - query='', fragment='') + ParseResult(scheme='', netloc='', path='help/Python.html', params='', query='', fragment='') The *scheme* argument gives the default addressing scheme, to be used only if the URL does not specify one. It should be the same type @@ -162,11 +155,9 @@ or on combining URL components into a URL string. >>> from urllib.parse import urlparse >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') >>> u - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') >>> u._replace(scheme='http') - ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') .. versionchanged:: 3.2 @@ -358,21 +349,22 @@ or on combining URL components into a URL string. with an empty query; the RFC states that these are equivalent). -.. function:: urljoin(base, url, allow_fragments=True, classes=set()) +.. function:: urljoin(base, url, allow_fragments=True, classes=SchemeClass.NONE) - Construct a full ("absolute") URL by combining a "base URL" (*base*) with - another URL (*url*), and with behavior given by a set of ``SchemeClass`` - enums. Informally, this uses components of the base URL, in particular the - addressing scheme, the network location and (part of) the path, to provide - missing components in the relative URL. For example: + Construct a full ("absolute") URL by combining a "base URL" + (*base*) with another URL (*url*), and with behavior given by a + ``SchemeClass`` enum. Informally, this uses components of the base + URL, in particular the addressing scheme, the network location and + (part of) the path, to provide missing components in the relative + URL. For example: >>> from urllib.parse import urljoin >>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', 'FAQ.html') 'http://www.cwi.nl/%7Eguido/FAQ.html' - The *allow_fragments* argument has the same meaning and default as for - :func:`urlparse`. As in :func:`urlparse`, a ``SchemeClass`` set may be given - to override behavior inferred by the scheme. + The *allow_fragments* argument has the same meaning and default as + for :func:`urlparse`. As in :func:`urlparse`, a ``SchemeClass`` may + be given to override behavior inferred by the scheme. .. note:: @@ -593,7 +585,7 @@ behavior as that of HTTP: >>> from urllib.parse import urljoin, SchemeClass >>> urljoin( ... 'my-protocol://example.org/post/x/', '../y/', - ... classes=[SchemeClass.NETLOC, SchemeClass.RELATIVE]) + ... classes=(SchemeClass.NETLOC | SchemeClass.RELATIVE)) 'my-protocol://example.org/post/y/' For reference, the following three scheme classes are present (exactly diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 38accdda5dc92c9..923d5ef98f5cb2a 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -213,7 +213,7 @@ def _encode(t): split = (scheme,) + split self.checkRoundtrips(url, parsed, split) - def checkJoin(self, base, relurl, expected, classes=[]): + def checkJoin(self, base, relurl, expected, classes=urllib.parse.SchemeClass.NONE): str_components = (base, relurl, expected) self.assertEqual(urllib.parse.urljoin(base, relurl, classes=classes), expected) bytes_components = baseb, relurlb, expectedb = [ @@ -420,7 +420,7 @@ def test_urljoins(self): self.checkJoin( 'nonsensebase://net.loc/url/', '..', 'nonsensebase://net.loc/', - classes=[urllib.parse.SchemeClass.RELATIVE, urllib.parse.SchemeClass.NETLOC], + classes=(urllib.parse.SchemeClass.RELATIVE | urllib.parse.SchemeClass.NETLOC), ) # XXX: The following tests are no longer compatible with RFC3986 diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 9cc778e2939e8db..64f341d81d8385e 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -28,7 +28,7 @@ """ from collections import namedtuple -from enum import Enum +from enum import Flag, auto import functools import re import sys @@ -46,11 +46,17 @@ # The empty string classifies URLs with no scheme specified, # being the default value returned by “urlsplit” and “urlparse”. -"""SchemeClass is an enum with the members RELATIVE, NETLOC, and PARAMS. These -describe methods for URL resolution, usually by scheme. These resolution classes -determine, namely, whether a scheme supports, respectively, relative addressing, -preserving the netloc (domain name), and preserving the parameters.""" -SchemeClass = Enum('SchemeClass', 'RELATIVE NETLOC PARAMS') +class SchemeClass(Flag): + """SchemeClass is an enum with the members RELATIVE, NETLOC, and + PARAMS. These describe methods for URL resolution, usually by + scheme. These resolution classes determine, namely, whether a + scheme supports, respectively, relative addressing, preserving the + netloc (domain name), and preserving the parameters. + """ + NONE = 0 + RELATIVE = auto() + NETLOC = auto() + PARAMS = auto() uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', @@ -68,27 +74,24 @@ 'mms', 'sftp', 'tel'] -def _scheme_classes(scheme, overrides=set()): +def _scheme_classes(scheme, overrides=SchemeClass.NONE): """Find out what scheme classes a given scheme fits in. This consults the variables uses_relative, uses_netloc, and uses_params. It returns a set of all the classes that apply, with at least the unique classes specified by the optional overrides parameter. - """ - scheme_classes = set(overrides) - if scheme in uses_relative: - scheme_classes.add(SchemeClass.RELATIVE) + overrides |= SchemeClass.RELATIVE if scheme in uses_netloc: - scheme_classes.add(SchemeClass.NETLOC) + overrides |= SchemeClass.NETLOC if scheme in uses_params: - scheme_classes.add(SchemeClass.PARAMS) + overrides |= SchemeClass.PARAMS - return scheme_classes + return overrides # These are not actually used anymore, but should stay for backwards @@ -394,7 +397,7 @@ def _fix_result_transcoding(): _fix_result_transcoding() del _fix_result_transcoding -def urlparse(url, scheme='', allow_fragments=True, classes=set()): +def urlparse(url, scheme='', allow_fragments=True, classes=SchemeClass.NONE): """Parse a URL into 6 components: :///;?# @@ -418,7 +421,7 @@ def urlparse(url, scheme='', allow_fragments=True, classes=set()): splitresult = urlsplit(url, scheme, allow_fragments) scheme, netloc, url, query, fragment = splitresult scheme_classes = _scheme_classes(scheme, overrides=classes) - if SchemeClass.PARAMS in scheme_classes and ';' in url: + if SchemeClass.PARAMS & scheme_classes and ';' in url: url, params = _splitparams(url) else: params = '' @@ -534,7 +537,7 @@ def urlunsplit(components): _coerce_args(*components)) scheme_classes = _scheme_classes(scheme) - if netloc or (scheme and SchemeClass.NETLOC in scheme_classes and url[:2] != '//'): + if netloc or (scheme and (SchemeClass.NETLOC & scheme_classes) and url[:2] != '//'): if url and url[:1] != '/': url = '/' + url url = '//' + (netloc or '') + url if scheme: @@ -545,7 +548,7 @@ def urlunsplit(components): url = url + '#' + fragment return _coerce_result(url) -def urljoin(base, url, allow_fragments=True, classes=set()): +def urljoin(base, url, allow_fragments=True, classes=SchemeClass.NONE): """Join a base URL and a possibly relative URL to form an absolute interpretation of the latter. Some logic may be enabled by setting the classes variable.""" @@ -562,7 +565,7 @@ def urljoin(base, url, allow_fragments=True, classes=set()): scheme_classes = _scheme_classes(scheme, overrides=classes) - if scheme != bscheme or SchemeClass.RELATIVE not in scheme_classes: + if scheme != bscheme or not (SchemeClass.RELATIVE & scheme_classes): return _coerce_result(url) if SchemeClass.NETLOC in scheme_classes: if netloc: From 5f81d161b37cf42d007c7550c0d170f65d54692c Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Tue, 29 Mar 2022 19:20:57 -0600 Subject: [PATCH 11/22] urllib: rename SchemeClass to SchemeFlag --- Doc/library/urllib.parse.rst | 16 ++++++++-------- Lib/test/test_urlparse.py | 8 ++++---- Lib/urllib/parse.py | 34 +++++++++++++++++----------------- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 6620b0412788d1a..c1ad56e256828e5 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -51,17 +51,17 @@ or on combining URL components into a URL string. The scheme of the URL determines whether or not parameters are parsed as distinct from the path. To override the scheme and parse parameters anyway, - pass a SchemeClass containing instances of ``SchemeClass.PARAMS``. + pass the corresponding SchemeFlag. For example: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse, SchemeClass + >>> from urllib.parse import urlparse, SchemeFlag >>> urlparse("scheme://netloc/path;parameters?query#fragment") ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='', query='query', fragment='fragment') - >>> urlparse("scheme://netloc/path;parameters?query#fragment", classes=SchemeClass.PARAMS) + >>> urlparse("scheme://netloc/path;parameters?query#fragment", flags=SchemeFlag.PARAMS) ParseResult(scheme='scheme', netloc='netloc', path='/path', params='parameters', query='query', fragment='fragment') >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?" ... "highlight=params#url-parsing") @@ -582,18 +582,18 @@ In addition, any function that takes a ``classes`` parameter (for instance, lists, for instance, parsing a custom or widely unused scheme with the same behavior as that of HTTP: - >>> from urllib.parse import urljoin, SchemeClass + >>> from urllib.parse import urljoin, SchemeFlag >>> urljoin( ... 'my-protocol://example.org/post/x/', '../y/', - ... classes=(SchemeClass.NETLOC | SchemeClass.RELATIVE)) + ... flags=(SchemeFlag.NETLOC | SchemeFlag.RELATIVE)) 'my-protocol://example.org/post/y/' For reference, the following three scheme classes are present (exactly corresponding to the uses lists): -* ``urllib.SchemeClass.RELATIVE`` -* ``urllib.SchemeClass.NETLOC`` -* ``urllib.SchemeClass.PARAMS`` +* ``urllib.SchemeFlag.RELATIVE`` +* ``urllib.SchemeFlag.NETLOC`` +* ``urllib.SchemeFlag.PARAMS`` URL Quoting ----------- diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 923d5ef98f5cb2a..517ec6031c16158 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -213,12 +213,12 @@ def _encode(t): split = (scheme,) + split self.checkRoundtrips(url, parsed, split) - def checkJoin(self, base, relurl, expected, classes=urllib.parse.SchemeClass.NONE): + def checkJoin(self, base, relurl, expected, flags=urllib.parse.SchemeFlag.NONE): str_components = (base, relurl, expected) - self.assertEqual(urllib.parse.urljoin(base, relurl, classes=classes), expected) + self.assertEqual(urllib.parse.urljoin(base, relurl, flags=flags), expected) bytes_components = baseb, relurlb, expectedb = [ x.encode('ascii') for x in str_components] - self.assertEqual(urllib.parse.urljoin(baseb, relurlb, classes=classes), expectedb) + self.assertEqual(urllib.parse.urljoin(baseb, relurlb, flags=flags), expectedb) def test_unparse_parse(self): str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] @@ -420,7 +420,7 @@ def test_urljoins(self): self.checkJoin( 'nonsensebase://net.loc/url/', '..', 'nonsensebase://net.loc/', - classes=(urllib.parse.SchemeClass.RELATIVE | urllib.parse.SchemeClass.NETLOC), + flags=(urllib.parse.SchemeFlag.RELATIVE | urllib.parse.SchemeFlag.NETLOC), ) # XXX: The following tests are no longer compatible with RFC3986 diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 64f341d81d8385e..dfe72ae8947504c 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -39,15 +39,15 @@ "urlsplit", "urlunsplit", "urlencode", "parse_qs", "parse_qsl", "quote", "quote_plus", "quote_from_bytes", "unquote", "unquote_plus", "unquote_to_bytes", - "DefragResult", "ParseResult", "SplitResult", "SchemeClass", + "DefragResult", "ParseResult", "SplitResult", "SchemeFlag", "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] # A classification of schemes. # The empty string classifies URLs with no scheme specified, # being the default value returned by “urlsplit” and “urlparse”. -class SchemeClass(Flag): - """SchemeClass is an enum with the members RELATIVE, NETLOC, and +class SchemeFlag(Flag): + """SchemeFlag is an enum with the members RELATIVE, NETLOC, and PARAMS. These describe methods for URL resolution, usually by scheme. These resolution classes determine, namely, whether a scheme supports, respectively, relative addressing, preserving the @@ -74,7 +74,7 @@ class SchemeClass(Flag): 'mms', 'sftp', 'tel'] -def _scheme_classes(scheme, overrides=SchemeClass.NONE): +def _scheme_classes(scheme, overrides=SchemeFlag.NONE): """Find out what scheme classes a given scheme fits in. This consults the variables uses_relative, uses_netloc, and @@ -83,13 +83,13 @@ def _scheme_classes(scheme, overrides=SchemeClass.NONE): parameter. """ if scheme in uses_relative: - overrides |= SchemeClass.RELATIVE + overrides |= SchemeFlag.RELATIVE if scheme in uses_netloc: - overrides |= SchemeClass.NETLOC + overrides |= SchemeFlag.NETLOC if scheme in uses_params: - overrides |= SchemeClass.PARAMS + overrides |= SchemeFlag.PARAMS return overrides @@ -397,7 +397,7 @@ def _fix_result_transcoding(): _fix_result_transcoding() del _fix_result_transcoding -def urlparse(url, scheme='', allow_fragments=True, classes=SchemeClass.NONE): +def urlparse(url, scheme='', allow_fragments=True, flags=SchemeFlag.NONE): """Parse a URL into 6 components: :///;?# @@ -420,8 +420,8 @@ def urlparse(url, scheme='', allow_fragments=True, classes=SchemeClass.NONE): url, scheme, _coerce_result = _coerce_args(url, scheme) splitresult = urlsplit(url, scheme, allow_fragments) scheme, netloc, url, query, fragment = splitresult - scheme_classes = _scheme_classes(scheme, overrides=classes) - if SchemeClass.PARAMS & scheme_classes and ';' in url: + scheme_classes = _scheme_classes(scheme, overrides=flags) + if SchemeFlag.PARAMS & scheme_classes and ';' in url: url, params = _splitparams(url) else: params = '' @@ -537,7 +537,7 @@ def urlunsplit(components): _coerce_args(*components)) scheme_classes = _scheme_classes(scheme) - if netloc or (scheme and (SchemeClass.NETLOC & scheme_classes) and url[:2] != '//'): + if netloc or (scheme and (SchemeFlag.NETLOC & scheme_classes) and url[:2] != '//'): if url and url[:1] != '/': url = '/' + url url = '//' + (netloc or '') + url if scheme: @@ -548,7 +548,7 @@ def urlunsplit(components): url = url + '#' + fragment return _coerce_result(url) -def urljoin(base, url, allow_fragments=True, classes=SchemeClass.NONE): +def urljoin(base, url, allow_fragments=True, flags=SchemeFlag.NONE): """Join a base URL and a possibly relative URL to form an absolute interpretation of the latter. Some logic may be enabled by setting the classes variable.""" @@ -559,15 +559,15 @@ def urljoin(base, url, allow_fragments=True, classes=SchemeClass.NONE): base, url, _coerce_result = _coerce_args(base, url) bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ - urlparse(base, '', allow_fragments, classes=classes) + urlparse(base, '', allow_fragments, flags=flags) scheme, netloc, path, params, query, fragment = \ - urlparse(url, bscheme, allow_fragments, classes=classes) + urlparse(url, bscheme, allow_fragments, flags=flags) - scheme_classes = _scheme_classes(scheme, overrides=classes) + scheme_classes = _scheme_classes(scheme, overrides=flags) - if scheme != bscheme or not (SchemeClass.RELATIVE & scheme_classes): + if scheme != bscheme or not (SchemeFlag.RELATIVE & scheme_classes): return _coerce_result(url) - if SchemeClass.NETLOC in scheme_classes: + if SchemeFlag.NETLOC in scheme_classes: if netloc: return _coerce_result(urlunparse((scheme, netloc, path, params, query, fragment))) From 226bbe96d71e48217f52aba7ead1b8526429a7d7 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Thu, 31 Mar 2022 13:02:27 -0600 Subject: [PATCH 12/22] urllib: remove SchemeFlag.NONE SchemeFlag(0) is equivalent. --- Lib/test/test_urlparse.py | 2 +- Lib/urllib/parse.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 517ec6031c16158..e1c40e5919f9585 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -213,7 +213,7 @@ def _encode(t): split = (scheme,) + split self.checkRoundtrips(url, parsed, split) - def checkJoin(self, base, relurl, expected, flags=urllib.parse.SchemeFlag.NONE): + def checkJoin(self, base, relurl, expected, flags=urllib.parse.SchemeFlag(0)): str_components = (base, relurl, expected) self.assertEqual(urllib.parse.urljoin(base, relurl, flags=flags), expected) bytes_components = baseb, relurlb, expectedb = [ diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index dfe72ae8947504c..9336747082502b4 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -53,7 +53,6 @@ class SchemeFlag(Flag): scheme supports, respectively, relative addressing, preserving the netloc (domain name), and preserving the parameters. """ - NONE = 0 RELATIVE = auto() NETLOC = auto() PARAMS = auto() @@ -74,7 +73,7 @@ class SchemeFlag(Flag): 'mms', 'sftp', 'tel'] -def _scheme_classes(scheme, overrides=SchemeFlag.NONE): +def _scheme_classes(scheme, overrides=SchemeFlag(0)): """Find out what scheme classes a given scheme fits in. This consults the variables uses_relative, uses_netloc, and @@ -397,7 +396,7 @@ def _fix_result_transcoding(): _fix_result_transcoding() del _fix_result_transcoding -def urlparse(url, scheme='', allow_fragments=True, flags=SchemeFlag.NONE): +def urlparse(url, scheme='', allow_fragments=True, flags=SchemeFlag(0)): """Parse a URL into 6 components: :///;?# @@ -548,7 +547,7 @@ def urlunsplit(components): url = url + '#' + fragment return _coerce_result(url) -def urljoin(base, url, allow_fragments=True, flags=SchemeFlag.NONE): +def urljoin(base, url, allow_fragments=True, flags=SchemeFlag(0)): """Join a base URL and a possibly relative URL to form an absolute interpretation of the latter. Some logic may be enabled by setting the classes variable.""" From ff888811ee68f2c623b265a514e213c6f42cfda9 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Thu, 31 Mar 2022 13:04:18 -0600 Subject: [PATCH 13/22] doc/urllib: remove old references to SchemeClass This was renamed to urllib; docs should reflect that. --- Doc/library/urllib.parse.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index c1ad56e256828e5..50b5a8005329f4f 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -349,11 +349,11 @@ or on combining URL components into a URL string. with an empty query; the RFC states that these are equivalent). -.. function:: urljoin(base, url, allow_fragments=True, classes=SchemeClass.NONE) +.. function:: urljoin(base, url, allow_fragments=True, classes=SchemeFlag.NONE) Construct a full ("absolute") URL by combining a "base URL" (*base*) with another URL (*url*), and with behavior given by a - ``SchemeClass`` enum. Informally, this uses components of the base + ``SchemeFlag`` flag. Informally, this uses components of the base URL, in particular the addressing scheme, the network location and (part of) the path, to provide missing components in the relative URL. For example: @@ -363,7 +363,7 @@ or on combining URL components into a URL string. 'http://www.cwi.nl/%7Eguido/FAQ.html' The *allow_fragments* argument has the same meaning and default as - for :func:`urlparse`. As in :func:`urlparse`, a ``SchemeClass`` may + for :func:`urlparse`. As in :func:`urlparse`, a ``SchemeFlag`` may be given to override behavior inferred by the scheme. .. note:: @@ -547,8 +547,8 @@ operating on :class:`bytes` or :class:`bytearray` objects: .. versionadded:: 3.2 -Special URL Behaviors and Scheme Classes ----------------------------------------- +Special URL Behaviors and Scheme Flags +-------------------------------------- :mod:`urllib.parse` recognizes three special properties of URLs, namely relative addressing (used in, for instance, the ``ftp``, ``http``, or ``gopher`` From bf64df0eaef28ce54448b4ba3b49a88054fb3f8a Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Thu, 31 Mar 2022 13:06:47 -0600 Subject: [PATCH 14/22] news: update news file to use SchemeFlag name --- .../next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst b/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst index c2d311527ddb174..5af5eccc0aa9694 100644 --- a/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst +++ b/Misc/NEWS.d/next/Library/2022-01-10-22-03-02.bpo-46337.qkFZAw.rst @@ -1 +1 @@ -Expose scheme-specific URL options in ``urllib.parse.urljoin`` and ``urllib.parse.urlparse`` via the enum ``urllib.parse.SchemeClass``. +Expose scheme-specific URL options in ``urllib.parse.urljoin`` and ``urllib.parse.urlparse`` via the enum ``urllib.parse.SchemeFlag``. From 6a09c3802c8f54079405acac5c22d4f81c7c99f8 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Thu, 31 Mar 2022 15:50:34 -0600 Subject: [PATCH 15/22] doc/urllib: fix formatting and wording --- Doc/library/urllib.parse.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 50b5a8005329f4f..09d6e6a6ee10a0e 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -26,7 +26,7 @@ Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, ``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtspu``, ``sftp``, ``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, ``telnet``, ``wais``, ``ws``, ``wss``. The behavior of other schemes may be controlled with -a collection of ``UrlClass`` enums passed to dependent functions. +a ``UrlFlag`` passed to dependent functions. The :mod:`urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in @@ -349,7 +349,7 @@ or on combining URL components into a URL string. with an empty query; the RFC states that these are equivalent). -.. function:: urljoin(base, url, allow_fragments=True, classes=SchemeFlag.NONE) +.. function:: urljoin(base, url, allow_fragments=True, classes=SchemeFlag(0)) Construct a full ("absolute") URL by combining a "base URL" (*base*) with another URL (*url*), and with behavior given by a @@ -577,10 +577,10 @@ each are specified by three lists in :mod:`urllib.parse`: * ``urllib.uses_netloc`` * ``urllib.uses_params`` -In addition, any function that takes a ``classes`` parameter (for instance, -:func:`urlparse` and :func:`urljoin`) may override the behavior of the uses -lists, for instance, parsing a custom or widely unused scheme with the same -behavior as that of HTTP: +In addition, any function that takes a ``flags`` parameter (for +instance, :func:`urlparse` and :func:`urljoin`) may override the +behavior of the ``uses`` lists, for instance, parsing a custom or +widely unused scheme with the same behavior as that of HTTP: >>> from urllib.parse import urljoin, SchemeFlag >>> urljoin( @@ -589,7 +589,7 @@ behavior as that of HTTP: 'my-protocol://example.org/post/y/' For reference, the following three scheme classes are present (exactly -corresponding to the uses lists): +corresponding to the ``uses`` lists): * ``urllib.SchemeFlag.RELATIVE`` * ``urllib.SchemeFlag.NETLOC`` From 9d7cfb50d0842b65c562cf2938a0044c6f8cb0ac Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Thu, 31 Mar 2022 16:27:33 -0600 Subject: [PATCH 16/22] urllib: expose enums SchemeFlag variants directly --- Doc/library/urllib.parse.rst | 16 ++++++++-------- Lib/urllib/parse.py | 25 +++++++++++++++++-------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 09d6e6a6ee10a0e..6182c8b33cc8a9d 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -38,7 +38,7 @@ URL Parsing The URL parsing functions focus on splitting a URL string into its components, or on combining URL components into a URL string. -.. function:: urlparse(urlstring, scheme='', allow_fragments=True, classes=set()) +.. function:: urlparse(urlstring, scheme='', allow_fragments=True, flags=SchemeFlag(0)) Parse a URL into six components with respect to given scheme classes, returning a 6-item :term:`named tuple`. This corresponds to the general @@ -58,10 +58,10 @@ or on combining URL components into a URL string. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse, SchemeFlag + >>> from urllib.parse import urlparse, PARAMS >>> urlparse("scheme://netloc/path;parameters?query#fragment") ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='', query='query', fragment='fragment') - >>> urlparse("scheme://netloc/path;parameters?query#fragment", flags=SchemeFlag.PARAMS) + >>> urlparse("scheme://netloc/path;parameters?query#fragment", flags=PARAMS) ParseResult(scheme='scheme', netloc='netloc', path='/path', params='parameters', query='query', fragment='fragment') >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?" ... "highlight=params#url-parsing") @@ -582,18 +582,18 @@ instance, :func:`urlparse` and :func:`urljoin`) may override the behavior of the ``uses`` lists, for instance, parsing a custom or widely unused scheme with the same behavior as that of HTTP: - >>> from urllib.parse import urljoin, SchemeFlag + >>> from urllib.parse import urljoin, NETLOC, RELATIVE >>> urljoin( ... 'my-protocol://example.org/post/x/', '../y/', - ... flags=(SchemeFlag.NETLOC | SchemeFlag.RELATIVE)) + ... flags=(NETLOC | RELATIVE)) 'my-protocol://example.org/post/y/' For reference, the following three scheme classes are present (exactly corresponding to the ``uses`` lists): -* ``urllib.SchemeFlag.RELATIVE`` -* ``urllib.SchemeFlag.NETLOC`` -* ``urllib.SchemeFlag.PARAMS`` +* ``urllib.RELATIVE`` +* ``urllib.NETLOC`` +* ``urllib.PARAMS`` URL Quoting ----------- diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 9336747082502b4..9bb5ee6dcc51502 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -39,7 +39,8 @@ "urlsplit", "urlunsplit", "urlencode", "parse_qs", "parse_qsl", "quote", "quote_plus", "quote_from_bytes", "unquote", "unquote_plus", "unquote_to_bytes", - "DefragResult", "ParseResult", "SplitResult", "SchemeFlag", + "DefragResult", "ParseResult", "SplitResult", + "SchemeFlag", "RELATIVE", "NETLOC", "PARAMS", "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] # A classification of schemes. @@ -57,6 +58,14 @@ class SchemeFlag(Flag): NETLOC = auto() PARAMS = auto() + def __repr__(self): + return f'{self.__module__}.{self._name_}' + + __str__ = __repr__ + +RELATIVE, NETLOC, PARAMS = SchemeFlag + + uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', 'prospero', 'rtsp', 'rtspu', 'sftp', @@ -82,13 +91,13 @@ def _scheme_classes(scheme, overrides=SchemeFlag(0)): parameter. """ if scheme in uses_relative: - overrides |= SchemeFlag.RELATIVE + overrides |= RELATIVE if scheme in uses_netloc: - overrides |= SchemeFlag.NETLOC + overrides |= NETLOC if scheme in uses_params: - overrides |= SchemeFlag.PARAMS + overrides |= PARAMS return overrides @@ -420,7 +429,7 @@ def urlparse(url, scheme='', allow_fragments=True, flags=SchemeFlag(0)): splitresult = urlsplit(url, scheme, allow_fragments) scheme, netloc, url, query, fragment = splitresult scheme_classes = _scheme_classes(scheme, overrides=flags) - if SchemeFlag.PARAMS & scheme_classes and ';' in url: + if PARAMS in scheme_classes and ';' in url: url, params = _splitparams(url) else: params = '' @@ -536,7 +545,7 @@ def urlunsplit(components): _coerce_args(*components)) scheme_classes = _scheme_classes(scheme) - if netloc or (scheme and (SchemeFlag.NETLOC & scheme_classes) and url[:2] != '//'): + if netloc or (scheme and (NETLOC in scheme_classes) and url[:2] != '//'): if url and url[:1] != '/': url = '/' + url url = '//' + (netloc or '') + url if scheme: @@ -564,9 +573,9 @@ def urljoin(base, url, allow_fragments=True, flags=SchemeFlag(0)): scheme_classes = _scheme_classes(scheme, overrides=flags) - if scheme != bscheme or not (SchemeFlag.RELATIVE & scheme_classes): + if scheme != bscheme or RELATIVE not in scheme_classes: return _coerce_result(url) - if SchemeFlag.NETLOC in scheme_classes: + if NETLOC in scheme_classes: if netloc: return _coerce_result(urlunparse((scheme, netloc, path, params, query, fragment))) From 81d3414fdd49737f6b4cef90af37147861ec04c9 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Thu, 31 Mar 2022 16:55:03 -0600 Subject: [PATCH 17/22] urllib: add UNIVERSAL SchemeFlag --- Doc/library/urllib.parse.rst | 21 +++++++++++++-------- Lib/urllib/parse.py | 5 ++++- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 6182c8b33cc8a9d..b3a011a4be06988 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -573,9 +573,9 @@ Without specifying optional parameters or modifying global variables, Python will guess what parameters to apply based on the scheme. Schemes associated with each are specified by three lists in :mod:`urllib.parse`: -* ``urllib.uses_relative`` -* ``urllib.uses_netloc`` -* ``urllib.uses_params`` +* ``urllib.parse.uses_relative`` +* ``urllib.parse.uses_netloc`` +* ``urllib.parse.uses_params`` In addition, any function that takes a ``flags`` parameter (for instance, :func:`urlparse` and :func:`urljoin`) may override the @@ -588,12 +588,17 @@ widely unused scheme with the same behavior as that of HTTP: ... flags=(NETLOC | RELATIVE)) 'my-protocol://example.org/post/y/' -For reference, the following three scheme classes are present (exactly -corresponding to the ``uses`` lists): +Also provided is the ``UNIVERSAL`` flag, which will use all +recognizable elements of a URL (``RELATIVE``, ``NETLOC``, and +``PARAMS``). It is exactly equivalent to the logical or of all other +flags. -* ``urllib.RELATIVE`` -* ``urllib.NETLOC`` -* ``urllib.PARAMS`` +For reference, the following scheme classes are present: + +* ``urllib.parse.RELATIVE`` +* ``urllib.parse.NETLOC`` +* ``urllib.parse.PARAMS`` +* ``urllib.parse.UNIVERSAL`` URL Quoting ----------- diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 9bb5ee6dcc51502..00a7d179a1cf507 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -40,7 +40,7 @@ "parse_qsl", "quote", "quote_plus", "quote_from_bytes", "unquote", "unquote_plus", "unquote_to_bytes", "DefragResult", "ParseResult", "SplitResult", - "SchemeFlag", "RELATIVE", "NETLOC", "PARAMS", + "SchemeFlag", "RELATIVE", "NETLOC", "PARAMS", "UNIVERSAL", "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] # A classification of schemes. @@ -57,6 +57,7 @@ class SchemeFlag(Flag): RELATIVE = auto() NETLOC = auto() PARAMS = auto() + UNIVERSAL = RELATIVE | NETLOC | PARAMS def __repr__(self): return f'{self.__module__}.{self._name_}' @@ -64,6 +65,8 @@ def __repr__(self): __str__ = __repr__ RELATIVE, NETLOC, PARAMS = SchemeFlag +# UNIVERSAL must be assigned separately as it's a combination of other variants. +UNIVERSAL = SchemeFlag.UNIVERSAL uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', From 677ed1aac3534a07eec4cb6ea6ea2f1352f5fd2a Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Wed, 20 Apr 2022 19:30:01 -0600 Subject: [PATCH 18/22] use None rather than SchemeFlag in public API --- Doc/library/urllib.parse.rst | 2 +- Lib/urllib/parse.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index b3a011a4be06988..db73326bd0d3d96 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -38,7 +38,7 @@ URL Parsing The URL parsing functions focus on splitting a URL string into its components, or on combining URL components into a URL string. -.. function:: urlparse(urlstring, scheme='', allow_fragments=True, flags=SchemeFlag(0)) +.. function:: urlparse(urlstring, scheme='', allow_fragments=True, flags=None) Parse a URL into six components with respect to given scheme classes, returning a 6-item :term:`named tuple`. This corresponds to the general diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 00a7d179a1cf507..988c9daf1585713 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -85,7 +85,7 @@ def __repr__(self): 'mms', 'sftp', 'tel'] -def _scheme_classes(scheme, overrides=SchemeFlag(0)): +def _scheme_classes(scheme, overrides=None): """Find out what scheme classes a given scheme fits in. This consults the variables uses_relative, uses_netloc, and @@ -93,6 +93,9 @@ def _scheme_classes(scheme, overrides=SchemeFlag(0)): at least the unique classes specified by the optional overrides parameter. """ + if overrides is None: + overrides = SchemeFlag(0) + if scheme in uses_relative: overrides |= RELATIVE @@ -408,7 +411,7 @@ def _fix_result_transcoding(): _fix_result_transcoding() del _fix_result_transcoding -def urlparse(url, scheme='', allow_fragments=True, flags=SchemeFlag(0)): +def urlparse(url, scheme='', allow_fragments=True, flags=None): """Parse a URL into 6 components: :///;?# @@ -559,7 +562,7 @@ def urlunsplit(components): url = url + '#' + fragment return _coerce_result(url) -def urljoin(base, url, allow_fragments=True, flags=SchemeFlag(0)): +def urljoin(base, url, allow_fragments=True, flags=None): """Join a base URL and a possibly relative URL to form an absolute interpretation of the latter. Some logic may be enabled by setting the classes variable.""" From 0ec4a4edf6d33c85cbcd439ee579283480d87240 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Wed, 20 Apr 2022 19:53:22 -0600 Subject: [PATCH 19/22] do not import from enum --- Lib/urllib/parse.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 988c9daf1585713..9d3e91054c278dc 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -28,7 +28,7 @@ """ from collections import namedtuple -from enum import Flag, auto +import enum import functools import re import sys @@ -47,16 +47,16 @@ # The empty string classifies URLs with no scheme specified, # being the default value returned by “urlsplit” and “urlparse”. -class SchemeFlag(Flag): +class SchemeFlag(enum.Flag): """SchemeFlag is an enum with the members RELATIVE, NETLOC, and PARAMS. These describe methods for URL resolution, usually by scheme. These resolution classes determine, namely, whether a scheme supports, respectively, relative addressing, preserving the netloc (domain name), and preserving the parameters. """ - RELATIVE = auto() - NETLOC = auto() - PARAMS = auto() + RELATIVE = enum.auto() + NETLOC = enum.auto() + PARAMS = enum.auto() UNIVERSAL = RELATIVE | NETLOC | PARAMS def __repr__(self): From b25e0e88105497d67c734075d971eb6b9a62d8a5 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Wed, 20 Apr 2022 20:25:03 -0600 Subject: [PATCH 20/22] doc: correct urljoin signature --- Doc/library/urllib.parse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index db73326bd0d3d96..d336402e11a4925 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -349,7 +349,7 @@ or on combining URL components into a URL string. with an empty query; the RFC states that these are equivalent). -.. function:: urljoin(base, url, allow_fragments=True, classes=SchemeFlag(0)) +.. function:: urljoin(base, url, allow_fragments=True, flags=None) Construct a full ("absolute") URL by combining a "base URL" (*base*) with another URL (*url*), and with behavior given by a From 2123ad7e11089b8fcc2a8f77bceb3be4b523669a Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Wed, 20 Apr 2022 20:35:45 -0600 Subject: [PATCH 21/22] make flags parameter keyword-only --- Doc/library/urllib.parse.rst | 4 ++-- Lib/urllib/parse.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index d336402e11a4925..77cfcc08d7c7a1f 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -38,7 +38,7 @@ URL Parsing The URL parsing functions focus on splitting a URL string into its components, or on combining URL components into a URL string. -.. function:: urlparse(urlstring, scheme='', allow_fragments=True, flags=None) +.. function:: urlparse(urlstring, scheme='', allow_fragments=True, *, flags=None) Parse a URL into six components with respect to given scheme classes, returning a 6-item :term:`named tuple`. This corresponds to the general @@ -349,7 +349,7 @@ or on combining URL components into a URL string. with an empty query; the RFC states that these are equivalent). -.. function:: urljoin(base, url, allow_fragments=True, flags=None) +.. function:: urljoin(base, url, allow_fragments=True, *, flags=None) Construct a full ("absolute") URL by combining a "base URL" (*base*) with another URL (*url*), and with behavior given by a diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 9d3e91054c278dc..a3a51037406004c 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -411,7 +411,7 @@ def _fix_result_transcoding(): _fix_result_transcoding() del _fix_result_transcoding -def urlparse(url, scheme='', allow_fragments=True, flags=None): +def urlparse(url, scheme='', allow_fragments=True, *, flags=None): """Parse a URL into 6 components: :///;?# @@ -562,7 +562,7 @@ def urlunsplit(components): url = url + '#' + fragment return _coerce_result(url) -def urljoin(base, url, allow_fragments=True, flags=None): +def urljoin(base, url, allow_fragments=True, *, flags=None): """Join a base URL and a possibly relative URL to form an absolute interpretation of the latter. Some logic may be enabled by setting the classes variable.""" From 9f50dfb0f19504977faa7c0095e221b829143e08 Mon Sep 17 00:00:00 2001 From: "lincoln auster [they/them]" Date: Wed, 20 Apr 2022 20:38:40 -0600 Subject: [PATCH 22/22] s/classes/flags --- Lib/urllib/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index a3a51037406004c..b65a067d93a73b0 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -565,7 +565,7 @@ def urlunsplit(components): def urljoin(base, url, allow_fragments=True, *, flags=None): """Join a base URL and a possibly relative URL to form an absolute interpretation of the latter. Some logic may be enabled by setting - the classes variable.""" + the flags variable.""" if not base: return url if not url: