From dafd047f30c352813e27dcc572ba7a87d0386933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 22 Apr 2018 16:53:21 -0700 Subject: [PATCH 1/3] [lib2to3] Synchronize token.py and tokenize.py with stdlib lib2to3's token.py and tokenize.py were initially copies of the respective files from the standard library. They were copied to allow Python 3 to read Python 2's grammar. Since 2006, lib2to3 grew to be widely used as a Concrete Syntax Tree, also for parsing Python 3 code. Additions to support Python 3 grammar were added but sadly, the main token.py and tokenize.py diverged. This change brings them back together, minimizing the differences to the bare minimum that is in fact required by lib2to3. Before this change, almost every line in lib2to3/pgen2/tokenize.py was different from tokenize.py. After this change, the diff between the two files is only 200 lines long and is entirely filled with relevant Python 2 compatibility bits. Merging the implementations, there's numerous fixes to the lib2to3 tokenizer: + docstrings made as similar as possible + ported `TokenInfo` + ported `tokenize.tokenize()` and `tokenize.open()` + removed Python 2-only implementation cruft + made Unicode identifier handling the same + made string prefix handling the same + added Ellipsis to the Special group + Untokenizer backported bugfixes: - 5e6db313686c200da425a54d2e0c95fa40107b1d - 9dc3a36c849c15c227a8af218cfb215abe7b3c48 - 5b8d2c3af76e704926cf5915ad0e6af59a232e61 - e411b6629fb5f7bc01bec89df75737875ce6d8f5 - BPO-2495 + `detect_encoding` tries to figure out a filename and `find_cookie` uses the filename in error messages, if available + `find_cookie` bugfix: BPO-14990 + BPO-16152: tokenizer doesn't crash on missing newline at the end of the stream (added \Z (end of string) to PseudoExtras) Improvements to token.py: + taken from the current Lib/token.py + tokens renumbered to match Lib/token.py + `__all__` properly defined + ASYNC, AWAIT and BACKQUOTE exist under different numbers (100 + old number) + ELLIPSIS added + ENCODING added --- Lib/lib2to3/Grammar.txt | 2 +- Lib/lib2to3/pgen2/grammar.py | 1 + Lib/lib2to3/pgen2/token.py | 93 ++-- Lib/lib2to3/pgen2/tokenize.py | 669 +++++++++++++++++++-------- Lib/lib2to3/tests/test_all_fixers.py | 6 +- 5 files changed, 522 insertions(+), 249 deletions(-) diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index 4905c91e635e8a4..983cc76bf6898a2 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -110,7 +110,7 @@ atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictsetmaker] '}' | '`' testlist1 '`' | - NAME | NUMBER | STRING+ | '.' '.' '.') + NAME | NUMBER | STRING+ | '...') listmaker: (test|star_expr) ( old_comp_for | (',' (test|star_expr))* [','] ) testlist_gexp: (test|star_expr) ( old_comp_for | (',' (test|star_expr))* [','] ) lambdef: 'lambda' [varargslist] ':' test diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py index c00cb2248d56760..018c2370e0bdffa 100644 --- a/Lib/lib2to3/pgen2/grammar.py +++ b/Lib/lib2to3/pgen2/grammar.py @@ -179,6 +179,7 @@ def report(self): // DOUBLESLASH //= DOUBLESLASHEQUAL -> RARROW +... ELLIPSIS """ opmap = {} diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py index 1a679554d2db4e8..c594016f0c6c11b 100755 --- a/Lib/lib2to3/pgen2/token.py +++ b/Lib/lib2to3/pgen2/token.py @@ -1,9 +1,9 @@ -#! /usr/bin/env python3 - """Token constants (from "token.h").""" -# Taken from Python (r53757) and modified to include some tokens -# originally monkeypatched in by pgen2.tokenize +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] + +# Taken from Lib/token.py and extended with tokens removed from the current +# version of Python. #--start constants-- ENDMARKER = 0 @@ -31,49 +31,54 @@ EQUAL = 22 DOT = 23 PERCENT = 24 -BACKQUOTE = 25 -LBRACE = 26 -RBRACE = 27 -EQEQUAL = 28 -NOTEQUAL = 29 -LESSEQUAL = 30 -GREATEREQUAL = 31 -TILDE = 32 -CIRCUMFLEX = 33 -LEFTSHIFT = 34 -RIGHTSHIFT = 35 -DOUBLESTAR = 36 -PLUSEQUAL = 37 -MINEQUAL = 38 -STAREQUAL = 39 -SLASHEQUAL = 40 -PERCENTEQUAL = 41 -AMPEREQUAL = 42 -VBAREQUAL = 43 -CIRCUMFLEXEQUAL = 44 -LEFTSHIFTEQUAL = 45 -RIGHTSHIFTEQUAL = 46 -DOUBLESTAREQUAL = 47 -DOUBLESLASH = 48 -DOUBLESLASHEQUAL = 49 -AT = 50 -ATEQUAL = 51 -OP = 52 -COMMENT = 53 -NL = 54 -RARROW = 55 -AWAIT = 56 -ASYNC = 57 -ERRORTOKEN = 58 -N_TOKENS = 59 +LBRACE = 25 +RBRACE = 26 +EQEQUAL = 27 +NOTEQUAL = 28 +LESSEQUAL = 29 +GREATEREQUAL = 30 +TILDE = 31 +CIRCUMFLEX = 32 +LEFTSHIFT = 33 +RIGHTSHIFT = 34 +DOUBLESTAR = 35 +PLUSEQUAL = 36 +MINEQUAL = 37 +STAREQUAL = 38 +SLASHEQUAL = 39 +PERCENTEQUAL = 40 +AMPEREQUAL = 41 +VBAREQUAL = 42 +CIRCUMFLEXEQUAL = 43 +LEFTSHIFTEQUAL = 44 +RIGHTSHIFTEQUAL = 45 +DOUBLESTAREQUAL = 46 +DOUBLESLASH = 47 +DOUBLESLASHEQUAL = 48 +AT = 49 +ATEQUAL = 50 +RARROW = 51 +ELLIPSIS = 52 +# Don't forget to update the table _PyParser_TokenNames in tokenizer.c! +OP = 53 +ERRORTOKEN = 54 +# These aren't used by the C tokenizer but are needed for tokenize.py +COMMENT = 55 +NL = 56 +ENCODING = 57 +N_TOKENS = 58 +# Special definitions for cooperation with parser NT_OFFSET = 256 +# Tokens no longer present in the current Python +BACKQUOTE = 125 +AWAIT = 156 +ASYNC = 157 #--end constants-- -tok_name = {} -for _name, _value in list(globals().items()): - if type(_value) is type(0): - tok_name[_value] = _name - +tok_name = {value: name + for name, value in globals().items() + if isinstance(value, int) and not name.startswith('_')} +__all__.extend(tok_name.values()) def ISTERMINAL(x): return x < NT_OFFSET diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index bf83ab8c7d9d2e4..5d21e875e659474 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -1,12 +1,12 @@ -# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation. -# All rights reserved. - """Tokenization help for Python programs. -generate_tokens(readline) is a generator that breaks a stream of -text into Python tokens. It accepts a readline-like method which is called -repeatedly to get the next line of input (or "" for EOF). It generates -5-tuples with these members: +tokenize(readline) is a generator that breaks a stream of bytes into +Python tokens. It decodes the bytes according to PEP-0263 for +determining source file encoding. + +It accepts a readline-like method which is called repeatedly to get the +next line of input (or b"" for EOF). It generates 5-tuples with these +members: the token type (see token.py) the token (a string) @@ -16,60 +16,141 @@ It is designed to match the working of the Python tokenizer exactly, except that it produces COMMENT tokens for comments and gives type OP for all -operators - -Older entry points - tokenize_loop(readline, tokeneater) - tokenize(readline, tokeneater=printtoken) -are the same, except instead of generating tokens, tokeneater is a callback -function to which the 5 fields described above are passed as 5 arguments, -each time a new token is found.""" +operators. Additionally, all token lists start with an ENCODING token +which tells you which encoding was used to decode the bytes stream. +""" __author__ = 'Ka-Ping Yee ' -__credits__ = \ - 'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro' +__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' + 'Skip Montanaro, Raymond Hettinger, Trent Nelson, ' + 'Michael Foord') +from builtins import open as _builtin_open +from codecs import lookup, BOM_UTF8 +import collections +from io import TextIOWrapper +import itertools as _itertools +import re +import sys +from .token import * -import string, re -from codecs import BOM_UTF8, lookup -from lib2to3.pgen2.token import * +cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) +blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) from . import token -__all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize", - "generate_tokens", "untokenize"] +__all__ = token.__all__ + ["tokenize", "detect_encoding", + "untokenize", "TokenInfo", + "generate_tokens"] del token -try: - bytes -except NameError: - # Support bytes type in Python <= 2.5, so 2to3 turns itself into - # valid Python 3 code. - bytes = str +EXACT_TOKEN_TYPES = { + '(': LPAR, + ')': RPAR, + '[': LSQB, + ']': RSQB, + ':': COLON, + ',': COMMA, + ';': SEMI, + '+': PLUS, + '-': MINUS, + '*': STAR, + '/': SLASH, + '|': VBAR, + '&': AMPER, + '<': LESS, + '>': GREATER, + '=': EQUAL, + '.': DOT, + '%': PERCENT, + '{': LBRACE, + '}': RBRACE, + '==': EQEQUAL, + '!=': NOTEQUAL, + '<=': LESSEQUAL, + '>=': GREATEREQUAL, + '~': TILDE, + '^': CIRCUMFLEX, + '<<': LEFTSHIFT, + '>>': RIGHTSHIFT, + '**': DOUBLESTAR, + '+=': PLUSEQUAL, + '-=': MINEQUAL, + '*=': STAREQUAL, + '/=': SLASHEQUAL, + '%=': PERCENTEQUAL, + '&=': AMPEREQUAL, + '|=': VBAREQUAL, + '^=': CIRCUMFLEXEQUAL, + '<<=': LEFTSHIFTEQUAL, + '>>=': RIGHTSHIFTEQUAL, + '**=': DOUBLESTAREQUAL, + '//': DOUBLESLASH, + '//=': DOUBLESLASHEQUAL, + '...': ELLIPSIS, + '->': RARROW, + '@': AT, + '@=': ATEQUAL, +} + +class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): + def __repr__(self): + annotated_type = '%d (%s)' % (self.type, tok_name[self.type]) + return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % + self._replace(type=annotated_type)) + + @property + def exact_type(self): + if self.type == OP and self.string in EXACT_TOKEN_TYPES: + return EXACT_TOKEN_TYPES[self.string] + else: + return self.type def group(*choices): return '(' + '|'.join(choices) + ')' def any(*choices): return group(*choices) + '*' def maybe(*choices): return group(*choices) + '?' -def _combinations(*l): - return set( - x + y for x in l for y in l + ("",) if x.casefold() != y.casefold() - ) +# Note: we use unicode matching for names ("\w") but ascii matching for +# number literals. Whitespace = r'[ \f\t]*' Comment = r'#[^\r\n]*' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) -Name = r'[a-zA-Z_]\w*' - -Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' -Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' -Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' -Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') -Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+(?:_\d+)*' -Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) -Expfloat = r'\d+(?:_\d+)*' + Exponent +Name = r'\w+' + +Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+[lL]?' +Binnumber = r'0[bB](?:_?[01])+' +Octnumber = r'0[oO]?(?:_?[0-7])+[lL]?' +Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)[lL]?' +Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) +Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*' +Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?', + r'\.[0-9](?:_?[0-9])*') + maybe(Exponent) +Expfloat = r'[0-9](?:_?[0-9])*' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) +# Return the empty string, plus all of the valid string prefixes. +def _all_string_prefixes(): + # The valid string prefixes. Only contain the lower case versions, + # and don't contain any permuations (include 'fr', but not + # 'rf'). The various permutations will be generated. + _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + # if we add binary f-strings, add: ['fb', 'fbr'] + result = {'', 'ur', 'Ur', 'uR', 'UR'} + for prefix in _valid_string_prefixes: + for t in _itertools.permutations(prefix): + # create a list with upper and lower versions of each + # character + for u in _itertools.product(*[(c, c.upper()) for c in t]): + result.add(''.join(u)) + return result + +def _compile(expr): + return re.compile(expr, re.UNICODE) + +# Note that since _all_string_prefixes includes the empty string, +# StringPrefix can be the empty string (making it optional). +StringPrefix = group(*_all_string_prefixes()) + # Tail end of ' string. Single = r"[^'\\]*(?:\\.[^'\\]*)*'" # Tail end of " string. @@ -78,60 +159,54 @@ def _combinations(*l): Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" # Tail end of """ string. Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -_litprefix = r"(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?" -Triple = group(_litprefix + "'''", _litprefix + '"""') +Triple = group(StringPrefix + "'''", StringPrefix + '"""') # Single-line ' or " string. -String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') +String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') # Because of leftmost-then-longest match semantics, be sure to put the # longest operators first (e.g., if = came before ==, == would get # recognized as two instances of =). -Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", +Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", + r"<>", r"//=?", r"->", r"[+\-*/%&@|^=<>]=?", r"~") Bracket = '[][(){}]' -Special = group(r'\r?\n', r'[:;.,`@]') +Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@`]') Funny = group(Operator, Bracket, Special) PlainToken = group(Number, Funny, String, Name) Token = Ignore + PlainToken # First (or only) line of ' or " string. -ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + +ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), - _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) -tokenprog, pseudoprog, single3prog, double3prog = list(map( - re.compile, (Token, PseudoToken, Single3, Double3))) - -_strprefixes = ( - _combinations('r', 'R', 'f', 'F') | - _combinations('r', 'R', 'b', 'B') | - {'u', 'U', 'ur', 'uR', 'Ur', 'UR'} -) - -endprogs = {"'": re.compile(Single), '"': re.compile(Double), - "'''": single3prog, '"""': double3prog, - **{f"{prefix}'''": single3prog for prefix in _strprefixes}, - **{f'{prefix}"""': double3prog for prefix in _strprefixes}, - **{prefix: None for prefix in _strprefixes}} - -triple_quoted = ( - {"'''", '"""'} | - {f"{prefix}'''" for prefix in _strprefixes} | - {f'{prefix}"""' for prefix in _strprefixes} -) -single_quoted = ( - {"'", '"'} | - {f"{prefix}'" for prefix in _strprefixes} | - {f'{prefix}"' for prefix in _strprefixes} -) +# For a given string prefix plus quotes, endpats maps it to a regex +# to match the remainder of that string. _prefix can be empty, for +# a normal single or triple quoted string (with no prefix). +endpats = {} +for _prefix in _all_string_prefixes(): + endpats[_prefix + "'"] = Single + endpats[_prefix + '"'] = Double + endpats[_prefix + "'''"] = Single3 + endpats[_prefix + '"""'] = Double3 + +# A set of all of the single and triple quoted string prefixes, +# including the opening quotes. +single_quoted = set() +triple_quoted = set() +for t in _all_string_prefixes(): + for u in (t + '"', t + "'"): + single_quoted.add(u) + for u in (t + '"""', t + "'''"): + triple_quoted.add(u) tabsize = 8 @@ -139,34 +214,6 @@ class TokenError(Exception): pass class StopTokenizing(Exception): pass -def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing - (srow, scol) = xxx_todo_changeme - (erow, ecol) = xxx_todo_changeme1 - print("%d,%d-%d,%d:\t%s\t%s" % \ - (srow, scol, erow, ecol, tok_name[type], repr(token))) - -def tokenize(readline, tokeneater=printtoken): - """ - The tokenize() function accepts two parameters: one representing the - input stream, and one providing an output mechanism for tokenize(). - - The first parameter, readline, must be a callable object which provides - the same interface as the readline() method of built-in file objects. - Each call to the function should return one line of input as a string. - - The second parameter, tokeneater, must also be a callable object. It is - called once for each token, with five arguments, corresponding to the - tuples generated by generate_tokens(). - """ - try: - tokenize_loop(readline, tokeneater) - except StopTokenizing: - pass - -# backwards compatible interface -def tokenize_loop(readline, tokeneater): - for token_info in generate_tokens(readline): - tokeneater(*token_info) class Untokenizer: @@ -174,20 +221,50 @@ def __init__(self): self.tokens = [] self.prev_row = 1 self.prev_col = 0 + self.encoding = None def add_whitespace(self, start): row, col = start - assert row <= self.prev_row + if row < self.prev_row or row == self.prev_row and col < self.prev_col: + raise ValueError("start ({},{}) precedes previous end ({},{})" + .format(row, col, self.prev_row, self.prev_col)) + row_offset = row - self.prev_row + if row_offset: + self.tokens.append("\\\n" * row_offset) + self.prev_col = 0 col_offset = col - self.prev_col if col_offset: self.tokens.append(" " * col_offset) def untokenize(self, iterable): - for t in iterable: + it = iter(iterable) + indents = [] + startline = False + for t in it: if len(t) == 2: - self.compat(t, iterable) + self.compat(t, it) break tok_type, token, start, end, line = t + if tok_type == ENCODING: + self.encoding = token + continue + if tok_type == ENDMARKER: + break + if tok_type == INDENT: + indents.append(token) + continue + elif tok_type == DEDENT: + indents.pop() + self.prev_row, self.prev_col = end + continue + elif tok_type in (NEWLINE, NL): + startline = True + elif startline and indents: + indent = indents[-1] + if start[1] >= len(indent): + self.tokens.append(indent) + self.prev_col = len(indent) + startline = False self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end @@ -197,20 +274,28 @@ def untokenize(self, iterable): return "".join(self.tokens) def compat(self, token, iterable): - startline = False indents = [] toks_append = self.tokens.append - toknum, tokval = token - if toknum in (NAME, NUMBER): - tokval += ' ' - if toknum in (NEWLINE, NL): - startline = True - for tok in iterable: + startline = token[0] in (NEWLINE, NL) + prevstring = False + + for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] + if toknum == ENCODING: + self.encoding = tokval + continue if toknum in (NAME, NUMBER, ASYNC, AWAIT): tokval += ' ' + # Insert a space between two consecutive strings + if toknum == STRING: + if prevstring: + tokval = ' ' + tokval + prevstring = True + else: + prevstring = False + if toknum == INDENT: indents.append(tokval) continue @@ -224,8 +309,33 @@ def compat(self, token, iterable): startline = False toks_append(tokval) -cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) -blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) + +def untokenize(iterable): + """Transform tokens back into Python source code. + It returns a bytes object, encoded using the ENCODING + token, which is the first token sequence output by tokenize. + + Each element returned by the iterable must be a token sequence + with at least two elements, a token number and token value. If + only two tokens are passed, the resulting output is poor. + + Round-trip invariant for full input: + Untokenized source will match input source exactly + + Round-trip invariant for limited input: + # Output bytes will tokenize back to the input + t1 = [tok[:2] for tok in tokenize(f.readline)] + newcode = untokenize(t1) + readline = BytesIO(newcode).readline + t2 = [tok[:2] for tok in tokenize(readline)] + assert t1 == t2 + """ + ut = Untokenizer() + out = ut.untokenize(iterable) + if ut.encoding is not None: + out = out.encode(ut.encoding) + return out + def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" @@ -241,21 +351,24 @@ def _get_normal_name(orig_enc): def detect_encoding(readline): """ The detect_encoding() function is used to detect the encoding that should - be used to decode a Python source file. It requires one argument, readline, + be used to decode a Python source file. It requires one argument, readline, in the same way as the tokenize() generator. It will call readline a maximum of twice, and return the encoding used - (as a string) and a list of any lines (left as bytes) it has read - in. + (as a string) and a list of any lines (left as bytes) it has read in. It detects the encoding from the presence of a utf-8 bom or an encoding - cookie as specified in pep-0263. If both a bom and a cookie are present, but - disagree, a SyntaxError will be raised. If the encoding cookie is an invalid - charset, raise a SyntaxError. Note that if a utf-8 bom is found, + cookie as specified in pep-0263. If both a bom and a cookie are present, + but disagree, a SyntaxError will be raised. If the encoding cookie is an + invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, 'utf-8-sig' is returned. If no encoding is specified, then the default of 'utf-8' will be returned. """ + try: + filename = readline.__self__.name + except AttributeError: + filename = None bom_found = False encoding = None default = 'utf-8' @@ -263,13 +376,20 @@ def read_or_stop(): try: return readline() except StopIteration: - return bytes() + return b'' def find_cookie(line): try: - line_string = line.decode('ascii') + # Decode as UTF-8. Either the line is an encoding declaration, + # in which case it should be pure ASCII, or it must be UTF-8 + # per default encoding. + line_string = line.decode('utf-8') except UnicodeDecodeError: - return None + msg = "invalid or missing encoding declaration" + if filename is not None: + msg = '{} for {!r}'.format(msg, filename) + raise SyntaxError(msg) + match = cookie_re.match(line_string) if not match: return None @@ -278,12 +398,21 @@ def find_cookie(line): codec = lookup(encoding) except LookupError: # This behaviour mimics the Python interpreter - raise SyntaxError("unknown encoding: " + encoding) + if filename is None: + msg = "unknown encoding: " + encoding + else: + msg = "unknown encoding for {!r}: {}".format(filename, + encoding) + raise SyntaxError(msg) if bom_found: - if codec.name != 'utf-8': + if encoding != 'utf-8': # This behaviour mimics the Python interpreter - raise SyntaxError('encoding problem: utf-8') + if filename is None: + msg = 'encoding problem: utf-8' + else: + msg = 'encoding problem for {!r}: utf-8'.format(filename) + raise SyntaxError(msg) encoding += '-sig' return encoding @@ -311,45 +440,65 @@ def find_cookie(line): return default, [first, second] -def untokenize(iterable): - """Transform tokens back into Python source code. - - Each element returned by the iterable must be a token sequence - with at least two elements, a token number and token value. If - only two tokens are passed, the resulting output is poor. - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited intput: - # Output text will tokenize the back to the input - t1 = [tok[:2] for tok in generate_tokens(f.readline)] - newcode = untokenize(t1) - readline = iter(newcode.splitlines(1)).next - t2 = [tok[:2] for tokin generate_tokens(readline)] - assert t1 == t2 +def open(filename): + """Open a file in read only mode using the encoding detected by + detect_encoding(). """ - ut = Untokenizer() - return ut.untokenize(iterable) + buffer = _builtin_open(filename, 'rb') + try: + encoding, lines = detect_encoding(buffer.readline) + buffer.seek(0) + text = TextIOWrapper(buffer, encoding, line_buffering=True) + text.mode = 'r' + return text + except: + buffer.close() + raise -def generate_tokens(readline): + +def tokenize(readline, tokeneater=None): """ - The generate_tokens() generator requires one argument, readline, which + The tokenize() generator requires one argument, readline, which must be a callable object which provides the same interface as the - readline() method of built-in file objects. Each call to the function - should return one line of input as a string. Alternately, readline + readline() method of built-in file objects. Each call to the function + should return one line of input as bytes. Alternatively, readline can be a callable function terminating with StopIteration: - readline = open(myfile).next # Example of alternate readline + readline = open(myfile, 'rb').__next__ # Example of alternate readline The generator produces 5-tuples with these members: the token type; the token string; a 2-tuple (srow, scol) of ints specifying the row and column where the token begins in the source; a 2-tuple (erow, ecol) of ints specifying the row and column where the token ends in the source; - and the line on which the token was found. The line passed is the + and the line on which the token was found. The line passed is the logical line; continuation lines are included. + + The first token sequence will always be an ENCODING token + which tells you which encoding was used to decode the bytes stream. + + The `tokeneater` argument is deprecated and intentionally undocumented. """ + if tokeneater: + import warnings + warnings.warn(PendingDeprecationWarning, + "The `tokeneater` argument to tokenize() is deprecated. " + "Use `for token in tokenize(readline): tokeneater` " + "instead. Note: readline should return bytes.") + try: + tokenize_loop(readline, tokeneater) + except StopTokenizing: + pass + return + + encoding, consumed = detect_encoding(readline) + empty = _itertools.repeat(b"") + rl_gen = _itertools.chain(consumed, iter(readline, b""), empty) + return _tokenize(rl_gen.__next__, encoding) + + +def _tokenize(readline, encoding): lnum = parenlev = continued = 0 - namechars, numchars = string.ascii_letters + '_', '0123456789' + numchars = '0123456789' contstr, needcont = '', 0 contline = None indents = [0] @@ -360,12 +509,20 @@ def generate_tokens(readline): async_def_indent = 0 async_def_nl = False - while 1: # loop over lines in stream + if encoding is not None: + if encoding == "utf-8-sig": + # BOM will already have been stripped. + encoding = "utf-8" + yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') + while True: # loop over lines in stream try: line = readline() except StopIteration: - line = '' - lnum = lnum + 1 + line = b'' + + if encoding is not None: + line = line.decode(encoding) + lnum += 1 pos, max = 0, len(line) if contstr: # continued string @@ -374,12 +531,12 @@ def generate_tokens(readline): endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) - yield (STRING, contstr + line[:end], + yield TokenInfo(STRING, contstr + line[:end], strstart, (lnum, end), contline + line) contstr, needcont = '', 0 contline = None elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': - yield (ERRORTOKEN, contstr + line, + yield TokenInfo(ERRORTOKEN, contstr + line, strstart, (lnum, len(line)), contline) contstr = '' contline = None @@ -393,12 +550,17 @@ def generate_tokens(readline): if not line: break column = 0 while pos < max: # measure leading whitespace - if line[pos] == ' ': column = column + 1 - elif line[pos] == '\t': column = (column//tabsize + 1)*tabsize - elif line[pos] == '\f': column = 0 - else: break - pos = pos + 1 - if pos == max: break + if line[pos] == ' ': + column += 1 + elif line[pos] == '\t': + column = (column//tabsize + 1)*tabsize + elif line[pos] == '\f': + column = 0 + else: + break + pos += 1 + if pos == max: + break if stashed: yield stashed @@ -407,19 +569,16 @@ def generate_tokens(readline): if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': comment_token = line[pos:].rstrip('\r\n') - nl_pos = pos + len(comment_token) - yield (COMMENT, comment_token, + yield TokenInfo(COMMENT, comment_token, (lnum, pos), (lnum, pos + len(comment_token)), line) - yield (NL, line[nl_pos:], - (lnum, nl_pos), (lnum, len(line)), line) - else: - yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], + pos += len(comment_token) + yield TokenInfo(NL, line[pos:], (lnum, pos), (lnum, len(line)), line) continue if column > indents[-1]: # count indents or dedents indents.append(column) - yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) + yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) while column < indents[-1]: if column not in indents: raise IndentationError( @@ -432,7 +591,7 @@ def generate_tokens(readline): async_def_nl = False async_def_indent = 0 - yield (DEDENT, '', (lnum, pos), (lnum, pos), line) + yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line) if async_def and async_def_nl and async_def_indent >= indents[-1]: async_def = False @@ -445,15 +604,17 @@ def generate_tokens(readline): continued = 0 while pos < max: - pseudomatch = pseudoprog.match(line, pos) + pseudomatch = _compile(PseudoToken).match(line, pos) if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (lnum, start), (lnum, end), end + if start == end: + continue token, initial = line[start:end], line[start] - if initial in numchars or \ - (initial == '.' and token != '.'): # ordinary number - yield (NUMBER, token, spos, epos, line) + if (initial in numchars or # ordinary number + (initial == '.' and token != '.' and token != '...')): + yield TokenInfo(NUMBER, token, spos, epos, line) elif initial in '\r\n': newline = NEWLINE if parenlev > 0: @@ -463,16 +624,17 @@ def generate_tokens(readline): if stashed: yield stashed stashed = None - yield (newline, token, spos, epos, line) + yield TokenInfo(newline, token, spos, epos, line) elif initial == '#': assert not token.endswith("\n") if stashed: yield stashed stashed = None - yield (COMMENT, token, spos, epos, line) + yield TokenInfo(COMMENT, token, spos, epos, line) + elif token in triple_quoted: - endprog = endprogs[token] + endprog = _compile(endpats[token]) endmatch = endprog.match(line, pos) if endmatch: # all on one line pos = endmatch.end(0) @@ -480,19 +642,37 @@ def generate_tokens(readline): if stashed: yield stashed stashed = None - yield (STRING, token, spos, (lnum, pos), line) + yield TokenInfo(STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines contstr = line[start:] contline = line break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: + + # Check up to the first 3 chars of the token to see if + # they're in the single_quoted set. If so, they start + # a string. + # We're using the first 3, because we're looking for + # "rb'" (for example) at the start of the token. If + # we switch to longer prefixes, this needs to be + # adjusted. + # Note that initial == token[:1]. + # Also note that single quote checking must come after + # triple quote checking (above). + elif (initial in single_quoted or + token[:2] in single_quoted or + token[:3] in single_quoted): if token[-1] == '\n': # continued string strstart = (lnum, start) - endprog = (endprogs[initial] or endprogs[token[1]] or - endprogs[token[2]]) + # Again, using the first 3 chars of the + # token. This is looking for the matching end + # regex for the correct type of quote + # character. So it's really looking for + # endpats["'"] or endpats['"'], by trying to + # skip string prefix characters, if any. + endprog = _compile(endpats.get(initial) or + endpats.get(token[1]) or + endpats.get(token[2])) contstr, needcont = line[start:], 1 contline = line break @@ -500,15 +680,15 @@ def generate_tokens(readline): if stashed: yield stashed stashed = None - yield (STRING, token, spos, epos, line) - elif initial in namechars: # ordinary name + yield TokenInfo(STRING, token, spos, epos, line) + elif initial.isidentifier(): # ordinary name if token in ('async', 'await'): if async_def: - yield (ASYNC if token == 'async' else AWAIT, + yield TokenInfo(ASYNC if token == 'async' else AWAIT, token, spos, epos, line) continue - tok = (NAME, token, spos, epos, line) + tok = TokenInfo(NAME, token, spos, epos, line) if token == 'async' and not stashed: stashed = tok continue @@ -521,7 +701,7 @@ def generate_tokens(readline): async_def = True async_def_indent = indents[-1] - yield (ASYNC, stashed[1], + yield TokenInfo(ASYNC, stashed[1], stashed[2], stashed[3], stashed[4]) stashed = None @@ -536,29 +716,112 @@ def generate_tokens(readline): if stashed: yield stashed stashed = None - yield (NL, token, spos, (lnum, pos), line) + yield TokenInfo(NL, token, spos, (lnum, pos), line) continued = 1 else: - if initial in '([{': parenlev = parenlev + 1 - elif initial in ')]}': parenlev = parenlev - 1 + if initial in '([{': + parenlev += 1 + elif initial in ')]}': + parenlev -= 1 if stashed: yield stashed stashed = None - yield (OP, token, spos, epos, line) + yield TokenInfo(OP, token, spos, epos, line) else: - yield (ERRORTOKEN, line[pos], + yield TokenInfo(ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) - pos = pos + 1 + pos += 1 if stashed: yield stashed stashed = None for indent in indents[1:]: # pop remaining indent levels - yield (DEDENT, '', (lnum, 0), (lnum, 0), '') - yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') + yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') + yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') + + +# An undocumented, backwards compatible, API for all the places in the standard +# library that expect to be able to use tokenize with strings +def generate_tokens(readline): + return _tokenize(readline, None) -if __name__ == '__main__': # testing - import sys - if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline) - else: tokenize(sys.stdin.readline) + +# An undocumented, backwards compatible, API for users looping over tokens +# with a callback. +def tokenize_loop(readline, tokeneater): + for token_info in generate_tokens(readline): + tokeneater(*token_info) + + +# An undocumented, backwards compatible token eater. +def printtoken(type, token, spos, epos, line): + srow, scol = spos + erow, ecol = epos + print("%d,%d-%d,%d:\t%s\t%s" % \ + (srow, scol, erow, ecol, tok_name[type], repr(token))) + + +def main(): + import argparse + + # Helper error handling routines + def perror(message): + sys.stderr.write(message) + sys.stderr.write('\n') + + def error(message, filename=None, location=None): + if location: + args = (filename,) + location + (message,) + perror("%s:%d:%d: error: %s" % args) + elif filename: + perror("%s: error: %s" % (filename, message)) + else: + perror("error: %s" % message) + sys.exit(1) + + # Parse the arguments and options + parser = argparse.ArgumentParser(prog='python -m tokenize') + parser.add_argument(dest='filename', nargs='?', + metavar='filename.py', + help='the file to tokenize; defaults to stdin') + parser.add_argument('-e', '--exact', dest='exact', action='store_true', + help='display token names using the exact type') + args = parser.parse_args() + + try: + # Tokenize the input + if args.filename: + filename = args.filename + with _builtin_open(filename, 'rb') as f: + tokens = list(tokenize(f.readline)) + else: + filename = "" + tokens = _tokenize(sys.stdin.readline, None) + + # Output the tokenization + for token in tokens: + token_type = token.type + if args.exact: + token_type = token.exact_type + token_range = "%d,%d-%d,%d:" % (token.start + token.end) + print("%-20s%-15s%-15r" % + (token_range, tok_name[token_type], token.string)) + except IndentationError as err: + line, column = err.args[1][1:3] + error(err.args[0], filename, (line, column)) + except TokenError as err: + line, column = err.args[1] + error(err.args[0], filename, (line, column)) + except SyntaxError as err: + error(err, filename) + except OSError as err: + error(err) + except KeyboardInterrupt: + print("interrupted\n") + except Exception as err: + perror("unexpected error: %s" % err) + raise + +if __name__ == "__main__": + main() diff --git a/Lib/lib2to3/tests/test_all_fixers.py b/Lib/lib2to3/tests/test_all_fixers.py index c0507cf3fbcbfe0..b397fc69b69527b 100644 --- a/Lib/lib2to3/tests/test_all_fixers.py +++ b/Lib/lib2to3/tests/test_all_fixers.py @@ -21,7 +21,11 @@ def setUp(self): def test_all_project_files(self): for filepath in support.all_project_files(): - self.refactor.refactor_file(filepath) + try: + self.refactor.refactor_file(filepath) + except Exception: + print('Refactoring failed on', filepath) + raise if __name__ == '__main__': unittest.main() From 77ae1de028f221683175fdd63bc1da335df92b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 22 Apr 2018 18:27:20 -0700 Subject: [PATCH 2/3] NEWS.d --- .../next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst diff --git a/Misc/NEWS.d/next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst b/Misc/NEWS.d/next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst new file mode 100644 index 000000000000000..b9c5331e44fd93f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst @@ -0,0 +1,8 @@ +lib2to3's tokenizer was synchronized with the built-in tokenizer, providing +many wins, including: + +- the ``tokenize.tokenize()`` API is now mimicked in ``lib2to3``, including +automatic encoding detection and yielding ENCODING tokens; - Unicode +identifiers are now supported; - ELLIPSIS is its own token type now; - +Untokenizer improved with backports of 5e6db31, 9dc3a36, 5b8d2c3, e411b66, +and BPO-2495. From a6a3957eb456b959b05894acb04e536342b4259a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 23 Apr 2018 00:37:52 -0700 Subject: [PATCH 3/3] D'oh. Blurb hates bullet points. --- .../2018-04-22-18-27-13.bpo-33338.185_om.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst b/Misc/NEWS.d/next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst index b9c5331e44fd93f..5127639b2ab7513 100644 --- a/Misc/NEWS.d/next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst +++ b/Misc/NEWS.d/next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst @@ -1,8 +1,6 @@ -lib2to3's tokenizer was synchronized with the built-in tokenizer, providing -many wins, including: - -- the ``tokenize.tokenize()`` API is now mimicked in ``lib2to3``, including -automatic encoding detection and yielding ENCODING tokens; - Unicode -identifiers are now supported; - ELLIPSIS is its own token type now; - -Untokenizer improved with backports of 5e6db31, 9dc3a36, 5b8d2c3, e411b66, -and BPO-2495. +lib2to3's tokenizer was synchronized with the built-in tokenizer. This +provides multiple wins. The ``tokenize.tokenize()`` API is now mimicked +in ``lib2to3``, including automatic encoding detection and yielding of +the ENCODING token. Unicode identifiers are now supported. ELLIPSIS is +its own token type now. Untokenizer improved with backports of 5e6db31, +9dc3a36, 5b8d2c3, e411b66, and BPO-2495.