diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index 4905c91e635e8a4..983cc76bf6898a2 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -110,7 +110,7 @@ atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictsetmaker] '}' | '`' testlist1 '`' | - NAME | NUMBER | STRING+ | '.' '.' '.') + NAME | NUMBER | STRING+ | '...') listmaker: (test|star_expr) ( old_comp_for | (',' (test|star_expr))* [','] ) testlist_gexp: (test|star_expr) ( old_comp_for | (',' (test|star_expr))* [','] ) lambdef: 'lambda' [varargslist] ':' test diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py index c00cb2248d56760..018c2370e0bdffa 100644 --- a/Lib/lib2to3/pgen2/grammar.py +++ b/Lib/lib2to3/pgen2/grammar.py @@ -179,6 +179,7 @@ def report(self): // DOUBLESLASH //= DOUBLESLASHEQUAL -> RARROW +... ELLIPSIS """ opmap = {} diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py index 1a679554d2db4e8..c594016f0c6c11b 100755 --- a/Lib/lib2to3/pgen2/token.py +++ b/Lib/lib2to3/pgen2/token.py @@ -1,9 +1,9 @@ -#! /usr/bin/env python3 - """Token constants (from "token.h").""" -# Taken from Python (r53757) and modified to include some tokens -# originally monkeypatched in by pgen2.tokenize +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] + +# Taken from Lib/token.py and extended with tokens removed from the current +# version of Python. #--start constants-- ENDMARKER = 0 @@ -31,49 +31,54 @@ EQUAL = 22 DOT = 23 PERCENT = 24 -BACKQUOTE = 25 -LBRACE = 26 -RBRACE = 27 -EQEQUAL = 28 -NOTEQUAL = 29 -LESSEQUAL = 30 -GREATEREQUAL = 31 -TILDE = 32 -CIRCUMFLEX = 33 -LEFTSHIFT = 34 -RIGHTSHIFT = 35 -DOUBLESTAR = 36 -PLUSEQUAL = 37 -MINEQUAL = 38 -STAREQUAL = 39 -SLASHEQUAL = 40 -PERCENTEQUAL = 41 -AMPEREQUAL = 42 -VBAREQUAL = 43 -CIRCUMFLEXEQUAL = 44 -LEFTSHIFTEQUAL = 45 -RIGHTSHIFTEQUAL = 46 -DOUBLESTAREQUAL = 47 -DOUBLESLASH = 48 -DOUBLESLASHEQUAL = 49 -AT = 50 -ATEQUAL = 51 -OP = 52 -COMMENT = 53 -NL = 54 -RARROW = 55 -AWAIT = 56 -ASYNC = 57 -ERRORTOKEN = 58 -N_TOKENS = 59 +LBRACE = 25 +RBRACE = 26 +EQEQUAL = 27 +NOTEQUAL = 28 +LESSEQUAL = 29 +GREATEREQUAL = 30 +TILDE = 31 +CIRCUMFLEX = 32 +LEFTSHIFT = 33 +RIGHTSHIFT = 34 +DOUBLESTAR = 35 +PLUSEQUAL = 36 +MINEQUAL = 37 +STAREQUAL = 38 +SLASHEQUAL = 39 +PERCENTEQUAL = 40 +AMPEREQUAL = 41 +VBAREQUAL = 42 +CIRCUMFLEXEQUAL = 43 +LEFTSHIFTEQUAL = 44 +RIGHTSHIFTEQUAL = 45 +DOUBLESTAREQUAL = 46 +DOUBLESLASH = 47 +DOUBLESLASHEQUAL = 48 +AT = 49 +ATEQUAL = 50 +RARROW = 51 +ELLIPSIS = 52 +# Don't forget to update the table _PyParser_TokenNames in tokenizer.c! +OP = 53 +ERRORTOKEN = 54 +# These aren't used by the C tokenizer but are needed for tokenize.py +COMMENT = 55 +NL = 56 +ENCODING = 57 +N_TOKENS = 58 +# Special definitions for cooperation with parser NT_OFFSET = 256 +# Tokens no longer present in the current Python +BACKQUOTE = 125 +AWAIT = 156 +ASYNC = 157 #--end constants-- -tok_name = {} -for _name, _value in list(globals().items()): - if type(_value) is type(0): - tok_name[_value] = _name - +tok_name = {value: name + for name, value in globals().items() + if isinstance(value, int) and not name.startswith('_')} +__all__.extend(tok_name.values()) def ISTERMINAL(x): return x < NT_OFFSET diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index bf83ab8c7d9d2e4..5d21e875e659474 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -1,12 +1,12 @@ -# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation. -# All rights reserved. - """Tokenization help for Python programs. -generate_tokens(readline) is a generator that breaks a stream of -text into Python tokens. It accepts a readline-like method which is called -repeatedly to get the next line of input (or "" for EOF). It generates -5-tuples with these members: +tokenize(readline) is a generator that breaks a stream of bytes into +Python tokens. It decodes the bytes according to PEP-0263 for +determining source file encoding. + +It accepts a readline-like method which is called repeatedly to get the +next line of input (or b"" for EOF). It generates 5-tuples with these +members: the token type (see token.py) the token (a string) @@ -16,60 +16,141 @@ It is designed to match the working of the Python tokenizer exactly, except that it produces COMMENT tokens for comments and gives type OP for all -operators - -Older entry points - tokenize_loop(readline, tokeneater) - tokenize(readline, tokeneater=printtoken) -are the same, except instead of generating tokens, tokeneater is a callback -function to which the 5 fields described above are passed as 5 arguments, -each time a new token is found.""" +operators. Additionally, all token lists start with an ENCODING token +which tells you which encoding was used to decode the bytes stream. +""" __author__ = 'Ka-Ping Yee ' -__credits__ = \ - 'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro' +__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' + 'Skip Montanaro, Raymond Hettinger, Trent Nelson, ' + 'Michael Foord') +from builtins import open as _builtin_open +from codecs import lookup, BOM_UTF8 +import collections +from io import TextIOWrapper +import itertools as _itertools +import re +import sys +from .token import * -import string, re -from codecs import BOM_UTF8, lookup -from lib2to3.pgen2.token import * +cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) +blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) from . import token -__all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize", - "generate_tokens", "untokenize"] +__all__ = token.__all__ + ["tokenize", "detect_encoding", + "untokenize", "TokenInfo", + "generate_tokens"] del token -try: - bytes -except NameError: - # Support bytes type in Python <= 2.5, so 2to3 turns itself into - # valid Python 3 code. - bytes = str +EXACT_TOKEN_TYPES = { + '(': LPAR, + ')': RPAR, + '[': LSQB, + ']': RSQB, + ':': COLON, + ',': COMMA, + ';': SEMI, + '+': PLUS, + '-': MINUS, + '*': STAR, + '/': SLASH, + '|': VBAR, + '&': AMPER, + '<': LESS, + '>': GREATER, + '=': EQUAL, + '.': DOT, + '%': PERCENT, + '{': LBRACE, + '}': RBRACE, + '==': EQEQUAL, + '!=': NOTEQUAL, + '<=': LESSEQUAL, + '>=': GREATEREQUAL, + '~': TILDE, + '^': CIRCUMFLEX, + '<<': LEFTSHIFT, + '>>': RIGHTSHIFT, + '**': DOUBLESTAR, + '+=': PLUSEQUAL, + '-=': MINEQUAL, + '*=': STAREQUAL, + '/=': SLASHEQUAL, + '%=': PERCENTEQUAL, + '&=': AMPEREQUAL, + '|=': VBAREQUAL, + '^=': CIRCUMFLEXEQUAL, + '<<=': LEFTSHIFTEQUAL, + '>>=': RIGHTSHIFTEQUAL, + '**=': DOUBLESTAREQUAL, + '//': DOUBLESLASH, + '//=': DOUBLESLASHEQUAL, + '...': ELLIPSIS, + '->': RARROW, + '@': AT, + '@=': ATEQUAL, +} + +class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): + def __repr__(self): + annotated_type = '%d (%s)' % (self.type, tok_name[self.type]) + return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % + self._replace(type=annotated_type)) + + @property + def exact_type(self): + if self.type == OP and self.string in EXACT_TOKEN_TYPES: + return EXACT_TOKEN_TYPES[self.string] + else: + return self.type def group(*choices): return '(' + '|'.join(choices) + ')' def any(*choices): return group(*choices) + '*' def maybe(*choices): return group(*choices) + '?' -def _combinations(*l): - return set( - x + y for x in l for y in l + ("",) if x.casefold() != y.casefold() - ) +# Note: we use unicode matching for names ("\w") but ascii matching for +# number literals. Whitespace = r'[ \f\t]*' Comment = r'#[^\r\n]*' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) -Name = r'[a-zA-Z_]\w*' - -Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' -Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' -Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' -Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') -Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+(?:_\d+)*' -Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) -Expfloat = r'\d+(?:_\d+)*' + Exponent +Name = r'\w+' + +Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+[lL]?' +Binnumber = r'0[bB](?:_?[01])+' +Octnumber = r'0[oO]?(?:_?[0-7])+[lL]?' +Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)[lL]?' +Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) +Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*' +Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?', + r'\.[0-9](?:_?[0-9])*') + maybe(Exponent) +Expfloat = r'[0-9](?:_?[0-9])*' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) +# Return the empty string, plus all of the valid string prefixes. +def _all_string_prefixes(): + # The valid string prefixes. Only contain the lower case versions, + # and don't contain any permuations (include 'fr', but not + # 'rf'). The various permutations will be generated. + _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + # if we add binary f-strings, add: ['fb', 'fbr'] + result = {'', 'ur', 'Ur', 'uR', 'UR'} + for prefix in _valid_string_prefixes: + for t in _itertools.permutations(prefix): + # create a list with upper and lower versions of each + # character + for u in _itertools.product(*[(c, c.upper()) for c in t]): + result.add(''.join(u)) + return result + +def _compile(expr): + return re.compile(expr, re.UNICODE) + +# Note that since _all_string_prefixes includes the empty string, +# StringPrefix can be the empty string (making it optional). +StringPrefix = group(*_all_string_prefixes()) + # Tail end of ' string. Single = r"[^'\\]*(?:\\.[^'\\]*)*'" # Tail end of " string. @@ -78,60 +159,54 @@ def _combinations(*l): Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" # Tail end of """ string. Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -_litprefix = r"(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?" -Triple = group(_litprefix + "'''", _litprefix + '"""') +Triple = group(StringPrefix + "'''", StringPrefix + '"""') # Single-line ' or " string. -String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') +String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') # Because of leftmost-then-longest match semantics, be sure to put the # longest operators first (e.g., if = came before ==, == would get # recognized as two instances of =). -Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", +Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", + r"<>", r"//=?", r"->", r"[+\-*/%&@|^=<>]=?", r"~") Bracket = '[][(){}]' -Special = group(r'\r?\n', r'[:;.,`@]') +Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@`]') Funny = group(Operator, Bracket, Special) PlainToken = group(Number, Funny, String, Name) Token = Ignore + PlainToken # First (or only) line of ' or " string. -ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + +ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), - _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) -tokenprog, pseudoprog, single3prog, double3prog = list(map( - re.compile, (Token, PseudoToken, Single3, Double3))) - -_strprefixes = ( - _combinations('r', 'R', 'f', 'F') | - _combinations('r', 'R', 'b', 'B') | - {'u', 'U', 'ur', 'uR', 'Ur', 'UR'} -) - -endprogs = {"'": re.compile(Single), '"': re.compile(Double), - "'''": single3prog, '"""': double3prog, - **{f"{prefix}'''": single3prog for prefix in _strprefixes}, - **{f'{prefix}"""': double3prog for prefix in _strprefixes}, - **{prefix: None for prefix in _strprefixes}} - -triple_quoted = ( - {"'''", '"""'} | - {f"{prefix}'''" for prefix in _strprefixes} | - {f'{prefix}"""' for prefix in _strprefixes} -) -single_quoted = ( - {"'", '"'} | - {f"{prefix}'" for prefix in _strprefixes} | - {f'{prefix}"' for prefix in _strprefixes} -) +# For a given string prefix plus quotes, endpats maps it to a regex +# to match the remainder of that string. _prefix can be empty, for +# a normal single or triple quoted string (with no prefix). +endpats = {} +for _prefix in _all_string_prefixes(): + endpats[_prefix + "'"] = Single + endpats[_prefix + '"'] = Double + endpats[_prefix + "'''"] = Single3 + endpats[_prefix + '"""'] = Double3 + +# A set of all of the single and triple quoted string prefixes, +# including the opening quotes. +single_quoted = set() +triple_quoted = set() +for t in _all_string_prefixes(): + for u in (t + '"', t + "'"): + single_quoted.add(u) + for u in (t + '"""', t + "'''"): + triple_quoted.add(u) tabsize = 8 @@ -139,34 +214,6 @@ class TokenError(Exception): pass class StopTokenizing(Exception): pass -def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing - (srow, scol) = xxx_todo_changeme - (erow, ecol) = xxx_todo_changeme1 - print("%d,%d-%d,%d:\t%s\t%s" % \ - (srow, scol, erow, ecol, tok_name[type], repr(token))) - -def tokenize(readline, tokeneater=printtoken): - """ - The tokenize() function accepts two parameters: one representing the - input stream, and one providing an output mechanism for tokenize(). - - The first parameter, readline, must be a callable object which provides - the same interface as the readline() method of built-in file objects. - Each call to the function should return one line of input as a string. - - The second parameter, tokeneater, must also be a callable object. It is - called once for each token, with five arguments, corresponding to the - tuples generated by generate_tokens(). - """ - try: - tokenize_loop(readline, tokeneater) - except StopTokenizing: - pass - -# backwards compatible interface -def tokenize_loop(readline, tokeneater): - for token_info in generate_tokens(readline): - tokeneater(*token_info) class Untokenizer: @@ -174,20 +221,50 @@ def __init__(self): self.tokens = [] self.prev_row = 1 self.prev_col = 0 + self.encoding = None def add_whitespace(self, start): row, col = start - assert row <= self.prev_row + if row < self.prev_row or row == self.prev_row and col < self.prev_col: + raise ValueError("start ({},{}) precedes previous end ({},{})" + .format(row, col, self.prev_row, self.prev_col)) + row_offset = row - self.prev_row + if row_offset: + self.tokens.append("\\\n" * row_offset) + self.prev_col = 0 col_offset = col - self.prev_col if col_offset: self.tokens.append(" " * col_offset) def untokenize(self, iterable): - for t in iterable: + it = iter(iterable) + indents = [] + startline = False + for t in it: if len(t) == 2: - self.compat(t, iterable) + self.compat(t, it) break tok_type, token, start, end, line = t + if tok_type == ENCODING: + self.encoding = token + continue + if tok_type == ENDMARKER: + break + if tok_type == INDENT: + indents.append(token) + continue + elif tok_type == DEDENT: + indents.pop() + self.prev_row, self.prev_col = end + continue + elif tok_type in (NEWLINE, NL): + startline = True + elif startline and indents: + indent = indents[-1] + if start[1] >= len(indent): + self.tokens.append(indent) + self.prev_col = len(indent) + startline = False self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end @@ -197,20 +274,28 @@ def untokenize(self, iterable): return "".join(self.tokens) def compat(self, token, iterable): - startline = False indents = [] toks_append = self.tokens.append - toknum, tokval = token - if toknum in (NAME, NUMBER): - tokval += ' ' - if toknum in (NEWLINE, NL): - startline = True - for tok in iterable: + startline = token[0] in (NEWLINE, NL) + prevstring = False + + for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] + if toknum == ENCODING: + self.encoding = tokval + continue if toknum in (NAME, NUMBER, ASYNC, AWAIT): tokval += ' ' + # Insert a space between two consecutive strings + if toknum == STRING: + if prevstring: + tokval = ' ' + tokval + prevstring = True + else: + prevstring = False + if toknum == INDENT: indents.append(tokval) continue @@ -224,8 +309,33 @@ def compat(self, token, iterable): startline = False toks_append(tokval) -cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) -blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) + +def untokenize(iterable): + """Transform tokens back into Python source code. + It returns a bytes object, encoded using the ENCODING + token, which is the first token sequence output by tokenize. + + Each element returned by the iterable must be a token sequence + with at least two elements, a token number and token value. If + only two tokens are passed, the resulting output is poor. + + Round-trip invariant for full input: + Untokenized source will match input source exactly + + Round-trip invariant for limited input: + # Output bytes will tokenize back to the input + t1 = [tok[:2] for tok in tokenize(f.readline)] + newcode = untokenize(t1) + readline = BytesIO(newcode).readline + t2 = [tok[:2] for tok in tokenize(readline)] + assert t1 == t2 + """ + ut = Untokenizer() + out = ut.untokenize(iterable) + if ut.encoding is not None: + out = out.encode(ut.encoding) + return out + def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" @@ -241,21 +351,24 @@ def _get_normal_name(orig_enc): def detect_encoding(readline): """ The detect_encoding() function is used to detect the encoding that should - be used to decode a Python source file. It requires one argument, readline, + be used to decode a Python source file. It requires one argument, readline, in the same way as the tokenize() generator. It will call readline a maximum of twice, and return the encoding used - (as a string) and a list of any lines (left as bytes) it has read - in. + (as a string) and a list of any lines (left as bytes) it has read in. It detects the encoding from the presence of a utf-8 bom or an encoding - cookie as specified in pep-0263. If both a bom and a cookie are present, but - disagree, a SyntaxError will be raised. If the encoding cookie is an invalid - charset, raise a SyntaxError. Note that if a utf-8 bom is found, + cookie as specified in pep-0263. If both a bom and a cookie are present, + but disagree, a SyntaxError will be raised. If the encoding cookie is an + invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, 'utf-8-sig' is returned. If no encoding is specified, then the default of 'utf-8' will be returned. """ + try: + filename = readline.__self__.name + except AttributeError: + filename = None bom_found = False encoding = None default = 'utf-8' @@ -263,13 +376,20 @@ def read_or_stop(): try: return readline() except StopIteration: - return bytes() + return b'' def find_cookie(line): try: - line_string = line.decode('ascii') + # Decode as UTF-8. Either the line is an encoding declaration, + # in which case it should be pure ASCII, or it must be UTF-8 + # per default encoding. + line_string = line.decode('utf-8') except UnicodeDecodeError: - return None + msg = "invalid or missing encoding declaration" + if filename is not None: + msg = '{} for {!r}'.format(msg, filename) + raise SyntaxError(msg) + match = cookie_re.match(line_string) if not match: return None @@ -278,12 +398,21 @@ def find_cookie(line): codec = lookup(encoding) except LookupError: # This behaviour mimics the Python interpreter - raise SyntaxError("unknown encoding: " + encoding) + if filename is None: + msg = "unknown encoding: " + encoding + else: + msg = "unknown encoding for {!r}: {}".format(filename, + encoding) + raise SyntaxError(msg) if bom_found: - if codec.name != 'utf-8': + if encoding != 'utf-8': # This behaviour mimics the Python interpreter - raise SyntaxError('encoding problem: utf-8') + if filename is None: + msg = 'encoding problem: utf-8' + else: + msg = 'encoding problem for {!r}: utf-8'.format(filename) + raise SyntaxError(msg) encoding += '-sig' return encoding @@ -311,45 +440,65 @@ def find_cookie(line): return default, [first, second] -def untokenize(iterable): - """Transform tokens back into Python source code. - - Each element returned by the iterable must be a token sequence - with at least two elements, a token number and token value. If - only two tokens are passed, the resulting output is poor. - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited intput: - # Output text will tokenize the back to the input - t1 = [tok[:2] for tok in generate_tokens(f.readline)] - newcode = untokenize(t1) - readline = iter(newcode.splitlines(1)).next - t2 = [tok[:2] for tokin generate_tokens(readline)] - assert t1 == t2 +def open(filename): + """Open a file in read only mode using the encoding detected by + detect_encoding(). """ - ut = Untokenizer() - return ut.untokenize(iterable) + buffer = _builtin_open(filename, 'rb') + try: + encoding, lines = detect_encoding(buffer.readline) + buffer.seek(0) + text = TextIOWrapper(buffer, encoding, line_buffering=True) + text.mode = 'r' + return text + except: + buffer.close() + raise -def generate_tokens(readline): + +def tokenize(readline, tokeneater=None): """ - The generate_tokens() generator requires one argument, readline, which + The tokenize() generator requires one argument, readline, which must be a callable object which provides the same interface as the - readline() method of built-in file objects. Each call to the function - should return one line of input as a string. Alternately, readline + readline() method of built-in file objects. Each call to the function + should return one line of input as bytes. Alternatively, readline can be a callable function terminating with StopIteration: - readline = open(myfile).next # Example of alternate readline + readline = open(myfile, 'rb').__next__ # Example of alternate readline The generator produces 5-tuples with these members: the token type; the token string; a 2-tuple (srow, scol) of ints specifying the row and column where the token begins in the source; a 2-tuple (erow, ecol) of ints specifying the row and column where the token ends in the source; - and the line on which the token was found. The line passed is the + and the line on which the token was found. The line passed is the logical line; continuation lines are included. + + The first token sequence will always be an ENCODING token + which tells you which encoding was used to decode the bytes stream. + + The `tokeneater` argument is deprecated and intentionally undocumented. """ + if tokeneater: + import warnings + warnings.warn(PendingDeprecationWarning, + "The `tokeneater` argument to tokenize() is deprecated. " + "Use `for token in tokenize(readline): tokeneater` " + "instead. Note: readline should return bytes.") + try: + tokenize_loop(readline, tokeneater) + except StopTokenizing: + pass + return + + encoding, consumed = detect_encoding(readline) + empty = _itertools.repeat(b"") + rl_gen = _itertools.chain(consumed, iter(readline, b""), empty) + return _tokenize(rl_gen.__next__, encoding) + + +def _tokenize(readline, encoding): lnum = parenlev = continued = 0 - namechars, numchars = string.ascii_letters + '_', '0123456789' + numchars = '0123456789' contstr, needcont = '', 0 contline = None indents = [0] @@ -360,12 +509,20 @@ def generate_tokens(readline): async_def_indent = 0 async_def_nl = False - while 1: # loop over lines in stream + if encoding is not None: + if encoding == "utf-8-sig": + # BOM will already have been stripped. + encoding = "utf-8" + yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') + while True: # loop over lines in stream try: line = readline() except StopIteration: - line = '' - lnum = lnum + 1 + line = b'' + + if encoding is not None: + line = line.decode(encoding) + lnum += 1 pos, max = 0, len(line) if contstr: # continued string @@ -374,12 +531,12 @@ def generate_tokens(readline): endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) - yield (STRING, contstr + line[:end], + yield TokenInfo(STRING, contstr + line[:end], strstart, (lnum, end), contline + line) contstr, needcont = '', 0 contline = None elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': - yield (ERRORTOKEN, contstr + line, + yield TokenInfo(ERRORTOKEN, contstr + line, strstart, (lnum, len(line)), contline) contstr = '' contline = None @@ -393,12 +550,17 @@ def generate_tokens(readline): if not line: break column = 0 while pos < max: # measure leading whitespace - if line[pos] == ' ': column = column + 1 - elif line[pos] == '\t': column = (column//tabsize + 1)*tabsize - elif line[pos] == '\f': column = 0 - else: break - pos = pos + 1 - if pos == max: break + if line[pos] == ' ': + column += 1 + elif line[pos] == '\t': + column = (column//tabsize + 1)*tabsize + elif line[pos] == '\f': + column = 0 + else: + break + pos += 1 + if pos == max: + break if stashed: yield stashed @@ -407,19 +569,16 @@ def generate_tokens(readline): if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': comment_token = line[pos:].rstrip('\r\n') - nl_pos = pos + len(comment_token) - yield (COMMENT, comment_token, + yield TokenInfo(COMMENT, comment_token, (lnum, pos), (lnum, pos + len(comment_token)), line) - yield (NL, line[nl_pos:], - (lnum, nl_pos), (lnum, len(line)), line) - else: - yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], + pos += len(comment_token) + yield TokenInfo(NL, line[pos:], (lnum, pos), (lnum, len(line)), line) continue if column > indents[-1]: # count indents or dedents indents.append(column) - yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) + yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) while column < indents[-1]: if column not in indents: raise IndentationError( @@ -432,7 +591,7 @@ def generate_tokens(readline): async_def_nl = False async_def_indent = 0 - yield (DEDENT, '', (lnum, pos), (lnum, pos), line) + yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line) if async_def and async_def_nl and async_def_indent >= indents[-1]: async_def = False @@ -445,15 +604,17 @@ def generate_tokens(readline): continued = 0 while pos < max: - pseudomatch = pseudoprog.match(line, pos) + pseudomatch = _compile(PseudoToken).match(line, pos) if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (lnum, start), (lnum, end), end + if start == end: + continue token, initial = line[start:end], line[start] - if initial in numchars or \ - (initial == '.' and token != '.'): # ordinary number - yield (NUMBER, token, spos, epos, line) + if (initial in numchars or # ordinary number + (initial == '.' and token != '.' and token != '...')): + yield TokenInfo(NUMBER, token, spos, epos, line) elif initial in '\r\n': newline = NEWLINE if parenlev > 0: @@ -463,16 +624,17 @@ def generate_tokens(readline): if stashed: yield stashed stashed = None - yield (newline, token, spos, epos, line) + yield TokenInfo(newline, token, spos, epos, line) elif initial == '#': assert not token.endswith("\n") if stashed: yield stashed stashed = None - yield (COMMENT, token, spos, epos, line) + yield TokenInfo(COMMENT, token, spos, epos, line) + elif token in triple_quoted: - endprog = endprogs[token] + endprog = _compile(endpats[token]) endmatch = endprog.match(line, pos) if endmatch: # all on one line pos = endmatch.end(0) @@ -480,19 +642,37 @@ def generate_tokens(readline): if stashed: yield stashed stashed = None - yield (STRING, token, spos, (lnum, pos), line) + yield TokenInfo(STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines contstr = line[start:] contline = line break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: + + # Check up to the first 3 chars of the token to see if + # they're in the single_quoted set. If so, they start + # a string. + # We're using the first 3, because we're looking for + # "rb'" (for example) at the start of the token. If + # we switch to longer prefixes, this needs to be + # adjusted. + # Note that initial == token[:1]. + # Also note that single quote checking must come after + # triple quote checking (above). + elif (initial in single_quoted or + token[:2] in single_quoted or + token[:3] in single_quoted): if token[-1] == '\n': # continued string strstart = (lnum, start) - endprog = (endprogs[initial] or endprogs[token[1]] or - endprogs[token[2]]) + # Again, using the first 3 chars of the + # token. This is looking for the matching end + # regex for the correct type of quote + # character. So it's really looking for + # endpats["'"] or endpats['"'], by trying to + # skip string prefix characters, if any. + endprog = _compile(endpats.get(initial) or + endpats.get(token[1]) or + endpats.get(token[2])) contstr, needcont = line[start:], 1 contline = line break @@ -500,15 +680,15 @@ def generate_tokens(readline): if stashed: yield stashed stashed = None - yield (STRING, token, spos, epos, line) - elif initial in namechars: # ordinary name + yield TokenInfo(STRING, token, spos, epos, line) + elif initial.isidentifier(): # ordinary name if token in ('async', 'await'): if async_def: - yield (ASYNC if token == 'async' else AWAIT, + yield TokenInfo(ASYNC if token == 'async' else AWAIT, token, spos, epos, line) continue - tok = (NAME, token, spos, epos, line) + tok = TokenInfo(NAME, token, spos, epos, line) if token == 'async' and not stashed: stashed = tok continue @@ -521,7 +701,7 @@ def generate_tokens(readline): async_def = True async_def_indent = indents[-1] - yield (ASYNC, stashed[1], + yield TokenInfo(ASYNC, stashed[1], stashed[2], stashed[3], stashed[4]) stashed = None @@ -536,29 +716,112 @@ def generate_tokens(readline): if stashed: yield stashed stashed = None - yield (NL, token, spos, (lnum, pos), line) + yield TokenInfo(NL, token, spos, (lnum, pos), line) continued = 1 else: - if initial in '([{': parenlev = parenlev + 1 - elif initial in ')]}': parenlev = parenlev - 1 + if initial in '([{': + parenlev += 1 + elif initial in ')]}': + parenlev -= 1 if stashed: yield stashed stashed = None - yield (OP, token, spos, epos, line) + yield TokenInfo(OP, token, spos, epos, line) else: - yield (ERRORTOKEN, line[pos], + yield TokenInfo(ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) - pos = pos + 1 + pos += 1 if stashed: yield stashed stashed = None for indent in indents[1:]: # pop remaining indent levels - yield (DEDENT, '', (lnum, 0), (lnum, 0), '') - yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') + yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') + yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') + + +# An undocumented, backwards compatible, API for all the places in the standard +# library that expect to be able to use tokenize with strings +def generate_tokens(readline): + return _tokenize(readline, None) -if __name__ == '__main__': # testing - import sys - if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline) - else: tokenize(sys.stdin.readline) + +# An undocumented, backwards compatible, API for users looping over tokens +# with a callback. +def tokenize_loop(readline, tokeneater): + for token_info in generate_tokens(readline): + tokeneater(*token_info) + + +# An undocumented, backwards compatible token eater. +def printtoken(type, token, spos, epos, line): + srow, scol = spos + erow, ecol = epos + print("%d,%d-%d,%d:\t%s\t%s" % \ + (srow, scol, erow, ecol, tok_name[type], repr(token))) + + +def main(): + import argparse + + # Helper error handling routines + def perror(message): + sys.stderr.write(message) + sys.stderr.write('\n') + + def error(message, filename=None, location=None): + if location: + args = (filename,) + location + (message,) + perror("%s:%d:%d: error: %s" % args) + elif filename: + perror("%s: error: %s" % (filename, message)) + else: + perror("error: %s" % message) + sys.exit(1) + + # Parse the arguments and options + parser = argparse.ArgumentParser(prog='python -m tokenize') + parser.add_argument(dest='filename', nargs='?', + metavar='filename.py', + help='the file to tokenize; defaults to stdin') + parser.add_argument('-e', '--exact', dest='exact', action='store_true', + help='display token names using the exact type') + args = parser.parse_args() + + try: + # Tokenize the input + if args.filename: + filename = args.filename + with _builtin_open(filename, 'rb') as f: + tokens = list(tokenize(f.readline)) + else: + filename = "" + tokens = _tokenize(sys.stdin.readline, None) + + # Output the tokenization + for token in tokens: + token_type = token.type + if args.exact: + token_type = token.exact_type + token_range = "%d,%d-%d,%d:" % (token.start + token.end) + print("%-20s%-15s%-15r" % + (token_range, tok_name[token_type], token.string)) + except IndentationError as err: + line, column = err.args[1][1:3] + error(err.args[0], filename, (line, column)) + except TokenError as err: + line, column = err.args[1] + error(err.args[0], filename, (line, column)) + except SyntaxError as err: + error(err, filename) + except OSError as err: + error(err) + except KeyboardInterrupt: + print("interrupted\n") + except Exception as err: + perror("unexpected error: %s" % err) + raise + +if __name__ == "__main__": + main() diff --git a/Lib/lib2to3/tests/test_all_fixers.py b/Lib/lib2to3/tests/test_all_fixers.py index c0507cf3fbcbfe0..b397fc69b69527b 100644 --- a/Lib/lib2to3/tests/test_all_fixers.py +++ b/Lib/lib2to3/tests/test_all_fixers.py @@ -21,7 +21,11 @@ def setUp(self): def test_all_project_files(self): for filepath in support.all_project_files(): - self.refactor.refactor_file(filepath) + try: + self.refactor.refactor_file(filepath) + except Exception: + print('Refactoring failed on', filepath) + raise if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst b/Misc/NEWS.d/next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst new file mode 100644 index 000000000000000..5127639b2ab7513 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-22-18-27-13.bpo-33338.185_om.rst @@ -0,0 +1,6 @@ +lib2to3's tokenizer was synchronized with the built-in tokenizer. This +provides multiple wins. The ``tokenize.tokenize()`` API is now mimicked +in ``lib2to3``, including automatic encoding detection and yielding of +the ENCODING token. Unicode identifiers are now supported. ELLIPSIS is +its own token type now. Untokenizer improved with backports of 5e6db31, +9dc3a36, 5b8d2c3, e411b66, and BPO-2495.