Commit ae5028c5 authored by adam j hartz's avatar adam j hartz

add support for f-strings (including in py3.5)

resolves #23
parent c3eaae90
......@@ -32,8 +32,9 @@ from ast import (Module, Num, Expr, Str, Bytes, UnaryOp, UAdd, USub, Invert,
Del, Pass, Raise, Import, alias, ImportFrom, Continue, Break, Yield,
YieldFrom, Return, IfExp, Lambda, arguments, arg, Call, keyword,
Attribute, Global, Nonlocal, If, While, For, withitem, With, Try,
ExceptHandler, FunctionDef, ClassDef, Starred, NodeTransformer,
Interactive, Expression, Index, literal_eval, dump, walk, increment_lineno)
ExceptHandler, FunctionDef, ClassDef, Starred, NodeTransformer, NodeVisitor,
Interactive, Expression, Index, literal_eval, dump, walk, increment_lineno,
parse)
from ast import Ellipsis # pylint: disable=redefined-builtin
# pylint: enable=unused-import
import textwrap
......@@ -120,13 +121,6 @@ def get_id(node, default=None):
return getattr(node, 'id', default)
def gather_names(node):
"""Returns the set of all names present in the node's tree."""
rtn = set(map(get_id, walk(node)))
rtn.discard(None)
return rtn
def has_elts(x):
"""Tests if x is an AST node with elements."""
return isinstance(x, AST) and hasattr(x, 'elts')
......@@ -413,6 +407,43 @@ class CtxAwareTransformer(NodeTransformer):
return node
def gather_names(node):
ng = NameGatherer()
ng.visit(node)
ng.names.discard(None)
return ng.names
class NameGatherer(NodeVisitor):
def __init__(self):
self.names = set()
NodeVisitor.__init__(self)
def generic_visit(self, node):
self.names.add(get_id(node))
NodeVisitor.generic_visit(self, node)
def visit_ListComp(self, node):
defined_names = set()
needed_names = gather_names(node.elt)
for g in node.generators:
defined_names |= gather_names(g)
for i in g.ifs:
needed_names |= gather_names(i)
self.names |= {i for i in needed_names if i not in defined_names}
def visit_DictComp(self, node):
defined_names = set()
needed_names = gather_names(node.key)
needed_names |= gather_names(node.value)
for g in node.generators:
defined_names |= gather_names(g)
for i in g.ifs:
needed_names |= gather_names(i)
self.names |= {i for i in needed_names if i not in defined_names}
visit_SetComp = visit_ListComp
def pdump(s, **kwargs):
"""performs a pretty dump of an AST node."""
......
......@@ -229,7 +229,7 @@ def get_script_subproc_command(fname, args):
raise PermissionError
if not os.access(fname, os.R_OK):
# on some systems, some importnat programs (e.g. sudo) will have
# on some systems, some important programs (e.g. sudo) will have
# execute permissions but not read/write permisions. This enables
# things with the SUID set to be run. Needs to come before _is_binary()
# is called, because that function tries to read the file.
......
......@@ -24,6 +24,7 @@
import os
import re
import sys
import string
from collections import Iterable, Sequence, Mapping
from takoshell.ply import yacc
......@@ -35,6 +36,9 @@ from takoshell.platform import PYTHON_VERSION_INFO
from takoshell.tokenize import SearchPath
from takoshell.parsers.context_check import check_contexts
_FORMATTER = string.Formatter()
class Location(object):
"""Location in a file."""
......@@ -1795,9 +1799,30 @@ class BaseParser(object):
def p_string_literal(self, p):
"""string_literal : string_tok"""
p1 = p[1]
s = ast.literal_eval(p1.value)
cls = ast.Bytes if p1.value.startswith('b') else ast.Str
p[0] = cls(s=s, lineno=p1.lineno, col_offset=p1.lexpos)
for index, val in enumerate(p1.value):
if val in {'"', "'"}:
break
prefix = p1.value[:index]
thestr = p1.value[index:]
if 'f' in prefix:
# this is a hack; maybe come back to improve it later by actually
# using 3.6's machinery for f-strings but it is
# backward-compatible.
prefix = prefix.replace('f', '')
s = ast.literal_eval(prefix + thestr)
try:
included_names = {i[1] for i in _FORMATTER.parse(s) if i[1] is not None}
except Exception:
included_names = set()
# omg this is such a hack. but it should work, i think.
# maybe screws with error reporting
thething = '%r.format(**{i: eval(i) for i in %r})'
thething = thething % (s, included_names)
p[0] = ast.parse(thething, mode='eval').body
else:
s = ast.literal_eval(p1.value)
cls = ast.Bytes if isinstance(s, bytes) else ast.Str
p[0] = cls(s=s, lineno=p1.lineno, col_offset=p1.lexpos)
def p_string_literal_list(self, p):
"""string_literal_list : string_literal
......
......@@ -273,47 +273,25 @@ PseudoToken = Whitespace + group(PseudoExtras, IORedirect, Number, Funny,
def _compile(expr):
return re.compile(expr, re.UNICODE)
endpats = {"'": Single, '"': Double,
"'''": Single3, '"""': Double3,
"r'''": Single3, 'r"""': Double3,
"b'''": Single3, 'b"""': Double3,
"R'''": Single3, 'R"""': Double3,
"B'''": Single3, 'B"""': Double3,
"br'''": Single3, 'br"""': Double3,
"bR'''": Single3, 'bR"""': Double3,
"Br'''": Single3, 'Br"""': Double3,
"BR'''": Single3, 'BR"""': Double3,
"rb'''": Single3, 'rb"""': Double3,
"Rb'''": Single3, 'Rb"""': Double3,
"rB'''": Single3, 'rB"""': Double3,
"RB'''": Single3, 'RB"""': Double3,
"u'''": Single3, 'u"""': Double3,
"U'''": Single3, 'U"""': Double3,
'r': None, 'R': None, 'b': None, 'B': None,
'u': None, 'U': None}
triple_quoted = {}
for t in ("'''", '"""',
"r'''", 'r"""', "R'''", 'R"""',
"b'''", 'b"""', "B'''", 'B"""',
"br'''", 'br"""', "Br'''", 'Br"""',
"bR'''", 'bR"""', "BR'''", 'BR"""',
"rb'''", 'rb"""', "rB'''", 'rB"""',
"Rb'''", 'Rb"""', "RB'''", 'RB"""',
"u'''", 'u"""', "U'''", 'U"""',
):
triple_quoted[t] = t
single_quoted = {}
for t in ("'", '"',
"r'", 'r"', "R'", 'R"',
"b'", 'b"', "B'", 'B"',
"br'", 'br"', "Br'", 'Br"',
"bR'", 'bR"', "BR'", 'BR"' ,
"rb'", 'rb"', "rB'", 'rB"',
"Rb'", 'Rb"', "RB'", 'RB"' ,
"u'", 'u"', "U'", 'U"',
):
single_quoted[t] = t
# For a given string prefix plus quotes, endpats maps it to a regex
# to match the remainder of that string. _prefix can be empty, for
# a normal single or triple quoted string (with no prefix).
endpats = {}
for _prefix in _all_string_prefixes():
endpats[_prefix + "'"] = Single
endpats[_prefix + '"'] = Double
endpats[_prefix + "'''"] = Single3
endpats[_prefix + '"""'] = Double3
# A set of all of the single and triple quoted string prefixes,
# including the opening quotes.
single_quoted = set()
triple_quoted = set()
for t in _all_string_prefixes():
for u in (t + '"', t + "'"):
single_quoted.add(u)
for u in (t + '"""', t + "'''"):
triple_quoted.add(u)
tabsize = 8
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment