Skip to content
Snippets Groups Projects
Commit a4ae51aa authored by Daniel Silverstone's avatar Daniel Silverstone
Browse files

Foldme!

parent 96f8c019
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
buildstream/_variables.py
+ 72
− 211
View file @ a4ae51aa
......@@ -25,7 +25,7 @@ from . import _yaml
# Variables are allowed to have dashes here
#
_VARIABLE_MATCH = r'\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}'
PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}")
# The Variables helper object will resolve the variable references in
......@@ -46,7 +46,7 @@ class Variables():
def __init__(self, node):
self.original = node
self.newexp = self._resolve_exp(node)
self.newexp = self._resolve(node)
self._sanity_check()
def get(self, var):
......@@ -69,130 +69,35 @@ class Variables():
# LoadError, if the string contains unresolved variable references.
#
def subst(self, string):
# real = self.real_subst(string)
exp = self.subst_exp(string)
# if real != exp:
# raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, "Oh dear!")
# return real
return exp
def real_subst(self, string):
substitute, unmatched, _ = self._subst(string, self.variables)
unmatched = list(set(unmatched))
if unmatched:
if len(unmatched) == 1:
message = "Unresolved variable '{var}'".format(var=unmatched[0])
else:
message = "Unresolved variables: "
for unmatch in unmatched:
if unmatched.index(unmatch) > 0:
message += ', '
message += unmatch
raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
return substitute
def _subst(self, string, variables):
def subst_callback(match):
nonlocal variables
nonlocal unmatched
nonlocal matched
token = match.group(0)
varname = match.group(1)
exp = _parse_expstr(string)
value = _yaml.node_get(variables, str, varname, default_value=None)
if value is not None:
# We have to check if the inner string has variables
# and return unmatches for those
unmatched += re.findall(_VARIABLE_MATCH, value)
matched += [varname]
else:
# Return unmodified token
unmatched += [varname]
value = token
try:
return _expand_expstr(self.newexp, exp)
except KeyError:
unmatched = []
return value
for v in exp[1][1::2]:
if v not in self.newexp:
unmatched.append(v)
matched = []
unmatched = []
replacement = re.sub(_VARIABLE_MATCH, subst_callback, string)
if unmatched:
if len(unmatched) == 1:
message = "Unresolved variable '{var}'".format(var=unmatched[0])
else:
message = "Unresolved variables: "
for unmatch in unmatched:
if unmatched.index(unmatch) > 0:
message += ', '
message += unmatch
return (replacement, unmatched, matched)
raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
raise
# Variable resolving code
#
# Here we substitute variables for values (resolve variables) repeatedly
# in a dictionary, each time creating a new dictionary until there is no
# more unresolved variables to resolve, or, until resolving further no
# longer resolves anything, in which case we throw an exception.
# Here we resolve all of our inputs into a dictionary, ready for use
# in subst()
def _resolve(self, node):
variables = node
# Special case, if notparallel is specified in the variables for this
# element, then override max-jobs to be 1.
# Initialize it as a string as all variables are processed as strings.
#
if _yaml.node_get(variables, bool, 'notparallel', default_value=False):
variables['max-jobs'] = str(1)
# Resolve the dictionary once, reporting the new dictionary with things
# substituted in it, and reporting unmatched tokens.
#
def resolve_one(variables):
unmatched = []
resolved = {}
for key, value in _yaml.node_items(variables):
# Ensure stringness of the value before substitution
value = _yaml.node_get(variables, str, key)
resolved_var, item_unmatched, matched = self._subst(value, variables)
if _wrap_variable(key) in resolved_var:
referenced_through = find_recursive_variable(key, matched, variables)
raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE,
"{}: ".format(_yaml.node_get_provenance(variables, key)) +
("Variable '{}' expands to contain a reference to itself. " +
"Perhaps '{}' contains '{}").format(key, referenced_through, _wrap_variable(key)))
resolved[key] = resolved_var
unmatched += item_unmatched
# Carry over provenance
resolved[_yaml.PROVENANCE_KEY] = variables[_yaml.PROVENANCE_KEY]
return (resolved, unmatched)
# Resolve it until it's resolved or broken
#
resolved = variables
unmatched = ['dummy']
last_unmatched = ['dummy']
while unmatched:
resolved, unmatched = resolve_one(resolved)
# Lists of strings can be compared like this
if unmatched == last_unmatched:
# We've got the same result twice without matching everything,
# something is undeclared or cyclic, compose a summary.
#
summary = ''
for unmatch in set(unmatched):
for var, provenance in self._find_references(unmatch):
line = " unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}\n"
summary += line.format(unmatched=unmatch, variable=var, provenance=provenance)
raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE,
"Failed to resolve one or more variable:\n{}".format(summary))
last_unmatched = unmatched
return resolved
def _resolve_exp(self, node):
# Special case, if notparallel is specified in the variables for this
# element, then override max-jobs to be 1.
# Initialize it as a string as all variables are processed as strings.
......@@ -206,13 +111,17 @@ class Variables():
ret[sys.intern(key)] = _parse_expstr(value)
return ret
# Sanity checks on the resolved variables dictionary
#
# Here we check for loops and for missing inputs which might cause the
# resolved dictionary of inputs to not be sufficient.
def _sanity_check(self):
# Ensure that every variable we could expand here has its targets resolved
# First the check for anything unresolvable
summary = []
wants = {}
for k, es in self.newexp.items():
wants[k] = wants.get(k, set())
for var in _exp_deps(es):
for var in es[1][1::2]:
wants[k].add(var)
if var not in self.newexp:
line = " unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}"
......@@ -222,6 +131,7 @@ class Variables():
raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE,
"Failed to resolve one or more variable:\n{}\n".format("\n".join(summary)))
# And now the cycle checks
def cycle_check(exp, visited, cleared):
for var in _exp_deps(exp):
if var in cleared:
......@@ -230,116 +140,67 @@ class Variables():
raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE,
"{}: ".format(_yaml.node_get_provenance(self.original, var)) +
("Variable '{}' expands to contain a reference to itself. " +
"Perhaps '{}' contains '{}").format(var, visited[-1], _wrap_variable(var)))
"Perhaps '{}' contains '%{{{}}}").format(var, visited[-1], var))
visited.append(var)
cycle_check(self.newexp[var], visited, cleared)
visited.pop()
cleared.add(var)
cleared = set()
for k,es in self.newexp.items():
for k, es in self.newexp.items():
if k not in cleared:
cycle_check(es, [k], cleared)
# resolved():
#
# Perform any substitutions necessary and return a dictionary of all
# inputs fully resolved.
#
# Returns:
# (dict): A dictionary mapping variable name to resolved content
#
def resolved(self):
# Make a fresh dict of the resolved expansions
ret = {}
for k, es in self.newexp.items():
ret[k] = _expand_expstr(self.newexp, es)
return ret
def subst_exp(self, string):
exp = _parse_expstr(string)
try:
return _expand_expstr(self.newexp, exp)
except KeyError:
unmatched = []
for v in _exp_deps(exp):
if v not in self.newexp:
unmatched.append(v)
if unmatched:
if len(unmatched) == 1:
message = "Unresolved variable '{var}'".format(var=unmatched[0])
else:
message = "Unresolved variables: "
for unmatch in unmatched:
if unmatched.index(unmatch) > 0:
message += ', '
message += unmatch
raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
raise
# Helper function to fetch information about the node referring to a variable
#
def _find_references(self, varname):
fullname = _wrap_variable(varname)
for key, value in _yaml.node_items(self.original):
if fullname in value:
provenance = _yaml.node_get_provenance(self.original, key)
yield (key, provenance)
def find_recursive_variable(variable, matched_variables, all_vars):
matched_values = (_yaml.node_get(all_vars, str, key) for key in matched_variables)
for key, value in zip(matched_variables, matched_values):
if _wrap_variable(variable) in value:
return key
# We failed to find a recursive variable
return None
def _wrap_variable(var):
return "%{" + var + "}"
## Stuff for new style expansion strings
PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}")
def __parse_expstr(s):
spl = PARSE_EXPANSION.split(s)
if spl[-1] == '':
spl = spl[:-1]
return (len(spl), [sys.intern(s) for s in spl])
def _exp_deps(es):
for i, s in enumerate(es[1]):
if i % 2 == 1:
yield s
# Cache for the parsed expansion strings. While this is nominally
# something which might "waste" memory, in reality each of these
# will live as long as the element which uses it, which is the
# vast majority of the memory usage across the execution of BuildStream.
PARSE_CACHE = {}
def _parse_expstr(s):
# Helper to parse a string into an expansion string tuple, caching
# the results so that future parse requests don't need to think about
# the string
def _parse_expstr(instr):
try:
return PARSE_CACHE[s]
return PARSE_CACHE[instr]
except KeyError:
PARSE_CACHE[s] = __parse_expstr(s)
return PARSE_CACHE[s]
spl = PARSE_EXPANSION.split(instr)
if spl[-1] == '':
spl = spl[:-1]
PARSE_CACHE[instr] = (len(spl), [sys.intern(s) for s in spl])
return PARSE_CACHE[instr]
#def _expand_expstr(content, value):
# ret = []
# expstack = [(value,0)]
# while expstack:
# ((elen,expanded), idx) = expstack.pop()
# ret.append(expanded[idx])
# idx += 1
# if idx < elen:
# if elen > idx+1:
# expstack.append(((elen,expanded), idx+1))
# expstack.append((content[expanded[idx]], 0))
# return "".join(ret)
def __expand(content, value):
(elen, bits) = value
idx = 0
while idx < elen:
yield bits[idx]
idx += 1
if idx < elen:
yield from __expand(content, content[bits[idx]])
idx += 1
def _expand_expstr(content, value):
return "".join(__expand(content, value))
# Helper to expand a given top level expansion string tuple in the context
# of the given dictionary of expansion strings.
#
# Note: Will raise KeyError if any expansion is missing
def _expand_expstr(content, topvalue):
def __expand(value):
(elen, bits) = value
idx = 0
while idx < elen:
yield bits[idx]
idx += 1
if idx < elen:
yield from __expand(content[bits[idx]])
idx += 1
return "".join(__expand(topvalue))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment