diff --git a/buildstream/_variables.py b/buildstream/_variables.py
index aa988c7e5a607880adfe5111372e851a971cdd92..04d5a99e0d951f3d27f1fa8f8d36919a3011e0b9 100644
--- a/buildstream/_variables.py
+++ b/buildstream/_variables.py
@@ -25,7 +25,7 @@ from . import _yaml
# Variables are allowed to have dashes here
#
-_VARIABLE_MATCH = r'\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}'
+PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}")
# The Variables helper object will resolve the variable references in
@@ -46,7 +46,7 @@ class Variables():
def __init__(self, node):
self.original = node
- self.newexp = self._resolve_exp(node)
+ self.newexp = self._resolve(node)
self._sanity_check()
def get(self, var):
@@ -69,130 +69,35 @@ class Variables():
# LoadError, if the string contains unresolved variable references.
#
def subst(self, string):
-# real = self.real_subst(string)
- exp = self.subst_exp(string)
-# if real != exp:
-# raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, "Oh dear!")
-# return real
- return exp
-
- def real_subst(self, string):
- substitute, unmatched, _ = self._subst(string, self.variables)
- unmatched = list(set(unmatched))
- if unmatched:
- if len(unmatched) == 1:
- message = "Unresolved variable '{var}'".format(var=unmatched[0])
- else:
- message = "Unresolved variables: "
- for unmatch in unmatched:
- if unmatched.index(unmatch) > 0:
- message += ', '
- message += unmatch
-
- raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
-
- return substitute
-
- def _subst(self, string, variables):
-
- def subst_callback(match):
- nonlocal variables
- nonlocal unmatched
- nonlocal matched
-
- token = match.group(0)
- varname = match.group(1)
+ exp = _parse_expstr(string)
- value = _yaml.node_get(variables, str, varname, default_value=None)
- if value is not None:
- # We have to check if the inner string has variables
- # and return unmatches for those
- unmatched += re.findall(_VARIABLE_MATCH, value)
- matched += [varname]
- else:
- # Return unmodified token
- unmatched += [varname]
- value = token
+ try:
+ return _expand_expstr(self.newexp, exp)
+ except KeyError:
+ unmatched = []
- return value
+ for v in exp[1][1::2]:
+ if v not in self.newexp:
+ unmatched.append(v)
- matched = []
- unmatched = []
- replacement = re.sub(_VARIABLE_MATCH, subst_callback, string)
+ if unmatched:
+ if len(unmatched) == 1:
+ message = "Unresolved variable '{var}'".format(var=unmatched[0])
+ else:
+ message = "Unresolved variables: "
+ for unmatch in unmatched:
+ if unmatched.index(unmatch) > 0:
+ message += ', '
+ message += unmatch
- return (replacement, unmatched, matched)
+ raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
+ raise
# Variable resolving code
#
- # Here we substitute variables for values (resolve variables) repeatedly
- # in a dictionary, each time creating a new dictionary until there is no
- # more unresolved variables to resolve, or, until resolving further no
- # longer resolves anything, in which case we throw an exception.
+ # Here we resolve all of our inputs into a dictionary, ready for use
+ # in subst()
def _resolve(self, node):
- variables = node
-
- # Special case, if notparallel is specified in the variables for this
- # element, then override max-jobs to be 1.
- # Initialize it as a string as all variables are processed as strings.
- #
- if _yaml.node_get(variables, bool, 'notparallel', default_value=False):
- variables['max-jobs'] = str(1)
-
- # Resolve the dictionary once, reporting the new dictionary with things
- # substituted in it, and reporting unmatched tokens.
- #
- def resolve_one(variables):
- unmatched = []
- resolved = {}
-
- for key, value in _yaml.node_items(variables):
-
- # Ensure stringness of the value before substitution
- value = _yaml.node_get(variables, str, key)
-
- resolved_var, item_unmatched, matched = self._subst(value, variables)
-
- if _wrap_variable(key) in resolved_var:
- referenced_through = find_recursive_variable(key, matched, variables)
- raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE,
- "{}: ".format(_yaml.node_get_provenance(variables, key)) +
- ("Variable '{}' expands to contain a reference to itself. " +
- "Perhaps '{}' contains '{}").format(key, referenced_through, _wrap_variable(key)))
-
- resolved[key] = resolved_var
- unmatched += item_unmatched
-
- # Carry over provenance
- resolved[_yaml.PROVENANCE_KEY] = variables[_yaml.PROVENANCE_KEY]
- return (resolved, unmatched)
-
- # Resolve it until it's resolved or broken
- #
- resolved = variables
- unmatched = ['dummy']
- last_unmatched = ['dummy']
- while unmatched:
- resolved, unmatched = resolve_one(resolved)
-
- # Lists of strings can be compared like this
- if unmatched == last_unmatched:
- # We've got the same result twice without matching everything,
- # something is undeclared or cyclic, compose a summary.
- #
- summary = ''
- for unmatch in set(unmatched):
- for var, provenance in self._find_references(unmatch):
- line = " unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}\n"
- summary += line.format(unmatched=unmatch, variable=var, provenance=provenance)
-
- raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE,
- "Failed to resolve one or more variable:\n{}".format(summary))
-
- last_unmatched = unmatched
-
- return resolved
-
- def _resolve_exp(self, node):
# Special case, if notparallel is specified in the variables for this
# element, then override max-jobs to be 1.
# Initialize it as a string as all variables are processed as strings.
@@ -206,13 +111,17 @@ class Variables():
ret[sys.intern(key)] = _parse_expstr(value)
return ret
+ # Sanity checks on the resolved variables dictionary
+ #
+ # Here we check for loops and for missing inputs which might cause the
+ # resolved dictionary of inputs to not be sufficient.
def _sanity_check(self):
- # Ensure that every variable we could expand here has its targets resolved
+ # First the check for anything unresolvable
summary = []
wants = {}
for k, es in self.newexp.items():
wants[k] = wants.get(k, set())
- for var in _exp_deps(es):
+ for var in es[1][1::2]:
wants[k].add(var)
if var not in self.newexp:
line = " unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}"
@@ -222,6 +131,7 @@ class Variables():
raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE,
"Failed to resolve one or more variable:\n{}\n".format("\n".join(summary)))
+ # And now the cycle checks
def cycle_check(exp, visited, cleared):
for var in _exp_deps(exp):
if var in cleared:
@@ -230,116 +140,67 @@ class Variables():
raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE,
"{}: ".format(_yaml.node_get_provenance(self.original, var)) +
("Variable '{}' expands to contain a reference to itself. " +
- "Perhaps '{}' contains '{}").format(var, visited[-1], _wrap_variable(var)))
+ "Perhaps '{}' contains '%{{{}}}").format(var, visited[-1], var))
visited.append(var)
cycle_check(self.newexp[var], visited, cleared)
visited.pop()
cleared.add(var)
cleared = set()
- for k,es in self.newexp.items():
+ for k, es in self.newexp.items():
if k not in cleared:
cycle_check(es, [k], cleared)
+ # resolved():
+ #
+ # Perform any substitutions necessary and return a dictionary of all
+ # inputs fully resolved.
+ #
+ # Returns:
+ # (dict): A dictionary mapping variable name to resolved content
+ #
def resolved(self):
+ # Make a fresh dict of the resolved expansions
ret = {}
for k, es in self.newexp.items():
ret[k] = _expand_expstr(self.newexp, es)
return ret
-
- def subst_exp(self, string):
- exp = _parse_expstr(string)
-
- try:
- return _expand_expstr(self.newexp, exp)
- except KeyError:
- unmatched = []
- for v in _exp_deps(exp):
- if v not in self.newexp:
- unmatched.append(v)
-
- if unmatched:
- if len(unmatched) == 1:
- message = "Unresolved variable '{var}'".format(var=unmatched[0])
- else:
- message = "Unresolved variables: "
- for unmatch in unmatched:
- if unmatched.index(unmatch) > 0:
- message += ', '
- message += unmatch
-
- raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
- raise
-
- # Helper function to fetch information about the node referring to a variable
- #
- def _find_references(self, varname):
- fullname = _wrap_variable(varname)
- for key, value in _yaml.node_items(self.original):
- if fullname in value:
- provenance = _yaml.node_get_provenance(self.original, key)
- yield (key, provenance)
-
-
-def find_recursive_variable(variable, matched_variables, all_vars):
- matched_values = (_yaml.node_get(all_vars, str, key) for key in matched_variables)
- for key, value in zip(matched_variables, matched_values):
- if _wrap_variable(variable) in value:
- return key
- # We failed to find a recursive variable
- return None
-
-
-def _wrap_variable(var):
- return "%{" + var + "}"
-
-## Stuff for new style expansion strings
-
-PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}")
-
-def __parse_expstr(s):
- spl = PARSE_EXPANSION.split(s)
- if spl[-1] == '':
- spl = spl[:-1]
- return (len(spl), [sys.intern(s) for s in spl])
-
-def _exp_deps(es):
- for i, s in enumerate(es[1]):
- if i % 2 == 1:
- yield s
+# Cache for the parsed expansion strings. While this is nominally
+# something which might "waste" memory, in reality each of these
+# will live as long as the element which uses it, which is the
+# vast majority of the memory usage across the execution of BuildStream.
PARSE_CACHE = {}
-def _parse_expstr(s):
+
+# Helper to parse a string into an expansion string tuple, caching
+# the results so that future parse requests don't need to think about
+# the string
+def _parse_expstr(instr):
try:
- return PARSE_CACHE[s]
+ return PARSE_CACHE[instr]
except KeyError:
- PARSE_CACHE[s] = __parse_expstr(s)
- return PARSE_CACHE[s]
+ spl = PARSE_EXPANSION.split(instr)
+ if spl[-1] == '':
+ spl = spl[:-1]
+ PARSE_CACHE[instr] = (len(spl), [sys.intern(s) for s in spl])
+ return PARSE_CACHE[instr]
-#def _expand_expstr(content, value):
-# ret = []
-# expstack = [(value,0)]
-# while expstack:
-# ((elen,expanded), idx) = expstack.pop()
-# ret.append(expanded[idx])
-# idx += 1
-# if idx < elen:
-# if elen > idx+1:
-# expstack.append(((elen,expanded), idx+1))
-# expstack.append((content[expanded[idx]], 0))
-# return "".join(ret)
-def __expand(content, value):
- (elen, bits) = value
- idx = 0
- while idx < elen:
- yield bits[idx]
- idx += 1
- if idx < elen:
- yield from __expand(content, content[bits[idx]])
- idx += 1
-
-def _expand_expstr(content, value):
- return "".join(__expand(content, value))
+# Helper to expand a given top level expansion string tuple in the context
+# of the given dictionary of expansion strings.
+#
+# Note: Will raise KeyError if any expansion is missing
+def _expand_expstr(content, topvalue):
+ def __expand(value):
+ (elen, bits) = value
+ idx = 0
+ while idx < elen:
+ yield bits[idx]
+ idx += 1
+ if idx < elen:
+ yield from __expand(content[bits[idx]])
+ idx += 1
+
+ return "".join(__expand(topvalue))