diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py
index 5feae93f454cf24c3c4f3208ecee26f522ae2f78..3c69a6b79cf90f2ed40c5b4e16c54ebfe3cd35a6 100644
--- a/buildstream/_artifactcache/artifactcache.py
+++ b/buildstream/_artifactcache/artifactcache.py
@@ -147,6 +147,7 @@ class ArtifactCache():
has_remote_caches = True
if use_config:
for project in self.context.get_projects():
+ project.ensure_fully_loaded()
artifact_caches = _configured_remote_artifact_cache_specs(self.context, project)
if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances
self._set_remotes(artifact_caches, project=project)
diff --git a/buildstream/_exceptions.py b/buildstream/_exceptions.py
index c86b6780c8bb5754865981caf70eced471047f06..f9923b40433b9544e68cbbbb4b08595374287215 100644
--- a/buildstream/_exceptions.py
+++ b/buildstream/_exceptions.py
@@ -205,6 +205,9 @@ class LoadErrorReason(Enum):
# Try to load a directory not a yaml file
LOADING_DIRECTORY = 18
+ # A recursive include has been encountered.
+ RECURSIVE_INCLUDE = 19
+
# LoadError
#
diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py
index dab8cab56c2c2149aa0b9318730b823021ee65ac..8584780c025383e616d6c881b41722a48be584bb 100644
--- a/buildstream/_frontend/widget.py
+++ b/buildstream/_frontend/widget.py
@@ -480,8 +480,11 @@ class LogLine(Widget):
text += '\n'
# Plugins
- text += self._format_plugins(project._element_factory.loaded_dependencies,
- project._source_factory.loaded_dependencies)
+ text += self._format_plugins(project.first_pass_config.element_factory.loaded_dependencies,
+ project.first_pass_config.source_factory.loaded_dependencies)
+ if project.config.element_factory and project.config.source_factory:
+ text += self._format_plugins(project.config.element_factory.loaded_dependencies,
+ project.config.source_factory.loaded_dependencies)
# Pipeline state
text += self.content_profile.fmt("Pipeline\n", bold=True)
diff --git a/buildstream/_includes.py b/buildstream/_includes.py
new file mode 100644
index 0000000000000000000000000000000000000000..e4afeaf82984c0ff9b3fd0578cad46c3dbf1f477
--- /dev/null
+++ b/buildstream/_includes.py
@@ -0,0 +1,128 @@
+import os
+from collections import Mapping
+from . import _yaml
+from ._exceptions import LoadError, LoadErrorReason
+
+
+# Includes()
+#
+# This takes care of processing include directives "(@)".
+#
+# Args:
+# loader (Loader): The Loader object
+class Includes:
+
+ def __init__(self, loader):
+ self._loader = loader
+ self._loaded = {}
+
+ # process()
+ #
+ # Process recursively include directives in a YAML node.
+ #
+ # Args:
+ # node (dict): A YAML node
+ # included (set): Fail for recursion if trying to load any files in this set
+ # current_loader (Loader): Use alternative loader (for junction files)
+ # only_local (bool): Whether to ignore junction files
+ def process(self, node, *,
+ included=set(),
+ current_loader=None,
+ only_local=False):
+ if current_loader is None:
+ current_loader = self._loader
+
+ if isinstance(node.get('(@)'), str):
+ includes = [_yaml.node_get(node, str, '(@)')]
+ else:
+ includes = _yaml.node_get(node, list, '(@)', default_value=None)
+ if '(@)' in node:
+ del node['(@)']
+
+ if includes:
+ for include in reversed(includes):
+ if only_local and ':' in include:
+ continue
+ include_node, file_path, sub_loader = self._include_file(include,
+ current_loader)
+ if file_path in included:
+ provenance = _yaml.node_get_provenance(node)
+ raise LoadError(LoadErrorReason.RECURSIVE_INCLUDE,
+ "{}: trying to recursively include {}". format(provenance,
+ file_path))
+ # Because the included node will be modified, we need
+ # to copy it so that we do not modify the toplevel
+ # node of the provenance.
+ include_node = _yaml.node_chain_copy(include_node)
+
+ try:
+ included.add(file_path)
+ self.process(include_node, included=included,
+ current_loader=sub_loader,
+ only_local=only_local)
+ finally:
+ included.remove(file_path)
+
+ _yaml.composite(include_node, node)
+ to_delete = [key for key, _ in _yaml.node_items(node) if key not in include_node]
+ for key, value in include_node.items():
+ node[key] = value
+ for key in to_delete:
+ del node[key]
+
+ for _, value in _yaml.node_items(node):
+ self._process_value(value,
+ included=included,
+ current_loader=current_loader,
+ only_local=only_local)
+
+ # _include_file()
+ #
+ # Load include YAML file from with a loader.
+ #
+ # Args:
+ # include (str): file path relative to loader's project directory.
+ # Can be prefixed with junctio name.
+ # loader (Loader): Loader for the current project.
+ def _include_file(self, include, loader):
+ shortname = include
+ if ':' in include:
+ junction, include = include.split(':', 1)
+ junction_loader = loader._get_loader(junction, fetch_subprojects=True)
+ current_loader = junction_loader
+ else:
+ current_loader = loader
+ project = current_loader.project
+ directory = project.directory
+ file_path = os.path.join(directory, include)
+ key = (current_loader, file_path)
+ if file_path not in self._loaded:
+ self._loaded[key] = _yaml.load(os.path.join(directory, include),
+ shortname=shortname,
+ project=project)
+ return self._loaded[key], file_path, current_loader
+
+ # _process_value()
+ #
+ # Select processing for value that could be a list or a dictionary.
+ #
+ # Args:
+ # value: Value to process. Can be a list or a dictionary.
+ # included (set): Fail for recursion if trying to load any files in this set
+ # current_loader (Loader): Use alternative loader (for junction files)
+ # only_local (bool): Whether to ignore junction files
+ def _process_value(self, value, *,
+ included=set(),
+ current_loader=None,
+ only_local=False):
+ if isinstance(value, Mapping):
+ self.process(value,
+ included=included,
+ current_loader=current_loader,
+ only_local=only_local)
+ elif isinstance(value, list):
+ for v in value:
+ self._process_value(v,
+ included=included,
+ current_loader=current_loader,
+ only_local=only_local)
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 280805981ab5fa90a401af1eb5241b7e9df5bbeb..6e46197ab63a56a50ebd9951e503e0693b9cdd5a 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -29,6 +29,7 @@ from .. import _yaml
from ..element import Element
from .._profile import Topics, profile_start, profile_end
from .._platform import Platform
+from .._includes import Includes
from .types import Symbol, Dependency
from .loadelement import LoadElement
@@ -69,6 +70,7 @@ class Loader():
self._context = context
self._options = project.options # Project options (OptionPool)
self._basedir = basedir # Base project directory
+ self._first_pass_options = project.first_pass_config.options # Project options (OptionPool)
self._tempdir = tempdir # A directory to cleanup
self._parent = parent # The parent loader
@@ -76,6 +78,8 @@ class Loader():
self._elements = {} # Dict of elements
self._loaders = {} # Dict of junction loaders
+ self._includes = Includes(self)
+
# load():
#
# Loads the project based on the parameters given to the constructor
@@ -215,7 +219,7 @@ class Loader():
# Load the data and process any conditional statements therein
fullpath = os.path.join(self._basedir, filename)
try:
- node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable)
+ node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable, project=self.project)
except LoadError as e:
if e.reason == LoadErrorReason.MISSING_FILE:
# If we can't find the file, try to suggest plausible
@@ -241,7 +245,15 @@ class Loader():
message, detail=detail) from e
else:
raise
- self._options.process_node(node)
+ kind = _yaml.node_get(node, str, Symbol.KIND)
+ if kind == "junction":
+ self._first_pass_options.process_node(node)
+ else:
+ self.project.ensure_fully_loaded()
+
+ self._includes.process(node)
+
+ self._options.process_node(node)
element = LoadElement(node, filename, self)
@@ -433,7 +445,8 @@ class Loader():
_yaml.node_get(node, Mapping, Symbol.ENVIRONMENT, default_value={}),
_yaml.node_get(node, list, Symbol.ENV_NOCACHE, default_value=[]),
_yaml.node_get(node, Mapping, Symbol.PUBLIC, default_value={}),
- _yaml.node_get(node, Mapping, Symbol.SANDBOX, default_value={}))
+ _yaml.node_get(node, Mapping, Symbol.SANDBOX, default_value={}),
+ element_kind == 'junction')
# Cache it now, make sure it's already there before recursing
self._meta_elements[element_name] = meta_element
diff --git a/buildstream/_loader/metaelement.py b/buildstream/_loader/metaelement.py
index 16788e92b9788237857e8865c1bc27bd2684cce0..c13d5591e90edd2038212a8f299ec52c38e995ab 100644
--- a/buildstream/_loader/metaelement.py
+++ b/buildstream/_loader/metaelement.py
@@ -36,9 +36,11 @@ class MetaElement():
# env_nocache: List of environment vars which should not be considered in cache keys
# public: Public domain data dictionary
# sandbox: Configuration specific to the sandbox environment
+ # first_pass: The element is to be loaded with first pass configuration (junction)
#
def __init__(self, project, name, kind, provenance, sources, config,
- variables, environment, env_nocache, public, sandbox):
+ variables, environment, env_nocache, public, sandbox,
+ first_pass):
self.project = project
self.name = name
self.kind = kind
@@ -52,3 +54,4 @@ class MetaElement():
self.sandbox = sandbox
self.build_dependencies = []
self.dependencies = []
+ self.first_pass = first_pass
diff --git a/buildstream/_loader/metasource.py b/buildstream/_loader/metasource.py
index 3bcc21ec657c91c500f2339bec8d848e5150a64a..da2c0e2927585907f6fb7052958ab68f37080772 100644
--- a/buildstream/_loader/metasource.py
+++ b/buildstream/_loader/metasource.py
@@ -30,6 +30,7 @@ class MetaSource():
# element_kind: The kind of the owning element
# kind: The kind of the source
# config: The configuration data for the source
+ # first_pass: This source will be used with first project pass configuration (used for junctions).
#
def __init__(self, element_name, element_index, element_kind, kind, config, directory):
self.element_name = element_name
@@ -38,3 +39,4 @@ class MetaSource():
self.kind = kind
self.config = config
self.directory = directory
+ self.first_pass = False
diff --git a/buildstream/_options/optionpool.py b/buildstream/_options/optionpool.py
index f90fd820cdb33c3a725aaf1ef64ec985ac76823d..b53e87a3d00759d691f180407b162393cb77aaf3 100644
--- a/buildstream/_options/optionpool.py
+++ b/buildstream/_options/optionpool.py
@@ -107,16 +107,19 @@ class OptionPool():
#
# Args:
# cli_options (list): A list of (str, str) tuples
+ # ignore_unknown (bool): Whether to silently ignore unknown options.
#
- def load_cli_values(self, cli_options):
+ def load_cli_values(self, cli_options, *, ignore_unknown=False):
for option_name, option_value in cli_options:
try:
option = self._options[option_name]
except KeyError as e:
- raise LoadError(LoadErrorReason.INVALID_DATA,
- "Unknown option '{}' specified on the command line"
- .format(option_name)) from e
- option.set_value(option_value)
+ if not ignore_unknown:
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "Unknown option '{}' specified on the command line"
+ .format(option_name)) from e
+ else:
+ option.set_value(option_value)
# resolve()
#
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 6d8bca008fa7d181c4d36e0ad676b7e8b4edff40..3f7f949dd2e1a6f21ea2ad79b509a09ed3e57411 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -36,6 +36,7 @@ from ._versions import BST_FORMAT_VERSION
from ._loader import Loader
from .element import Element
from ._message import Message, MessageType
+from ._includes import Includes
# Project Configuration file
@@ -64,6 +65,20 @@ class HostMount():
self.host_path = self.path
+# Represents project configuration that can have different values for junctions.
+class ProjectConfig:
+ def __init__(self):
+ self.element_factory = None
+ self.source_factory = None
+ self.options = None # OptionPool
+ self.base_variables = {} # The base set of variables
+ self.element_overrides = {} # Element specific configurations
+ self.source_overrides = {} # Source specific configurations
+ self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs.
+ self.default_mirror = None # The name of the preferred mirror.
+ self._aliases = {} # Aliases dictionary
+
+
# Project()
#
# The Project Configuration
@@ -86,23 +101,21 @@ class Project():
self.refs = ProjectRefs(self.directory, 'project.refs')
self.junction_refs = ProjectRefs(self.directory, 'junction.refs')
- self.options = None # OptionPool
+ self.config = ProjectConfig()
+ self.first_pass_config = ProjectConfig()
+
self.junction = junction # The junction Element object, if this is a subproject
self.fail_on_overlap = False # Whether overlaps are treated as errors
self.ref_storage = None # ProjectRefStorage setting
- self.base_variables = {} # The base set of variables
self.base_environment = {} # The base set of environment variables
self.base_env_nocache = None # The base nocache mask (list) for the environment
- self.element_overrides = {} # Element specific configurations
- self.source_overrides = {} # Source specific configurations
- self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs.
- self.default_mirror = default_mirror # The name of the preferred mirror.
#
# Private Members
#
self._context = context # The invocation Context
- self._aliases = {} # Aliases dictionary
+
+ self._default_mirror = default_mirror # The name of the preferred mirror.
self._cli_options = cli_options
self._cache_key = None
@@ -111,18 +124,37 @@ class Project():
self._shell_environment = {} # Statically set environment vars
self._shell_host_files = [] # A list of HostMount objects
- self._element_factory = None
- self._source_factory = None
+ self.artifact_cache_specs = None
+ self._sandbox = None
+ self._splits = None
+
+ self._context.add_project(self)
+
+ self._partially_loaded = False
+ self._fully_loaded = False
+ self._project_includes = None
profile_start(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
- self._load()
+ self._load(parent_loader=parent_loader, tempdir=tempdir)
profile_end(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
- self._context.add_project(self)
+ self._partially_loaded = True
- self.loader = Loader(self._context, self,
- parent=parent_loader,
- tempdir=tempdir)
+ @property
+ def options(self):
+ return self.config.options
+
+ @property
+ def base_variables(self):
+ return self.config.base_variables
+
+ @property
+ def element_overrides(self):
+ return self.config.element_overrides
+
+ @property
+ def source_overrides(self):
+ return self.config.source_overrides
# translate_url():
#
@@ -138,10 +170,15 @@ class Project():
# This method is provided for :class:`.Source` objects to resolve
# fully qualified urls based on the shorthand which is allowed
# to be specified in the YAML
- def translate_url(self, url):
+ def translate_url(self, url, *, first_pass=False):
+ if first_pass:
+ config = self.first_pass_config
+ else:
+ config = self.config
+
if url and utils._ALIAS_SEPARATOR in url:
url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
- alias_url = self._aliases.get(url_alias)
+ alias_url = config._aliases.get(url_alias)
if alias_url:
url = alias_url + url_body
@@ -186,8 +223,11 @@ class Project():
# Returns:
# (Element): A newly created Element object of the appropriate kind
#
- def create_element(self, artifacts, meta):
- return self._element_factory.create(self._context, self, artifacts, meta)
+ def create_element(self, artifacts, meta, *, first_pass=False):
+ if first_pass:
+ return self.first_pass_config.element_factory.create(self._context, self, artifacts, meta)
+ else:
+ return self.config.element_factory.create(self._context, self, artifacts, meta)
# create_source()
#
@@ -199,8 +239,11 @@ class Project():
# Returns:
# (Source): A newly created Source object of the appropriate kind
#
- def create_source(self, meta):
- return self._source_factory.create(self._context, self, meta)
+ def create_source(self, meta, *, first_pass=False):
+ if first_pass:
+ return self.first_pass_config.source_factory.create(self._context, self, meta)
+ else:
+ return self.config.source_factory.create(self._context, self, meta)
# get_alias_uri()
#
@@ -212,24 +255,34 @@ class Project():
# Returns:
# str: The URI for the given alias; or None: if there is no URI for
# that alias.
- def get_alias_uri(self, alias):
- return self._aliases.get(alias)
+ def get_alias_uri(self, alias, *, first_pass=False):
+ if first_pass:
+ config = self.first_pass_config
+ else:
+ config = self.config
+
+ return config._aliases.get(alias)
# get_alias_uris()
#
# Returns a list of every URI to replace an alias with
- def get_alias_uris(self, alias):
- if not alias or alias not in self._aliases:
+ def get_alias_uris(self, alias, *, first_pass=False):
+ if first_pass:
+ config = self.first_pass_config
+ else:
+ config = self.config
+
+ if not alias or alias not in config._aliases:
return [None]
mirror_list = []
- for key, alias_mapping in self.mirrors.items():
+ for key, alias_mapping in config.mirrors.items():
if alias in alias_mapping:
- if key == self.default_mirror:
+ if key == config.default_mirror:
mirror_list = alias_mapping[alias] + mirror_list
else:
mirror_list += alias_mapping[alias]
- mirror_list.append(self._aliases[alias])
+ mirror_list.append(config._aliases[alias])
return mirror_list
# load_elements()
@@ -291,32 +344,24 @@ class Project():
#
# Raises: LoadError if there was a problem with the project.conf
#
- def _load(self):
+ def _load(self, parent_loader=None, tempdir=None):
# Load builtin default
projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE)
- config = _yaml.load(_site.default_project_config)
+ self._default_config_node = _yaml.load(_site.default_project_config)
# Load project local config and override the builtin
try:
- project_conf = _yaml.load(projectfile)
+ self._project_conf = _yaml.load(projectfile)
except LoadError as e:
# Raise a more specific error here
raise LoadError(LoadErrorReason.MISSING_PROJECT_CONF, str(e))
- _yaml.composite(config, project_conf)
-
- # Element and Source type configurations will be composited later onto
- # element/source types, so we delete it from here and run our final
- # assertion after.
- self.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
- self.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
- config.pop('elements', None)
- config.pop('sources', None)
- _yaml.node_final_assertions(config)
+ pre_config_node = _yaml.node_copy(self._default_config_node)
+ _yaml.composite(pre_config_node, self._project_conf)
# Assert project's format version early, before validating toplevel keys
- format_version = _yaml.node_get(config, int, 'format-version')
+ format_version = _yaml.node_get(pre_config_node, int, 'format-version')
if BST_FORMAT_VERSION < format_version:
major, minor = utils.get_bst_version()
raise LoadError(
@@ -324,84 +369,80 @@ class Project():
"Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
.format(format_version, major, minor, BST_FORMAT_VERSION))
- _yaml.node_validate(config, [
- 'format-version',
- 'element-path', 'variables',
- 'environment', 'environment-nocache',
- 'split-rules', 'elements', 'plugins',
- 'aliases', 'name',
- 'artifacts', 'options',
- 'fail-on-overlap', 'shell',
- 'ref-storage', 'sandbox', 'mirrors',
- ])
-
# The project name, element path and option declarations
# are constant and cannot be overridden by option conditional statements
- self.name = _yaml.node_get(config, str, 'name')
+ self.name = _yaml.node_get(pre_config_node, str, 'name')
# Validate that project name is a valid symbol name
- _yaml.assert_symbol_name(_yaml.node_get_provenance(config, 'name'),
+ _yaml.assert_symbol_name(_yaml.node_get_provenance(pre_config_node, 'name'),
self.name, "project name")
self.element_path = os.path.join(
self.directory,
- _yaml.node_get(config, str, 'element-path')
+ _yaml.node_get(pre_config_node, str, 'element-path')
)
- # Load project options
- options_node = _yaml.node_get(config, Mapping, 'options', default_value={})
- self.options = OptionPool(self.element_path)
- self.options.load(options_node)
- if self.junction:
- # load before user configuration
- self.options.load_yaml_values(self.junction.options, transform=self.junction._subst_string)
+ self.config.options = OptionPool(self.element_path)
+ self.first_pass_config.options = OptionPool(self.element_path)
- # Collect option values specified in the user configuration
- overrides = self._context.get_overrides(self.name)
- override_options = _yaml.node_get(overrides, Mapping, 'options', default_value={})
- self.options.load_yaml_values(override_options)
- if self._cli_options:
- self.options.load_cli_values(self._cli_options)
+ self.loader = Loader(self._context, self,
+ parent=parent_loader,
+ tempdir=tempdir)
- # We're done modifying options, now we can use them for substitutions
- self.options.resolve()
+ self._project_includes = Includes(self.loader)
- #
- # Now resolve any conditionals in the remaining configuration,
- # any conditionals specified for project option declarations,
- # or conditionally specifying the project name; will be ignored.
- #
- self.options.process_node(config)
+ project_conf_first_pass = _yaml.node_copy(self._project_conf)
+ self._project_includes.process(project_conf_first_pass, only_local=True)
+ config_no_include = _yaml.node_copy(self._default_config_node)
+ _yaml.composite(config_no_include, project_conf_first_pass)
- # Override default_mirror if not set by command-line
- if not self.default_mirror:
- self.default_mirror = _yaml.node_get(overrides, str, 'default-mirror', default_value=None)
+ self._load_pass(config_no_include, self.first_pass_config, True)
- #
- # Now all YAML composition is done, from here on we just load
- # the values from our loaded configuration dictionary.
- #
+ # Use separate file for storing source references
+ self.ref_storage = _yaml.node_get(pre_config_node, str, 'ref-storage')
+ if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]:
+ p = _yaml.node_get_provenance(pre_config_node, 'ref-storage')
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Invalid value '{}' specified for ref-storage"
+ .format(p, self.ref_storage))
- # Load artifacts pull/push configuration for this project
- self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
+ if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
+ self.junction_refs.load(self.first_pass_config.options)
- self._load_plugin_factories(config)
+ def ensure_fully_loaded(self):
+ if self._fully_loaded:
+ return
+ assert self._partially_loaded
+ self._fully_loaded = True
- # Source url aliases
- self._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
+ if self.junction:
+ self.junction._get_project().ensure_fully_loaded()
- # Load base variables
- self.base_variables = _yaml.node_get(config, Mapping, 'variables')
+ project_conf_second_pass = _yaml.node_copy(self._project_conf)
+ self._project_includes.process(project_conf_second_pass)
+ config = _yaml.node_copy(self._default_config_node)
+ _yaml.composite(config, project_conf_second_pass)
- # Add the project name as a default variable
- self.base_variables['project-name'] = self.name
+ self._load_pass(config, self.config, False)
- # Extend variables with automatic variables and option exports
- # Initialize it as a string as all variables are processed as strings.
- self.base_variables['max-jobs'] = str(multiprocessing.cpu_count())
+ _yaml.node_validate(config, [
+ 'format-version',
+ 'element-path', 'variables',
+ 'environment', 'environment-nocache',
+ 'split-rules', 'elements', 'plugins',
+ 'aliases', 'name',
+ 'artifacts', 'options',
+ 'fail-on-overlap', 'shell',
+ 'ref-storage', 'sandbox', 'mirrors'
+ ])
- # Export options into variables, if that was requested
- self.options.export_variables(self.base_variables)
+ #
+ # Now all YAML composition is done, from here on we just load
+ # the values from our loaded configuration dictionary.
+ #
+
+ # Load artifacts pull/push configuration for this project
+ self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
# Load sandbox environment variables
self.base_environment = _yaml.node_get(config, Mapping, 'environment')
@@ -416,18 +457,9 @@ class Project():
# Fail on overlap
self.fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap')
- # Use separate file for storing source references
- self.ref_storage = _yaml.node_get(config, str, 'ref-storage')
- if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]:
- p = _yaml.node_get_provenance(config, 'ref-storage')
- raise LoadError(LoadErrorReason.INVALID_DATA,
- "{}: Invalid value '{}' specified for ref-storage"
- .format(p, self.ref_storage))
-
# Load project.refs if it exists, this may be ignored.
if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
self.refs.load(self.options)
- self.junction_refs.load(self.options)
# Parse shell options
shell_options = _yaml.node_get(config, Mapping, 'shell')
@@ -459,6 +491,70 @@ class Project():
self._shell_host_files.append(mount)
+ # _load_pass():
+ #
+ # Loads parts of the project configuration that are different
+ # for first and second pass configurations.
+ #
+ # Args:
+ # config (dict) - YaML node of the configuration file.
+ # output (ProjectConfig) - ProjectConfig to load configuration onto.
+ # ignore_unknown (bool) - Whether option loader shoud ignore unknown options.
+ #
+ def _load_pass(self, config, output, ignore_unknown):
+
+ # Element and Source type configurations will be composited later onto
+ # element/source types, so we delete it from here and run our final
+ # assertion after.
+ output.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
+ output.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
+ config.pop('elements', None)
+ config.pop('sources', None)
+ _yaml.node_final_assertions(config)
+
+ self._load_plugin_factories(config, output)
+
+ # Load project options
+ options_node = _yaml.node_get(config, Mapping, 'options', default_value={})
+ output.options.load(options_node)
+ if self.junction:
+ # load before user configuration
+ output.options.load_yaml_values(self.junction.options, transform=self.junction._subst_string)
+
+ # Collect option values specified in the user configuration
+ overrides = self._context.get_overrides(self.name)
+ override_options = _yaml.node_get(overrides, Mapping, 'options', default_value={})
+ output.options.load_yaml_values(override_options)
+ if self._cli_options:
+ output.options.load_cli_values(self._cli_options, ignore_unknown=ignore_unknown)
+
+ # We're done modifying options, now we can use them for substitutions
+ output.options.resolve()
+
+ #
+ # Now resolve any conditionals in the remaining configuration,
+ # any conditionals specified for project option declarations,
+ # or conditionally specifying the project name; will be ignored.
+ #
+ output.options.process_node(config)
+
+ # Load base variables
+ output.base_variables = _yaml.node_get(config, Mapping, 'variables')
+
+ # Add the project name as a default variable
+ output.base_variables['project-name'] = self.name
+
+ # Extend variables with automatic variables and option exports
+ # Initialize it as a string as all variables are processed as strings.
+ output.base_variables['max-jobs'] = str(multiprocessing.cpu_count())
+
+ # Export options into variables, if that was requested
+ output.options.export_variables(output.base_variables)
+
+ # Override default_mirror if not set by command-line
+ output.default_mirror = self._default_mirror or _yaml.node_get(overrides, str,
+ 'default-mirror', default_value=None)
+
mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
for mirror in mirrors:
allowed_mirror_fields = [
@@ -470,11 +566,38 @@ class Project():
for alias_mapping, uris in _yaml.node_items(mirror['aliases']):
assert isinstance(uris, list)
alias_mappings[alias_mapping] = list(uris)
- self.mirrors[mirror_name] = alias_mappings
- if not self.default_mirror:
- self.default_mirror = mirror_name
+ output.mirrors[mirror_name] = alias_mappings
+ if not output.default_mirror:
+ output.default_mirror = mirror_name
+
+ # Source url aliases
+ output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
- def _load_plugin_factories(self, config):
+ # _ensure_project_dir()
+ #
+ # Returns path of the project directory, if a configuration file is found
+ # in given directory or any of its parent directories.
+ #
+ # Args:
+ # directory (str) - directory from where the command was invoked
+ #
+ # Raises:
+ # LoadError if project.conf is not found
+ #
+ def _ensure_project_dir(self, directory):
+ directory = os.path.abspath(directory)
+ while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
+ parent_dir = os.path.dirname(directory)
+ if directory == parent_dir:
+ raise LoadError(
+ LoadErrorReason.MISSING_PROJECT_CONF,
+ '{} not found in current directory or any of its parent directories'
+ .format(_PROJECT_CONF_FILE))
+ directory = parent_dir
+
+ return directory
+
+ def _load_plugin_factories(self, config, output):
plugin_source_origins = [] # Origins of custom sources
plugin_element_origins = [] # Origins of custom elements
@@ -521,12 +644,12 @@ class Project():
self._store_origin(origin, 'elements', plugin_element_origins)
pluginbase = PluginBase(package='buildstream.plugins')
- self._element_factory = ElementFactory(pluginbase,
- plugin_origins=plugin_element_origins,
- format_versions=element_format_versions)
- self._source_factory = SourceFactory(pluginbase,
- plugin_origins=plugin_source_origins,
- format_versions=source_format_versions)
+ output.element_factory = ElementFactory(pluginbase,
+ plugin_origins=plugin_element_origins,
+ format_versions=element_format_versions)
+ output.source_factory = SourceFactory(pluginbase,
+ plugin_origins=plugin_source_origins,
+ format_versions=source_format_versions)
# _store_origin()
#
@@ -558,27 +681,3 @@ class Project():
# paths are passed in relative to the project, but must be absolute
origin_dict['path'] = os.path.join(self.directory, origin_dict['path'])
destination.append(origin_dict)
-
- # _ensure_project_dir()
- #
- # Returns path of the project directory, if a configuration file is found
- # in given directory or any of its parent directories.
- #
- # Args:
- # directory (str) - directory from where the command was invoked
- #
- # Raises:
- # LoadError if project.conf is not found
- #
- def _ensure_project_dir(self, directory):
- directory = os.path.abspath(directory)
- while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
- parent_dir = os.path.dirname(directory)
- if directory == parent_dir:
- raise LoadError(
- LoadErrorReason.MISSING_PROJECT_CONF,
- '{} not found in current directory or any of its parent directories'
- .format(_PROJECT_CONF_FILE))
- directory = parent_dir
-
- return directory
diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py
index 0e090e2e776936a69cdb60b6245fccff537a31cf..e34357762ec3aecc6c77842c9837cb85993eedfa 100644
--- a/buildstream/_yaml.py
+++ b/buildstream/_yaml.py
@@ -37,6 +37,19 @@ RoundTripConstructor.add_constructor(u'tag:yaml.org,2002:float', RoundTripConstr
PROVENANCE_KEY = '__bst_provenance_info'
+# Provides information about file for provenance
+#
+# Args:
+# name (str): Full path to the file
+# shortname (str): Relative path to the file
+# project (Project): Project where the shortname is relative from
+class ProvenanceFile():
+ def __init__(self, name, shortname, project):
+ self.name = name
+ self.shortname = shortname
+ self.project = project
+
+
# Provenance tracks the origin of a given node in the parsed dictionary.
#
# Args:
@@ -56,7 +69,7 @@ class Provenance():
# Convert a Provenance to a string for error reporting
def __str__(self):
- return "{} [line {:d} column {:d}]".format(self.filename, self.line, self.col)
+ return "{} [line {:d} column {:d}]".format(self.filename.shortname, self.line, self.col)
# Abstract method
def clone(self):
@@ -174,13 +187,15 @@ class CompositeTypeError(CompositeError):
#
# Raises: LoadError
#
-def load(filename, shortname=None, copy_tree=False):
+def load(filename, shortname=None, copy_tree=False, *, project=None):
if not shortname:
shortname = filename
+ file = ProvenanceFile(filename, shortname, project)
+
try:
with open(filename) as f:
- return load_data(f, shortname=shortname, copy_tree=copy_tree)
+ return load_data(f, file, copy_tree=copy_tree)
except FileNotFoundError as e:
raise LoadError(LoadErrorReason.MISSING_FILE,
"Could not find file at {}".format(filename)) from e
@@ -192,7 +207,7 @@ def load(filename, shortname=None, copy_tree=False):
# Like load(), but doesnt require the data to be in a file
#
-def load_data(data, shortname=None, copy_tree=False):
+def load_data(data, file=None, copy_tree=False):
try:
contents = yaml.load(data, yaml.loader.RoundTripLoader, preserve_quotes=True)
@@ -207,9 +222,9 @@ def load_data(data, shortname=None, copy_tree=False):
else:
raise LoadError(LoadErrorReason.INVALID_YAML,
"YAML file has content of type '{}' instead of expected type 'dict': {}"
- .format(type(contents).__name__, shortname))
+ .format(type(contents).__name__, file.name))
- return node_decorated_copy(shortname, contents, copy_tree=copy_tree)
+ return node_decorated_copy(file, contents, copy_tree=copy_tree)
# Dumps a previously loaded YAML node to a file
@@ -416,7 +431,7 @@ def node_items(node):
def ensure_provenance(node):
provenance = node.get(PROVENANCE_KEY)
if not provenance:
- provenance = DictProvenance('', node, node)
+ provenance = DictProvenance(ProvenanceFile('', '', None), node, node)
node[PROVENANCE_KEY] = provenance
return provenance
diff --git a/buildstream/element.py b/buildstream/element.py
index 49cc934e1832286512d5963e96bd0ce2cf5a03d8..ec5b91c9ee9fcde922f49d0956d2174344e51041 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -199,6 +199,11 @@ class Element(Plugin):
super().__init__(meta.name, context, project, meta.provenance, "element")
+ self.__is_junction = meta.kind == "junction"
+
+ if not self.__is_junction:
+ project.ensure_fully_loaded()
+
self.normal_name = os.path.splitext(self.name.replace(os.sep, '-'))[0]
"""A normalized element name
@@ -889,16 +894,20 @@ class Element(Plugin):
@classmethod
def _new_from_meta(cls, meta, artifacts):
+ if not meta.first_pass:
+ meta.project.ensure_fully_loaded()
+
if meta in cls.__instantiated_elements:
return cls.__instantiated_elements[meta]
- project = meta.project
- element = project.create_element(artifacts, meta)
+ element = meta.project.create_element(artifacts, meta, first_pass=meta.first_pass)
cls.__instantiated_elements[meta] = element
# Instantiate sources
for meta_source in meta.sources:
- source = project.create_source(meta_source)
+ meta_source.first_pass = meta.kind == "junction"
+ source = meta.project.create_source(meta_source,
+ first_pass=meta.first_pass)
redundant_ref = source._load_ref()
element.__sources.append(source)
@@ -2060,16 +2069,21 @@ class Element(Plugin):
def __compose_default_splits(self, defaults):
project = self._get_project()
- project_splits = _yaml.node_chain_copy(project._splits)
element_public = _yaml.node_get(defaults, Mapping, 'public', default_value={})
element_bst = _yaml.node_get(element_public, Mapping, 'bst', default_value={})
element_splits = _yaml.node_get(element_bst, Mapping, 'split-rules', default_value={})
- # Extend project wide split rules with any split rules defined by the element
- _yaml.composite(project_splits, element_splits)
+ if self.__is_junction:
+ splits = _yaml.node_chain_copy(element_splits)
+ else:
+ assert project._splits is not None
+
+ splits = _yaml.node_chain_copy(project._splits)
+ # Extend project wide split rules with any split rules defined by the element
+ _yaml.composite(splits, element_splits)
- element_bst['split-rules'] = project_splits
+ element_bst['split-rules'] = splits
element_public['bst'] = element_bst
defaults['public'] = element_public
@@ -2093,7 +2107,11 @@ class Element(Plugin):
# Override the element's defaults with element specific
# overrides from the project.conf
project = self._get_project()
- elements = project.element_overrides
+ if self.__is_junction:
+ elements = project.first_pass_config.element_overrides
+ else:
+ elements = project.element_overrides
+
overrides = elements.get(self.get_kind())
if overrides:
_yaml.composite(defaults, overrides)
@@ -2106,10 +2124,14 @@ class Element(Plugin):
# creating sandboxes for this element
#
def __extract_environment(self, meta):
- project = self._get_project()
default_env = _yaml.node_get(self.__defaults, Mapping, 'environment', default_value={})
- environment = _yaml.node_chain_copy(project.base_environment)
+ if self.__is_junction:
+ environment = {}
+ else:
+ project = self._get_project()
+ environment = _yaml.node_chain_copy(project.base_environment)
+
_yaml.composite(environment, default_env)
_yaml.composite(environment, meta.environment)
_yaml.node_final_assertions(environment)
@@ -2122,8 +2144,13 @@ class Element(Plugin):
return final_env
def __extract_env_nocache(self, meta):
- project = self._get_project()
- project_nocache = project.base_env_nocache
+ if self.__is_junction:
+ project_nocache = []
+ else:
+ project = self._get_project()
+ project.ensure_fully_loaded()
+ project_nocache = project.base_env_nocache
+
default_nocache = _yaml.node_get(self.__defaults, list, 'environment-nocache', default_value=[])
element_nocache = meta.env_nocache
@@ -2138,10 +2165,15 @@ class Element(Plugin):
# substituting command strings to be run in the sandbox
#
def __extract_variables(self, meta):
- project = self._get_project()
default_vars = _yaml.node_get(self.__defaults, Mapping, 'variables', default_value={})
- variables = _yaml.node_chain_copy(project.base_variables)
+ project = self._get_project()
+ if self.__is_junction:
+ variables = _yaml.node_chain_copy(project.first_pass_config.base_variables)
+ else:
+ project.ensure_fully_loaded()
+ variables = _yaml.node_chain_copy(project.base_variables)
+
_yaml.composite(variables, default_vars)
_yaml.composite(variables, meta.variables)
_yaml.node_final_assertions(variables)
@@ -2165,13 +2197,18 @@ class Element(Plugin):
# Sandbox-specific configuration data, to be passed to the sandbox's constructor.
#
def __extract_sandbox_config(self, meta):
- project = self._get_project()
+ if self.__is_junction:
+ sandbox_config = {'build-uid': 0,
+ 'build-gid': 0}
+ else:
+ project = self._get_project()
+ project.ensure_fully_loaded()
+ sandbox_config = _yaml.node_chain_copy(project._sandbox)
# The default config is already composited with the project overrides
sandbox_defaults = _yaml.node_get(self.__defaults, Mapping, 'sandbox', default_value={})
sandbox_defaults = _yaml.node_chain_copy(sandbox_defaults)
- sandbox_config = _yaml.node_chain_copy(project._sandbox)
_yaml.composite(sandbox_config, sandbox_defaults)
_yaml.composite(sandbox_config, meta.sandbox)
_yaml.node_final_assertions(sandbox_config)
diff --git a/buildstream/source.py b/buildstream/source.py
index 2f3f1c2818f9e2ba4280466f56a9471fdf45aba0..7cd4d837ec33ed1491a962415e2c5d4396ebd93d 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -227,8 +227,10 @@ class Source(Plugin):
# Collect the composited element configuration and
# ask the element to configure itself.
- self.__init_defaults()
+ self.__init_defaults(meta)
self.__config = self.__extract_config(meta)
+ self.__first_pass = meta.first_pass
+
self.configure(self.__config)
COMMON_CONFIG_KEYS = ['kind', 'directory']
@@ -454,7 +456,7 @@ class Source(Plugin):
self.__expected_alias = url_alias
project = self._get_project()
- return project.translate_url(url)
+ return project.translate_url(url, first_pass=self.__first_pass)
def get_project_directory(self):
"""Fetch the project base directory
@@ -524,7 +526,7 @@ class Source(Plugin):
for fetcher in source_fetchers:
alias = fetcher._get_alias()
success = False
- for uri in project.get_alias_uris(alias):
+ for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
try:
fetcher.fetch(uri)
# FIXME: Need to consider temporary vs. permanent failures,
@@ -538,13 +540,17 @@ class Source(Plugin):
raise last_error
else:
alias = self._get_alias()
- if not project.mirrors or not alias:
+ if self.__first_pass:
+ mirrors = project.config.mirrors
+ else:
+ mirrors = project.first_pass_config.mirrors
+ if not mirrors or not alias:
self.fetch()
return
context = self._get_context()
source_kind = type(self)
- for uri in project.get_alias_uris(alias):
+ for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
new_source = source_kind(context, project, self.__meta,
alias_override=(alias, uri))
new_source._preflight()
@@ -739,24 +745,29 @@ class Source(Plugin):
#
# Step 3 - Apply the change in project data
#
- if project is toplevel:
- if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
- do_save_refs(toplevel_refs)
- else:
+ if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
+ do_save_refs(toplevel_refs)
+ else:
+ if provenance.filename.project is toplevel:
# Save the ref in the originating file
#
- fullname = os.path.join(toplevel.element_path, provenance.filename)
try:
- _yaml.dump(provenance.toplevel, fullname)
+ _yaml.dump(_yaml.node_sanitize(provenance.toplevel), provenance.filename.name)
except OSError as e:
raise SourceError("{}: Error saving source reference to '{}': {}"
- .format(self, provenance.filename, e),
+ .format(self, provenance.filename.name, e),
reason="save-ref-error") from e
- else:
- if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
- do_save_refs(toplevel_refs)
- else:
+ elif provenance.filename.project is project:
self.warn("{}: Not persisting new reference in junctioned project".format(self))
+ elif provenance.filename.project is None:
+ assert provenance.filename.name == ''
+ assert provenance.filename.shortname == ''
+ raise SourceError("{}: Error saving source reference to synthetic node."
+ .format(self))
+ else:
+ raise SourceError("{}: Cannot track source in a fragment from a junction"
+ .format(provenance.filename.shortname),
+ reason="tracking-junction-fragment")
return changed
@@ -779,7 +790,7 @@ class Source(Plugin):
def _get_alias(self):
alias = self.__expected_alias
project = self._get_project()
- if project.get_alias_uri(alias):
+ if project.get_alias_uri(alias, first_pass=self.__first_pass):
# The alias must already be defined in the project's aliases
# otherwise http://foo gets treated like it contains an alias
return alias
@@ -795,7 +806,11 @@ class Source(Plugin):
project = self._get_project()
# If there are no mirrors, or no aliases to replace, there's nothing to do here.
alias = self._get_alias()
- if not project.mirrors or not alias:
+ if self.__first_pass:
+ mirrors = project.config.mirrors
+ else:
+ mirrors = project.first_pass_config.mirrors
+ if not mirrors or not alias:
return self.track()
context = self._get_context()
@@ -803,7 +818,7 @@ class Source(Plugin):
# NOTE: We are assuming here that tracking only requires substituting the
# first alias used
- for uri in reversed(project.get_alias_uris(alias)):
+ for uri in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass)):
new_source = source_kind(context, project, self.__meta,
alias_override=(alias, uri))
new_source._preflight()
@@ -831,10 +846,13 @@ class Source(Plugin):
reason="ensure-stage-dir-fail") from e
return directory
- def __init_defaults(self):
+ def __init_defaults(self, meta):
if not self.__defaults_set:
project = self._get_project()
- sources = project.source_overrides
+ if meta.first_pass:
+ sources = project.first_pass_config.source_overrides
+ else:
+ sources = project.source_overrides
type(self).__defaults = sources.get(self.get_kind(), {})
type(self).__defaults_set = True
diff --git a/doc/source/format_intro.rst b/doc/source/format_intro.rst
index b1780f9dc457c51047ae20bdfe6ef0228540eb51..fb82ddf418ed50a37240c090629e452f88ecbb29 100644
--- a/doc/source/format_intro.rst
+++ b/doc/source/format_intro.rst
@@ -289,3 +289,40 @@ free form and not validated.
# This element's `make install` is broken, replace it.
(=):
- cp src/program %{bindir}
+
+(@) Include
+~~~~~~~~~~~
+Indicates that content should be loaded from files.
+
+The include directive expects a list of strings. Those are file names
+relative to project directory. Or they can be prefixed with a
+:mod:`junction <elements.junction>` name and a colon (':'). In that
+case, the remain of the string is a file name relative to the project
+of the junction.
+
+The include directive can be used in :ref:`project.conf <projectconf>`
+or in a :ref:`.bst <format_basics>` file. It can also be used in a
+file included by another include directive.
+
+Including files are composed into included files from last to first,
+at the level of the directive. That is the including file has higher
+priority. Included files are listed in increasing order of
+priority. Files including other files should take care of composition
+using :ref:`list directives <format_directives_list_prepend>`.
+
+Junction elements never use values from included junctions files from
+:ref:`project.conf <projectconf>`. Variables, :ref:`element overrides
+<project_element_overrides>` and :ref:`source overrides
+<project_source_overrides>` required by junctions should all be
+directly in the ref:`project.conf <projectconf>` or included from a
+file local to the project.
+
+Junction elements cannot use the include directives.
+
+**Example:**
+
+.. code:: yaml
+
+ elements:
+ (@):
+ - junction.bst:includes/element-overrides.bst
diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py
index 079e511efefaa04d83f339c6ef68e5e1311ecdea..f594747085c4eb455a6217516f9433a35d17a15c 100644
--- a/tests/artifactcache/config.py
+++ b/tests/artifactcache/config.py
@@ -98,6 +98,7 @@ def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user
context = Context()
context.load(config=user_config_file)
project = Project(str(project_dir), context)
+ project.ensure_fully_loaded()
# Use the helper from the artifactcache module to parse our configuration.
parsed_cache_specs = _configured_remote_artifact_cache_specs(context, project)
diff --git a/tests/format/include.py b/tests/format/include.py
new file mode 100644
index 0000000000000000000000000000000000000000..36e723ed05b074d502ddf55b7234e008ff09cacc
--- /dev/null
+++ b/tests/format/include.py
@@ -0,0 +1,263 @@
+import os
+import pytest
+from buildstream import _yaml
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
+from tests.testutils import cli, generate_junction, create_repo
+
+
+# Project directory
+DATA_DIR = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)),
+ 'include'
+)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_project_file(cli, datafiles):
+ project = os.path.join(str(datafiles), 'file')
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert loaded['included'] == 'True'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_junction_file(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'junction')
+
+ generate_junction(tmpdir,
+ os.path.join(project, 'subproject'),
+ os.path.join(project, 'junction.bst'),
+ store_ref=True)
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert loaded['included'] == 'True'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_junction_options(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'options')
+
+ result = cli.run(project=project, args=[
+ '-o', 'build_arch', 'x86_64',
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert loaded['build_arch'] == 'x86_64'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_element_partial_project_project(cli, tmpdir, datafiles):
+ """
+ Junction elements never depend on fully include processed project.
+ """
+
+ project = os.path.join(str(datafiles), 'junction')
+
+ subproject_path = os.path.join(project, 'subproject')
+ junction_path = os.path.join(project, 'junction.bst')
+
+ repo = create_repo('git', str(tmpdir))
+
+ ref = repo.create(subproject_path)
+
+ element = {
+ 'kind': 'junction',
+ 'sources': [
+ repo.source_config(ref=ref)
+ ]
+ }
+ _yaml.dump(element, junction_path)
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'junction.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'included' not in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_element_not_partial_project_file(cli, tmpdir, datafiles):
+ """
+ Junction elements never depend on fully include processed project.
+ """
+
+ project = os.path.join(str(datafiles), 'file_with_subproject')
+
+ subproject_path = os.path.join(project, 'subproject')
+ junction_path = os.path.join(project, 'junction.bst')
+
+ repo = create_repo('git', str(tmpdir))
+
+ ref = repo.create(subproject_path)
+
+ element = {
+ 'kind': 'junction',
+ 'sources': [
+ repo.source_config(ref=ref)
+ ]
+ }
+ _yaml.dump(element, junction_path)
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'junction.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'included' in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'overrides')
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'manual_main_override' in loaded
+ assert 'manual_included_override' in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides_composition(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'overrides')
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{config}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'build-commands' in loaded
+ assert loaded['build-commands'] == ['first', 'second']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides_sub_include(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'sub-include')
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'included' in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_do_not_use_included_overrides(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'overrides-junction')
+
+ generate_junction(tmpdir,
+ os.path.join(project, 'subproject'),
+ os.path.join(project, 'junction.bst'),
+ store_ref=True)
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'junction.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'main_override' in loaded
+ assert 'included_override' not in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_conditional_in_fragment(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'conditional')
+
+ result = cli.run(project=project, args=[
+ '-o', 'build_arch', 'x86_64',
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'size' in loaded
+ assert loaded['size'] == '8'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_inner(cli, datafiles):
+ project = os.path.join(str(datafiles), 'inner')
+ result = cli.run(project=project, args=[
+ '-o', 'build_arch', 'x86_64',
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert loaded['build_arch'] == 'x86_64'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_recusive_include(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'recursive')
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.RECURSIVE_INCLUDE)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_local_to_junction(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'local_to_junction')
+
+ generate_junction(tmpdir,
+ os.path.join(project, 'subproject'),
+ os.path.join(project, 'junction.bst'),
+ store_ref=True)
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert loaded['included'] == 'True'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_project_file(cli, datafiles):
+ project = os.path.join(str(datafiles), 'string')
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert loaded['included'] == 'True'
diff --git a/tests/format/include/conditional/element.bst b/tests/format/include/conditional/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/conditional/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/conditional/extra_conf.yml b/tests/format/include/conditional/extra_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..dd58c98556fbce10501440c44eae81b488ecbb78
--- /dev/null
+++ b/tests/format/include/conditional/extra_conf.yml
@@ -0,0 +1,6 @@
+variables:
+ (?):
+ - build_arch == "i586":
+ size: "4"
+ - build_arch == "x86_64":
+ size: "8"
diff --git a/tests/format/include/conditional/project.conf b/tests/format/include/conditional/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..cb54779d368f140c9c0427bbf7e02d5c345853b1
--- /dev/null
+++ b/tests/format/include/conditional/project.conf
@@ -0,0 +1,13 @@
+name: test
+
+options:
+ build_arch:
+ type: arch
+ description: Architecture
+ variable: build_arch
+ values:
+ - i586
+ - x86_64
+
+(@):
+ - extra_conf.yml
diff --git a/tests/format/include/file/element.bst b/tests/format/include/file/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/file/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/file/extra_conf.yml b/tests/format/include/file/extra_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..404ecd6dd761c4323b666bc6ce606f99abfdbf30
--- /dev/null
+++ b/tests/format/include/file/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+ included: 'True'
diff --git a/tests/format/include/file/project.conf b/tests/format/include/file/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..a7791a4163e25b9351c7eb4d6cec341a8afa8c4e
--- /dev/null
+++ b/tests/format/include/file/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/element.bst b/tests/format/include/file_with_subproject/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/file_with_subproject/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/file_with_subproject/extra_conf.yml b/tests/format/include/file_with_subproject/extra_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..404ecd6dd761c4323b666bc6ce606f99abfdbf30
--- /dev/null
+++ b/tests/format/include/file_with_subproject/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+ included: 'True'
diff --git a/tests/format/include/file_with_subproject/project.bst b/tests/format/include/file_with_subproject/project.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4836c5f8b924cd8f0de0528c3c0a30f3c03977de
--- /dev/null
+++ b/tests/format/include/file_with_subproject/project.bst
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - junction.bst:extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/project.conf b/tests/format/include/file_with_subproject/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..a7791a4163e25b9351c7eb4d6cec341a8afa8c4e
--- /dev/null
+++ b/tests/format/include/file_with_subproject/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/subproject/project.conf b/tests/format/include/file_with_subproject/subproject/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..7a66554218de2e2d32d2b645639b9ad36bc5dfda
--- /dev/null
+++ b/tests/format/include/file_with_subproject/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/inner/element.bst b/tests/format/include/inner/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/inner/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/inner/extra_conf.yml b/tests/format/include/inner/extra_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..4c1847b5f6da2dd5f8d6b661955e90f550b27eef
--- /dev/null
+++ b/tests/format/include/inner/extra_conf.yml
@@ -0,0 +1,7 @@
+build_arch:
+ type: arch
+ description: Architecture
+ variable: build_arch
+ values:
+ - i586
+ - x86_64
diff --git a/tests/format/include/inner/project.conf b/tests/format/include/inner/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..8bdfc428a002c83a002a18225666c37e4ef3b8ef
--- /dev/null
+++ b/tests/format/include/inner/project.conf
@@ -0,0 +1,5 @@
+name: test
+
+options:
+ (@):
+ - extra_conf.yml
diff --git a/tests/format/include/junction/element.bst b/tests/format/include/junction/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/junction/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/junction/project.conf b/tests/format/include/junction/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..4836c5f8b924cd8f0de0528c3c0a30f3c03977de
--- /dev/null
+++ b/tests/format/include/junction/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - junction.bst:extra_conf.yml
diff --git a/tests/format/include/junction/subproject/extra_conf.yml b/tests/format/include/junction/subproject/extra_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..404ecd6dd761c4323b666bc6ce606f99abfdbf30
--- /dev/null
+++ b/tests/format/include/junction/subproject/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+ included: 'True'
diff --git a/tests/format/include/junction/subproject/project.conf b/tests/format/include/junction/subproject/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..7a66554218de2e2d32d2b645639b9ad36bc5dfda
--- /dev/null
+++ b/tests/format/include/junction/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/local_to_junction/element.bst b/tests/format/include/local_to_junction/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/local_to_junction/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/local_to_junction/project.conf b/tests/format/include/local_to_junction/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..4836c5f8b924cd8f0de0528c3c0a30f3c03977de
--- /dev/null
+++ b/tests/format/include/local_to_junction/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - junction.bst:extra_conf.yml
diff --git a/tests/format/include/local_to_junction/subproject/extra_conf.yml b/tests/format/include/local_to_junction/subproject/extra_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..1c0b8ccdd6462e5b3b383f1ef99105426011914f
--- /dev/null
+++ b/tests/format/include/local_to_junction/subproject/extra_conf.yml
@@ -0,0 +1,2 @@
+(@):
+ - internal.yml
diff --git a/tests/format/include/local_to_junction/subproject/internal.yml b/tests/format/include/local_to_junction/subproject/internal.yml
new file mode 100644
index 0000000000000000000000000000000000000000..404ecd6dd761c4323b666bc6ce606f99abfdbf30
--- /dev/null
+++ b/tests/format/include/local_to_junction/subproject/internal.yml
@@ -0,0 +1,2 @@
+variables:
+ included: 'True'
diff --git a/tests/format/include/local_to_junction/subproject/project.conf b/tests/format/include/local_to_junction/subproject/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..7a66554218de2e2d32d2b645639b9ad36bc5dfda
--- /dev/null
+++ b/tests/format/include/local_to_junction/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/options/element.bst b/tests/format/include/options/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/options/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/options/extra_conf.yml b/tests/format/include/options/extra_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..ad1401e0a01b60c10f0e93b53d1702588e6ab6a7
--- /dev/null
+++ b/tests/format/include/options/extra_conf.yml
@@ -0,0 +1,8 @@
+options:
+ build_arch:
+ type: arch
+ description: Architecture
+ variable: build_arch
+ values:
+ - i586
+ - x86_64
diff --git a/tests/format/include/options/project.conf b/tests/format/include/options/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..a7791a4163e25b9351c7eb4d6cec341a8afa8c4e
--- /dev/null
+++ b/tests/format/include/options/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - extra_conf.yml
diff --git a/tests/format/include/overrides-junction/element.bst b/tests/format/include/overrides-junction/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/overrides-junction/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/overrides-junction/project.conf b/tests/format/include/overrides-junction/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..d03bec63475d072338070e1a2892cc07ef1a8666
--- /dev/null
+++ b/tests/format/include/overrides-junction/project.conf
@@ -0,0 +1,20 @@
+name: test
+
+elements:
+ junction:
+ variables:
+ main_override: True
+ manual:
+ variables:
+ manual_main_override: True
+ config:
+ build-commands:
+ - "first"
+
+sources:
+ git:
+ variables:
+ from_main: True
+
+(@):
+ - junction.bst:extra_conf.yml
diff --git a/tests/format/include/overrides-junction/subproject/extra_conf.yml b/tests/format/include/overrides-junction/subproject/extra_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..3cd3530c540fb08aa26afbf1c1aad7980f3377e9
--- /dev/null
+++ b/tests/format/include/overrides-junction/subproject/extra_conf.yml
@@ -0,0 +1,16 @@
+elements:
+ junction:
+ variables:
+ included_override: True
+ manual:
+ variables:
+ manual_included_override: True
+ config:
+ build-commands:
+ (>):
+ - "second"
+
+sources:
+ git:
+ variables:
+ from_included: True
diff --git a/tests/format/include/overrides-junction/subproject/project.conf b/tests/format/include/overrides-junction/subproject/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..7a66554218de2e2d32d2b645639b9ad36bc5dfda
--- /dev/null
+++ b/tests/format/include/overrides-junction/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/overrides/element.bst b/tests/format/include/overrides/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/overrides/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/overrides/extra_conf.yml b/tests/format/include/overrides/extra_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..ccb874bd7b53c33dc6bc4bdf6cfb9384bc4159d6
--- /dev/null
+++ b/tests/format/include/overrides/extra_conf.yml
@@ -0,0 +1,15 @@
+elements:
+ junction:
+ variables:
+ included_override: True
+ manual:
+ variables:
+ manual_included_override: True
+ config:
+ build-commands:
+ - "ignored"
+
+sources:
+ git:
+ variables:
+ from_included: True
diff --git a/tests/format/include/overrides/extra_conf2.yml b/tests/format/include/overrides/extra_conf2.yml
new file mode 100644
index 0000000000000000000000000000000000000000..750abd7251be01b43a1dbab875a6f5f7bb8a6511
--- /dev/null
+++ b/tests/format/include/overrides/extra_conf2.yml
@@ -0,0 +1,5 @@
+elements:
+ manual:
+ config:
+ build-commands:
+ - "first"
diff --git a/tests/format/include/overrides/project.conf b/tests/format/include/overrides/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..fa3c757039e94f2be4fcdcbf1706753935d8759d
--- /dev/null
+++ b/tests/format/include/overrides/project.conf
@@ -0,0 +1,22 @@
+name: test
+
+elements:
+ junction:
+ variables:
+ main_override: True
+ manual:
+ variables:
+ manual_main_override: True
+ config:
+ build-commands:
+ (>):
+ - "second"
+
+sources:
+ git:
+ variables:
+ from_main: True
+
+(@):
+ - extra_conf.yml
+ - extra_conf2.yml
diff --git a/tests/format/include/overrides/subproject/project.conf b/tests/format/include/overrides/subproject/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..7a66554218de2e2d32d2b645639b9ad36bc5dfda
--- /dev/null
+++ b/tests/format/include/overrides/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/recursive/element.bst b/tests/format/include/recursive/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/recursive/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/recursive/extra_conf.yml b/tests/format/include/recursive/extra_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..57db0d3c6cde42067c797668f772624d211f9228
--- /dev/null
+++ b/tests/format/include/recursive/extra_conf.yml
@@ -0,0 +1,2 @@
+(@):
+ - extra_conf2.yml
diff --git a/tests/format/include/recursive/extra_conf2.yml b/tests/format/include/recursive/extra_conf2.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e8dd5e2edbdf6a5866d9df9acf934fa0c9595c2c
--- /dev/null
+++ b/tests/format/include/recursive/extra_conf2.yml
@@ -0,0 +1,2 @@
+(@):
+ - extra_conf.yml
diff --git a/tests/format/include/recursive/project.conf b/tests/format/include/recursive/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..a7791a4163e25b9351c7eb4d6cec341a8afa8c4e
--- /dev/null
+++ b/tests/format/include/recursive/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - extra_conf.yml
diff --git a/tests/format/include/string/element.bst b/tests/format/include/string/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/string/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/string/extra_conf.yml b/tests/format/include/string/extra_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..404ecd6dd761c4323b666bc6ce606f99abfdbf30
--- /dev/null
+++ b/tests/format/include/string/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+ included: 'True'
diff --git a/tests/format/include/string/project.conf b/tests/format/include/string/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..6ee9988e984d384fe9acfac1430816fbdae6e324
--- /dev/null
+++ b/tests/format/include/string/project.conf
@@ -0,0 +1,3 @@
+name: test
+
+(@): extra_conf.yml
diff --git a/tests/format/include/sub-include/element.bst b/tests/format/include/sub-include/element.bst
new file mode 100644
index 0000000000000000000000000000000000000000..4d7f7026665231e5e58625bbbe9e4f3619163b13
--- /dev/null
+++ b/tests/format/include/sub-include/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/sub-include/manual_conf.yml b/tests/format/include/sub-include/manual_conf.yml
new file mode 100644
index 0000000000000000000000000000000000000000..9c2c0dd3448cbd19ef2e6bfe8bdaa45a4d3cc9eb
--- /dev/null
+++ b/tests/format/include/sub-include/manual_conf.yml
@@ -0,0 +1,2 @@
+variables:
+ included: True
diff --git a/tests/format/include/sub-include/project.conf b/tests/format/include/sub-include/project.conf
new file mode 100644
index 0000000000000000000000000000000000000000..7f7df84c8ad6db9350ac1a48f3a3205f525d6380
--- /dev/null
+++ b/tests/format/include/sub-include/project.conf
@@ -0,0 +1,6 @@
+name: test
+
+elements:
+ manual:
+ (@):
+ - manual_conf.yml
diff --git a/tests/format/include_composition.py b/tests/format/include_composition.py
new file mode 100644
index 0000000000000000000000000000000000000000..b73fca392bc84a26d7488d05784eef2d066117a9
--- /dev/null
+++ b/tests/format/include_composition.py
@@ -0,0 +1,131 @@
+import os
+from buildstream._context import Context
+from buildstream._project import Project
+from buildstream._includes import Includes
+from buildstream import _yaml
+
+
+def make_includes(basedir):
+ _yaml.dump({'name': 'test'},
+ os.path.join(basedir, 'project.conf'))
+ context = Context()
+ project = Project(basedir, context)
+ loader = project.loader
+ return Includes(loader)
+
+
+def test_main_has_prority(tmpdir):
+ includes = make_includes(str(tmpdir))
+
+ _yaml.dump({'(@)': ['a.yml'],
+ 'test': ['main']},
+ str(tmpdir.join('main.yml')))
+
+ main = _yaml.load(str(tmpdir.join('main.yml')))
+
+ _yaml.dump({'test': ['a']},
+ str(tmpdir.join('a.yml')))
+
+ includes.process(main)
+
+ assert main['test'] == ['main']
+
+
+def test_include_cannot_append(tmpdir):
+ includes = make_includes(str(tmpdir))
+
+ _yaml.dump({'(@)': ['a.yml'],
+ 'test': ['main']},
+ str(tmpdir.join('main.yml')))
+ main = _yaml.load(str(tmpdir.join('main.yml')))
+
+ _yaml.dump({'test': {'(>)': ['a']}},
+ str(tmpdir.join('a.yml')))
+
+ includes.process(main)
+
+ assert main['test'] == ['main']
+
+
+def test_main_can_append(tmpdir):
+ includes = make_includes(str(tmpdir))
+
+ _yaml.dump({'(@)': ['a.yml'],
+ 'test': {'(>)': ['main']}},
+ str(tmpdir.join('main.yml')))
+ main = _yaml.load(str(tmpdir.join('main.yml')))
+
+ _yaml.dump({'test': ['a']},
+ str(tmpdir.join('a.yml')))
+
+ includes.process(main)
+
+ assert main['test'] == ['a', 'main']
+
+
+def test_sibling_cannot_append_backward(tmpdir):
+ includes = make_includes(str(tmpdir))
+
+ _yaml.dump({'(@)': ['a.yml', 'b.yml']},
+ str(tmpdir.join('main.yml')))
+ main = _yaml.load(str(tmpdir.join('main.yml')))
+
+ _yaml.dump({'test': {'(>)': ['a']}},
+ str(tmpdir.join('a.yml')))
+ _yaml.dump({'test': ['b']},
+ str(tmpdir.join('b.yml')))
+
+ includes.process(main)
+
+ assert main['test'] == ['b']
+
+
+def test_sibling_can_append_forward(tmpdir):
+ includes = make_includes(str(tmpdir))
+
+ _yaml.dump({'(@)': ['a.yml', 'b.yml']},
+ str(tmpdir.join('main.yml')))
+ main = _yaml.load(str(tmpdir.join('main.yml')))
+
+ _yaml.dump({'test': ['a']},
+ str(tmpdir.join('a.yml')))
+ _yaml.dump({'test': {'(>)': ['b']}},
+ str(tmpdir.join('b.yml')))
+
+ includes.process(main)
+
+ assert main['test'] == ['a', 'b']
+
+
+def test_lastest_sibling_has_priority(tmpdir):
+ includes = make_includes(str(tmpdir))
+
+ _yaml.dump({'(@)': ['a.yml', 'b.yml']},
+ str(tmpdir.join('main.yml')))
+ main = _yaml.load(str(tmpdir.join('main.yml')))
+
+ _yaml.dump({'test': ['a']},
+ str(tmpdir.join('a.yml')))
+ _yaml.dump({'test': ['b']},
+ str(tmpdir.join('b.yml')))
+
+ includes.process(main)
+
+ assert main['test'] == ['b']
+
+
+def test_main_keeps_keys(tmpdir):
+ includes = make_includes(str(tmpdir))
+
+ _yaml.dump({'(@)': ['a.yml'],
+ 'something': 'else'},
+ str(tmpdir.join('main.yml')))
+ main = _yaml.load(str(tmpdir.join('main.yml')))
+
+ _yaml.dump({'test': ['a']},
+ str(tmpdir.join('a.yml')))
+
+ includes.process(main)
+
+ assert main['test'] == ['a']
+ assert main['something'] == 'else'
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index 4e10598242cd84ae25ba885b383346b50b54c5c5..1cf962f88d55ffbee58c577c86e538d5538a654d 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -480,3 +480,135 @@ def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'buildable'
assert os.path.exists(os.path.join(project, 'project.refs'))
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_track_include(cli, tmpdir, datafiles, ref_storage, kind):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ dev_files_path = os.path.join(project, 'files', 'dev-files')
+ element_path = os.path.join(project, 'elements')
+ element_name = 'track-test-{}.bst'.format(kind)
+
+ configure_project(project, {
+ 'ref-storage': ref_storage
+ })
+
+ # Create our repo object of the given source type with
+ # the dev files, and then collect the initial ref.
+ #
+ repo = create_repo(kind, str(tmpdir))
+ ref = repo.create(dev_files_path)
+
+ # Generate the element
+ element = {
+ 'kind': 'import',
+ '(@)': ['elements/sources.yml']
+ }
+ sources = {
+ 'sources': [
+ repo.source_config()
+ ]
+ }
+
+ _yaml.dump(element, os.path.join(element_path, element_name))
+ _yaml.dump(sources, os.path.join(element_path, 'sources.yml'))
+
+ # Assert that a fetch is needed
+ assert cli.get_element_state(project, element_name) == 'no reference'
+
+ # Now first try to track it
+ result = cli.run(project=project, args=['track', element_name])
+ result.assert_success()
+
+ # And now fetch it: The Source has probably already cached the
+ # latest ref locally, but it is not required to have cached
+ # the associated content of the latest ref at track time, that
+ # is the job of fetch.
+ result = cli.run(project=project, args=['fetch', element_name])
+ result.assert_success()
+
+ # Assert that we are now buildable because the source is
+ # now cached.
+ assert cli.get_element_state(project, element_name) == 'buildable'
+
+ # Assert there was a project.refs created, depending on the configuration
+ if ref_storage == 'project.refs':
+ assert os.path.exists(os.path.join(project, 'project.refs'))
+ else:
+ assert not os.path.exists(os.path.join(project, 'project.refs'))
+ new_sources = _yaml.load(os.path.join(element_path, 'sources.yml'))
+ assert 'sources' in new_sources
+ assert len(new_sources['sources']) == 1
+ assert 'ref' in new_sources['sources'][0]
+ assert ref == new_sources['sources'][0]['ref']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ dev_files_path = os.path.join(project, 'files', 'dev-files')
+ element_path = os.path.join(project, 'elements')
+ element_name = 'track-test-{}.bst'.format(kind)
+ subproject_path = os.path.join(project, 'files', 'sub-project')
+ sub_element_path = os.path.join(subproject_path, 'elements')
+ junction_path = os.path.join(element_path, 'junction.bst')
+
+ configure_project(project, {
+ 'ref-storage': ref_storage
+ })
+
+ # Create our repo object of the given source type with
+ # the dev files, and then collect the initial ref.
+ #
+ repo = create_repo(kind, str(tmpdir.join('element_repo')))
+ ref = repo.create(dev_files_path)
+
+ # Generate the element
+ element = {
+ 'kind': 'import',
+ '(@)': ['junction.bst:elements/sources.yml']
+ }
+ sources = {
+ 'sources': [
+ repo.source_config()
+ ]
+ }
+
+ _yaml.dump(element, os.path.join(element_path, element_name))
+ _yaml.dump(sources, os.path.join(sub_element_path, 'sources.yml'))
+
+ generate_junction(str(tmpdir.join('junction_repo')),
+ subproject_path, junction_path, store_ref=True)
+
+ result = cli.run(project=project, args=['track', 'junction.bst'])
+ result.assert_success()
+
+ # Assert that a fetch is needed
+ assert cli.get_element_state(project, element_name) == 'no reference'
+
+ # Now first try to track it
+ result = cli.run(project=project, args=['track', element_name])
+
+ # Assert there was a project.refs created, depending on the configuration
+ if ref_storage == 'inline':
+ # FIXME: We should expect an error. But only a warning is emitted
+ # result.assert_main_error(ErrorDomain.SOURCE, 'tracking-junction-fragment')
+
+ assert 'junction.bst:elements/sources.yml: Cannot track source in a fragment from a junction' in result.stderr
+ else:
+ assert os.path.exists(os.path.join(project, 'project.refs'))
+
+ # And now fetch it: The Source has probably already cached the
+ # latest ref locally, but it is not required to have cached
+ # the associated content of the latest ref at track time, that
+ # is the job of fetch.
+ result = cli.run(project=project, args=['fetch', element_name])
+ result.assert_success()
+
+ # Assert that we are now buildable because the source is
+ # now cached.
+ assert cli.get_element_state(project, element_name) == 'buildable'
diff --git a/tests/yaml/yaml.py b/tests/yaml/yaml.py
index 3b9f385ed3e4516ebbcf0736389b2cd4dfadf706..78176371730f23388764325401cfba11f7888ad7 100644
--- a/tests/yaml/yaml.py
+++ b/tests/yaml/yaml.py
@@ -33,7 +33,7 @@ def assert_provenance(filename, line, col, node, key=None, indices=[]):
else:
assert(isinstance(provenance, _yaml.DictProvenance))
- assert(provenance.filename == filename)
+ assert(provenance.filename.shortname == filename)
assert(provenance.line == line)
assert(provenance.col == col)