_project.py 38.7 KB
Newer Older
1
#
2
#  Copyright (C) 2016-2018 Codethink Limited
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2 of the License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
#  Authors:
#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
19
#        Tiago Gomes <tiago.gomes@codethink.co.uk>
20

21
import gc
22
import os
23
import sys
24 25
from collections import OrderedDict
from collections.abc import Mapping
26
from pathlib import Path
27
from pluginbase import PluginBase
28
from . import utils
29
from . import _cachekey
30 31
from . import _site
from . import _yaml
32
from ._artifactelement import ArtifactElement
33
from ._profile import Topics, PROFILER
34
from ._exceptions import LoadError, LoadErrorReason
35
from ._options import OptionPool
36
from ._artifactcache import ArtifactCache
37
from ._sourcecache import SourceCache
38
from .sandbox import SandboxRemote
39 40
from ._elementfactory import ElementFactory
from ._sourcefactory import SourceFactory
41
from .types import CoreWarnings
42
from ._projectrefs import ProjectRefs, ProjectRefStorage
43
from ._versions import BST_FORMAT_VERSION
44 45 46
from ._loader import Loader
from .element import Element
from ._message import Message, MessageType
47
from ._includes import Includes
48
from ._platform import Platform
49
from ._workspaces import WORKSPACE_PROJECT_FILE
50

51

52 53 54
# Project Configuration file
_PROJECT_CONF_FILE = 'project.conf'

55

56 57 58 59 60 61 62 63 64
# HostMount()
#
# A simple object describing the behavior of
# a host mount.
#
class HostMount():

    def __init__(self, path, host_path=None, optional=False):

65 66 67 68 69
        # Support environment variable expansion in host mounts
        path = os.path.expandvars(path)
        if host_path is not None:
            host_path = os.path.expandvars(host_path)

70 71 72 73 74 75 76 77
        self.path = path              # Path inside the sandbox
        self.host_path = host_path    # Path on the host
        self.optional = optional      # Optional mounts do not incur warnings or errors

        if self.host_path is None:
            self.host_path = self.path


78 79 80 81 82 83 84 85 86 87 88 89 90 91
# Represents project configuration that can have different values for junctions.
class ProjectConfig:
    def __init__(self):
        self.element_factory = None
        self.source_factory = None
        self.options = None                      # OptionPool
        self.base_variables = {}                 # The base set of variables
        self.element_overrides = {}              # Element specific configurations
        self.source_overrides = {}               # Source specific configurations
        self.mirrors = OrderedDict()             # contains dicts of alias-mappings to URIs.
        self.default_mirror = None               # The name of the preferred mirror.
        self._aliases = {}                       # Aliases dictionary


92 93 94 95
# Project()
#
# The Project Configuration
#
96 97
class Project():

98
    def __init__(self, directory, context, *, junction=None, cli_options=None,
99 100
                 default_mirror=None, parent_loader=None,
                 search_for_project=True):
101

102
        # The project name
103 104
        self.name = None

105 106
        self._context = context  # The invocation Context, a private member

107 108 109 110 111
        if search_for_project:
            self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory)
        else:
            self.directory = directory
            self._invoked_from_workspace_element = None
112

113 114
        self._absolute_directory_path = Path(self.directory).resolve()

115
        # Absolute path to where elements are loaded from within the project
116 117
        self.element_path = None

118 119 120
        # Default target elements
        self._default_targets = None

121 122 123 124
        # ProjectRefs for the main refs and also for junctions
        self.refs = ProjectRefs(self.directory, 'project.refs')
        self.junction_refs = ProjectRefs(self.directory, 'junction.refs')

125 126 127
        self.config = ProjectConfig()
        self.first_pass_config = ProjectConfig()

128
        self.junction = junction                 # The junction Element object, if this is a subproject
129

130 131 132 133 134 135 136
        self.ref_storage = None                  # ProjectRefStorage setting
        self.base_environment = {}               # The base set of environment variables
        self.base_env_nocache = None             # The base nocache mask (list) for the environment

        #
        # Private Members
        #
137 138

        self._default_mirror = default_mirror    # The name of the preferred mirror.
139

140
        self._cli_options = cli_options
141
        self._cache_key = None
142

143 144
        self._fatal_warnings = []             # A list of warnings which should trigger an error

145
        self._shell_command = []      # The default interactive shell command
146
        self._shell_environment = {}  # Statically set environment vars
147
        self._shell_host_files = []   # A list of HostMount objects
148

149
        self.artifact_cache_specs = None
150
        self.source_cache_specs = None
151
        self.remote_execution_specs = None
152 153 154 155 156 157 158 159
        self._sandbox = None
        self._splits = None

        self._context.add_project(self)

        self._partially_loaded = False
        self._fully_loaded = False
        self._project_includes = None
160

161 162
        with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-')):
            self._load(parent_loader=parent_loader)
163

164
        self._partially_loaded = True
165

166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
    @property
    def options(self):
        return self.config.options

    @property
    def base_variables(self):
        return self.config.base_variables

    @property
    def element_overrides(self):
        return self.config.element_overrides

    @property
    def source_overrides(self):
        return self.config.source_overrides
181

182 183 184 185 186 187 188
    # translate_url():
    #
    # Translates the given url which may be specified with an alias
    # into a fully qualified url.
    #
    # Args:
    #    url (str): A url, which may be using an alias
189
    #    first_pass (bool): Whether to use first pass configuration (for junctions)
190 191 192 193 194 195 196
    #
    # Returns:
    #    str: The fully qualified url, with aliases resolved
    #
    # This method is provided for :class:`.Source` objects to resolve
    # fully qualified urls based on the shorthand which is allowed
    # to be specified in the YAML
197 198 199 200 201 202
    def translate_url(self, url, *, first_pass=False):
        if first_pass:
            config = self.first_pass_config
        else:
            config = self.config

203 204
        if url and utils._ALIAS_SEPARATOR in url:
            url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
205
            alias_url = _yaml.node_get(config._aliases, str, url_alias, default_value=None)
206 207 208 209 210
            if alias_url:
                url = alias_url + url_body

        return url

211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
    # get_shell_config()
    #
    # Gets the project specified shell configuration
    #
    # Returns:
    #    (list): The shell command
    #    (dict): The shell environment
    #    (list): The list of HostMount objects
    #
    def get_shell_config(self):
        return (self._shell_command, self._shell_environment, self._shell_host_files)

    # get_cache_key():
    #
    # Returns the cache key, calculating it if necessary
    #
    # Returns:
    #    (str): A hex digest cache key for the Context
    #
    def get_cache_key(self):
        if self._cache_key is None:

            # Anything that alters the build goes into the unique key
            # (currently nothing here)
235
            self._cache_key = _cachekey.generate_key(_yaml.new_empty_node())
236 237 238

        return self._cache_key

239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
    # get_path_from_node()
    #
    # Fetches the project path from a dictionary node and validates it
    #
    # Paths are asserted to never lead to a directory outside of the project
    # directory. In addition, paths can not point to symbolic links, fifos,
    # sockets and block/character devices.
    #
    # The `check_is_file` and `check_is_dir` parameters can be used to
    # perform additional validations on the path. Note that an exception
    # will always be raised if both parameters are set to ``True``.
    #
    # Args:
    #    node (dict): A dictionary loaded from YAML
    #    key (str): The key whose value contains a path to validate
    #    check_is_file (bool): If ``True`` an error will also be raised
    #                          if path does not point to a regular file.
    #                          Defaults to ``False``
    #    check_is_dir (bool): If ``True`` an error will be also raised
    #                         if path does not point to a directory.
    #                         Defaults to ``False``
    # Returns:
    #    (str): The project path
    #
    # Raises:
    #    (LoadError): In case that the project path is not valid or does not
    #                 exist
    #
    def get_path_from_node(self, node, key, *,
                           check_is_file=False, check_is_dir=False):
        path_str = _yaml.node_get(node, str, key)
        path = Path(path_str)
271
        full_path = self._absolute_directory_path / path
272 273 274

        provenance = _yaml.node_get_provenance(node, key=key)

275
        if full_path.is_symlink():
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
            raise LoadError(LoadErrorReason.PROJ_PATH_INVALID_KIND,
                            "{}: Specified path '{}' must not point to "
                            "symbolic links "
                            .format(provenance, path_str))

        if path.parts and path.parts[0] == '..':
            raise LoadError(LoadErrorReason.PROJ_PATH_INVALID,
                            "{}: Specified path '{}' first component must "
                            "not be '..'"
                            .format(provenance, path_str))

        try:
            if sys.version_info[0] == 3 and sys.version_info[1] < 6:
                full_resolved_path = full_path.resolve()
            else:
                full_resolved_path = full_path.resolve(strict=True)  # pylint: disable=unexpected-keyword-arg
        except FileNotFoundError:
            raise LoadError(LoadErrorReason.MISSING_FILE,
                            "{}: Specified path '{}' does not exist"
                            .format(provenance, path_str))

297 298
        is_inside = self._absolute_directory_path in full_resolved_path.parents or (
            full_resolved_path == self._absolute_directory_path)
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331

        if not is_inside:
            raise LoadError(LoadErrorReason.PROJ_PATH_INVALID,
                            "{}: Specified path '{}' must not lead outside of the "
                            "project directory"
                            .format(provenance, path_str))

        if path.is_absolute():
            raise LoadError(LoadErrorReason.PROJ_PATH_INVALID,
                            "{}: Absolute path: '{}' invalid.\n"
                            "Please specify a path relative to the project's root."
                            .format(provenance, path))

        if full_resolved_path.is_socket() or (
                full_resolved_path.is_fifo() or
                full_resolved_path.is_block_device()):
            raise LoadError(LoadErrorReason.PROJ_PATH_INVALID_KIND,
                            "{}: Specified path '{}' points to an unsupported "
                            "file kind"
                            .format(provenance, path_str))

        if check_is_file and not full_resolved_path.is_file():
            raise LoadError(LoadErrorReason.PROJ_PATH_INVALID_KIND,
                            "{}: Specified path '{}' is not a regular file"
                            .format(provenance, path_str))

        if check_is_dir and not full_resolved_path.is_dir():
            raise LoadError(LoadErrorReason.PROJ_PATH_INVALID_KIND,
                            "{}: Specified path '{}' is not a directory"
                            .format(provenance, path_str))

        return path_str

332 333 334 335 336 337
    def _validate_node(self, node):
        _yaml.node_validate(node, [
            'format-version',
            'element-path', 'variables',
            'environment', 'environment-nocache',
            'split-rules', 'elements', 'plugins',
338
            'aliases', 'name', 'defaults',
339 340 341
            'artifacts', 'options',
            'fail-on-overlap', 'shell', 'fatal-warnings',
            'ref-storage', 'sandbox', 'mirrors', 'remote-execution',
342
            'sources', 'source-caches', '(@)'
343 344
        ])

345 346 347 348 349
    # create_element()
    #
    # Instantiate and return an element
    #
    # Args:
350
    #    meta (MetaElement): The loaded MetaElement
351
    #    first_pass (bool): Whether to use first pass configuration (for junctions)
352 353 354 355
    #
    # Returns:
    #    (Element): A newly created Element object of the appropriate kind
    #
356
    def create_element(self, meta, *, first_pass=False):
357
        if first_pass:
358
            return self.first_pass_config.element_factory.create(self._context, self, meta)
359
        else:
360
            return self.config.element_factory.create(self._context, self, meta)
361

362 363 364 365 366 367 368 369 370 371 372 373 374
    # create_artifact_element()
    #
    # Instantiate and return an ArtifactElement
    #
    # Args:
    #    ref (str): A string of the artifact ref
    #
    # Returns:
    #    (ArtifactElement): A newly created ArtifactElement object of the appropriate kind
    #
    def create_artifact_element(self, ref):
        return ArtifactElement(self._context, ref)

375 376 377 378 379
    # create_source()
    #
    # Instantiate and return a Source
    #
    # Args:
380
    #    meta (MetaSource): The loaded MetaSource
381
    #    first_pass (bool): Whether to use first pass configuration (for junctions)
382 383 384 385
    #
    # Returns:
    #    (Source): A newly created Source object of the appropriate kind
    #
386 387 388 389 390
    def create_source(self, meta, *, first_pass=False):
        if first_pass:
            return self.first_pass_config.source_factory.create(self._context, self, meta)
        else:
            return self.config.source_factory.create(self._context, self, meta)
391

392 393 394 395 396 397
    # get_alias_uri()
    #
    # Returns the URI for a given alias, if it exists
    #
    # Args:
    #    alias (str): The alias.
398
    #    first_pass (bool): Whether to use first pass configuration (for junctions)
399 400 401 402
    #
    # Returns:
    #    str: The URI for the given alias; or None: if there is no URI for
    #         that alias.
403 404 405 406 407 408
    def get_alias_uri(self, alias, *, first_pass=False):
        if first_pass:
            config = self.first_pass_config
        else:
            config = self.config

409
        return _yaml.node_get(config._aliases, str, alias, default_value=None)
410 411 412

    # get_alias_uris()
    #
413 414 415 416
    # Args:
    #    alias (str): The alias.
    #    first_pass (bool): Whether to use first pass configuration (for junctions)
    #
417
    # Returns a list of every URI to replace an alias with
418 419 420 421 422 423
    def get_alias_uris(self, alias, *, first_pass=False):
        if first_pass:
            config = self.first_pass_config
        else:
            config = self.config

424
        if not alias or alias not in config._aliases:
425 426 427
            return [None]

        mirror_list = []
428
        for key, alias_mapping in config.mirrors.items():
429
            if alias in alias_mapping:
430
                if key == config.default_mirror:
431 432 433
                    mirror_list = alias_mapping[alias] + mirror_list
                else:
                    mirror_list += alias_mapping[alias]
434
        mirror_list.append(_yaml.node_get(config._aliases, str, alias))
435 436
        return mirror_list

437 438 439 440 441 442 443 444 445 446 447 448 449 450
    # load_elements()
    #
    # Loads elements from target names.
    #
    # Args:
    #    targets (list): Target names
    #    rewritable (bool): Whether the loaded files should be rewritable
    #                       this is a bit more expensive due to deep copies
    #    fetch_subprojects (bool): Whether we should fetch subprojects as a part of the
    #                              loading process, if they are not yet locally cached
    #
    # Returns:
    #    (list): A list of loaded Element
    #
451
    def load_elements(self, targets, *,
452 453 454 455 456 457
                      rewritable=False, fetch_subprojects=False):
        with self._context.timed_activity("Loading elements", silent_nested=True):
            meta_elements = self.loader.load(targets, rewritable=rewritable,
                                             ticker=None,
                                             fetch_subprojects=fetch_subprojects)

458 459 460
        # Loading elements generates a lot of garbage, clear it now
        gc.collect()

461 462
        with self._context.timed_activity("Resolving elements"):
            elements = [
463
                Element._new_from_meta(meta)
464 465 466
                for meta in meta_elements
            ]

467 468
        Element._clear_meta_elements_cache()

469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
        # Now warn about any redundant source references which may have
        # been discovered in the resolve() phase.
        redundant_refs = Element._get_redundant_source_refs()
        if redundant_refs:
            detail = "The following inline specified source references will be ignored:\n\n"
            lines = [
                "{}:{}".format(source._get_provenance(), ref)
                for source, ref in redundant_refs
            ]
            detail += "\n".join(lines)
            self._context.message(
                Message(None, MessageType.WARN, "Ignoring redundant source references", detail=detail))

        return elements

484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
    # ensure_fully_loaded()
    #
    # Ensure project has finished loading. At first initialization, a
    # project can only load junction elements. Other elements require
    # project to be fully loaded.
    #
    def ensure_fully_loaded(self):
        if self._fully_loaded:
            return
        assert self._partially_loaded
        self._fully_loaded = True

        if self.junction:
            self.junction._get_project().ensure_fully_loaded()

        self._load_second_pass()

501 502 503 504 505 506 507 508
    # invoked_from_workspace_element()
    #
    # Returns the element whose workspace was used to invoke buildstream
    # if buildstream was invoked from an external workspace
    #
    def invoked_from_workspace_element(self):
        return self._invoked_from_workspace_element

509 510 511 512 513 514 515 516
    # cleanup()
    #
    # Cleans up resources used loading elements
    #
    def cleanup(self):
        # Reset the element loader state
        Element._reset_load_state()

517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
    # get_default_target()
    #
    # Attempts to interpret which element the user intended to run a command on.
    # This is for commands that only accept a single target element and thus,
    # this only uses the workspace element (if invoked from workspace directory)
    # and does not use the project default targets.
    #
    def get_default_target(self):
        return self._invoked_from_workspace_element

    # get_default_targets()
    #
    # Attempts to interpret which elements the user intended to run a command on.
    # This is for commands that accept multiple target elements.
    #
    def get_default_targets(self):

        # If _invoked_from_workspace_element has a value,
        # a workspace element was found before a project config
        # Therefore the workspace does not contain a project
        if self._invoked_from_workspace_element:
            return (self._invoked_from_workspace_element,)

        # Default targets from project configuration
        if self._default_targets:
            return tuple(self._default_targets)

        # If default targets are not configured, default to all project elements
        default_targets = []
        for root, _, files in os.walk(self.element_path):
            for file in files:
                if file.endswith(".bst"):
                    rel_dir = os.path.relpath(root, self.element_path)
                    rel_file = os.path.join(rel_dir, file).lstrip("./")
                    default_targets.append(rel_file)

        return tuple(default_targets)

555
    # _load():
556
    #
557 558
    # Loads the project configuration file in the project
    # directory process the first pass.
559 560 561
    #
    # Raises: LoadError if there was a problem with the project.conf
    #
562
    def _load(self, parent_loader=None):
563

564
        # Load builtin default
565
        projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE)
566
        self._default_config_node = _yaml.load(_site.default_project_config)
567 568

        # Load project local config and override the builtin
569
        try:
570
            self._project_conf = _yaml.load(projectfile)
571 572
        except LoadError as e:
            # Raise a more specific error here
573 574 575 576
            if e.reason == LoadErrorReason.MISSING_FILE:
                raise LoadError(LoadErrorReason.MISSING_PROJECT_CONF, str(e)) from e
            else:
                raise
577

578 579
        pre_config_node = _yaml.node_copy(self._default_config_node)
        _yaml.composite(pre_config_node, self._project_conf)
580 581

        # Assert project's format version early, before validating toplevel keys
582
        format_version = _yaml.node_get(pre_config_node, int, 'format-version')
583 584 585 586 587 588 589
        if BST_FORMAT_VERSION < format_version:
            major, minor = utils.get_bst_version()
            raise LoadError(
                LoadErrorReason.UNSUPPORTED_PROJECT,
                "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
                .format(format_version, major, minor, BST_FORMAT_VERSION))

590 591
        self._validate_node(pre_config_node)

592 593
        # The project name, element path and option declarations
        # are constant and cannot be overridden by option conditional statements
594
        self.name = _yaml.node_get(self._project_conf, str, 'name')
595 596

        # Validate that project name is a valid symbol name
597
        _yaml.assert_symbol_name(_yaml.node_get_provenance(pre_config_node, 'name'),
598 599
                                 self.name, "project name")

600 601
        self.element_path = os.path.join(
            self.directory,
602 603
            self.get_path_from_node(pre_config_node, 'element-path',
                                    check_is_dir=True)
604 605
        )

606 607
        self.config.options = OptionPool(self.element_path)
        self.first_pass_config.options = OptionPool(self.element_path)
608

609 610 611 612
        defaults = _yaml.node_get(pre_config_node, Mapping, 'defaults')
        _yaml.node_validate(defaults, ['targets'])
        self._default_targets = _yaml.node_get(defaults, list, "targets")

613 614 615
        # Fatal warnings
        self._fatal_warnings = _yaml.node_get(pre_config_node, list, 'fatal-warnings', default_value=[])

616
        self.loader = Loader(self._context, self,
617
                             parent=parent_loader)
618

619
        self._project_includes = Includes(self.loader, copy_tree=False)
620

621 622 623 624
        project_conf_first_pass = _yaml.node_copy(self._project_conf)
        self._project_includes.process(project_conf_first_pass, only_local=True)
        config_no_include = _yaml.node_copy(self._default_config_node)
        _yaml.composite(config_no_include, project_conf_first_pass)
625

626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
        self._load_pass(config_no_include, self.first_pass_config,
                        ignore_unknown=True)

        # Use separate file for storing source references
        self.ref_storage = _yaml.node_get(pre_config_node, str, 'ref-storage')
        if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]:
            p = _yaml.node_get_provenance(pre_config_node, 'ref-storage')
            raise LoadError(LoadErrorReason.INVALID_DATA,
                            "{}: Invalid value '{}' specified for ref-storage"
                            .format(p, self.ref_storage))

        if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
            self.junction_refs.load(self.first_pass_config.options)

    # _load_second_pass()
    #
    # Process the second pass of loading the project configuration.
    #
    def _load_second_pass(self):
        project_conf_second_pass = _yaml.node_copy(self._project_conf)
        self._project_includes.process(project_conf_second_pass)
        config = _yaml.node_copy(self._default_config_node)
        _yaml.composite(config, project_conf_second_pass)

        self._load_pass(config, self.config)

652
        self._validate_node(config)
653

654 655 656 657
        #
        # Now all YAML composition is done, from here on we just load
        # the values from our loaded configuration dictionary.
        #
658

659
        # Load artifacts pull/push configuration for this project
660 661 662 663 664 665 666 667 668
        project_specs = ArtifactCache.specs_from_config_node(config, self.directory)
        override_specs = ArtifactCache.specs_from_config_node(
            self._context.get_overrides(self.name), self.directory)

        self.artifact_cache_specs = override_specs + project_specs

        if self.junction:
            parent = self.junction._get_project()
            self.artifact_cache_specs = parent.artifact_cache_specs + self.artifact_cache_specs
669

670 671 672
        # Load source caches with pull/push config
        self.source_cache_specs = SourceCache.specs_from_config_node(config, self.directory)

673
        # Load remote-execution configuration for this project
674 675 676 677 678 679 680 681 682 683
        project_specs = SandboxRemote.specs_from_config_node(config, self.directory)
        override_specs = SandboxRemote.specs_from_config_node(
            self._context.get_overrides(self.name), self.directory)

        if override_specs is not None:
            self.remote_execution_specs = override_specs
        elif project_specs is not None:
            self.remote_execution_specs = project_specs
        else:
            self.remote_execution_specs = self._context.remote_execution_specs
684

685
        # Load sandbox environment variables
686 687
        self.base_environment = _yaml.node_get(config, Mapping, 'environment')
        self.base_env_nocache = _yaml.node_get(config, list, 'environment-nocache')
688

689 690 691
        # Load sandbox configuration
        self._sandbox = _yaml.node_get(config, Mapping, 'sandbox')

692
        # Load project split rules
693
        self._splits = _yaml.node_get(config, Mapping, 'split-rules')
694

695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710
        # Support backwards compatibility for fail-on-overlap
        fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap', default_value=None)

        if (CoreWarnings.OVERLAPS not in self._fatal_warnings) and fail_on_overlap:
            self._fatal_warnings.append(CoreWarnings.OVERLAPS)

        # Deprecation check
        if fail_on_overlap is not None:
            self._context.message(
                Message(
                    None,
                    MessageType.WARN,
                    "Use of fail-on-overlap within project.conf " +
                    "is deprecated. Consider using fatal-warnings instead."
                )
            )
711

712
        # Load project.refs if it exists, this may be ignored.
713 714
        if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
            self.refs.load(self.options)
715

716
        # Parse shell options
717
        shell_options = _yaml.node_get(config, Mapping, 'shell')
718
        _yaml.node_validate(shell_options, ['command', 'environment', 'host-files'])
719
        self._shell_command = _yaml.node_get(shell_options, list, 'command')
720

721 722
        # Perform environment expansion right away
        shell_environment = _yaml.node_get(shell_options, Mapping, 'environment', default_value={})
723
        for key in _yaml.node_keys(shell_environment):
724 725 726
            value = _yaml.node_get(shell_environment, str, key)
            self._shell_environment[key] = os.path.expandvars(value)

727 728 729 730
        # Host files is parsed as a list for convenience
        host_files = _yaml.node_get(shell_options, list, 'host-files', default_value=[])
        for host_file in host_files:
            if isinstance(host_file, str):
731
                mount = HostMount(host_file)
732
            else:
733
                # Some validation
734 735
                index = host_files.index(host_file)
                host_file_desc = _yaml.node_get(shell_options, Mapping, 'host-files', indices=[index])
736 737 738 739
                _yaml.node_validate(host_file_desc, ['path', 'host_path', 'optional'])

                # Parse the host mount
                path = _yaml.node_get(host_file_desc, str, 'path')
740
                host_path = _yaml.node_get(host_file_desc, str, 'host_path', default_value=None)
741 742 743 744
                optional = _yaml.node_get(host_file_desc, bool, 'optional', default_value=False)
                mount = HostMount(path, host_path, optional)

            self._shell_host_files.append(mount)
745

746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763
    # _load_pass():
    #
    # Loads parts of the project configuration that are different
    # for first and second pass configurations.
    #
    # Args:
    #    config (dict) - YaML node of the configuration file.
    #    output (ProjectConfig) - ProjectConfig to load configuration onto.
    #    ignore_unknown (bool) - Whether option loader shoud ignore unknown options.
    #
    def _load_pass(self, config, output, *,
                   ignore_unknown=False):

        # Element and Source  type configurations will be composited later onto
        # element/source types, so we delete it from here and run our final
        # assertion after.
        output.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
        output.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
764 765
        _yaml.node_del(config, 'elements', safe=True)
        _yaml.node_del(config, 'sources', safe=True)
766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
        _yaml.node_final_assertions(config)

        self._load_plugin_factories(config, output)

        # Load project options
        options_node = _yaml.node_get(config, Mapping, 'options', default_value={})
        output.options.load(options_node)
        if self.junction:
            # load before user configuration
            output.options.load_yaml_values(self.junction.options, transform=self.junction._subst_string)

        # Collect option values specified in the user configuration
        overrides = self._context.get_overrides(self.name)
        override_options = _yaml.node_get(overrides, Mapping, 'options', default_value={})
        output.options.load_yaml_values(override_options)
        if self._cli_options:
            output.options.load_cli_values(self._cli_options, ignore_unknown=ignore_unknown)

        # We're done modifying options, now we can use them for substitutions
        output.options.resolve()

        #
        # Now resolve any conditionals in the remaining configuration,
        # any conditionals specified for project option declarations,
        # or conditionally specifying the project name; will be ignored.
        #
792
        # Don't forget to also resolve options in the element and source overrides.
793
        output.options.process_node(config)
794 795
        output.options.process_node(output.element_overrides)
        output.options.process_node(output.source_overrides)
796 797 798 799 800

        # Load base variables
        output.base_variables = _yaml.node_get(config, Mapping, 'variables')

        # Add the project name as a default variable
801
        _yaml.node_set(output.base_variables, 'project-name', self.name)
802 803 804

        # Extend variables with automatic variables and option exports
        # Initialize it as a string as all variables are processed as strings.
805 806 807
        # Based on some testing (mainly on AWS), maximum effective
        # max-jobs value seems to be around 8-10 if we have enough cores
        # users should set values based on workload and build infrastructure
808
        platform = Platform.get_platform()
809
        _yaml.node_set(output.base_variables, 'max-jobs', str(platform.get_cpu_count(8)))
810 811 812 813 814 815 816 817

        # Export options into variables, if that was requested
        output.options.export_variables(output.base_variables)

        # Override default_mirror if not set by command-line
        output.default_mirror = self._default_mirror or _yaml.node_get(overrides, str,
                                                                       'default-mirror', default_value=None)

818 819 820 821 822 823 824 825
        mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
        for mirror in mirrors:
            allowed_mirror_fields = [
                'name', 'aliases'
            ]
            _yaml.node_validate(mirror, allowed_mirror_fields)
            mirror_name = _yaml.node_get(mirror, str, 'name')
            alias_mappings = {}
826
            for alias_mapping, uris in _yaml.node_items(_yaml.node_get(mirror, Mapping, 'aliases')):
827 828
                assert isinstance(uris, list)
                alias_mappings[alias_mapping] = list(uris)
829 830 831
            output.mirrors[mirror_name] = alias_mappings
            if not output.default_mirror:
                output.default_mirror = mirror_name
832

833 834 835
        # Source url aliases
        output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})

836
    # _find_project_dir()
837 838 839 840 841 842 843 844 845 846
    #
    # Returns path of the project directory, if a configuration file is found
    # in given directory or any of its parent directories.
    #
    # Args:
    #    directory (str) - directory from where the command was invoked
    #
    # Raises:
    #    LoadError if project.conf is not found
    #
847 848 849 850
    # Returns:
    #    (str) - the directory that contains the project, and
    #    (str) - the name of the element required to find the project, or None
    #
851
    def _find_project_dir(self, directory):
852
        workspace_element = None
853
        config_filenames = [_PROJECT_CONF_FILE, WORKSPACE_PROJECT_FILE]
854
        found_directory, filename = utils._search_upward_for_files(
855
            directory, config_filenames
856 857 858 859 860 861 862 863 864 865 866 867
        )
        if filename == _PROJECT_CONF_FILE:
            project_directory = found_directory
        elif filename == WORKSPACE_PROJECT_FILE:
            workspace_project_cache = self._context.get_workspace_project_cache()
            workspace_project = workspace_project_cache.get(found_directory)
            if workspace_project:
                project_directory = workspace_project.get_default_project_path()
                workspace_element = workspace_project.get_default_element()
        else:
            raise LoadError(
                LoadErrorReason.MISSING_PROJECT_CONF,
868 869
                "None of {names} found in '{path}' or any of its parent directories"
                .format(names=config_filenames, path=directory))
870

871
        return project_directory, workspace_element
872 873

    def _load_plugin_factories(self, config, output):
874 875 876 877 878 879 880 881 882 883 884 885 886 887 888
        plugin_source_origins = []   # Origins of custom sources
        plugin_element_origins = []  # Origins of custom elements

        # Plugin origins and versions
        origins = _yaml.node_get(config, list, 'plugins', default_value=[])
        source_format_versions = {}
        element_format_versions = {}
        for origin in origins:
            allowed_origin_fields = [
                'origin', 'sources', 'elements',
                'package-name', 'path',
            ]
            allowed_origins = ['core', 'local', 'pip']
            _yaml.node_validate(origin, allowed_origin_fields)

889 890
            origin_value = _yaml.node_get(origin, str, 'origin')
            if origin_value not in allowed_origins:
891 892 893
                raise LoadError(
                    LoadErrorReason.INVALID_YAML,
                    "Origin '{}' is not one of the allowed types"
894
                    .format(origin_value))
895 896 897

            # Store source versions for checking later
            source_versions = _yaml.node_get(origin, Mapping, 'sources', default_value={})
898
            for key in _yaml.node_keys(source_versions):
899 900 901 902 903 904 905 906
                if key in source_format_versions:
                    raise LoadError(
                        LoadErrorReason.INVALID_YAML,
                        "Duplicate listing of source '{}'".format(key))
                source_format_versions[key] = _yaml.node_get(source_versions, int, key)

            # Store element versions for checking later
            element_versions = _yaml.node_get(origin, Mapping, 'elements', default_value={})
907
            for key in _yaml.node_keys(element_versions):
908 909 910 911 912 913 914 915 916 917 918 919 920
                if key in element_format_versions:
                    raise LoadError(
                        LoadErrorReason.INVALID_YAML,
                        "Duplicate listing of element '{}'".format(key))
                element_format_versions[key] = _yaml.node_get(element_versions, int, key)

            # Store the origins if they're not 'core'.
            # core elements are loaded by default, so storing is unnecessary.
            if _yaml.node_get(origin, str, 'origin') != 'core':
                self._store_origin(origin, 'sources', plugin_source_origins)
                self._store_origin(origin, 'elements', plugin_element_origins)

        pluginbase = PluginBase(package='buildstream.plugins')
921 922 923 924 925 926
        output.element_factory = ElementFactory(pluginbase,
                                                plugin_origins=plugin_element_origins,
                                                format_versions=element_format_versions)
        output.source_factory = SourceFactory(pluginbase,
                                              plugin_origins=plugin_source_origins,
                                              format_versions=source_format_versions)
927

928 929 930 931 932
    # _store_origin()
    #
    # Helper function to store plugin origins
    #
    # Args:
933
    #    origin (node) - a node indicating the origin of a group of
934 935 936
    #                    plugins.
    #    plugin_group (str) - The name of the type of plugin that is being
    #                         loaded
937
    #    destination (list) - A list of nodes to store the origins in
938 939 940 941 942 943 944 945 946
    #
    # Raises:
    #    LoadError if 'origin' is an unexpected value
    def _store_origin(self, origin, plugin_group, destination):
        expected_groups = ['sources', 'elements']
        if plugin_group not in expected_groups:
            raise LoadError(LoadErrorReason.INVALID_DATA,
                            "Unexpected plugin group: {}, expecting {}"
                            .format(plugin_group, expected_groups))
947
        node_keys = [key for key in _yaml.node_keys(origin)]
948 949
        if plugin_group in node_keys:
            origin_node = _yaml.node_copy(origin)
950
            plugins = _yaml.node_get(origin, Mapping, plugin_group, default_value={})
951
            _yaml.node_set(origin_node, 'plugins', [k for k in _yaml.node_keys(plugins)])
952
            for group in expected_groups:
953
                if group in origin_node:
954 955 956
                    _yaml.node_del(origin_node, group)

            if _yaml.node_get(origin_node, str, 'origin') == 'local':
957 958
                path = self.get_path_from_node(origin, 'path',
                                               check_is_dir=True)
959
                # paths are passed in relative to the project, but must be absolute
960 961
                _yaml.node_set(origin_node, 'path', os.path.join(self.directory, path))
            destination.append(origin_node)
962 963 964 965 966 967 968 969 970 971 972 973 974 975

    # _warning_is_fatal():
    #
    # Returns true if the warning in question should be considered fatal based on
    # the project configuration.
    #
    # Args:
    #   warning_str (str): The warning configuration string to check against
    #
    # Returns:
    #    (bool): True if the warning should be considered fatal and cause an error.
    #
    def _warning_is_fatal(self, warning_str):
        return warning_str in self._fatal_warnings