.gitlab-ci.yml

@@ -166,6 +166,12 @@ docs:
     BST_EXT_REF: 1d6ab71151b93c8cbc0a91a36ffe9270f3b835f1 # 0.5.1
     FD_SDK_REF: 88d7c22c2281b987faa02edd57df80d430eecf1f # 18.08.11-35-g88d7c22c
   before_script:
+  - |
+    mkdir -p "${HOME}/.config"
+    cat <<EOF >"${HOME}/.config/buildstream.conf"
+    scheduler:
+      fetchers: 2
+    EOF
   - (cd dist && ./unpack.sh && cd buildstream && pip3 install .)
   - pip3 install --user -e ${BST_EXT_URL}@${BST_EXT_REF}#egg=bst_ext
   - git clone https://gitlab.com/freedesktop-sdk/freedesktop-sdk.git

NEWS

@@ -2,6 +2,15 @@
 buildstream 1.3.1
 =================
 
+  o `bst shell` learned the `-e` option for staging multiple elements
+    provided the element's kind implements `BST_GRANULAR_STAGE`.
+
+  o BREAKING CHANGE: The 'manual' element lost its default 'MAKEFLAGS' and 'V'
+    environment variables. There is already a 'make' element with the same
+    variables. Note that this is a breaking change, it will require users to
+    make changes to their .bst files if they are expecting these environment
+    variables to be set.
+
   o Failed builds are included in the cache as well.
     `bst checkout` will provide anything in `%{install-root}`.
     A build including cached fails will cause any dependant elements
@@ -31,6 +40,15 @@ buildstream 1.3.1
     new the `conf-root` variable to make the process easier. And there has been
     a bug fix to workspaces so they can be build in workspaces too.
 
+  o Creating a build shell through the interactive mode or `bst shell --build`
+    will now use the cached build tree. It is now easier to debug local build
+    failures.
+
+  o `bst shell --sysroot` now takes any directory that contains a sysroot,
+    instead of just a specially-formatted build-root with a `root` and `scratch`
+    subdirectory.
+
+
 =================
 buildstream 1.1.5
 =================

buildstream/_artifactcache/artifactcache.py

@@ -17,17 +17,22 @@
 #  Authors:
 #        Tristan Maat <tristan.maat@codethink.co.uk>
 
+import multiprocessing
 import os
+import signal
 import string
 from collections import namedtuple
 from collections.abc import Mapping
 
 from ..types import _KeyStrength
-from .._exceptions import ArtifactError, ImplError, LoadError, LoadErrorReason
+from .._exceptions import ArtifactError, CASError, LoadError, LoadErrorReason
 from .._message import Message, MessageType
+from .. import _signals
 from .. import utils
 from .. import _yaml
 
+from .cascache import CASCache, CASRemote
+
 
 CACHE_SIZE_FILE = "cache_size"
 
@@ -93,7 +98,8 @@ class ArtifactCache():
     def __init__(self, context):
         self.context = context
         self.extractdir = os.path.join(context.artifactdir, 'extract')
-        self.tmpdir = os.path.join(context.artifactdir, 'tmp')
+
+        self.cas = CASCache(context.artifactdir)
 
         self.global_remote_specs = []
         self.project_remote_specs = {}
@@ -104,12 +110,15 @@ class ArtifactCache():
         self._cache_lower_threshold = None    # The target cache size for a cleanup
         self._remotes_setup = False           # Check to prevent double-setup of remotes
 
+        # Per-project list of _CASRemote instances.
+        self._remotes = {}
+
+        self._has_fetch_remotes = False
+        self._has_push_remotes = False
+
         os.makedirs(self.extractdir, exist_ok=True)
-        os.makedirs(self.tmpdir, exist_ok=True)
 
-    ################################################
-    #  Methods implemented on the abstract class   #
-    ################################################
+        self._calculate_cache_quota()
 
     # get_artifact_fullname()
     #
@@ -240,8 +249,10 @@ class ArtifactCache():
             for key in (strong_key, weak_key):
                 if key:
                     try:
-                        self.update_mtime(element, key)
-                    except ArtifactError:
+                        ref = self.get_artifact_fullname(element, key)
+
+                        self.cas.update_mtime(ref)
+                    except CASError:
                         pass
 
     # clean():
@@ -252,7 +263,7 @@ class ArtifactCache():
     #    (int): The size of the cache after having cleaned up
     #
     def clean(self):
-        artifacts = self.list_artifacts()  # pylint: disable=assignment-from-no-return
+        artifacts = self.list_artifacts()
 
         # Build a set of the cache keys which are required
         # based on the required elements at cleanup time
@@ -294,7 +305,7 @@ class ArtifactCache():
             if key not in required_artifacts:
 
                 # Remove the actual artifact, if it's not required.
-                size = self.remove(to_remove)  # pylint: disable=assignment-from-no-return
+                size = self.remove(to_remove)
 
                 # Remove the size from the removed size
                 self.set_cache_size(self._cache_size - size)
@@ -311,7 +322,7 @@ class ArtifactCache():
     #    (int): The size of the artifact cache.
     #
     def compute_cache_size(self):
-        self._cache_size = self.calculate_cache_size()  # pylint: disable=assignment-from-no-return
+        self._cache_size = self.cas.calculate_cache_size()
 
         return self._cache_size
 
@@ -380,28 +391,12 @@ class ArtifactCache():
     def has_quota_exceeded(self):
         return self.get_cache_size() > self._cache_quota
 
-    ################################################
-    # Abstract methods for subclasses to implement #
-    ################################################
-
     # preflight():
     #
     # Preflight check.
     #
     def preflight(self):
-        pass
-
-    # update_mtime()
-    #
-    # Update the mtime of an artifact.
-    #
-    # Args:
-    #     element (Element): The Element to update
-    #     key (str): The key of the artifact.
-    #
-    def update_mtime(self, element, key):
-        raise ImplError("Cache '{kind}' does not implement update_mtime()"
-                        .format(kind=type(self).__name__))
+        self.cas.preflight()
 
     # initialize_remotes():
     #
@@ -411,7 +406,59 @@ class ArtifactCache():
     #     on_failure (callable): Called if we fail to contact one of the caches.
     #
     def initialize_remotes(self, *, on_failure=None):
-        pass
+        remote_specs = self.global_remote_specs
+
+        for project in self.project_remote_specs:
+            remote_specs += self.project_remote_specs[project]
+
+        remote_specs = list(utils._deduplicate(remote_specs))
+
+        remotes = {}
+        q = multiprocessing.Queue()
+        for remote_spec in remote_specs:
+            # Use subprocess to avoid creation of gRPC threads in main BuildStream process
+            # See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
+            p = multiprocessing.Process(target=self.cas.initialize_remote, args=(remote_spec, q))
+
+            try:
+                # Keep SIGINT blocked in the child process
+                with _signals.blocked([signal.SIGINT], ignore=False):
+                    p.start()
+
+                error = q.get()
+                p.join()
+            except KeyboardInterrupt:
+                utils._kill_process_tree(p.pid)
+                raise
+
+            if error and on_failure:
+                on_failure(remote_spec.url, error)
+            elif error:
+                raise ArtifactError(error)
+            else:
+                self._has_fetch_remotes = True
+                if remote_spec.push:
+                    self._has_push_remotes = True
+
+                remotes[remote_spec.url] = CASRemote(remote_spec)
+
+        for project in self.context.get_projects():
+            remote_specs = self.global_remote_specs
+            if project in self.project_remote_specs:
+                remote_specs = list(utils._deduplicate(remote_specs + self.project_remote_specs[project]))
+
+            project_remotes = []
+
+            for remote_spec in remote_specs:
+                # Errors are already handled in the loop above,
+                # skip unreachable remotes here.
+                if remote_spec.url not in remotes:
+                    continue
+
+                remote = remotes[remote_spec.url]
+                project_remotes.append(remote)
+
+            self._remotes[project] = project_remotes
 
     # contains():
     #
@@ -425,8 +472,9 @@ class ArtifactCache():
     # Returns: True if the artifact is in the cache, False otherwise
     #
     def contains(self, element, key):
-        raise ImplError("Cache '{kind}' does not implement contains()"
-                        .format(kind=type(self).__name__))
+        ref = self.get_artifact_fullname(element, key)
+
+        return self.cas.contains(ref)
 
     # list_artifacts():
     #
@@ -437,8 +485,7 @@ class ArtifactCache():
     #               `ArtifactCache.get_artifact_fullname` in LRU order
     #
     def list_artifacts(self):
-        raise ImplError("Cache '{kind}' does not implement list_artifacts()"
-                        .format(kind=type(self).__name__))
+        return self.cas.list_refs()
 
     # remove():
     #
@@ -450,9 +497,31 @@ class ArtifactCache():
     #                          generated by
     #                          `ArtifactCache.get_artifact_fullname`)
     #
-    def remove(self, artifact_name):
-        raise ImplError("Cache '{kind}' does not implement remove()"
-                        .format(kind=type(self).__name__))
+    # Returns:
+    #    (int|None) The amount of space pruned from the repository in
+    #               Bytes, or None if defer_prune is True
+    #
+    def remove(self, ref):
+
+        # Remove extract if not used by other ref
+        tree = self.cas.resolve_ref(ref)
+        ref_name, ref_hash = os.path.split(ref)
+        extract = os.path.join(self.extractdir, ref_name, tree.hash)
+        keys_file = os.path.join(extract, 'meta', 'keys.yaml')
+        if os.path.exists(keys_file):
+            keys_meta = _yaml.load(keys_file)
+            keys = [keys_meta['strong'], keys_meta['weak']]
+            remove_extract = True
+            for other_hash in keys:
+                if other_hash == ref_hash:
+                    continue
+                remove_extract = False
+                break
+
+            if remove_extract:
+                utils._force_rmtree(extract)
+
+        return self.cas.remove(ref)
 
     # extract():
     #
@@ -472,8 +541,11 @@ class ArtifactCache():
     # Returns: path to extracted artifact
     #
     def extract(self, element, key):
-        raise ImplError("Cache '{kind}' does not implement extract()"
-                        .format(kind=type(self).__name__))
+        ref = self.get_artifact_fullname(element, key)
+
+        path = os.path.join(self.extractdir, element._get_project().name, element.normal_name)
+
+        return self.cas.extract(ref, path)
 
     # commit():
     #
@@ -485,8 +557,9 @@ class ArtifactCache():
     #     keys (list): The cache keys to use
     #
     def commit(self, element, content, keys):
-        raise ImplError("Cache '{kind}' does not implement commit()"
-                        .format(kind=type(self).__name__))
+        refs = [self.get_artifact_fullname(element, key) for key in keys]
+
+        self.cas.commit(refs, content)
 
     # diff():
     #
@@ -500,8 +573,10 @@ class ArtifactCache():
     #     subdir (str): A subdirectory to limit the comparison to
     #
     def diff(self, element, key_a, key_b, *, subdir=None):
-        raise ImplError("Cache '{kind}' does not implement diff()"
-                        .format(kind=type(self).__name__))
+        ref_a = self.get_artifact_fullname(element, key_a)
+        ref_b = self.get_artifact_fullname(element, key_b)
+
+        return self.cas.diff(ref_a, ref_b, subdir=subdir)
 
     # has_fetch_remotes():
     #
@@ -513,7 +588,16 @@ class ArtifactCache():
     # Returns: True if any remote repositories are configured, False otherwise
     #
     def has_fetch_remotes(self, *, element=None):
-        return False
+        if not self._has_fetch_remotes:
+            # No project has fetch remotes
+            return False
+        elif element is None:
+            # At least one (sub)project has fetch remotes
+            return True
+        else:
+            # Check whether the specified element's project has fetch remotes
+            remotes_for_project = self._remotes[element._get_project()]
+            return bool(remotes_for_project)
 
     # has_push_remotes():
     #
@@ -525,7 +609,16 @@ class ArtifactCache():
     # Returns: True if any remote repository is configured, False otherwise
     #
     def has_push_remotes(self, *, element=None):
-        return False
+        if not self._has_push_remotes:
+            # No project has push remotes
+            return False
+        elif element is None:
+            # At least one (sub)project has push remotes
+            return True
+        else:
+            # Check whether the specified element's project has push remotes
+            remotes_for_project = self._remotes[element._get_project()]
+            return any(remote.spec.push for remote in remotes_for_project)
 
     # push():
     #
@@ -542,8 +635,28 @@ class ArtifactCache():
     #   (ArtifactError): if there was an error
     #
     def push(self, element, keys):
-        raise ImplError("Cache '{kind}' does not implement push()"
-                        .format(kind=type(self).__name__))
+        refs = [self.get_artifact_fullname(element, key) for key in list(keys)]
+
+        project = element._get_project()
+
+        push_remotes = [r for r in self._remotes[project] if r.spec.push]
+
+        pushed = False
+
+        for remote in push_remotes:
+            remote.init()
+            display_key = element._get_brief_display_key()
+            element.status("Pushing artifact {} -> {}".format(display_key, remote.spec.url))
+
+            if self.cas.push(refs, remote):
+                element.info("Pushed artifact {} -> {}".format(display_key, remote.spec.url))
+                pushed = True
+            else:
+                element.info("Remote ({}) already has {} cached".format(
+                    remote.spec.url, element._get_brief_display_key()
+                ))
+
+        return pushed
 
     # pull():
     #
@@ -558,8 +671,130 @@ class ArtifactCache():
     #   (bool): True if pull was successful, False if artifact was not available
     #
     def pull(self, element, key, *, progress=None):
-        raise ImplError("Cache '{kind}' does not implement pull()"
-                        .format(kind=type(self).__name__))
+        ref = self.get_artifact_fullname(element, key)
+
+        project = element._get_project()
+
+        for remote in self._remotes[project]:
+            try:
+                display_key = element._get_brief_display_key()
+                element.status("Pulling artifact {} <- {}".format(display_key, remote.spec.url))
+
+                if self.cas.pull(ref, remote, progress=progress):
+                    element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
+                    # no need to pull from additional remotes
+                    return True
+                else:
+                    element.info("Remote ({}) does not have {} cached".format(
+                        remote.spec.url, element._get_brief_display_key()
+                    ))
+
+            except CASError as e:
+                raise ArtifactError("Failed to pull artifact {}: {}".format(
+                    element._get_brief_display_key(), e)) from e
+
+        return False
+
+    # pull_tree():
+    #
+    # Pull a single Tree rather than an artifact.
+    # Does not update local refs.
+    #
+    # Args:
+    #     project (Project): The current project
+    #     digest (Digest): The digest of the tree
+    #
+    def pull_tree(self, project, digest):
+        for remote in self._remotes[project]:
+            digest = self.cas.pull_tree(remote, digest)
+
+            if digest:
+                # no need to pull from additional remotes
+                return digest
+
+        return None
+
+    # push_directory():
+    #
+    # Push the given virtual directory to all remotes.
+    #
+    # Args:
+    #     project (Project): The current project
+    #     directory (Directory): A virtual directory object to push.
+    #
+    # Raises:
+    #     (ArtifactError): if there was an error
+    #
+    def push_directory(self, project, directory):
+        if self._has_push_remotes:
+            push_remotes = [r for r in self._remotes[project] if r.spec.push]
+        else:
+            push_remotes = []
+
+        if not push_remotes:
+            raise ArtifactError("push_directory was called, but no remote artifact " +
+                                "servers are configured as push remotes.")
+
+        if directory.ref is None:
+            return
+
+        for remote in push_remotes:
+            self.cas.push_directory(remote, directory)
+
+    # push_message():
+    #
+    # Push the given protobuf message to all remotes.
+    #
+    # Args:
+    #     project (Project): The current project
+    #     message (Message): A protobuf message to push.
+    #
+    # Raises:
+    #     (ArtifactError): if there was an error
+    #
+    def push_message(self, project, message):
+
+        if self._has_push_remotes:
+            push_remotes = [r for r in self._remotes[project] if r.spec.push]
+        else:
+            push_remotes = []
+
+        if not push_remotes:
+            raise ArtifactError("push_message was called, but no remote artifact " +
+                                "servers are configured as push remotes.")
+
+        for remote in push_remotes:
+            message_digest = self.cas.push_message(remote, message)
+
+        return message_digest
+
+    # verify_digest_pushed():
+    #
+    # Check whether the object is already on the server in which case
+    # there is no need to upload it.
+    #
+    # Args:
+    #     project (Project): The current project
+    #     digest (Digest): The object digest.
+    #
+    def verify_digest_pushed(self, project, digest):
+
+        if self._has_push_remotes:
+            push_remotes = [r for r in self._remotes[project] if r.spec.push]
+        else:
+            push_remotes = []
+
+        if not push_remotes:
+            raise ArtifactError("verify_digest_pushed was called, but no remote artifact " +
+                                "servers are configured as push remotes.")
+
+        pushed = False
+
+        for remote in push_remotes:
+            if self.cas.verify_digest_on_remote(remote, digest):
+                pushed = True
+
+        return pushed
 
     # link_key():
     #
@@ -571,19 +806,10 @@ class ArtifactCache():
     #     newkey (str): A new cache key for the artifact
     #
     def link_key(self, element, oldkey, newkey):
-        raise ImplError("Cache '{kind}' does not implement link_key()"
-                        .format(kind=type(self).__name__))
+        oldref = self.get_artifact_fullname(element, oldkey)
+        newref = self.get_artifact_fullname(element, newkey)
 
-    # calculate_cache_size()
-    #
-    # Return the real artifact cache size.
-    #
-    # Returns:
-    #    (int): The size of the artifact cache.
-    #
-    def calculate_cache_size(self):
-        raise ImplError("Cache '{kind}' does not implement calculate_cache_size()"
-                        .format(kind=type(self).__name__))
+        self.cas.link_ref(oldref, newref)
 
     ################################################
     #               Local Private Methods          #

buildstream/_artifactcache/casserver.py

@@ -32,8 +32,9 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remo
 from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
 from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
 
-from .._exceptions import ArtifactError
-from .._context import Context
+from .._exceptions import CASError
+
+from .cascache import CASCache
 
 
 # The default limit for gRPC messages is 4 MiB.
@@ -55,26 +56,23 @@ class ArtifactTooLargeException(Exception):
 #     enable_push (bool): Whether to allow blob uploads and artifact updates
 #
 def create_server(repo, *, enable_push):
-    context = Context()
-    context.artifactdir = os.path.abspath(repo)
-
-    artifactcache = context.artifactcache
+    cas = CASCache(os.path.abspath(repo))
 
     # Use max_workers default from Python 3.5+
     max_workers = (os.cpu_count() or 1) * 5
     server = grpc.server(futures.ThreadPoolExecutor(max_workers))
 
     bytestream_pb2_grpc.add_ByteStreamServicer_to_server(
-        _ByteStreamServicer(artifactcache, enable_push=enable_push), server)
+        _ByteStreamServicer(cas, enable_push=enable_push), server)
 
     remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(
-        _ContentAddressableStorageServicer(artifactcache, enable_push=enable_push), server)
+        _ContentAddressableStorageServicer(cas, enable_push=enable_push), server)
 
     remote_execution_pb2_grpc.add_CapabilitiesServicer_to_server(
         _CapabilitiesServicer(), server)
 
     buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server(
-        _ReferenceStorageServicer(artifactcache, enable_push=enable_push), server)
+        _ReferenceStorageServicer(cas, enable_push=enable_push), server)
 
     return server
 
@@ -333,7 +331,7 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
 
             response.digest.hash = tree.hash
             response.digest.size_bytes = tree.size_bytes
-        except ArtifactError:
+        except CASError:
             context.set_code(grpc.StatusCode.NOT_FOUND)
 
         return response
@@ -437,7 +435,7 @@ def _clean_up_cache(cas, object_size):
         return 0
 
     # obtain a list of LRP artifacts
-    LRP_artifacts = cas.list_artifacts()
+    LRP_artifacts = cas.list_refs()
 
     removed_size = 0  # in bytes
     while object_size - removed_size > free_disk_space:

buildstream/_context.py

@@ -31,7 +31,6 @@ from ._exceptions import LoadError, LoadErrorReason, BstError
 from ._message import Message, MessageType
 from ._profile import Topics, profile_start, profile_end
 from ._artifactcache import ArtifactCache
-from ._artifactcache.cascache import CASCache
 from ._workspaces import Workspaces
 from .plugin import _plugin_lookup
 
@@ -233,7 +232,7 @@ class Context():
     @property
     def artifactcache(self):
         if not self._artifactcache:
-            self._artifactcache = CASCache(self)
+            self._artifactcache = ArtifactCache(self)
 
         return self._artifactcache
 

buildstream/_elementfactory.py

@@ -47,7 +47,6 @@ class ElementFactory(PluginContext):
     # Args:
     #    context (object): The Context object for processing
     #    project (object): The project object
-    #    artifacts (ArtifactCache): The artifact cache
     #    meta (object): The loaded MetaElement
     #
     # Returns: A newly created Element object of the appropriate kind
@@ -56,9 +55,9 @@ class ElementFactory(PluginContext):
     #    PluginError (if the kind lookup failed)
     #    LoadError (if the element itself took issue with the config)
     #
-    def create(self, context, project, artifacts, meta):
+    def create(self, context, project, meta):
         element_type, default_config = self.lookup(meta.kind)
-        element = element_type(context, project, artifacts, meta, default_config)
+        element = element_type(context, project, meta, default_config)
         version = self._format_versions.get(meta.kind, 0)
         self._assert_plugin_format(element, version)
         return element

buildstream/_exceptions.py

@@ -90,6 +90,7 @@ class ErrorDomain(Enum):
     APP = 12
     STREAM = 13
     VIRTUAL_FS = 14
+    CAS = 15
 
 
 # BstError is an internal base exception class for BuildSream
@@ -111,10 +112,8 @@ class BstError(Exception):
         #
         self.detail = detail
 
-        # The build sandbox in which the error occurred, if the
-        # error occurred at element assembly time.
-        #
-        self.sandbox = None
+        # A sandbox can be created to debug this error
+        self.sandbox = False
 
         # When this exception occurred during the handling of a job, indicate
         # whether or not there is any point retrying the job.
@@ -276,6 +275,15 @@ class ArtifactError(BstError):
         super().__init__(message, detail=detail, domain=ErrorDomain.ARTIFACT, reason=reason, temporary=True)
 
 
+# CASError
+#
+# Raised when errors are encountered in the CAS
+#
+class CASError(BstError):
+    def __init__(self, message, *, detail=None, reason=None, temporary=False):
+        super().__init__(message, detail=detail, domain=ErrorDomain.CAS, reason=reason, temporary=True)
+
+
 # PipelineError
 #
 # Raised from pipeline operations

buildstream/_frontend/app.py

@@ -305,7 +305,6 @@ class App():
         directory = self._main_options['directory']
         directory = os.path.abspath(directory)
         project_path = os.path.join(directory, 'project.conf')
-        elements_path = os.path.join(directory, element_path)
 
         try:
             # Abort if the project.conf already exists, unless `--force` was specified in `bst init`
@@ -335,6 +334,7 @@ class App():
                 raise AppError("Error creating project directory {}: {}".format(directory, e)) from e
 
             # Create the elements sub-directory if it doesnt exist
+            elements_path = os.path.join(directory, element_path)
             try:
                 os.makedirs(elements_path, exist_ok=True)
             except IOError as e:
@@ -597,7 +597,7 @@ class App():
                     click.echo("\nDropping into an interactive shell in the failed build sandbox\n", err=True)
                     try:
                         prompt = self.shell_prompt(element)
-                        self.stream.shell(element, Scope.BUILD, prompt, directory=failure.sandbox, isolate=True)
+                        self.stream.shell(element, Scope.BUILD, prompt, isolate=True)
                     except BstError as e:
                         click.echo("Error while attempting to create interactive shell: {}".format(e), err=True)
                 elif choice == 'log':

buildstream/_frontend/cli.py

@@ -572,11 +572,13 @@ def show(app, elements, deps, except_, order, format_):
               help="Mount a file or directory into the sandbox")
 @click.option('--isolate', is_flag=True, default=False,
               help='Create an isolated build sandbox')
+@click.option('--element', '-e', 'elements', multiple=True,
+              type=click.Path(readable=False), required=False)
 @click.argument('element',
-                type=click.Path(readable=False))
+                type=click.Path(readable=False), required=False)
 @click.argument('command', type=click.STRING, nargs=-1)
 @click.pass_obj
-def shell(app, element, sysroot, mount, isolate, build_, command):
+def shell(app, element, elements, sysroot, mount, isolate, build_, command):
     """Run a command in the target element's sandbox environment
 
     This will stage a temporary sysroot for running the target
@@ -597,13 +599,21 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
     from .._project import HostMount
     from .._pipeline import PipelineSelection
 
+    if elements and element is not None:
+        command = (element,) + command
+        element = None
+    if not elements and element is not None:
+        elements = (element,)
+    if not elements:
+        raise AppError('No elements specified to open a shell in')
+
     if build_:
         scope = Scope.BUILD
     else:
         scope = Scope.RUN
 
     with app.initialized():
-        dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
+        dependencies = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
         element = dependencies[0]
         prompt = app.shell_prompt(element)
         mounts = [
@@ -611,7 +621,7 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
             for host_path, path in mount
         ]
         try:
-            exitcode = app.stream.shell(element, scope, prompt,
+            exitcode = app.stream.shell(dependencies, scope, prompt,
                                         directory=sysroot,
                                         mounts=mounts,
                                         isolate=isolate,

buildstream/_frontend/widget.py

@@ -668,17 +668,6 @@ class LogLine(Widget):
 
             extra_nl = True
 
-        if message.sandbox is not None:
-            sandbox = self._indent + 'Sandbox directory: ' + message.sandbox
-
-            text += '\n'
-            if message.message_type == MessageType.FAIL:
-                text += self._err_profile.fmt(sandbox, bold=True)
-            else:
-                text += self._detail_profile.fmt(sandbox)
-            text += '\n'
-            extra_nl = True
-
         if message.scheduler and message.message_type == MessageType.FAIL:
             text += '\n'
 

buildstream/_loader/loader.py

@@ -537,7 +537,7 @@ class Loader():
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             "{}: Expected junction but element kind is {}".format(filename, meta_element.kind))
 
-        element = Element._new_from_meta(meta_element, self._context.artifactcache)
+        element = Element._new_from_meta(meta_element)
         element._preflight()
 
         sources = list(element.sources())

buildstream/_message.py

@@ -70,7 +70,7 @@ class Message():
         self.elapsed = elapsed            # The elapsed time, in timed messages
         self.depth = depth                # The depth of a timed message
         self.logfile = logfile            # The log file path where commands took place
-        self.sandbox = sandbox            # The sandbox directory where an error occurred (if any)
+        self.sandbox = sandbox            # The error that caused this message used a sandbox
         self.pid = os.getpid()            # The process pid
         self.unique_id = unique_id        # The plugin object ID issueing the message
         self.task_id = task_id            # The plugin object ID of the task

buildstream/_pipeline.py

@@ -106,7 +106,7 @@ class Pipeline():
 
         profile_start(Topics.LOAD_PIPELINE, "_".join(t.replace(os.sep, '-') for t in targets))
 
-        elements = self._project.load_elements(targets, self._artifacts,
+        elements = self._project.load_elements(targets,
                                                rewritable=rewritable,
                                                fetch_subprojects=fetch_subprojects)
 

buildstream/_project.py

@@ -224,18 +224,17 @@ class Project():
     # Instantiate and return an element
     #
     # Args:
-    #    artifacts (ArtifactCache): The artifact cache
     #    meta (MetaElement): The loaded MetaElement
     #    first_pass (bool): Whether to use first pass configuration (for junctions)
     #
     # Returns:
     #    (Element): A newly created Element object of the appropriate kind
     #
-    def create_element(self, artifacts, meta, *, first_pass=False):
+    def create_element(self, meta, *, first_pass=False):
         if first_pass:
-            return self.first_pass_config.element_factory.create(self._context, self, artifacts, meta)
+            return self.first_pass_config.element_factory.create(self._context, self, meta)
         else:
-            return self.config.element_factory.create(self._context, self, artifacts, meta)
+            return self.config.element_factory.create(self._context, self, meta)
 
     # create_source()
     #
@@ -305,7 +304,6 @@ class Project():
     #
     # Args:
     #    targets (list): Target names
-    #    artifacts (ArtifactCache): Artifact cache
     #    rewritable (bool): Whether the loaded files should be rewritable
     #                       this is a bit more expensive due to deep copies
     #    fetch_subprojects (bool): Whether we should fetch subprojects as a part of the
@@ -314,7 +312,7 @@ class Project():
     # Returns:
     #    (list): A list of loaded Element
     #
-    def load_elements(self, targets, artifacts, *,
+    def load_elements(self, targets, *,
                       rewritable=False, fetch_subprojects=False):
         with self._context.timed_activity("Loading elements", silent_nested=True):
             meta_elements = self.loader.load(targets, rewritable=rewritable,
@@ -323,7 +321,7 @@ class Project():
 
         with self._context.timed_activity("Resolving elements"):
             elements = [
-                Element._new_from_meta(meta, artifacts)
+                Element._new_from_meta(meta)
                 for meta in meta_elements
             ]
 

buildstream/_stream.py

@@ -25,15 +25,17 @@ import stat
 import shlex
 import shutil
 import tarfile
-from contextlib import contextmanager
+from contextlib import contextmanager, ExitStack
 from tempfile import TemporaryDirectory
 
 from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
 from ._message import Message, MessageType
-from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
 from ._pipeline import Pipeline, PipelineSelection
+from ._platform import Platform
+from .sandbox._config import SandboxConfig
+from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
 from . import utils, _yaml, _site
-from . import Scope, Consistency
+from . import SandboxFlags, Scope, Consistency
 
 
 # Stream()
@@ -117,7 +119,7 @@ class Stream():
     # Run a shell
     #
     # Args:
-    #    element (Element): An Element object to run the shell for
+    #    elements (List of Element): Elements to run the shell for
     #    scope (Scope): The scope for the shell (Scope.BUILD or Scope.RUN)
     #    prompt (str): The prompt to display in the shell
     #    directory (str): A directory where an existing prestaged sysroot is expected, or None
@@ -128,7 +130,7 @@ class Stream():
     # Returns:
     #    (int): The exit code of the launched shell
     #
-    def shell(self, element, scope, prompt, *,
+    def shell(self, elements, scope, prompt, *,
               directory=None,
               mounts=None,
               isolate=False,
@@ -140,14 +142,114 @@ class Stream():
         if directory is None:
             missing_deps = [
                 dep._get_full_name()
-                for dep in self._pipeline.dependencies([element], scope)
+                for dep in self._pipeline.dependencies(elements, scope)
                 if not dep._cached()
             ]
             if missing_deps:
                 raise StreamError("Elements need to be built or downloaded before staging a shell environment",
                                   detail="\n".join(missing_deps))
 
-        return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command)
+        # Assert we're not mixing virtual directory compatible
+        # and non-virtual directory compatible elements
+        if any(e.BST_VIRTUAL_DIRECTORY for e in elements) and not all(e.BST_VIRTUAL_DIRECTORY for e in elements):
+            raise StreamError(
+                "Elements do not support multiple-element staging",
+                detail=("Multi-element staging is not supported" +
+                        " because elements {} support BST_VIRTUAL_DIRECTORY and {} do not.").format(
+                            ', '.join(e.name for e in elements if e.BST_VIRTUAL_DIRECTORY),
+                            ', '.join(e.name for e in elements if not e.BST_VIRTUAL_DIRECTORY)))
+
+        with ExitStack() as stack:
+            # Creation logic duplicated from Element.__sandbox
+            # since most of it is creating the tmpdir
+            # and deciding whether to make a remote sandbox,
+            # which we don't want to.
+            if directory is None:
+                os.makedirs(self._context.builddir, exist_ok=True)
+                rootdir = stack.enter_context(TemporaryDirectory(dir=self._context.builddir))
+            else:
+                rootdir = directory
+
+            # SandboxConfig comes from project, element defaults and MetaElement sandbox config
+            # In the absence of it being exposed to other APIs and a merging strategy
+            # just make it from the project sandbox config.
+            sandbox_config = SandboxConfig(_yaml.node_get(self._project._sandbox, int, 'build-uid'),
+                                           _yaml.node_get(self._project._sandbox, int, 'build-gid'))
+            platform = Platform.get_platform()
+            sandbox = platform.create_sandbox(context=self._context,
+                                              project=self._project,
+                                              directory=rootdir,
+                                              stdout=None, stderr=None, config=sandbox_config,
+                                              bare_directory=directory is not None,
+                                              allow_real_directory=not any(e.BST_VIRTUAL_DIRECTORY
+                                                                           for e in elements))
+
+            # Configure the sandbox with the last element taking precedence for config.
+            for e in elements:
+                e.configure_sandbox(sandbox)
+
+            # Stage contents if not passed --sysroot
+            if not directory:
+                if not all(e.BST_GRANULAR_STAGE for e in elements):
+                    if len(elements) > 1:
+                        raise StreamError(
+                            "Elements do not support multiple-element staging",
+                            detail=("Elements {} do not support multi-element staging " +
+                                    " because element kinds {} do not support BST_GRANULAR_STAGE").format(
+                                        ', '.join(e.name for e in elements if not e.BST_GRANULAR_STAGE),
+                                        ', '.join(set(e.get_kind() for e in elements))))
+                    elements[0].stage(sandbox)
+                else:
+                    visited = {}
+                    for e in elements:
+                        e.stage_dependency_artifacts(sandbox, scope, visited=visited)
+
+                    visited = {}
+                    for e in elements:
+                        e.integrate_dependencies(sandbox, scope, visited=visited)
+
+                    for e in elements:
+                        e.post_integration_staging(sandbox)
+
+            environment = {}
+            for e in elements:
+                environment.update(e.get_environment())
+            flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY
+            shell_command, shell_environment, shell_host_files = self._project.get_shell_config()
+            environment['PS1'] = prompt
+            # Special configurations for non-isolated sandboxes
+            if not isolate:
+
+                # Open the network, and reuse calling uid/gid
+                #
+                flags |= SandboxFlags.NETWORK_ENABLED | SandboxFlags.INHERIT_UID
+
+                # Apply project defined environment vars to set for a shell
+                for key, value in _yaml.node_items(shell_environment):
+                    environment[key] = value
+
+                # Setup any requested bind mounts
+                if mounts is None:
+                    mounts = []
+
+                for mount in shell_host_files + mounts:
+                    if not os.path.exists(mount.host_path):
+                        if not mount.optional:
+                            self._message(MessageType.WARN,
+                                          "Not mounting non-existing host file: {}".format(mount.host_path))
+                    else:
+                        sandbox.mark_directory(mount.path)
+                        sandbox._set_mount_source(mount.path, mount.host_path)
+
+            if command:
+                argv = list(command)
+            else:
+                argv = shell_command
+
+            self._message(MessageType.STATUS, "Running command", detail=" ".join(argv))
+
+            # Run shells with network enabled and readonly root.
+            return sandbox.run(argv, flags, env=environment)
 
     # build()
     #

buildstream/_versions.py

@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 17
+BST_FORMAT_VERSION = 18
 
 
 # The base BuildStream artifact version

buildstream/_yaml.py

@@ -1049,6 +1049,12 @@ class ChainMap(collections.ChainMap):
         for key in clearable:
             del self[key]
 
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
 
 def node_chain_copy(source):
     copy = ChainMap({}, source)

buildstream/buildelement.py

@@ -208,7 +208,6 @@ class BuildElement(Element):
         sandbox.set_environment(self.get_environment())
 
     def stage(self, sandbox):
-
         # Stage deps in the sandbox root
         with self.timed_activity("Staging dependencies", silent_nested=True):
             self.stage_dependency_artifacts(sandbox, Scope.BUILD)
@@ -216,9 +215,11 @@ class BuildElement(Element):
         # Run any integration commands provided by the dependencies
         # once they are all staged and ready
         with self.timed_activity("Integrating sandbox"):
-            for dep in self.dependencies(Scope.BUILD):
-                dep.integrate(sandbox)
+            self.integrate_dependencies(sandbox, Scope.BUILD)
+
+        self.post_integration_staging(sandbox)
 
+    def post_integration_staging(self, sandbox):
         # Stage sources in the build root
         self.stage_sources(sandbox, self.get_variable('build-root'))