HACKING.rst → CONTRIBUTING.rst

@@ -10,6 +10,12 @@ Major feature additions should be proposed on the
 before being considered for inclusion, we strongly recommend proposing
 in advance of commencing work.
 
+If you are experiencing an issue with BuildStream or would like to submit a small patch/feature, then
+you can open issue `here <https://gitlab.com/BuildStream/buildstream/issues/new?issue%5Bassignee_id%5D=&issue%5Bmilestone_id%5D=>`
+
+For policies on how to submit and issue and how to use our project labels, we recommend that you read the policies guide
+`here <https://gitlab.com/BuildStream/nosoftware/alignment/blob/master/BuildStream_policies.md>`
+
 New features must be well documented and tested either in our main
 test suite if possible, or otherwise in the integration tests.
 

MANIFEST.in

 # Basic toplevel package includes
 include BuildStream.doap
 include COPYING
-include HACKING.rst
+include CONTRIBUTING.rst
 include MAINTAINERS
 include NEWS
 include README.rst

buildstream/_artifactcache/artifactcache.py

@@ -51,7 +51,7 @@ class ArtifactCacheSpec(namedtuple('ArtifactCacheSpec', 'url push server_cert cl
         url = _yaml.node_get(spec_node, str, 'url')
         push = _yaml.node_get(spec_node, bool, 'push', default_value=False)
         if not url:
-            provenance = _yaml.node_get_provenance(spec_node)
+            provenance = _yaml.node_get_provenance(spec_node, 'url')
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             "{}: empty artifact cache URL".format(provenance))
 
@@ -67,6 +67,16 @@ class ArtifactCacheSpec(namedtuple('ArtifactCacheSpec', 'url push server_cert cl
         if client_cert and basedir:
             client_cert = os.path.join(basedir, client_cert)
 
+        if client_key and not client_cert:
+            provenance = _yaml.node_get_provenance(spec_node, 'client-key')
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: 'client-key' was specified without 'client-cert'".format(provenance))
+
+        if client_cert and not client_key:
+            provenance = _yaml.node_get_provenance(spec_node, 'client-cert')
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: 'client-cert' was specified without 'client-key'".format(provenance))
+
         return ArtifactCacheSpec(url, push, server_cert, client_key, client_cert)
 
 
@@ -87,10 +97,11 @@ class ArtifactCache():
         self.global_remote_specs = []
         self.project_remote_specs = {}
 
-        self._required_artifacts = set()      # The artifacts required for this session
+        self._required_elements = set()       # The elements required for this session
         self._cache_size = None               # The current cache size, sometimes it's an estimate
         self._cache_quota = None              # The cache quota
         self._cache_lower_threshold = None    # The target cache size for a cleanup
+        self._remotes_setup = False           # Check to prevent double-setup of remotes
 
         os.makedirs(self.extractdir, exist_ok=True)
         os.makedirs(self.tmpdir, exist_ok=True)
@@ -143,6 +154,10 @@ class ArtifactCache():
     #
     def setup_remotes(self, *, use_config=False, remote_url=None):
 
+        # Ensure we do not double-initialise since this can be expensive
+        assert(not self._remotes_setup)
+        self._remotes_setup = True
+
         # Initialize remote artifact caches. We allow the commandline to override
         # the user config in some cases (for example `bst push --remote=...`).
         has_remote_caches = False
@@ -189,33 +204,40 @@ class ArtifactCache():
                                   (str(provenance)))
         return cache_specs
 
-    # append_required_artifacts():
+    # mark_required_elements():
     #
-    # Append to the list of elements whose artifacts are required for
-    # the current run. Artifacts whose elements are in this list will
-    # be locked by the artifact cache and not touched for the duration
-    # of the current pipeline.
+    # Mark elements whose artifacts are required for the current run.
+    #
+    # Artifacts whose elements are in this list will be locked by the artifact
+    # cache and not touched for the duration of the current pipeline.
     #
     # Args:
     #     elements (iterable): A set of elements to mark as required
     #
-    def append_required_artifacts(self, elements):
-        # We lock both strong and weak keys - deleting one but not the
-        # other won't save space in most cases anyway, but would be a
-        # user inconvenience.
+    def mark_required_elements(self, elements):
+
+        # We risk calling this function with a generator, so we
+        # better consume it first.
+        #
+        elements = list(elements)
 
+        # Mark the elements as required. We cannot know that we know the
+        # cache keys yet, so we only check that later when deleting.
+        #
+        self._required_elements.update(elements)
+
+        # For the cache keys which were resolved so far, we bump
+        # the atime of them.
+        #
+        # This is just in case we have concurrent instances of
+        # BuildStream running with the same artifact cache, it will
+        # reduce the likelyhood of one instance deleting artifacts
+        # which are required by the other.
         for element in elements:
             strong_key = element._get_cache_key(strength=_KeyStrength.STRONG)
             weak_key = element._get_cache_key(strength=_KeyStrength.WEAK)
-
             for key in (strong_key, weak_key):
-                if key and key not in self._required_artifacts:
-                    self._required_artifacts.add(key)
-
-                    # We also update the usage times of any artifacts
-                    # we will be using, which helps preventing a
-                    # buildstream process that runs in parallel with
-                    # this one from removing artifacts in-use.
+                if key:
                     try:
                         self.update_atime(key)
                     except ArtifactError:
@@ -231,6 +253,18 @@ class ArtifactCache():
     def clean(self):
         artifacts = self.list_artifacts()
 
+        # Build a set of the cache keys which are required
+        # based on the required elements at cleanup time
+        #
+        # We lock both strong and weak keys - deleting one but not the
+        # other won't save space, but would be a user inconvenience.
+        required_artifacts = set()
+        for element in self._required_elements:
+            required_artifacts.update([
+                element._get_cache_key(strength=_KeyStrength.STRONG),
+                element._get_cache_key(strength=_KeyStrength.WEAK)
+            ])
+
         # Do a real computation of the cache size once, just in case
         self.compute_cache_size()
 
@@ -248,7 +282,7 @@ class ArtifactCache():
                           "Please increase the cache-quota in {}."
                           .format(self.context.config_origin or default_conf))
 
-                if self.get_quota_exceeded():
+                if self.has_quota_exceeded():
                     raise ArtifactError("Cache too full. Aborting.",
                                         detail=detail,
                                         reason="cache-too-full")
@@ -256,7 +290,7 @@ class ArtifactCache():
                     break
 
             key = to_remove.rpartition('/')[2]
-            if key not in self._required_artifacts:
+            if key not in required_artifacts:
 
                 # Remove the actual artifact, if it's not required.
                 size = self.remove(to_remove)
@@ -335,14 +369,14 @@ class ArtifactCache():
         self._cache_size = cache_size
         self._write_cache_size(self._cache_size)
 
-    # get_quota_exceeded()
+    # has_quota_exceeded()
     #
     # Checks if the current artifact cache size exceeds the quota.
     #
     # Returns:
     #    (bool): True of the quota is exceeded
     #
-    def get_quota_exceeded(self):
+    def has_quota_exceeded(self):
         return self.get_cache_size() > self._cache_quota
 
     ################################################

buildstream/_artifactcache/cascache.py

@@ -117,7 +117,7 @@ class CASCache(ArtifactCache):
     def commit(self, element, content, keys):
         refs = [self.get_artifact_fullname(element, key) for key in keys]
 
-        tree = self._create_tree(content)
+        tree = self._commit_directory(content)
 
         for ref in refs:
             self.set_ref(ref, tree)
@@ -665,7 +665,21 @@ class CASCache(ArtifactCache):
     def _refpath(self, ref):
         return os.path.join(self.casdir, 'refs', 'heads', ref)
 
-    def _create_tree(self, path, *, digest=None):
+    # _commit_directory():
+    #
+    # Adds local directory to content addressable store.
+    #
+    # Adds files, symbolic links and recursively other directories in
+    # a local directory to the content addressable store.
+    #
+    # Args:
+    #     path (str): Path to the directory to add.
+    #     dir_digest (Digest): An optional Digest object to use.
+    #
+    # Returns:
+    #     (Digest): Digest object for the directory added.
+    #
+    def _commit_directory(self, path, *, dir_digest=None):
         directory = remote_execution_pb2.Directory()
 
         for name in sorted(os.listdir(path)):
@@ -674,7 +688,7 @@ class CASCache(ArtifactCache):
             if stat.S_ISDIR(mode):
                 dirnode = directory.directories.add()
                 dirnode.name = name
-                self._create_tree(full_path, digest=dirnode.digest)
+                self._commit_directory(full_path, dir_digest=dirnode.digest)
             elif stat.S_ISREG(mode):
                 filenode = directory.files.add()
                 filenode.name = name
@@ -684,10 +698,14 @@ class CASCache(ArtifactCache):
                 symlinknode = directory.symlinks.add()
                 symlinknode.name = name
                 symlinknode.target = os.readlink(full_path)
+            elif stat.S_ISSOCK(mode):
+                # The process serving the socket can't be cached anyway
+                pass
             else:
                 raise ArtifactError("Unsupported file type for {}".format(full_path))
 
-        return self.add_object(digest=digest, buffer=directory.SerializeToString())
+        return self.add_object(digest=dir_digest,
+                               buffer=directory.SerializeToString())
 
     def _get_subdir(self, tree, subdir):
         head, name = os.path.split(subdir)
@@ -830,14 +848,26 @@ class CASCache(ArtifactCache):
 
         assert digest.size_bytes == os.fstat(stream.fileno()).st_size
 
-    def _fetch_directory(self, remote, tree):
-        objpath = self.objpath(tree)
+    # _fetch_directory():
+    #
+    # Fetches remote directory and adds it to content addressable store.
+    #
+    # Fetches files, symbolic links and recursively other directories in
+    # the remote directory and adds them to the content addressable
+    # store.
+    #
+    # Args:
+    #     remote (Remote): The remote to use.
+    #     dir_digest (Digest): Digest object for the directory to fetch.
+    #
+    def _fetch_directory(self, remote, dir_digest):
+        objpath = self.objpath(dir_digest)
         if os.path.exists(objpath):
             # already in local cache
             return
 
         with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
-            self._fetch_blob(remote, tree, out)
+            self._fetch_blob(remote, dir_digest, out)
 
             directory = remote_execution_pb2.Directory()
 
@@ -845,7 +875,7 @@ class CASCache(ArtifactCache):
                 directory.ParseFromString(f.read())
 
             for filenode in directory.files:
-                fileobjpath = self.objpath(tree)
+                fileobjpath = self.objpath(filenode.digest)
                 if os.path.exists(fileobjpath):
                     # already in local cache
                     continue
@@ -859,10 +889,11 @@ class CASCache(ArtifactCache):
             for dirnode in directory.directories:
                 self._fetch_directory(remote, dirnode.digest)
 
-            # place directory blob only in final location when we've downloaded
-            # all referenced blobs to avoid dangling references in the repository
+            # Place directory blob only in final location when we've
+            # downloaded all referenced blobs to avoid dangling
+            # references in the repository.
             digest = self.add_object(path=out.name)
-            assert digest.hash == tree.hash
+            assert digest.hash == dir_digest.hash
 
     def _fetch_tree(self, remote, digest):
         # download but do not store the Tree object

buildstream/_context.py

@@ -119,7 +119,6 @@ class Context():
         self._log_handle = None
         self._log_filename = None
         self.config_cache_quota = 'infinity'
-        self.artifactdir_volume = None
 
     # load()
     #

buildstream/_frontend/complete.py

 #
+#  Copyright (c) 2014 by Armin Ronacher.
 #  Copyright (C) 2016 Codethink Limited
 #
 #  This program is free software; you can redistribute it and/or
@@ -14,8 +15,22 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 #
-#  This module was forked from the python click library.
-
+#  This module was forked from the python click library, Included
+#  original copyright notice from the Click library and following disclaimer
+#  as per their LICENSE requirements.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
 import collections
 import copy
 import os

buildstream/_fuse/hardlinks.py

@@ -42,9 +42,10 @@ from .mount import Mount
 #
 class SafeHardlinks(Mount):
 
-    def __init__(self, directory, tempdir):
+    def __init__(self, directory, tempdir, fuse_mount_options={}):
         self.directory = directory
         self.tempdir = tempdir
+        super().__init__(fuse_mount_options=fuse_mount_options)
 
     def create_operations(self):
         return SafeHardlinkOps(self.directory, self.tempdir)
@@ -121,7 +122,7 @@ class SafeHardlinkOps(Operations):
         st = os.lstat(full_path)
         return dict((key, getattr(st, key)) for key in (
             'st_atime', 'st_ctime', 'st_gid', 'st_mode',
-            'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
+            'st_mtime', 'st_nlink', 'st_size', 'st_uid', 'st_rdev'))
 
     def readdir(self, path, fh):
         full_path = self._full_path(path)

buildstream/_fuse/mount.py

@@ -87,6 +87,9 @@ class Mount():
     #               User Facing API                #
     ################################################
 
+    def __init__(self, fuse_mount_options={}):
+        self._fuse_mount_options = fuse_mount_options
+
     # mount():
     #
     # User facing API for mounting a fuse subclass implementation
@@ -184,7 +187,8 @@ class Mount():
         # Run fuse in foreground in this child process, internally libfuse
         # will handle SIGTERM and gracefully exit it's own little main loop.
         #
-        FUSE(self.__operations, self.__mountpoint, nothreads=True, foreground=True, nonempty=True)
+        FUSE(self.__operations, self.__mountpoint, nothreads=True, foreground=True, nonempty=True,
+             **self._fuse_mount_options)
 
         # Explicit 0 exit code, if the operations crashed for some reason, the exit
         # code will not be 0, and we want to know about it.

buildstream/_project.py

@@ -385,7 +385,10 @@ class Project():
             self._project_conf = _yaml.load(projectfile)
         except LoadError as e:
             # Raise a more specific error here
-            raise LoadError(LoadErrorReason.MISSING_PROJECT_CONF, str(e))
+            if e.reason == LoadErrorReason.MISSING_FILE:
+                raise LoadError(LoadErrorReason.MISSING_PROJECT_CONF, str(e)) from e
+            else:
+                raise
 
         pre_config_node = _yaml.node_copy(self._default_config_node)
         _yaml.composite(pre_config_node, self._project_conf)
@@ -598,7 +601,10 @@ class Project():
         # any conditionals specified for project option declarations,
         # or conditionally specifying the project name; will be ignored.
         #
+        # Don't forget to also resolve options in the element and source overrides.
         output.options.process_node(config)
+        output.options.process_node(output.element_overrides)
+        output.options.process_node(output.source_overrides)
 
         # Load base variables
         output.base_variables = _yaml.node_get(config, Mapping, 'variables')

buildstream/_scheduler/jobs/__init__.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Maat <tristan.maat@codethink.co.uk>
+
 from .elementjob import ElementJob
 from .cachesizejob import CacheSizeJob
 from .cleanupjob import CleanupJob

buildstream/_scheduler/jobs/cleanupjob.py

@@ -21,9 +21,8 @@ from ..._platform import Platform
 
 
 class CleanupJob(Job):
-    def __init__(self, *args, complete_cb, **kwargs):
+    def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self._complete_cb = complete_cb
 
         platform = Platform.get_platform()
         self._artifacts = platform.artifactcache
@@ -34,9 +33,3 @@ class CleanupJob(Job):
     def parent_complete(self, success, result):
         if success:
             self._artifacts.set_cache_size(result)
-
-            if self._complete_cb:
-                self._complete_cb()
-
-    def child_process_data(self):
-        return {}

buildstream/_scheduler/queues/buildqueue.py

@@ -101,7 +101,7 @@ class BuildQueue(Queue):
         # If the estimated size outgrows the quota, ask the scheduler
         # to queue a job to actually check the real cache size.
         #
-        if artifacts.get_quota_exceeded():
+        if artifacts.has_quota_exceeded():
             self._scheduler.check_cache_size()
 
     def done(self, job, element, result, success):

buildstream/_scheduler/scheduler.py

@@ -351,14 +351,13 @@ class Scheduler():
         platform = Platform.get_platform()
         artifacts = platform.artifactcache
 
-        if not artifacts.get_quota_exceeded():
+        if not artifacts.has_quota_exceeded():
             return
 
         job = CleanupJob(self, 'cleanup', 'cleanup/cleanup',
                          resources=[ResourceType.CACHE,
                                     ResourceType.PROCESS],
-                         exclusive_resources=[ResourceType.CACHE],
-                         complete_cb=None)
+                         exclusive_resources=[ResourceType.CACHE])
         self.schedule_jobs([job])
 
     # _suspend_jobs()

buildstream/_stream.py

@@ -938,13 +938,10 @@ class Stream():
         # Set the "required" artifacts that should not be removed
         # while this pipeline is active
         #
-        # FIXME: The set of required artifacts is only really needed
-        #        for build and pull tasks.
+        # It must include all the artifacts which are required by the
+        # final product. Note that this is a superset of the build plan.
         #
-        #        It must include all the artifacts which are required by the
-        #        final product. Note that this is a superset of the build plan.
-        #
-        self._artifacts.append_required_artifacts((e for e in self._pipeline.dependencies(elements, Scope.ALL)))
+        self._artifacts.mark_required_elements(self._pipeline.dependencies(elements, Scope.ALL))
 
         if selection == PipelineSelection.PLAN and dynamic_plan:
             # We use a dynamic build plan, only request artifacts of top-level targets,

buildstream/_versions.py

@@ -33,4 +33,4 @@ BST_FORMAT_VERSION = 16
 # or if buildstream was changed in a way which can cause
 # the same cache key to produce something that is no longer
 # the same.
-BST_CORE_ARTIFACT_VERSION = 5
+BST_CORE_ARTIFACT_VERSION = 6

buildstream/element.py

@@ -200,7 +200,6 @@ class Element(Plugin):
         self.__strict_cache_key = None          # Our cached cache key for strict builds
         self.__artifacts = artifacts            # Artifact cache
         self.__consistency = Consistency.INCONSISTENT  # Cached overall consistency state
-        self.__cached = None                    # Whether we have a cached artifact
         self.__strong_cached = None             # Whether we have a cached artifact
         self.__weak_cached = None               # Whether we have a cached artifact
         self.__assemble_scheduled = False       # Element is scheduled to be assembled
@@ -1126,8 +1125,6 @@ class Element(Plugin):
 
         # Query caches now that the weak and strict cache keys are available
         key_for_cache_lookup = self.__strict_cache_key if context.get_strict() else self.__weak_cache_key
-        if not self.__cached:
-            self.__cached = self.__artifacts.contains(self, key_for_cache_lookup)
         if not self.__strong_cached:
             self.__strong_cached = self.__artifacts.contains(self, self.__strict_cache_key)
         if key_for_cache_lookup == self.__weak_cache_key:
@@ -1489,15 +1486,20 @@ class Element(Plugin):
             workspace.clear_running_files()
             self._get_context().get_workspaces().save_config()
 
-            # We also need to update the required artifacts, since
-            # workspaced dependencies do not have a fixed cache key
-            # when the build starts.
+            # This element will have already been marked as
+            # required, but we bump the atime again, in case
+            # we did not know the cache key until now.
             #
-            # This does *not* cause a race condition, because
-            # _assemble_done is called before a cleanup job may be
-            # launched.
+            # FIXME: This is not exactly correct, we should be
+            #        doing this at the time which we have discovered
+            #        a new cache key, this just happens to be the
+            #        last place where that can happen.
             #
-            self.__artifacts.append_required_artifacts([self])
+            #        Ultimately, we should be refactoring
+            #        Element._update_state() such that we know
+            #        when a cache key is actually discovered.
+            #
+            self.__artifacts.mark_required_elements([self])
 
     # _assemble():
     #
@@ -2079,7 +2081,7 @@ class Element(Plugin):
 
     def __is_cached(self, keystrength):
         if keystrength is None:
-            return self.__cached
+            keystrength = _KeyStrength.STRONG if self._get_context().get_strict() else _KeyStrength.WEAK
 
         return self.__strong_cached if keystrength == _KeyStrength.STRONG else self.__weak_cached
 

buildstream/sandbox/_mount.py

@@ -30,7 +30,7 @@ from .._fuse import SafeHardlinks
 # Helper data object representing a single mount point in the mount map
 #
 class Mount():
-    def __init__(self, sandbox, mount_point, safe_hardlinks):
+    def __init__(self, sandbox, mount_point, safe_hardlinks, fuse_mount_options={}):
         scratch_directory = sandbox._get_scratch_directory()
         # Getting _get_underlying_directory() here is acceptable as
         # we're part of the sandbox code. This will fail if our
@@ -39,6 +39,7 @@ class Mount():
 
         self.mount_point = mount_point
         self.safe_hardlinks = safe_hardlinks
+        self._fuse_mount_options = fuse_mount_options
 
         # FIXME: When the criteria for mounting something and it's parent
         #        mount is identical, then there is no need to mount an additional
@@ -82,7 +83,7 @@ class Mount():
     @contextmanager
     def mounted(self, sandbox):
         if self.safe_hardlinks:
-            mount = SafeHardlinks(self.mount_origin, self.mount_tempdir)
+            mount = SafeHardlinks(self.mount_origin, self.mount_tempdir, self._fuse_mount_options)
             with mount.mounted(self.mount_source):
                 yield
         else:
@@ -100,12 +101,12 @@ class Mount():
 #
 class MountMap():
 
-    def __init__(self, sandbox, root_readonly):
+    def __init__(self, sandbox, root_readonly, fuse_mount_options={}):
         # We will be doing the mounts in the order in which they were declared.
         self.mounts = OrderedDict()
 
         # We want safe hardlinks on rootfs whenever root is not readonly
-        self.mounts['/'] = Mount(sandbox, '/', not root_readonly)
+        self.mounts['/'] = Mount(sandbox, '/', not root_readonly, fuse_mount_options)
 
         for mark in sandbox._get_marked_directories():
             directory = mark['directory']
@@ -113,7 +114,7 @@ class MountMap():
 
             # We want safe hardlinks for any non-root directory where
             # artifacts will be staged to
-            self.mounts[directory] = Mount(sandbox, directory, artifact)
+            self.mounts[directory] = Mount(sandbox, directory, artifact, fuse_mount_options)
 
     # get_mount_source()
     #

buildstream/sandbox/_sandboxbwrap.py

@@ -63,20 +63,8 @@ class SandboxBwrap(Sandbox):
         # Fallback to the sandbox default settings for
         # the cwd and env.
         #
-        if cwd is None:
-            cwd = self._get_work_directory()
-
-        if env is None:
-            env = self._get_environment()
-
-        if cwd is None:
-            cwd = '/'
-
-        # Naive getcwd implementations can break when bind-mounts to different
-        # paths on the same filesystem are present. Letting the command know
-        # what directory it is in makes it unnecessary to call the faulty
-        # getcwd.
-        env['PWD'] = cwd
+        cwd = self._get_work_directory(cwd=cwd)
+        env = self._get_environment(cwd=cwd, env=env)
 
         if not self._has_command(command[0], env):
             raise SandboxError("Staged artifacts do not provide command "

buildstream/sandbox/_sandboxchroot.py

@@ -35,6 +35,9 @@ from . import Sandbox, SandboxFlags
 
 
 class SandboxChroot(Sandbox):
+
+    _FUSE_MOUNT_OPTIONS = {'dev': True}
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -48,21 +51,11 @@ class SandboxChroot(Sandbox):
 
     def run(self, command, flags, *, cwd=None, env=None):
 
-        # Default settings
-        if cwd is None:
-            cwd = self._get_work_directory()
-
-        if cwd is None:
-            cwd = '/'
-
-        if env is None:
-            env = self._get_environment()
-
-        # Naive getcwd implementations can break when bind-mounts to different
-        # paths on the same filesystem are present. Letting the command know
-        # what directory it is in makes it unnecessary to call the faulty
-        # getcwd.
-        env['PWD'] = cwd
+        # Fallback to the sandbox default settings for
+        # the cwd and env.
+        #
+        cwd = self._get_work_directory(cwd=cwd)
+        env = self._get_environment(cwd=cwd, env=env)
 
         if not self._has_command(command[0], env):
             raise SandboxError("Staged artifacts do not provide command "
@@ -77,7 +70,8 @@ class SandboxChroot(Sandbox):
 
         # Create the mount map, this will tell us where
         # each mount point needs to be mounted from and to
-        self.mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY)
+        self.mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY,
+                                  self._FUSE_MOUNT_OPTIONS)
         root_mount_source = self.mount_map.get_mount_source('/')
 
         # Create a sysroot and run the command inside it

buildstream/sandbox/_sandboxremote.py

@@ -27,7 +27,7 @@ from . import Sandbox
 from ..storage._filebaseddirectory import FileBasedDirectory
 from ..storage._casbaseddirectory import CasBasedDirectory
 from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
-from .._artifactcache.cascache import CASCache
+from .._platform import Platform
 
 
 class SandboxError(Exception):
@@ -43,7 +43,6 @@ class SandboxRemote(Sandbox):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.cascache = None
 
         url = urlparse(kwargs['server_url'])
         if not url.scheme or not url.hostname or not url.port:
@@ -56,12 +55,6 @@ class SandboxRemote(Sandbox):
 
         self.server_url = '{}:{}'.format(url.hostname, url.port)
 
-    def _get_cascache(self):
-        if self.cascache is None:
-            self.cascache = CASCache(self._get_context())
-            self.cascache.setup_remotes(use_config=True)
-        return self.cascache
-
     def run_remote_command(self, command, input_root_digest, working_directory, environment):
         # Sends an execution request to the remote execution server.
         #
@@ -78,8 +71,8 @@ class SandboxRemote(Sandbox):
                                                       output_files=[],
                                                       output_directories=[self._output_directory],
                                                       platform=None)
-
-        cascache = self._get_cascache()
+        platform = Platform.get_platform()
+        cascache = platform.artifactcache
         # Upload the Command message to the remote CAS server
         command_digest = cascache.push_message(self._get_project(), remote_command)
         if not command_digest or not cascache.verify_digest_pushed(self._get_project(), command_digest):
@@ -141,7 +134,8 @@ class SandboxRemote(Sandbox):
         if tree_digest is None or not tree_digest.hash:
             raise SandboxError("Output directory structure had no digest attached.")
 
-        cascache = self._get_cascache()
+        platform = Platform.get_platform()
+        cascache = platform.artifactcache
         # Now do a pull to ensure we have the necessary parts.
         dir_digest = cascache.pull_tree(self._get_project(), tree_digest)
         if dir_digest is None or not dir_digest.hash or not dir_digest.size_bytes:
@@ -176,7 +170,8 @@ class SandboxRemote(Sandbox):
 
         upload_vdir.recalculate_hash()
 
-        cascache = self._get_cascache()
+        platform = Platform.get_platform()
+        cascache = platform.artifactcache
         # Now, push that key (without necessarily needing a ref) to the remote.
         vdir_digest = cascache.push_directory(self._get_project(), upload_vdir)
         if not vdir_digest or not cascache.verify_digest_pushed(self._get_project(), vdir_digest):