HACKING.rst → CONTRIBUTING.rst

@@ -10,6 +10,12 @@ Major feature additions should be proposed on the
 before being considered for inclusion, we strongly recommend proposing
 in advance of commencing work.
 
+If you are experiencing an issue with BuildStream or would like to submit a small patch/feature, then
+you can open issue `here <https://gitlab.com/BuildStream/buildstream/issues/new?issue%5Bassignee_id%5D=&issue%5Bmilestone_id%5D=>`
+
+For policies on how to submit and issue and how to use our project labels, we recommend that you read the policies guide
+`here <https://gitlab.com/BuildStream/nosoftware/alignment/blob/master/BuildStream_policies.md>`
+
 New features must be well documented and tested either in our main
 test suite if possible, or otherwise in the integration tests.
 

MANIFEST.in

 # Basic toplevel package includes
 include BuildStream.doap
 include COPYING
-include HACKING.rst
+include CONTRIBUTING.rst
 include MAINTAINERS
 include NEWS
 include README.rst

buildstream/_artifactcache/artifactcache.py

@@ -91,6 +91,7 @@ class ArtifactCache():
         self._cache_size = None               # The current cache size, sometimes it's an estimate
         self._cache_quota = None              # The cache quota
         self._cache_lower_threshold = None    # The target cache size for a cleanup
+        self._remotes_setup = False           # Check to prevent double-setup of remotes
 
         os.makedirs(self.extractdir, exist_ok=True)
         os.makedirs(self.tmpdir, exist_ok=True)
@@ -143,6 +144,10 @@ class ArtifactCache():
     #
     def setup_remotes(self, *, use_config=False, remote_url=None):
 
+        # Ensure we do not double-initialise since this can be expensive
+        assert(not self._remotes_setup)
+        self._remotes_setup = True
+
         # Initialize remote artifact caches. We allow the commandline to override
         # the user config in some cases (for example `bst push --remote=...`).
         has_remote_caches = False
@@ -267,7 +272,7 @@ class ArtifactCache():
                           "Please increase the cache-quota in {}."
                           .format(self.context.config_origin or default_conf))
 
-                if self.get_quota_exceeded():
+                if self.has_quota_exceeded():
                     raise ArtifactError("Cache too full. Aborting.",
                                         detail=detail,
                                         reason="cache-too-full")
@@ -354,14 +359,14 @@ class ArtifactCache():
         self._cache_size = cache_size
         self._write_cache_size(self._cache_size)
 
-    # get_quota_exceeded()
+    # has_quota_exceeded()
     #
     # Checks if the current artifact cache size exceeds the quota.
     #
     # Returns:
     #    (bool): True of the quota is exceeded
     #
-    def get_quota_exceeded(self):
+    def has_quota_exceeded(self):
         return self.get_cache_size() > self._cache_quota
 
     ################################################

buildstream/_artifactcache/cascache.py

@@ -117,7 +117,7 @@ class CASCache(ArtifactCache):
     def commit(self, element, content, keys):
         refs = [self.get_artifact_fullname(element, key) for key in keys]
 
-        tree = self._create_tree(content)
+        tree = self._commit_directory(content)
 
         for ref in refs:
             self.set_ref(ref, tree)
@@ -665,7 +665,21 @@ class CASCache(ArtifactCache):
     def _refpath(self, ref):
         return os.path.join(self.casdir, 'refs', 'heads', ref)
 
-    def _create_tree(self, path, *, digest=None):
+    # _commit_directory():
+    #
+    # Adds local directory to content addressable store.
+    #
+    # Adds files, symbolic links and recursively other directories in
+    # a local directory to the content addressable store.
+    #
+    # Args:
+    #     path (str): Path to the directory to add.
+    #     dir_digest (Digest): An optional Digest object to use.
+    #
+    # Returns:
+    #     (Digest): Digest object for the directory added.
+    #
+    def _commit_directory(self, path, *, dir_digest=None):
         directory = remote_execution_pb2.Directory()
 
         for name in sorted(os.listdir(path)):
@@ -674,7 +688,7 @@ class CASCache(ArtifactCache):
             if stat.S_ISDIR(mode):
                 dirnode = directory.directories.add()
                 dirnode.name = name
-                self._create_tree(full_path, digest=dirnode.digest)
+                self._commit_directory(full_path, dir_digest=dirnode.digest)
             elif stat.S_ISREG(mode):
                 filenode = directory.files.add()
                 filenode.name = name
@@ -690,7 +704,8 @@ class CASCache(ArtifactCache):
             else:
                 raise ArtifactError("Unsupported file type for {}".format(full_path))
 
-        return self.add_object(digest=digest, buffer=directory.SerializeToString())
+        return self.add_object(digest=dir_digest,
+                               buffer=directory.SerializeToString())
 
     def _get_subdir(self, tree, subdir):
         head, name = os.path.split(subdir)
@@ -833,14 +848,26 @@ class CASCache(ArtifactCache):
 
         assert digest.size_bytes == os.fstat(stream.fileno()).st_size
 
-    def _fetch_directory(self, remote, tree):
-        objpath = self.objpath(tree)
+    # _fetch_directory():
+    #
+    # Fetches remote directory and adds it to content addressable store.
+    #
+    # Fetches files, symbolic links and recursively other directories in
+    # the remote directory and adds them to the content addressable
+    # store.
+    #
+    # Args:
+    #     remote (Remote): The remote to use.
+    #     dir_digest (Digest): Digest object for the directory to fetch.
+    #
+    def _fetch_directory(self, remote, dir_digest):
+        objpath = self.objpath(dir_digest)
         if os.path.exists(objpath):
             # already in local cache
             return
 
         with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
-            self._fetch_blob(remote, tree, out)
+            self._fetch_blob(remote, dir_digest, out)
 
             directory = remote_execution_pb2.Directory()
 
@@ -848,7 +875,7 @@ class CASCache(ArtifactCache):
                 directory.ParseFromString(f.read())
 
             for filenode in directory.files:
-                fileobjpath = self.objpath(tree)
+                fileobjpath = self.objpath(filenode.digest)
                 if os.path.exists(fileobjpath):
                     # already in local cache
                     continue
@@ -862,10 +889,11 @@ class CASCache(ArtifactCache):
             for dirnode in directory.directories:
                 self._fetch_directory(remote, dirnode.digest)
 
-            # place directory blob only in final location when we've downloaded
-            # all referenced blobs to avoid dangling references in the repository
+            # Place directory blob only in final location when we've
+            # downloaded all referenced blobs to avoid dangling
+            # references in the repository.
             digest = self.add_object(path=out.name)
-            assert digest.hash == tree.hash
+            assert digest.hash == dir_digest.hash
 
     def _fetch_tree(self, remote, digest):
         # download but do not store the Tree object

buildstream/_context.py

@@ -119,7 +119,6 @@ class Context():
         self._log_handle = None
         self._log_filename = None
         self.config_cache_quota = 'infinity'
-        self.artifactdir_volume = None
 
     # load()
     #

buildstream/_frontend/complete.py

 #
+#  Copyright (c) 2014 by Armin Ronacher.
 #  Copyright (C) 2016 Codethink Limited
 #
 #  This program is free software; you can redistribute it and/or
@@ -14,8 +15,22 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 #
-#  This module was forked from the python click library.
-
+#  This module was forked from the python click library, Included
+#  original copyright notice from the Click library and following disclaimer
+#  as per their LICENSE requirements.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
 import collections
 import copy
 import os

buildstream/_project.py

@@ -598,7 +598,10 @@ class Project():
         # any conditionals specified for project option declarations,
         # or conditionally specifying the project name; will be ignored.
         #
+        # Don't forget to also resolve options in the element and source overrides.
         output.options.process_node(config)
+        output.options.process_node(output.element_overrides)
+        output.options.process_node(output.source_overrides)
 
         # Load base variables
         output.base_variables = _yaml.node_get(config, Mapping, 'variables')

buildstream/_scheduler/jobs/__init__.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Maat <tristan.maat@codethink.co.uk>
+
 from .elementjob import ElementJob
 from .cachesizejob import CacheSizeJob
 from .cleanupjob import CleanupJob

buildstream/_scheduler/jobs/cleanupjob.py

@@ -21,9 +21,8 @@ from ..._platform import Platform
 
 
 class CleanupJob(Job):
-    def __init__(self, *args, complete_cb, **kwargs):
+    def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self._complete_cb = complete_cb
 
         platform = Platform.get_platform()
         self._artifacts = platform.artifactcache
@@ -34,9 +33,3 @@ class CleanupJob(Job):
     def parent_complete(self, success, result):
         if success:
             self._artifacts.set_cache_size(result)
-
-            if self._complete_cb:
-                self._complete_cb()
-
-    def child_process_data(self):
-        return {}

buildstream/_scheduler/queues/buildqueue.py

@@ -101,7 +101,7 @@ class BuildQueue(Queue):
         # If the estimated size outgrows the quota, ask the scheduler
         # to queue a job to actually check the real cache size.
         #
-        if artifacts.get_quota_exceeded():
+        if artifacts.has_quota_exceeded():
             self._scheduler.check_cache_size()
 
     def done(self, job, element, result, success):

buildstream/_scheduler/scheduler.py

@@ -351,14 +351,13 @@ class Scheduler():
         platform = Platform.get_platform()
         artifacts = platform.artifactcache
 
-        if not artifacts.get_quota_exceeded():
+        if not artifacts.has_quota_exceeded():
             return
 
         job = CleanupJob(self, 'cleanup', 'cleanup/cleanup',
                          resources=[ResourceType.CACHE,
                                     ResourceType.PROCESS],
-                         exclusive_resources=[ResourceType.CACHE],
-                         complete_cb=None)
+                         exclusive_resources=[ResourceType.CACHE])
         self.schedule_jobs([job])
 
     # _suspend_jobs()

buildstream/_versions.py

@@ -33,4 +33,4 @@ BST_FORMAT_VERSION = 16
 # or if buildstream was changed in a way which can cause
 # the same cache key to produce something that is no longer
 # the same.
-BST_CORE_ARTIFACT_VERSION = 5
+BST_CORE_ARTIFACT_VERSION = 6

buildstream/sandbox/_sandboxbwrap.py

@@ -63,20 +63,8 @@ class SandboxBwrap(Sandbox):
         # Fallback to the sandbox default settings for
         # the cwd and env.
         #
-        if cwd is None:
-            cwd = self._get_work_directory()
-
-        if env is None:
-            env = self._get_environment()
-
-        if cwd is None:
-            cwd = '/'
-
-        # Naive getcwd implementations can break when bind-mounts to different
-        # paths on the same filesystem are present. Letting the command know
-        # what directory it is in makes it unnecessary to call the faulty
-        # getcwd.
-        env['PWD'] = cwd
+        cwd = self._get_work_directory(cwd=cwd)
+        env = self._get_environment(cwd=cwd, env=env)
 
         if not self._has_command(command[0], env):
             raise SandboxError("Staged artifacts do not provide command "

buildstream/sandbox/_sandboxchroot.py

@@ -48,21 +48,11 @@ class SandboxChroot(Sandbox):
 
     def run(self, command, flags, *, cwd=None, env=None):
 
-        # Default settings
-        if cwd is None:
-            cwd = self._get_work_directory()
-
-        if cwd is None:
-            cwd = '/'
-
-        if env is None:
-            env = self._get_environment()
-
-        # Naive getcwd implementations can break when bind-mounts to different
-        # paths on the same filesystem are present. Letting the command know
-        # what directory it is in makes it unnecessary to call the faulty
-        # getcwd.
-        env['PWD'] = cwd
+        # Fallback to the sandbox default settings for
+        # the cwd and env.
+        #
+        cwd = self._get_work_directory(cwd=cwd)
+        env = self._get_environment(cwd=cwd, env=env)
 
         if not self._has_command(command[0], env):
             raise SandboxError("Staged artifacts do not provide command "

buildstream/sandbox/_sandboxremote.py

@@ -27,7 +27,7 @@ from . import Sandbox
 from ..storage._filebaseddirectory import FileBasedDirectory
 from ..storage._casbaseddirectory import CasBasedDirectory
 from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
-from .._artifactcache.cascache import CASCache
+from .._platform import Platform
 
 
 class SandboxError(Exception):
@@ -58,7 +58,7 @@ class SandboxRemote(Sandbox):
 
     def _get_cascache(self):
         if self.cascache is None:
-            self.cascache = CASCache(self._get_context())
+            self.cascache = Platform.get_platform().artifactcache
             self.cascache.setup_remotes(use_config=True)
         return self.cascache
 

buildstream/sandbox/sandbox.py

@@ -279,20 +279,38 @@ class Sandbox():
     # Fetches the environment variables for running commands
     # in the sandbox.
     #
+    # Args:
+    #    cwd (str): The working directory the command has been requested to run in, if any.
+    #    env (str): The environment the command has been requested to run in, if any.
+    #
     # Returns:
     #    (str): The sandbox work directory
-    def _get_environment(self):
-        return self.__env
+    def _get_environment(self, *, cwd=None, env=None):
+        cwd = self._get_work_directory(cwd=cwd)
+        if env is None:
+            env = self.__env
+
+        # Naive getcwd implementations can break when bind-mounts to different
+        # paths on the same filesystem are present. Letting the command know
+        # what directory it is in makes it unnecessary to call the faulty
+        # getcwd.
+        env = dict(env)
+        env['PWD'] = cwd
+
+        return env
 
     # _get_work_directory()
     #
     # Fetches the working directory for running commands
     # in the sandbox.
     #
+    # Args:
+    #    cwd (str): The working directory the command has been requested to run in, if any.
+    #
     # Returns:
     #    (str): The sandbox work directory
-    def _get_work_directory(self):
-        return self.__cwd
+    def _get_work_directory(self, *, cwd=None):
+        return cwd or self.__cwd or '/'
 
     # _get_scratch_directory()
     #

doc/bst2html.py

@@ -444,7 +444,7 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
 def run_bst(directory, force, source_cache, description, palette):
     """Run a bst command and capture stdout/stderr in html
 
-    This command normally takes a description yaml file, see the HACKING
+    This command normally takes a description yaml file, see the CONTRIBUTING
     file for information on it's format.
     """
     if not source_cache and os.environ.get('BST_SOURCE_CACHE'):

doc/source/CONTRIBUTING.rst

+../../CONTRIBUTING.rst
\ No newline at end of file

doc/source/HACKING.rst

-../../HACKING.rst
\ No newline at end of file

doc/source/index.rst

@@ -21,7 +21,7 @@ Later sections provide detailed information on BuildStream internals.
    main_install
    main_using
    main_core
-   HACKING
+   CONTRIBUTING
 
 
 Resources