.gitignore

@@ -34,3 +34,4 @@ doc/source/modules.rst
 doc/source/buildstream.rst
 doc/source/buildstream.*.rst
 doc/build/
+versioneer.pyc

.gitlab-ci.yml

-image: buildstream/testsuite-debian:9-master-114-4cab18e3
+image: buildstream/testsuite-debian:9-master-119-552f5fc6
 
 cache:
   key: "$CI_JOB_NAME-"
@@ -78,7 +78,7 @@ source_dist:
   # Go back to the toplevel and collect our reports
   - cd ../..
   - mkdir -p coverage-linux/
-  - cp dist/buildstream/.coverage.* coverage-linux/coverage."${CI_JOB_NAME}"
+  - cp dist/buildstream/.coverage coverage-linux/coverage."${CI_JOB_NAME}"
   except:
   - schedules
   artifacts:
@@ -86,25 +86,25 @@ source_dist:
     - coverage-linux/
 
 tests-debian-9:
-  image: buildstream/testsuite-debian:9-master-117-aa3a33b3
+  image: buildstream/testsuite-debian:9-master-119-552f5fc6
   <<: *linux-tests
 
 tests-fedora-27:
-  image: buildstream/testsuite-fedora:27-master-117-aa3a33b3
+  image: buildstream/testsuite-fedora:27-master-119-552f5fc6
   <<: *linux-tests
 
 tests-fedora-28:
-  image: buildstream/testsuite-fedora:28-master-117-aa3a33b3
+  image: buildstream/testsuite-fedora:28-master-119-552f5fc6
   <<: *linux-tests
 
 tests-ubuntu-18.04:
-  image: buildstream/testsuite-ubuntu:18.04-master-117-aa3a33b3
+  image: buildstream/testsuite-ubuntu:18.04-master-119-552f5fc6
   <<: *linux-tests
 
 tests-unix:
   # Use fedora here, to a) run a test on fedora and b) ensure that we
   # can get rid of ostree - this is not possible with debian-8
-  image: buildstream/testsuite-fedora:27-master-117-aa3a33b3
+  image: buildstream/testsuite-fedora:27-master-119-552f5fc6
   stage: test
   variables:
     BST_FORCE_BACKEND: "unix"
@@ -128,7 +128,7 @@ tests-unix:
     # Go back to the toplevel and collect our reports
     - cd ../..
     - mkdir -p coverage-unix/
-    - cp dist/buildstream/.coverage.* coverage-unix/coverage.unix
+    - cp dist/buildstream/.coverage coverage-unix/coverage.unix
   except:
   - schedules
   artifacts:

MANIFEST.in

@@ -8,19 +8,27 @@ include README.rst
 
 # Documentation package includes
 include doc/Makefile
+include doc/badges.py
+include doc/bst2html.py
 include doc/source/conf.py
-include doc/source/index.rst
+include doc/source/plugin.rsttemplate
+recursive-include doc/source *.rst
+recursive-include doc/source *.py
+recursive-include doc/source *.in
+recursive-include doc/source *.html
+recursive-include doc/examples *
 
 # Tests
-recursive-include tests *.py
-recursive-include tests *.yaml
-recursive-include tests *.bst
-recursive-include tests *.conf
-recursive-include tests *.sh
-recursive-include tests *.expected
+recursive-include tests *
+include conftest.py
+include .coveragerc
+include .pylintrc
 
 # Protocol Buffers
 recursive-include buildstream/_protos *.proto
 
 # Requirements files
 include dev-requirements.txt
+
+# Versioneer
+include versioneer.py

buildstream/_artifactcache/artifactcache.py

@@ -156,7 +156,7 @@ class ArtifactCache():
     def setup_remotes(self, *, use_config=False, remote_url=None):
 
         # Ensure we do not double-initialise since this can be expensive
-        assert(not self._remotes_setup)
+        assert not self._remotes_setup
         self._remotes_setup = True
 
         # Initialize remote artifact caches. We allow the commandline to override
@@ -252,7 +252,7 @@ class ArtifactCache():
     #    (int): The size of the cache after having cleaned up
     #
     def clean(self):
-        artifacts = self.list_artifacts()
+        artifacts = self.list_artifacts()  # pylint: disable=assignment-from-no-return
 
         # Build a set of the cache keys which are required
         # based on the required elements at cleanup time
@@ -294,7 +294,7 @@ class ArtifactCache():
             if key not in required_artifacts:
 
                 # Remove the actual artifact, if it's not required.
-                size = self.remove(to_remove)
+                size = self.remove(to_remove)  # pylint: disable=assignment-from-no-return
 
                 # Remove the size from the removed size
                 self.set_cache_size(self._cache_size - size)
@@ -311,7 +311,7 @@ class ArtifactCache():
     #    (int): The size of the artifact cache.
     #
     def compute_cache_size(self):
-        self._cache_size = self.calculate_cache_size()
+        self._cache_size = self.calculate_cache_size()  # pylint: disable=assignment-from-no-return
 
         return self._cache_size
 

buildstream/_artifactcache/cascache.py

@@ -33,11 +33,11 @@ import grpc
 
 from .. import _yaml
 
+from .._protos.google.rpc import code_pb2
 from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
 from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
 from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
 
-from .._message import MessageType, Message
 from .. import _signals, utils
 from .._exceptions import ArtifactError
 
@@ -81,8 +81,9 @@ class CASCache(ArtifactCache):
     ################################################
 
     def preflight(self):
-        if (not os.path.isdir(os.path.join(self.casdir, 'refs', 'heads')) or
-            not os.path.isdir(os.path.join(self.casdir, 'objects'))):
+        headdir = os.path.join(self.casdir, 'refs', 'heads')
+        objdir = os.path.join(self.casdir, 'objects')
+        if not (os.path.isdir(headdir) and os.path.isdir(objdir)):
             raise ArtifactError("CAS repository check failed for '{}'"
                                 .format(self.casdir))
 
@@ -918,7 +919,7 @@ class CASCache(ArtifactCache):
             # Skip download, already in local cache.
             pass
         elif (digest.size_bytes >= remote.max_batch_total_size_bytes or
-                not remote.batch_read_supported):
+              not remote.batch_read_supported):
             # Too large for batch request, download in independent request.
             self._ensure_blob(remote, digest)
             in_local_cache = True
@@ -958,7 +959,7 @@ class CASCache(ArtifactCache):
         batch = _CASBatchRead(remote)
 
         while len(fetch_queue) + len(fetch_next_queue) > 0:
-            if len(fetch_queue) == 0:
+            if not fetch_queue:
                 batch = self._fetch_directory_batch(remote, batch, fetch_queue, fetch_next_queue)
 
             dir_digest = fetch_queue.pop(0)
@@ -1087,6 +1088,10 @@ class _CASRemote():
         self.bytestream = None
         self.cas = None
         self.ref_storage = None
+        self.batch_update_supported = None
+        self.batch_read_supported = None
+        self.capabilities = None
+        self.max_batch_total_size_bytes = None
 
     def init(self):
         if not self._initialized:
@@ -1191,13 +1196,13 @@ class _CASBatchRead():
         assert not self._sent
         self._sent = True
 
-        if len(self._request.digests) == 0:
+        if not self._request.digests:
             return
 
         batch_response = self._remote.cas.BatchReadBlobs(self._request)
 
         for response in batch_response.responses:
-            if response.status.code != grpc.StatusCode.OK.value[0]:
+            if response.status.code != code_pb2.OK:
                 raise ArtifactError("Failed to download blob {}: {}".format(
                     response.digest.hash, response.status.code))
             if response.digest.size_bytes != len(response.data):
@@ -1236,13 +1241,13 @@ class _CASBatchUpdate():
         assert not self._sent
         self._sent = True
 
-        if len(self._request.requests) == 0:
+        if not self._request.requests:
             return
 
         batch_response = self._remote.cas.BatchUpdateBlobs(self._request)
 
         for response in batch_response.responses:
-            if response.status.code != grpc.StatusCode.OK.value[0]:
+            if response.status.code != code_pb2.OK:
                 raise ArtifactError("Failed to upload blob {}: {}".format(
                     response.digest.hash, response.status.code))
 

buildstream/_context.py

@@ -364,7 +364,6 @@ class Context():
         assert self._message_handler
 
         self._message_handler(message, context=self)
-        return
 
     # silence()
     #

buildstream/_frontend/app.py

@@ -20,7 +20,6 @@
 from contextlib import contextmanager
 import os
 import sys
-import resource
 import traceback
 import datetime
 from textwrap import TextWrapper

buildstream/_frontend/cli.py

@@ -662,6 +662,33 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
                             tar=tar)
 
 
+##################################################################
+#                  Source Checkout Command                      #
+##################################################################
+@cli.command(name='source-checkout', short_help='Checkout sources for an element')
+@click.option('--except', 'except_', multiple=True,
+              type=click.Path(readable=False),
+              help="Except certain dependencies")
+@click.option('--deps', '-d', default='none',
+              type=click.Choice(['build', 'none', 'run', 'all']),
+              help='The dependencies whose sources to checkout (default: none)')
+@click.option('--fetch', default=False, is_flag=True,
+              help='Fetch elements if they are not fetched')
+@click.argument('element',
+                type=click.Path(readable=False))
+@click.argument('location', type=click.Path())
+@click.pass_obj
+def source_checkout(app, element, location, deps, fetch, except_):
+    """Checkout sources of an element to the specified location
+    """
+    with app.initialized():
+        app.stream.source_checkout(element,
+                                   location=location,
+                                   deps=deps,
+                                   fetch=fetch,
+                                   except_targets=except_)
+
+
 ##################################################################
 #                      Workspace Command                         #
 ##################################################################

buildstream/_frontend/status.py

@@ -18,8 +18,8 @@
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 import os
 import sys
-import click
 import curses
+import click
 
 # Import a widget internal for formatting time codes
 from .widget import TimeCode

buildstream/_fuse/hardlinks.py

@@ -42,9 +42,11 @@ from .mount import Mount
 #
 class SafeHardlinks(Mount):
 
-    def __init__(self, directory, tempdir, fuse_mount_options={}):
+    def __init__(self, directory, tempdir, fuse_mount_options=None):
         self.directory = directory
         self.tempdir = tempdir
+        if fuse_mount_options is None:
+            fuse_mount_options = {}
         super().__init__(fuse_mount_options=fuse_mount_options)
 
     def create_operations(self):

buildstream/_fuse/mount.py

@@ -87,8 +87,8 @@ class Mount():
     #               User Facing API                #
     ################################################
 
-    def __init__(self, fuse_mount_options={}):
-        self._fuse_mount_options = fuse_mount_options
+    def __init__(self, fuse_mount_options=None):
+        self._fuse_mount_options = {} if fuse_mount_options is None else fuse_mount_options
 
     # mount():
     #
@@ -182,7 +182,7 @@ class Mount():
 
         # Ask the subclass to give us an Operations object
         #
-        self.__operations = self.create_operations()
+        self.__operations = self.create_operations()  # pylint: disable=assignment-from-no-return
 
         # Run fuse in foreground in this child process, internally libfuse
         # will handle SIGTERM and gracefully exit its own little main loop.

buildstream/_loader/loadelement.py

@@ -146,8 +146,8 @@ def _extract_depends_from_node(node, *, key=None):
     depends = _yaml.node_get(node, list, key, default_value=[])
     output_deps = []
 
-    for dep in depends:
-        dep_provenance = _yaml.node_get_provenance(node, key=key, indices=[depends.index(dep)])
+    for index, dep in enumerate(depends):
+        dep_provenance = _yaml.node_get_provenance(node, key=key, indices=[index])
 
         if isinstance(dep, str):
             dependency = Dependency(dep, provenance=dep_provenance, dep_type=default_dep_type)
@@ -177,10 +177,8 @@ def _extract_depends_from_node(node, *, key=None):
                                     provenance=dep_provenance)
 
         else:
-            index = depends.index(dep)
-            p = _yaml.node_get_provenance(node, key=key, indices=[index])
             raise LoadError(LoadErrorReason.INVALID_DATA,
-                            "{}: Dependency is not specified as a string or a dictionary".format(p))
+                            "{}: Dependency is not specified as a string or a dictionary".format(dep_provenance))
 
         output_deps.append(dependency)
 

buildstream/_options/optionbool.py

@@ -43,9 +43,9 @@ class OptionBool(Option):
             self.value = _yaml.node_get(node, bool, self.name)
 
     def set_value(self, value):
-        if value == 'True' or value == 'true':
+        if value in ('True', 'true'):
             self.value = True
-        elif value == 'False' or value == 'false':
+        elif value in ('False', 'false'):
             self.value = False
         else:
             raise LoadError(LoadErrorReason.INVALID_DATA,

buildstream/_pipeline.py

@@ -370,7 +370,7 @@ class Pipeline():
                 detail += "  Element: {} is inconsistent\n".format(element._get_full_name())
                 for source in element.sources():
                     if source._get_consistency() == Consistency.INCONSISTENT:
-                        detail += "    Source {} is missing ref\n".format(source)
+                        detail += "    {} is missing ref\n".format(source)
                 detail += '\n'
             detail += "Try tracking these elements first with `bst track`\n"
 
@@ -383,6 +383,33 @@ class Pipeline():
                 detail += "  " + element._get_full_name() + "\n"
             raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline-workspaced")
 
+    # assert_sources_cached()
+    #
+    # Asserts that sources for the given list of elements are cached.
+    #
+    # Args:
+    #    elements (list): The list of elements
+    #
+    def assert_sources_cached(self, elements):
+        uncached = []
+        with self._context.timed_activity("Checking sources"):
+            for element in elements:
+                if element._get_consistency() != Consistency.CACHED:
+                    uncached.append(element)
+
+        if uncached:
+            detail = "Sources are not cached for the following elements:\n\n"
+            for element in uncached:
+                detail += "  Following sources for element: {} are not cached:\n".format(element._get_full_name())
+                for source in element.sources():
+                    if source._get_consistency() != Consistency.CACHED:
+                        detail += "    {}\n".format(source)
+                detail += '\n'
+            detail += "Try fetching these elements first with `bst fetch`,\n" + \
+                      "or run this command with `--fetch` option\n"
+
+            raise PipelineError("Uncached sources", detail=detail, reason="uncached-sources")
+
     #############################################################
     #                     Private Methods                       #
     #############################################################

buildstream/_platform/darwin.py

@@ -16,9 +16,7 @@
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import resource
 
-from .._exceptions import PlatformError
 from ..sandbox import SandboxDummy
 
 from . import Platform
@@ -29,10 +27,6 @@ class Darwin(Platform):
     # This value comes from OPEN_MAX in syslimits.h
     OPEN_MAX = 10240
 
-    def __init__(self):
-
-        super().__init__()
-
     def create_sandbox(self, *args, **kwargs):
         kwargs['dummy_reason'] = \
             "OSXFUSE is not supported and there are no supported sandbox" + \

buildstream/_platform/linux.py

@@ -22,7 +22,6 @@ import subprocess
 
 from .. import _site
 from .. import utils
-from .._message import Message, MessageType
 from ..sandbox import SandboxDummy
 
 from . import Platform
@@ -112,8 +111,4 @@ class Linux(Platform):
         except subprocess.CalledProcessError:
             output = ''
 
-        if output == 'root':
-            return True
-
-        else:
-            return False
+        return output == 'root'

buildstream/_scheduler/jobs/job.py

@@ -414,7 +414,7 @@ class Job():
 
             try:
                 # Try the task action
-                result = self.child_process()
+                result = self.child_process()  # pylint: disable=assignment-from-no-return
             except SkipJob as e:
                 elapsed = datetime.datetime.now() - starttime
                 self.message(MessageType.SKIPPED, str(e),

buildstream/_scheduler/queues/pullqueue.py

@@ -57,7 +57,7 @@ class PullQueue(Queue):
     def done(self, _, element, result, success):
 
         if not success:
-            return False
+            return
 
         element._pull_done()
 

buildstream/_scheduler/queues/trackqueue.py

@@ -20,7 +20,6 @@
 
 # BuildStream toplevel imports
 from ...plugin import _plugin_lookup
-from ... import SourceError
 
 # Local imports
 from . import Queue, QueueStatus

buildstream/_stream.py

@@ -379,27 +379,7 @@ class Stream():
         elements, _ = self._load((target,), (), fetch_subprojects=True)
         target = elements[0]
 
-        if not tar:
-            try:
-                os.makedirs(location, exist_ok=True)
-            except OSError as e:
-                raise StreamError("Failed to create checkout directory: '{}'"
-                                  .format(e)) from e
-
-        if not tar:
-            if not os.access(location, os.W_OK):
-                raise StreamError("Checkout directory '{}' not writable"
-                                  .format(location))
-            if not force and os.listdir(location):
-                raise StreamError("Checkout directory '{}' not empty"
-                                  .format(location))
-        elif os.path.exists(location) and location != '-':
-            if not os.access(location, os.W_OK):
-                raise StreamError("Output file '{}' not writable"
-                                  .format(location))
-            if not force and os.path.exists(location):
-                raise StreamError("Output file '{}' already exists"
-                                  .format(location))
+        self._check_location_writable(location, force=force, tar=tar)
 
         # Stage deps into a temporary sandbox first
         try:
@@ -436,6 +416,42 @@ class Stream():
             raise StreamError("Error while staging dependencies into a sandbox"
                               ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
 
+    # source_checkout()
+    #
+    # Checkout sources of the target element to the specified location
+    #
+    # Args:
+    #    target (str): The target element whose sources to checkout
+    #    location (str): Location to checkout the sources to
+    #    deps (str): The dependencies to checkout
+    #    fetch (bool): Whether to fetch missing sources
+    #    except_targets (list): List of targets to except from staging
+    #
+    def source_checkout(self, target, *,
+                        location=None,
+                        deps='none',
+                        fetch=False,
+                        except_targets=()):
+
+        self._check_location_writable(location)
+
+        elements, _ = self._load((target,), (),
+                                 selection=deps,
+                                 except_targets=except_targets,
+                                 fetch_subprojects=True)
+
+        # Assert all sources are cached
+        if fetch:
+            self._fetch(elements)
+        self._pipeline.assert_sources_cached(elements)
+
+        # Stage all sources determined by scope
+        try:
+            self._write_element_sources(location, elements)
+        except BstError as e:
+            raise StreamError("Error while writing sources"
+                              ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
+
     # workspace_open
     #
     # Open a project workspace
@@ -719,7 +735,7 @@ class Stream():
                 if self._write_element_script(source_directory, element)
             ]
 
-            self._write_element_sources(tempdir, elements)
+            self._write_element_sources(os.path.join(tempdir, "source"), elements)
             self._write_build_script(tempdir, elements)
             self._collect_sources(tempdir, tar_location,
                                   target.normal_name, compression)
@@ -1061,6 +1077,39 @@ class Stream():
         self._enqueue_plan(fetch_plan)
         self._run()
 
+    # _check_location_writable()
+    #
+    # Check if given location is writable.
+    #
+    # Args:
+    #    location (str): Destination path
+    #    force (bool): Allow files to be overwritten
+    #    tar (bool): Whether destination is a tarball
+    #
+    # Raises:
+    #    (StreamError): If the destination is not writable
+    #
+    def _check_location_writable(self, location, force=False, tar=False):
+        if not tar:
+            try:
+                os.makedirs(location, exist_ok=True)
+            except OSError as e:
+                raise StreamError("Failed to create destination directory: '{}'"
+                                  .format(e)) from e
+            if not os.access(location, os.W_OK):
+                raise StreamError("Destination directory '{}' not writable"
+                                  .format(location))
+            if not force and os.listdir(location):
+                raise StreamError("Destination directory '{}' not empty"
+                                  .format(location))
+        elif os.path.exists(location) and location != '-':
+            if not os.access(location, os.W_OK):
+                raise StreamError("Output file '{}' not writable"
+                                  .format(location))
+            if not force and os.path.exists(location):
+                raise StreamError("Output file '{}' already exists"
+                                  .format(location))
+
     # Helper function for checkout()
     #
     def _checkout_hardlinks(self, sandbox_vroot, directory):
@@ -1082,11 +1131,10 @@ class Stream():
     # Write all source elements to the given directory
     def _write_element_sources(self, directory, elements):
         for element in elements:
-            source_dir = os.path.join(directory, "source")
-            element_source_dir = os.path.join(source_dir, element.normal_name)
-            os.makedirs(element_source_dir)
-
-            element._stage_sources_at(element_source_dir)
+            element_source_dir = self._get_element_dirname(directory, element)
+            if list(element.sources()):
+                os.makedirs(element_source_dir)
+                element._stage_sources_at(element_source_dir)
 
     # Write a master build script to the sandbox
     def _write_build_script(self, directory, elements):
@@ -1115,3 +1163,25 @@ class Stream():
 
             with tarfile.open(tar_name, permissions) as tar:
                 tar.add(directory, arcname=element_name)
+
+    # _get_element_dirname()
+    #
+    # Get path to directory for an element based on its normal name.
+    #
+    # For cross-junction elements, the path will be prefixed with the name
+    # of the junction element.
+    #
+    # Args:
+    #    directory (str): path to base directory
+    #    element (Element): the element
+    #
+    # Returns:
+    #    (str): Path to directory for this element
+    #
+    def _get_element_dirname(self, directory, element):
+        parts = [element.normal_name]
+        while element._get_project() != self._project:
+            element = element._get_project().junction
+            parts.append(element.normal_name)
+
+        return os.path.join(directory, *reversed(parts))