.gitignore

@@ -34,3 +34,4 @@ doc/source/modules.rst
 doc/source/buildstream.rst
 doc/source/buildstream.*.rst
 doc/build/
+versioneer.pyc

.gitlab-ci.yml

-image: buildstream/testsuite-debian:9-master-114-4cab18e3
+image: buildstream/testsuite-debian:9-master-119-552f5fc6
 
 cache:
   key: "$CI_JOB_NAME-"
@@ -78,7 +78,7 @@ source_dist:
   # Go back to the toplevel and collect our reports
   - cd ../..
   - mkdir -p coverage-linux/
-  - cp dist/buildstream/.coverage.* coverage-linux/coverage."${CI_JOB_NAME}"
+  - cp dist/buildstream/.coverage coverage-linux/coverage."${CI_JOB_NAME}"
   except:
   - schedules
   artifacts:
@@ -86,25 +86,25 @@ source_dist:
     - coverage-linux/
 
 tests-debian-9:
-  image: buildstream/testsuite-debian:9-master-117-aa3a33b3
+  image: buildstream/testsuite-debian:9-master-119-552f5fc6
   <<: *linux-tests
 
 tests-fedora-27:
-  image: buildstream/testsuite-fedora:27-master-117-aa3a33b3
+  image: buildstream/testsuite-fedora:27-master-119-552f5fc6
   <<: *linux-tests
 
 tests-fedora-28:
-  image: buildstream/testsuite-fedora:28-master-117-aa3a33b3
+  image: buildstream/testsuite-fedora:28-master-119-552f5fc6
   <<: *linux-tests
 
 tests-ubuntu-18.04:
-  image: buildstream/testsuite-ubuntu:18.04-master-117-aa3a33b3
+  image: buildstream/testsuite-ubuntu:18.04-master-119-552f5fc6
   <<: *linux-tests
 
 tests-unix:
   # Use fedora here, to a) run a test on fedora and b) ensure that we
   # can get rid of ostree - this is not possible with debian-8
-  image: buildstream/testsuite-fedora:27-master-117-aa3a33b3
+  image: buildstream/testsuite-fedora:27-master-119-552f5fc6
   stage: test
   variables:
     BST_FORCE_BACKEND: "unix"
@@ -128,7 +128,7 @@ tests-unix:
     # Go back to the toplevel and collect our reports
     - cd ../..
     - mkdir -p coverage-unix/
-    - cp dist/buildstream/.coverage.* coverage-unix/coverage.unix
+    - cp dist/buildstream/.coverage coverage-unix/coverage.unix
   except:
   - schedules
   artifacts:

MANIFEST.in

@@ -8,19 +8,29 @@ include README.rst
 
 # Documentation package includes
 include doc/Makefile
+include doc/badges.py
+include doc/bst2html.py
 include doc/source/conf.py
-include doc/source/index.rst
+include doc/source/plugin.rsttemplate
+recursive-include doc/source *.rst
+recursive-include doc/source *.py
+recursive-include doc/source *.in
+recursive-include doc/source *.html
+recursive-include doc/source *.odg
+recursive-include doc/source *.svg
+recursive-include doc/examples *
 
 # Tests
-recursive-include tests *.py
-recursive-include tests *.yaml
-recursive-include tests *.bst
-recursive-include tests *.conf
-recursive-include tests *.sh
-recursive-include tests *.expected
+recursive-include tests *
+include conftest.py
+include .coveragerc
+include .pylintrc
 
 # Protocol Buffers
 recursive-include buildstream/_protos *.proto
 
 # Requirements files
 include dev-requirements.txt
+
+# Versioneer
+include versioneer.py

NEWS

@@ -31,6 +31,15 @@ buildstream 1.3.1
     new the `conf-root` variable to make the process easier. And there has been
     a bug fix to workspaces so they can be build in workspaces too.
 
+  o Creating a build shell through the interactive mode or `bst shell --build`
+    will now use the cached build tree. It is now easier to debug local build
+    failures.
+
+  o `bst shell --sysroot` now takes any directory that contains a sysroot,
+    instead of just a specially-formatted build-root with a `root` and `scratch`
+    subdirectory.
+
+
 =================
 buildstream 1.1.5
 =================

buildstream/_artifactcache/artifactcache.py

@@ -156,7 +156,7 @@ class ArtifactCache():
     def setup_remotes(self, *, use_config=False, remote_url=None):
 
         # Ensure we do not double-initialise since this can be expensive
-        assert(not self._remotes_setup)
+        assert not self._remotes_setup
         self._remotes_setup = True
 
         # Initialize remote artifact caches. We allow the commandline to override
@@ -252,7 +252,7 @@ class ArtifactCache():
     #    (int): The size of the cache after having cleaned up
     #
     def clean(self):
-        artifacts = self.list_artifacts()
+        artifacts = self.list_artifacts()  # pylint: disable=assignment-from-no-return
 
         # Build a set of the cache keys which are required
         # based on the required elements at cleanup time
@@ -294,7 +294,7 @@ class ArtifactCache():
             if key not in required_artifacts:
 
                 # Remove the actual artifact, if it's not required.
-                size = self.remove(to_remove)
+                size = self.remove(to_remove)  # pylint: disable=assignment-from-no-return
 
                 # Remove the size from the removed size
                 self.set_cache_size(self._cache_size - size)
@@ -311,7 +311,7 @@ class ArtifactCache():
     #    (int): The size of the artifact cache.
     #
     def compute_cache_size(self):
-        self._cache_size = self.calculate_cache_size()
+        self._cache_size = self.calculate_cache_size()  # pylint: disable=assignment-from-no-return
 
         return self._cache_size
 

buildstream/_artifactcache/cascache.py

@@ -33,11 +33,11 @@ import grpc
 
 from .. import _yaml
 
+from .._protos.google.rpc import code_pb2
 from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
 from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
 from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
 
-from .._message import MessageType, Message
 from .. import _signals, utils
 from .._exceptions import ArtifactError
 
@@ -81,8 +81,9 @@ class CASCache(ArtifactCache):
     ################################################
 
     def preflight(self):
-        if (not os.path.isdir(os.path.join(self.casdir, 'refs', 'heads')) or
-            not os.path.isdir(os.path.join(self.casdir, 'objects'))):
+        headdir = os.path.join(self.casdir, 'refs', 'heads')
+        objdir = os.path.join(self.casdir, 'objects')
+        if not (os.path.isdir(headdir) and os.path.isdir(objdir)):
             raise ArtifactError("CAS repository check failed for '{}'"
                                 .format(self.casdir))
 
@@ -918,7 +919,7 @@ class CASCache(ArtifactCache):
             # Skip download, already in local cache.
             pass
         elif (digest.size_bytes >= remote.max_batch_total_size_bytes or
-                not remote.batch_read_supported):
+              not remote.batch_read_supported):
             # Too large for batch request, download in independent request.
             self._ensure_blob(remote, digest)
             in_local_cache = True
@@ -958,7 +959,7 @@ class CASCache(ArtifactCache):
         batch = _CASBatchRead(remote)
 
         while len(fetch_queue) + len(fetch_next_queue) > 0:
-            if len(fetch_queue) == 0:
+            if not fetch_queue:
                 batch = self._fetch_directory_batch(remote, batch, fetch_queue, fetch_next_queue)
 
             dir_digest = fetch_queue.pop(0)
@@ -1087,6 +1088,10 @@ class _CASRemote():
         self.bytestream = None
         self.cas = None
         self.ref_storage = None
+        self.batch_update_supported = None
+        self.batch_read_supported = None
+        self.capabilities = None
+        self.max_batch_total_size_bytes = None
 
     def init(self):
         if not self._initialized:
@@ -1191,13 +1196,13 @@ class _CASBatchRead():
         assert not self._sent
         self._sent = True
 
-        if len(self._request.digests) == 0:
+        if not self._request.digests:
             return
 
         batch_response = self._remote.cas.BatchReadBlobs(self._request)
 
         for response in batch_response.responses:
-            if response.status.code != grpc.StatusCode.OK.value[0]:
+            if response.status.code != code_pb2.OK:
                 raise ArtifactError("Failed to download blob {}: {}".format(
                     response.digest.hash, response.status.code))
             if response.digest.size_bytes != len(response.data):
@@ -1236,13 +1241,13 @@ class _CASBatchUpdate():
         assert not self._sent
         self._sent = True
 
-        if len(self._request.requests) == 0:
+        if not self._request.requests:
             return
 
         batch_response = self._remote.cas.BatchUpdateBlobs(self._request)
 
         for response in batch_response.responses:
-            if response.status.code != grpc.StatusCode.OK.value[0]:
+            if response.status.code != code_pb2.OK:
                 raise ArtifactError("Failed to upload blob {}: {}".format(
                     response.digest.hash, response.status.code))
 

buildstream/_context.py

@@ -364,7 +364,6 @@ class Context():
         assert self._message_handler
 
         self._message_handler(message, context=self)
-        return
 
     # silence()
     #

buildstream/_exceptions.py

@@ -111,10 +111,8 @@ class BstError(Exception):
         #
         self.detail = detail
 
-        # The build sandbox in which the error occurred, if the
-        # error occurred at element assembly time.
-        #
-        self.sandbox = None
+        # A sandbox can be created to debug this error
+        self.sandbox = False
 
         # When this exception occurred during the handling of a job, indicate
         # whether or not there is any point retrying the job.

buildstream/_frontend/app.py

@@ -20,7 +20,6 @@
 from contextlib import contextmanager
 import os
 import sys
-import resource
 import traceback
 import datetime
 from textwrap import TextWrapper
@@ -598,7 +597,7 @@ class App():
                     click.echo("\nDropping into an interactive shell in the failed build sandbox\n", err=True)
                     try:
                         prompt = self.shell_prompt(element)
-                        self.stream.shell(element, Scope.BUILD, prompt, directory=failure.sandbox, isolate=True)
+                        self.stream.shell(element, Scope.BUILD, prompt, isolate=True)
                     except BstError as e:
                         click.echo("Error while attempting to create interactive shell: {}".format(e), err=True)
                 elif choice == 'log':

buildstream/_frontend/status.py

@@ -18,8 +18,8 @@
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 import os
 import sys
-import click
 import curses
+import click
 
 # Import a widget internal for formatting time codes
 from .widget import TimeCode

buildstream/_frontend/widget.py

@@ -668,17 +668,6 @@ class LogLine(Widget):
 
             extra_nl = True
 
-        if message.sandbox is not None:
-            sandbox = self._indent + 'Sandbox directory: ' + message.sandbox
-
-            text += '\n'
-            if message.message_type == MessageType.FAIL:
-                text += self._err_profile.fmt(sandbox, bold=True)
-            else:
-                text += self._detail_profile.fmt(sandbox)
-            text += '\n'
-            extra_nl = True
-
         if message.scheduler and message.message_type == MessageType.FAIL:
             text += '\n'
 

buildstream/_fuse/hardlinks.py

@@ -42,9 +42,11 @@ from .mount import Mount
 #
 class SafeHardlinks(Mount):
 
-    def __init__(self, directory, tempdir, fuse_mount_options={}):
+    def __init__(self, directory, tempdir, fuse_mount_options=None):
         self.directory = directory
         self.tempdir = tempdir
+        if fuse_mount_options is None:
+            fuse_mount_options = {}
         super().__init__(fuse_mount_options=fuse_mount_options)
 
     def create_operations(self):

buildstream/_fuse/mount.py

@@ -87,8 +87,8 @@ class Mount():
     #               User Facing API                #
     ################################################
 
-    def __init__(self, fuse_mount_options={}):
-        self._fuse_mount_options = fuse_mount_options
+    def __init__(self, fuse_mount_options=None):
+        self._fuse_mount_options = {} if fuse_mount_options is None else fuse_mount_options
 
     # mount():
     #
@@ -182,7 +182,7 @@ class Mount():
 
         # Ask the subclass to give us an Operations object
         #
-        self.__operations = self.create_operations()
+        self.__operations = self.create_operations()  # pylint: disable=assignment-from-no-return
 
         # Run fuse in foreground in this child process, internally libfuse
         # will handle SIGTERM and gracefully exit its own little main loop.

buildstream/_loader/loadelement.py

@@ -146,8 +146,8 @@ def _extract_depends_from_node(node, *, key=None):
     depends = _yaml.node_get(node, list, key, default_value=[])
     output_deps = []
 
-    for dep in depends:
-        dep_provenance = _yaml.node_get_provenance(node, key=key, indices=[depends.index(dep)])
+    for index, dep in enumerate(depends):
+        dep_provenance = _yaml.node_get_provenance(node, key=key, indices=[index])
 
         if isinstance(dep, str):
             dependency = Dependency(dep, provenance=dep_provenance, dep_type=default_dep_type)
@@ -177,10 +177,8 @@ def _extract_depends_from_node(node, *, key=None):
                                     provenance=dep_provenance)
 
         else:
-            index = depends.index(dep)
-            p = _yaml.node_get_provenance(node, key=key, indices=[index])
             raise LoadError(LoadErrorReason.INVALID_DATA,
-                            "{}: Dependency is not specified as a string or a dictionary".format(p))
+                            "{}: Dependency is not specified as a string or a dictionary".format(dep_provenance))
 
         output_deps.append(dependency)
 

buildstream/_message.py

@@ -70,7 +70,7 @@ class Message():
         self.elapsed = elapsed            # The elapsed time, in timed messages
         self.depth = depth                # The depth of a timed message
         self.logfile = logfile            # The log file path where commands took place
-        self.sandbox = sandbox            # The sandbox directory where an error occurred (if any)
+        self.sandbox = sandbox            # The error that caused this message used a sandbox
         self.pid = os.getpid()            # The process pid
         self.unique_id = unique_id        # The plugin object ID issueing the message
         self.task_id = task_id            # The plugin object ID of the task

buildstream/_options/optionbool.py

@@ -43,9 +43,9 @@ class OptionBool(Option):
             self.value = _yaml.node_get(node, bool, self.name)
 
     def set_value(self, value):
-        if value == 'True' or value == 'true':
+        if value in ('True', 'true'):
             self.value = True
-        elif value == 'False' or value == 'false':
+        elif value in ('False', 'false'):
             self.value = False
         else:
             raise LoadError(LoadErrorReason.INVALID_DATA,

buildstream/_platform/darwin.py

@@ -16,9 +16,7 @@
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import resource
 
-from .._exceptions import PlatformError
 from ..sandbox import SandboxDummy
 
 from . import Platform
@@ -29,10 +27,6 @@ class Darwin(Platform):
     # This value comes from OPEN_MAX in syslimits.h
     OPEN_MAX = 10240
 
-    def __init__(self):
-
-        super().__init__()
-
     def create_sandbox(self, *args, **kwargs):
         kwargs['dummy_reason'] = \
             "OSXFUSE is not supported and there are no supported sandbox" + \

buildstream/_platform/linux.py

@@ -22,7 +22,6 @@ import subprocess
 
 from .. import _site
 from .. import utils
-from .._message import Message, MessageType
 from ..sandbox import SandboxDummy
 
 from . import Platform
@@ -112,8 +111,4 @@ class Linux(Platform):
         except subprocess.CalledProcessError:
             output = ''
 
-        if output == 'root':
-            return True
-
-        else:
-            return False
+        return output == 'root'

buildstream/_scheduler/jobs/job.py

@@ -414,7 +414,7 @@ class Job():
 
             try:
                 # Try the task action
-                result = self.child_process()
+                result = self.child_process()  # pylint: disable=assignment-from-no-return
             except SkipJob as e:
                 elapsed = datetime.datetime.now() - starttime
                 self.message(MessageType.SKIPPED, str(e),

buildstream/_scheduler/queues/pullqueue.py

@@ -57,7 +57,7 @@ class PullQueue(Queue):
     def done(self, _, element, result, success):
 
         if not success:
-            return False
+            return
 
         element._pull_done()