.gitignore

@@ -34,3 +34,4 @@ doc/source/modules.rst
 doc/source/buildstream.rst
 doc/source/buildstream.*.rst
 doc/build/
+versioneer.pyc

.gitlab-ci.yml

-image: buildstream/testsuite-debian:9-master-114-4cab18e3
+image: buildstream/testsuite-debian:9-master-119-552f5fc6
 
 cache:
   key: "$CI_JOB_NAME-"
@@ -78,7 +78,7 @@ source_dist:
   # Go back to the toplevel and collect our reports
   - cd ../..
   - mkdir -p coverage-linux/
-  - cp dist/buildstream/.coverage.* coverage-linux/coverage."${CI_JOB_NAME}"
+  - cp dist/buildstream/.coverage coverage-linux/coverage."${CI_JOB_NAME}"
   except:
   - schedules
   artifacts:
@@ -86,25 +86,25 @@ source_dist:
     - coverage-linux/
 
 tests-debian-9:
-  image: buildstream/testsuite-debian:9-master-117-aa3a33b3
+  image: buildstream/testsuite-debian:9-master-119-552f5fc6
   <<: *linux-tests
 
 tests-fedora-27:
-  image: buildstream/testsuite-fedora:27-master-117-aa3a33b3
+  image: buildstream/testsuite-fedora:27-master-119-552f5fc6
   <<: *linux-tests
 
 tests-fedora-28:
-  image: buildstream/testsuite-fedora:28-master-117-aa3a33b3
+  image: buildstream/testsuite-fedora:28-master-119-552f5fc6
   <<: *linux-tests
 
 tests-ubuntu-18.04:
-  image: buildstream/testsuite-ubuntu:18.04-master-117-aa3a33b3
+  image: buildstream/testsuite-ubuntu:18.04-master-119-552f5fc6
   <<: *linux-tests
 
 tests-unix:
   # Use fedora here, to a) run a test on fedora and b) ensure that we
   # can get rid of ostree - this is not possible with debian-8
-  image: buildstream/testsuite-fedora:27-master-117-aa3a33b3
+  image: buildstream/testsuite-fedora:27-master-119-552f5fc6
   stage: test
   variables:
     BST_FORCE_BACKEND: "unix"
@@ -128,7 +128,7 @@ tests-unix:
     # Go back to the toplevel and collect our reports
     - cd ../..
     - mkdir -p coverage-unix/
-    - cp dist/buildstream/.coverage.* coverage-unix/coverage.unix
+    - cp dist/buildstream/.coverage coverage-unix/coverage.unix
   except:
   - schedules
   artifacts:

MANIFEST.in

@@ -8,19 +8,27 @@ include README.rst
 
 # Documentation package includes
 include doc/Makefile
+include doc/badges.py
+include doc/bst2html.py
 include doc/source/conf.py
-include doc/source/index.rst
+include doc/source/plugin.rsttemplate
+recursive-include doc/source *.rst
+recursive-include doc/source *.py
+recursive-include doc/source *.in
+recursive-include doc/source *.html
+recursive-include doc/examples *
 
 # Tests
-recursive-include tests *.py
-recursive-include tests *.yaml
-recursive-include tests *.bst
-recursive-include tests *.conf
-recursive-include tests *.sh
-recursive-include tests *.expected
+recursive-include tests *
+include conftest.py
+include .coveragerc
+include .pylintrc
 
 # Protocol Buffers
 recursive-include buildstream/_protos *.proto
 
 # Requirements files
 include dev-requirements.txt
+
+# Versioneer
+include versioneer.py

buildstream/_artifactcache/artifactcache.py

@@ -156,7 +156,7 @@ class ArtifactCache():
     def setup_remotes(self, *, use_config=False, remote_url=None):
 
         # Ensure we do not double-initialise since this can be expensive
-        assert(not self._remotes_setup)
+        assert not self._remotes_setup
         self._remotes_setup = True
 
         # Initialize remote artifact caches. We allow the commandline to override
@@ -252,7 +252,7 @@ class ArtifactCache():
     #    (int): The size of the cache after having cleaned up
     #
     def clean(self):
-        artifacts = self.list_artifacts()
+        artifacts = self.list_artifacts()  # pylint: disable=assignment-from-no-return
 
         # Build a set of the cache keys which are required
         # based on the required elements at cleanup time
@@ -294,7 +294,7 @@ class ArtifactCache():
             if key not in required_artifacts:
 
                 # Remove the actual artifact, if it's not required.
-                size = self.remove(to_remove)
+                size = self.remove(to_remove)  # pylint: disable=assignment-from-no-return
 
                 # Remove the size from the removed size
                 self.set_cache_size(self._cache_size - size)
@@ -311,7 +311,7 @@ class ArtifactCache():
     #    (int): The size of the artifact cache.
     #
     def compute_cache_size(self):
-        self._cache_size = self.calculate_cache_size()
+        self._cache_size = self.calculate_cache_size()  # pylint: disable=assignment-from-no-return
 
         return self._cache_size
 

buildstream/_artifactcache/cascache.py

@@ -33,11 +33,11 @@ import grpc
 
 from .. import _yaml
 
+from .._protos.google.rpc import code_pb2
 from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
 from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
 from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
 
-from .._message import MessageType, Message
 from .. import _signals, utils
 from .._exceptions import ArtifactError
 
@@ -81,8 +81,9 @@ class CASCache(ArtifactCache):
     ################################################
 
     def preflight(self):
-        if (not os.path.isdir(os.path.join(self.casdir, 'refs', 'heads')) or
-            not os.path.isdir(os.path.join(self.casdir, 'objects'))):
+        headdir = os.path.join(self.casdir, 'refs', 'heads')
+        objdir = os.path.join(self.casdir, 'objects')
+        if not (os.path.isdir(headdir) and os.path.isdir(objdir)):
             raise ArtifactError("CAS repository check failed for '{}'"
                                 .format(self.casdir))
 
@@ -918,7 +919,7 @@ class CASCache(ArtifactCache):
             # Skip download, already in local cache.
             pass
         elif (digest.size_bytes >= remote.max_batch_total_size_bytes or
-                not remote.batch_read_supported):
+              not remote.batch_read_supported):
             # Too large for batch request, download in independent request.
             self._ensure_blob(remote, digest)
             in_local_cache = True
@@ -958,7 +959,7 @@ class CASCache(ArtifactCache):
         batch = _CASBatchRead(remote)
 
         while len(fetch_queue) + len(fetch_next_queue) > 0:
-            if len(fetch_queue) == 0:
+            if not fetch_queue:
                 batch = self._fetch_directory_batch(remote, batch, fetch_queue, fetch_next_queue)
 
             dir_digest = fetch_queue.pop(0)
@@ -1087,6 +1088,10 @@ class _CASRemote():
         self.bytestream = None
         self.cas = None
         self.ref_storage = None
+        self.batch_update_supported = None
+        self.batch_read_supported = None
+        self.capabilities = None
+        self.max_batch_total_size_bytes = None
 
     def init(self):
         if not self._initialized:
@@ -1191,13 +1196,13 @@ class _CASBatchRead():
         assert not self._sent
         self._sent = True
 
-        if len(self._request.digests) == 0:
+        if not self._request.digests:
             return
 
         batch_response = self._remote.cas.BatchReadBlobs(self._request)
 
         for response in batch_response.responses:
-            if response.status.code != grpc.StatusCode.OK.value[0]:
+            if response.status.code != code_pb2.OK:
                 raise ArtifactError("Failed to download blob {}: {}".format(
                     response.digest.hash, response.status.code))
             if response.digest.size_bytes != len(response.data):
@@ -1236,13 +1241,13 @@ class _CASBatchUpdate():
         assert not self._sent
         self._sent = True
 
-        if len(self._request.requests) == 0:
+        if not self._request.requests:
             return
 
         batch_response = self._remote.cas.BatchUpdateBlobs(self._request)
 
         for response in batch_response.responses:
-            if response.status.code != grpc.StatusCode.OK.value[0]:
+            if response.status.code != code_pb2.OK:
                 raise ArtifactError("Failed to upload blob {}: {}".format(
                     response.digest.hash, response.status.code))
 

buildstream/_context.py

@@ -364,7 +364,6 @@ class Context():
         assert self._message_handler
 
         self._message_handler(message, context=self)
-        return
 
     # silence()
     #

buildstream/_frontend/app.py

@@ -20,7 +20,6 @@
 from contextlib import contextmanager
 import os
 import sys
-import resource
 import traceback
 import datetime
 from textwrap import TextWrapper

buildstream/_frontend/status.py

@@ -18,8 +18,8 @@
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 import os
 import sys
-import click
 import curses
+import click
 
 # Import a widget internal for formatting time codes
 from .widget import TimeCode

buildstream/_fuse/hardlinks.py

@@ -42,9 +42,11 @@ from .mount import Mount
 #
 class SafeHardlinks(Mount):
 
-    def __init__(self, directory, tempdir, fuse_mount_options={}):
+    def __init__(self, directory, tempdir, fuse_mount_options=None):
         self.directory = directory
         self.tempdir = tempdir
+        if fuse_mount_options is None:
+            fuse_mount_options = {}
         super().__init__(fuse_mount_options=fuse_mount_options)
 
     def create_operations(self):

buildstream/_fuse/mount.py

@@ -87,8 +87,8 @@ class Mount():
     #               User Facing API                #
     ################################################
 
-    def __init__(self, fuse_mount_options={}):
-        self._fuse_mount_options = fuse_mount_options
+    def __init__(self, fuse_mount_options=None):
+        self._fuse_mount_options = {} if fuse_mount_options is None else fuse_mount_options
 
     # mount():
     #
@@ -182,7 +182,7 @@ class Mount():
 
         # Ask the subclass to give us an Operations object
         #
-        self.__operations = self.create_operations()
+        self.__operations = self.create_operations()  # pylint: disable=assignment-from-no-return
 
         # Run fuse in foreground in this child process, internally libfuse
         # will handle SIGTERM and gracefully exit its own little main loop.

buildstream/_options/optionbool.py

@@ -43,9 +43,9 @@ class OptionBool(Option):
             self.value = _yaml.node_get(node, bool, self.name)
 
     def set_value(self, value):
-        if value == 'True' or value == 'true':
+        if value in ('True', 'true'):
             self.value = True
-        elif value == 'False' or value == 'false':
+        elif value in ('False', 'false'):
             self.value = False
         else:
             raise LoadError(LoadErrorReason.INVALID_DATA,

buildstream/_platform/darwin.py

@@ -16,9 +16,7 @@
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import resource
 
-from .._exceptions import PlatformError
 from ..sandbox import SandboxDummy
 
 from . import Platform
@@ -29,10 +27,6 @@ class Darwin(Platform):
     # This value comes from OPEN_MAX in syslimits.h
     OPEN_MAX = 10240
 
-    def __init__(self):
-
-        super().__init__()
-
     def create_sandbox(self, *args, **kwargs):
         kwargs['dummy_reason'] = \
             "OSXFUSE is not supported and there are no supported sandbox" + \

buildstream/_platform/linux.py

@@ -22,7 +22,6 @@ import subprocess
 
 from .. import _site
 from .. import utils
-from .._message import Message, MessageType
 from ..sandbox import SandboxDummy
 
 from . import Platform
@@ -112,8 +111,4 @@ class Linux(Platform):
         except subprocess.CalledProcessError:
             output = ''
 
-        if output == 'root':
-            return True
-
-        else:
-            return False
+        return output == 'root'

buildstream/_scheduler/jobs/job.py

@@ -414,7 +414,7 @@ class Job():
 
             try:
                 # Try the task action
-                result = self.child_process()
+                result = self.child_process()  # pylint: disable=assignment-from-no-return
             except SkipJob as e:
                 elapsed = datetime.datetime.now() - starttime
                 self.message(MessageType.SKIPPED, str(e),

buildstream/_scheduler/queues/pullqueue.py

@@ -57,7 +57,7 @@ class PullQueue(Queue):
     def done(self, _, element, result, success):
 
         if not success:
-            return False
+            return
 
         element._pull_done()
 

buildstream/_scheduler/queues/trackqueue.py

@@ -20,7 +20,6 @@
 
 # BuildStream toplevel imports
 from ...plugin import _plugin_lookup
-from ... import SourceError
 
 # Local imports
 from . import Queue, QueueStatus

buildstream/_variables.py

@@ -191,8 +191,8 @@ def find_recursive_variable(variable, matched_variables, all_vars):
     for key, value in zip(matched_variables, matched_values):
         if _wrap_variable(variable) in value:
             return key
-    else:
-        return None
+    # We failed to find a recursive variable
+    return None
 
 
 def _wrap_variable(var):

buildstream/_workspaces.py

@@ -367,7 +367,7 @@ class Workspaces():
                 for element, config in _yaml.node_items(workspaces)
             }
 
-        elif version >= 1 and version <= BST_WORKSPACE_FORMAT_VERSION:
+        elif 1 <= version <= BST_WORKSPACE_FORMAT_VERSION:
             workspaces = _yaml.node_get(workspaces, dict, "workspaces", default_value={})
             res = {element: self._load_workspace(node)
                    for element, node in _yaml.node_items(workspaces)}

buildstream/_yaml.py

@@ -395,9 +395,9 @@ def node_get(node, expected_type, key, indices=None, default_value=_get_sentinel
         try:
             if (expected_type == bool and isinstance(value, str)):
                 # Dont coerce booleans to string, this makes "False" strings evaluate to True
-                if value == 'true' or value == 'True':
+                if value in ('True', 'true'):
                     value = True
-                elif value == 'false' or value == 'False':
+                elif value in ('False', 'false'):
                     value = False
                 else:
                     raise ValueError()
@@ -470,10 +470,11 @@ def node_get_project_path(node, key, project_dir, *,
                         .format(provenance, path_str))
 
     try:
+        full_path = (project_dir_path / path)
         if sys.version_info[0] == 3 and sys.version_info[1] < 6:
-            full_resolved_path = (project_dir_path / path).resolve()
+            full_resolved_path = full_path.resolve()
         else:
-            full_resolved_path = (project_dir_path / path).resolve(strict=True)
+            full_resolved_path = full_path.resolve(strict=True)  # pylint: disable=unexpected-keyword-arg
     except FileNotFoundError:
         raise LoadError(LoadErrorReason.MISSING_FILE,
                         "{}: Specified path '{}' does not exist"

buildstream/_yamlcache.py

@@ -27,9 +27,8 @@ import sys
 from contextlib import contextmanager
 from collections import namedtuple
 
-from ._cachekey import generate_key
 from ._context import Context
-from . import utils, _yaml
+from . import _yaml
 
 
 YAML_CACHE_FILENAME = "yaml_cache.pickle"
@@ -207,7 +206,7 @@ class YamlCache():
             filepath = os.path.relpath(full_path, project.directory)
         else:
             filepath = full_path
-        return full_path
+        return filepath
 
     # _calculate_key():
     #
@@ -329,7 +328,7 @@ class BstUnpickler(pickle.Unpickler):
                 if not project:
                     projects = [p.name for p in self._context.get_projects()]
                     raise pickle.UnpicklingError("No project with name {} found in {}"
-                                                 .format(key_id, projects))
+                                                 .format(project_tag, projects))
             else:
                 project = None
                 name = tagged_name