CONTRIBUTING.rst

@@ -1259,14 +1259,9 @@ into the ``setup.py``, as such, whenever the frontend command line
 interface changes, the static man pages should be regenerated and
 committed with that.
 
-To do this, first ensure you have ``click_man`` installed, possibly
-with::
+To do this, run the following from the the toplevel directory of BuildStream::
 
-  pip3 install --user click_man
-
-Then, in the toplevel directory of buildstream, run the following::
-
-  python3 setup.py --command-packages=click_man.commands man_pages
+  tox -e man
 
 And commit the result, ensuring that you have added anything in
 the ``man/`` subdirectory, which will be automatically included

NEWS

@@ -20,7 +20,7 @@ buildstream 1.3.1
     an element's sources and generated build scripts you can do the command
     `bst source-checkout --include-build-scripts --tar foo.bst some-file.tar`
 
-  o BREAKING CHANGE: `bst track` and `bst fetch` commands are now osbolete.
+  o BREAKING CHANGE: `bst track` and `bst fetch` commands are now obsolete.
     Their functionality is provided by `bst source track` and
     `bst source fetch` respectively.
 
@@ -50,6 +50,11 @@ buildstream 1.3.1
     an error message and a hint instead, to avoid bothering folks that just
     made a mistake.
 
+  o BREAKING CHANGE: The unconditional 'Are you sure?' prompts have been
+    removed. These would always ask you if you were sure when running
+    'bst workspace close --remove-dir' or 'bst workspace reset'. They got in
+    the way too often.
+
   o Failed builds are included in the cache as well.
     `bst checkout` will provide anything in `%{install-root}`.
     A build including cached fails will cause any dependant elements
@@ -87,12 +92,6 @@ buildstream 1.3.1
     instead of just a specially-formatted build-root with a `root` and `scratch`
     subdirectory.
 
-  o The buildstream.conf file learned new
-    'prompt.really-workspace-close-remove-dir' and
-    'prompt.really-workspace-reset-hard' options. These allow users to suppress
-    certain confirmation prompts, e.g. double-checking that the user meant to
-    run the command as typed.
-
   o Due to the element `build tree` being cached in the respective artifact their
     size in some cases has significantly increased. In *most* cases the build trees
     are not utilised when building targets, as such by default bst 'pull' & 'build'

buildstream/_artifactcache.py

@@ -98,6 +98,7 @@ class ArtifactCache():
         self._cache_size = None               # The current cache size, sometimes it's an estimate
         self._cache_quota = None              # The cache quota
         self._cache_quota_original = None     # The cache quota as specified by the user, in bytes
+        self._cache_quota_headroom = None     # The headroom in bytes before reaching the quota or full disk
         self._cache_lower_threshold = None    # The target cache size for a cleanup
         self._remotes_setup = False           # Check to prevent double-setup of remotes
 
@@ -314,7 +315,7 @@ class ArtifactCache():
                                   len(self._required_elements),
                                   (context.config_origin or default_conf)))
 
-                if self.has_quota_exceeded():
+                if self.full():
                     raise ArtifactError("Cache too full. Aborting.",
                                         detail=detail,
                                         reason="cache-too-full")
@@ -431,15 +432,25 @@ class ArtifactCache():
         self._cache_size = cache_size
         self._write_cache_size(self._cache_size)
 
-    # has_quota_exceeded()
+    # full()
     #
-    # Checks if the current artifact cache size exceeds the quota.
+    # Checks if the artifact cache is full, either
+    # because the user configured quota has been exceeded
+    # or because the underlying disk is almost full.
     #
     # Returns:
-    #    (bool): True of the quota is exceeded
+    #    (bool): True if the artifact cache is full
     #
-    def has_quota_exceeded(self):
-        return self.get_cache_size() > self._cache_quota
+    def full(self):
+
+        if self.get_cache_size() > self._cache_quota:
+            return True
+
+        _, volume_avail = self._get_cache_volume_size()
+        if volume_avail < self._cache_quota_headroom:
+            return True
+
+        return False
 
     # preflight():
     #
@@ -936,9 +947,9 @@ class ArtifactCache():
         # is taken from the user requested cache_quota.
         #
         if 'BST_TEST_SUITE' in os.environ:
-            headroom = 0
+            self._cache_quota_headroom = 0
         else:
-            headroom = 2e9
+            self._cache_quota_headroom = 2e9
 
         try:
             cache_quota = utils._parse_size(self.context.config_cache_quota,
@@ -960,27 +971,39 @@ class ArtifactCache():
         #
         if cache_quota is None:  # Infinity, set to max system storage
             cache_quota = cache_size + available_space
-        if cache_quota < headroom:  # Check minimum
+        if cache_quota < self._cache_quota_headroom:  # Check minimum
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
                             "BuildStream requires a minimum cache quota of 2G.")
-        elif cache_quota > cache_size + available_space:  # Check maximum
-            if '%' in self.context.config_cache_quota:
-                available = (available_space / total_size) * 100
-                available = '{}% of total disk space'.format(round(available, 1))
-            else:
-                available = utils._pretty_size(available_space)
-
+        elif cache_quota > total_size:
+            # A quota greater than the total disk size is certianly an error
             raise ArtifactError("Your system does not have enough available " +
                                 "space to support the cache quota specified.",
                                 detail=("You have specified a quota of {quota} total disk space.\n" +
                                         "The filesystem containing {local_cache_path} only " +
-                                        "has {available_size} available.")
+                                        "has {total_size} total disk space.")
                                 .format(
                                     quota=self.context.config_cache_quota,
                                     local_cache_path=self.context.artifactdir,
-                                    available_size=available),
+                                    total_size=utils._pretty_size(total_size)),
                                 reason='insufficient-storage-for-quota')
+        elif cache_quota > cache_size + available_space:
+            # The quota does not fit in the available space, this is a warning
+            if '%' in self.context.config_cache_quota:
+                available = (available_space / total_size) * 100
+                available = '{}% of total disk space'.format(round(available, 1))
+            else:
+                available = utils._pretty_size(available_space)
+
+            self._message(MessageType.WARN,
+                          "Your system does not have enough available " +
+                          "space to support the cache quota specified.",
+                          detail=("You have specified a quota of {quota} total disk space.\n" +
+                                  "The filesystem containing {local_cache_path} only " +
+                                  "has {available_size} available.")
+                          .format(quota=self.context.config_cache_quota,
+                                  local_cache_path=self.context.artifactdir,
+                                  available_size=available))
 
         # Place a slight headroom (2e9 (2GB) on the cache_quota) into
         # cache_quota to try and avoid exceptions.
@@ -990,7 +1013,7 @@ class ArtifactCache():
         # already really fuzzy.
         #
         self._cache_quota_original = cache_quota
-        self._cache_quota = cache_quota - headroom
+        self._cache_quota = cache_quota - self._cache_quota_headroom
         self._cache_lower_threshold = self._cache_quota / 2
 
     # _get_cache_volume_size()

buildstream/_cas/casserver.py

@@ -324,7 +324,7 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
             blob_response.digest.size_bytes = digest.size_bytes
 
             if len(blob_request.data) != digest.size_bytes:
-                blob_response.status.code = grpc.StatusCode.FAILED_PRECONDITION
+                blob_response.status.code = code_pb2.FAILED_PRECONDITION
                 continue
 
             try:
@@ -335,10 +335,10 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
                     out.flush()
                     server_digest = self.cas.add_object(path=out.name)
                     if server_digest.hash != digest.hash:
-                        blob_response.status.code = grpc.StatusCode.FAILED_PRECONDITION
+                        blob_response.status.code = code_pb2.FAILED_PRECONDITION
 
             except ArtifactTooLargeException:
-                blob_response.status.code = grpc.StatusCode.RESOURCE_EXHAUSTED
+                blob_response.status.code = code_pb2.RESOURCE_EXHAUSTED
 
         return response
 

buildstream/_context.py

@@ -121,18 +121,10 @@ class Context():
         # Whether or not to attempt to pull build trees globally
         self.pull_buildtrees = None
 
-        # Boolean, whether we double-check with the user that they meant to
-        # remove a workspace directory.
-        self.prompt_workspace_close_remove_dir = None
-
         # Boolean, whether we double-check with the user that they meant to
         # close the workspace when they're using it to access the project.
         self.prompt_workspace_close_project_inaccessible = None
 
-        # Boolean, whether we double-check with the user that they meant to do
-        # a hard reset of a workspace, potentially losing changes.
-        self.prompt_workspace_reset_hard = None
-
         # Whether elements must be rebuilt when their dependencies have changed
         self._strict_build_plan = None
 
@@ -260,16 +252,10 @@ class Context():
         prompt = _yaml.node_get(
             defaults, Mapping, 'prompt')
         _yaml.node_validate(prompt, [
-            'really-workspace-close-remove-dir',
             'really-workspace-close-project-inaccessible',
-            'really-workspace-reset-hard',
         ])
-        self.prompt_workspace_close_remove_dir = _node_get_option_str(
-            prompt, 'really-workspace-close-remove-dir', ['ask', 'yes']) == 'ask'
         self.prompt_workspace_close_project_inaccessible = _node_get_option_str(
             prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask'
-        self.prompt_workspace_reset_hard = _node_get_option_str(
-            prompt, 'really-workspace-reset-hard', ['ask', 'yes']) == 'ask'
 
         # Load per-projects overrides
         self._project_overrides = _yaml.node_get(defaults, Mapping, 'projects', default_value={})

buildstream/_frontend/cli.py

@@ -526,7 +526,7 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, command)
     else:
         scope = Scope.RUN
 
-    use_buildtree = False
+    use_buildtree = None
 
     with app.initialized():
         if not element:
@@ -534,7 +534,8 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, command)
             if not element:
                 raise AppError('Missing argument "ELEMENT".')
 
-        dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
+        dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE,
+                                                 use_artifact_config=True)
         element = dependencies[0]
         prompt = app.shell_prompt(element)
         mounts = [
@@ -543,20 +544,31 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, command)
         ]
 
         cached = element._cached_buildtree()
-        if cli_buildtree == "always":
-            if cached:
-                use_buildtree = True
-            else:
-                raise AppError("No buildtree is cached but the use buildtree option was specified")
-        elif cli_buildtree == "never":
-            pass
-        elif cli_buildtree == "try":
-            use_buildtree = cached
+        if cli_buildtree in ("always", "try"):
+            use_buildtree = cli_buildtree
+            if not cached and use_buildtree == "always":
+                click.echo("WARNING: buildtree is not cached locally, will attempt to pull from available remotes",
+                           err=True)
         else:
-            if app.interactive and cached:
-                use_buildtree = bool(click.confirm('Do you want to use the cached buildtree?'))
+            # If the value has defaulted to ask and in non interactive mode, don't consider the buildtree, this
+            # being the default behaviour of the command
+            if app.interactive and cli_buildtree == "ask":
+                if cached and bool(click.confirm('Do you want to use the cached buildtree?')):
+                    use_buildtree = "always"
+                elif not cached:
+                    try:
+                        choice = click.prompt("Do you want to pull & use a cached buildtree?",
+                                              type=click.Choice(['try', 'always', 'never']),
+                                              err=True, show_choices=True)
+                    except click.Abort:
+                        click.echo('Aborting', err=True)
+                        sys.exit(-1)
+
+                    if choice != "never":
+                        use_buildtree = choice
+
         if use_buildtree and not element._cached_success():
-            click.echo("Warning: using a buildtree from a failed build.")
+            click.echo("WARNING: using a buildtree from a failed build.", err=True)
 
         try:
             exitcode = app.stream.shell(element, scope, prompt,
@@ -829,11 +841,6 @@ def workspace_close(app, remove_dir, all_, elements):
         if nonexisting:
             raise AppError("Workspace does not exist", detail="\n".join(nonexisting))
 
-        if app.interactive and remove_dir and app.context.prompt_workspace_close_remove_dir:
-            if not click.confirm('This will remove all your changes, are you sure?'):
-                click.echo('Aborting', err=True)
-                sys.exit(-1)
-
         for element_name in elements:
             app.stream.workspace_close(element_name, remove_dir=remove_dir)
 
@@ -867,11 +874,6 @@ def workspace_reset(app, soft, track_, all_, elements):
         if all_ and not app.stream.workspace_exists():
             raise AppError("No open workspaces to reset")
 
-        if app.interactive and not soft and app.context.prompt_workspace_reset_hard:
-            if not click.confirm('This will remove all your changes, are you sure?'):
-                click.echo('Aborting', err=True)
-                sys.exit(-1)
-
         if all_:
             elements = tuple(element_name for element_name, _ in app.context.get_workspaces().list())
 

buildstream/_gitsourcebase.py

@@ -605,7 +605,7 @@ class _GitSourceBase(Source):
                 detail = "The ref provided for the element does not exist locally " + \
                          "in the provided track branch / tag '{}'.\n".format(self.tracking) + \
                          "You may wish to track the element to update the ref from '{}' ".format(self.tracking) + \
-                         "with `bst track`,\n" + \
+                         "with `bst source track`,\n" + \
                          "or examine the upstream at '{}' for the specific ref.".format(self.mirror.url)
 
                 self.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n"

buildstream/_loader/loader.py

@@ -557,7 +557,7 @@ class Loader():
                         ticker(filename, 'Fetching subproject from {} source'.format(source.get_kind()))
                     source._fetch(sources[0:idx])
                 else:
-                    detail = "Try fetching the project with `bst fetch {}`".format(filename)
+                    detail = "Try fetching the project with `bst source fetch {}`".format(filename)
                     raise LoadError(LoadErrorReason.SUBPROJECT_FETCH_NEEDED,
                                     "Subproject fetch needed for junction: {}".format(filename),
                                     detail=detail)
@@ -565,7 +565,7 @@ class Loader():
             # Handle the case where a subproject has no ref
             #
             elif source.get_consistency() == Consistency.INCONSISTENT:
-                detail = "Try tracking the junction element with `bst track {}`".format(filename)
+                detail = "Try tracking the junction element with `bst source track {}`".format(filename)
                 raise LoadError(LoadErrorReason.SUBPROJECT_INCONSISTENT,
                                 "Subproject has no ref for junction: {}".format(filename),
                                 detail=detail)

buildstream/_pipeline.py

@@ -373,7 +373,7 @@ class Pipeline():
                     if source._get_consistency() == Consistency.INCONSISTENT:
                         detail += "    {} is missing ref\n".format(source)
                 detail += '\n'
-            detail += "Try tracking these elements first with `bst track`\n"
+            detail += "Try tracking these elements first with `bst source track`\n"
 
             raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline")
 
@@ -406,7 +406,7 @@ class Pipeline():
                     if source._get_consistency() != Consistency.CACHED:
                         detail += "    {}\n".format(source)
                 detail += '\n'
-            detail += "Try fetching these elements first with `bst fetch`,\n" + \
+            detail += "Try fetching these elements first with `bst source fetch`,\n" + \
                       "or run this command with `--fetch` option\n"
 
             raise PipelineError("Uncached sources", detail=detail, reason="uncached-sources")

buildstream/_profile.py

 #
 #  Copyright (C) 2017 Codethink Limited
+#  Copyright (C) 2019 Bloomberg Finance LP
 #
 #  This program is free software; you can redistribute it and/or
 #  modify it under the terms of the GNU Lesser General Public
@@ -16,6 +17,7 @@
 #
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#        James Ennis <james.ennis@codethink.co.uk>
 
 import cProfile
 import pstats
@@ -46,6 +48,8 @@ class Topics():
     LOAD_CONTEXT = 'load-context'
     LOAD_PROJECT = 'load-project'
     LOAD_PIPELINE = 'load-pipeline'
+    LOAD_SELECTION = 'load-selection'
+    SCHEDULER = 'scheduler'
     SHOW = 'show'
     ARTIFACT_RECEIVE = 'artifact-receive'
     ALL = 'all'

buildstream/_scheduler/queues/buildqueue.py

@@ -100,7 +100,7 @@ class BuildQueue(Queue):
         # If the estimated size outgrows the quota, ask the scheduler
         # to queue a job to actually check the real cache size.
         #
-        if artifacts.has_quota_exceeded():
+        if artifacts.full():
             self._scheduler.check_cache_size()
 
     def done(self, job, element, result, status):

buildstream/_scheduler/scheduler.py

@@ -29,6 +29,7 @@ from contextlib import contextmanager
 # Local imports
 from .resources import Resources, ResourceType
 from .jobs import JobStatus, CacheSizeJob, CleanupJob
+from .._profile import Topics, profile_start, profile_end
 
 
 # A decent return code for Scheduler.run()
@@ -154,11 +155,16 @@ class Scheduler():
         # Check if we need to start with some cache maintenance
         self._check_cache_management()
 
+        # Start the profiler
+        profile_start(Topics.SCHEDULER, "_".join(queue.action_name for queue in self.queues))
+
         # Run the queues
         self._sched()
         self.loop.run_forever()
         self.loop.close()
 
+        profile_end(Topics.SCHEDULER, "_".join(queue.action_name for queue in self.queues))
+
         # Stop handling unix signals
         self._disconnect_signals()
 
@@ -297,7 +303,7 @@ class Scheduler():
         # starts while we are checking the cache.
         #
         artifacts = self.context.artifactcache
-        if artifacts.has_quota_exceeded():
+        if artifacts.full():
             self._sched_cache_size_job(exclusive=True)
 
     # _spawn_job()
@@ -308,10 +314,10 @@ class Scheduler():
     #    job (Job): The job to spawn
     #
     def _spawn_job(self, job):
-        job.spawn()
         self._active_jobs.append(job)
         if self._job_start_callback:
             self._job_start_callback(job)
+        job.spawn()
 
     # Callback for the cache size job
     def _cache_size_job_complete(self, status, cache_size):
@@ -332,7 +338,7 @@ class Scheduler():
         context = self.context
         artifacts = context.artifactcache
 
-        if artifacts.has_quota_exceeded():
+        if artifacts.full():
             self._cleanup_scheduled = True
 
     # Callback for the cleanup job

buildstream/_stream.py

@@ -32,6 +32,7 @@ from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
 from ._message import Message, MessageType
 from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
 from ._pipeline import Pipeline, PipelineSelection
+from ._profile import Topics, profile_start, profile_end
 from . import utils, _yaml, _site
 from . import Scope, Consistency
 
@@ -100,16 +101,25 @@ class Stream():
     #    targets (list of str): Targets to pull
     #    selection (PipelineSelection): The selection mode for the specified targets
     #    except_targets (list of str): Specified targets to except from fetching
+    #    use_artifact_config (bool): If artifact remote config should be loaded
     #
     # Returns:
     #    (list of Element): The selected elements
     def load_selection(self, targets, *,
                        selection=PipelineSelection.NONE,
-                       except_targets=()):
+                       except_targets=(),
+                       use_artifact_config=False):
+
+        profile_start(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, '-') for t in targets))
+
         elements, _ = self._load(targets, (),
                                  selection=selection,
                                  except_targets=except_targets,
-                                 fetch_subprojects=False)
+                                 fetch_subprojects=False,
+                                 use_artifact_config=use_artifact_config)
+
+        profile_end(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, '-') for t in targets))
+
         return elements
 
     # shell()
@@ -124,7 +134,7 @@ class Stream():
     #    mounts (list of HostMount): Additional directories to mount into the sandbox
     #    isolate (bool): Whether to isolate the environment like we do in builds
     #    command (list): An argv to launch in the sandbox, or None
-    #    usebuildtree (bool): Wheather to use a buildtree as the source.
+    #    usebuildtree (str): Whether to use a buildtree as the source, given cli option
     #
     # Returns:
     #    (int): The exit code of the launched shell
@@ -134,7 +144,7 @@ class Stream():
               mounts=None,
               isolate=False,
               command=None,
-              usebuildtree=False):
+              usebuildtree=None):
 
         # Assert we have everything we need built, unless the directory is specified
         # in which case we just blindly trust the directory, using the element
@@ -149,8 +159,31 @@ class Stream():
                 raise StreamError("Elements need to be built or downloaded before staging a shell environment",
                                   detail="\n".join(missing_deps))
 
+        buildtree = False
+        # Check if we require a pull queue attempt, with given artifact state and context
+        if usebuildtree:
+            if not element._cached_buildtree():
+                require_buildtree = self._buildtree_pull_required([element])
+                # Attempt a pull queue for the given element if remote and context allow it
+                if require_buildtree:
+                    self._message(MessageType.INFO, "Attempting to fetch missing artifact buildtree")
+                    self._add_queue(PullQueue(self._scheduler))
+                    self._enqueue_plan(require_buildtree)
+                    self._run()
+                    # Now check if the buildtree was successfully fetched
+                    if element._cached_buildtree():
+                        buildtree = True
+                if not buildtree:
+                    if usebuildtree == "always":
+                        raise StreamError("Buildtree is not cached locally or in available remotes")
+                    else:
+                        self._message(MessageType.INFO, """Buildtree is not cached locally or in available remotes,
+                                                        shell will be loaded without it""")
+            else:
+                buildtree = True
+
         return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command,
-                              usebuildtree=usebuildtree)
+                              usebuildtree=buildtree)
 
     # build()
     #

buildstream/data/userconfig.yaml

@@ -112,14 +112,6 @@ logging:
 #
 prompt:
 
-  # Whether to really proceed with 'bst workspace close --remove-dir' removing
-  # a workspace directory, potentially losing changes.
-  #
-  #  ask - Ask the user if they are sure.
-  #  yes - Always remove, without asking.
-  #
-  really-workspace-close-remove-dir: ask
-
   # Whether to really proceed with 'bst workspace close' when doing so would
   # stop them from running bst commands in this workspace.
   #
@@ -127,11 +119,3 @@ prompt:
   #  yes - Always close, without asking.
   #
   really-workspace-close-project-inaccessible: ask
-
-  # Whether to really proceed with 'bst workspace reset' doing a hard reset of
-  # a workspace, potentially losing changes.
-  #
-  #  ask - Ask the user if they are sure.
-  #  yes - Always hard reset, without asking.
-  #
-  really-workspace-reset-hard: ask

buildstream/element.py

@@ -2510,9 +2510,30 @@ class Element(Plugin):
             exclude = []
 
         # Ignore domains that dont apply to this element
-        #
-        include = [domain for domain in include if domain in element_domains]
-        exclude = [domain for domain in exclude if domain in element_domains]
+        #   - In the case of the filter element, we can explicitly
+        #   declare in domains we want to include/exclude.
+        unfound_domains = []
+        include_domains = []
+        for domain in include:
+            if domain not in element_domains:
+                unfound_domains.append(domain)
+            else:
+                include_domains.append(domain)
+
+        exclude_domains = []
+        for domain in exclude:
+            if domain not in element_domains:
+                unfound_domains.append(domain)
+            else:
+                exclude_domains.append(domain)
+
+        # Print a warning to the user if we have unfound domains
+        if unfound_domains:
+            warning_detail = "Split-domains: {} not provided by '{}'.\n" \
+                "Known split domains: {}".format(unfound_domains, self.name, element_domains)
+            self.warn("Split domains: {} not found.".format(unfound_domains),
+                      detail=warning_detail,
+                      warning_token=CoreWarnings.DOMAIN_NOT_FOUND)
 
         # FIXME: Instead of listing the paths in an extracted artifact,
         #        we should be using a manifest loaded from the artifact
@@ -2531,9 +2552,9 @@ class Element(Plugin):
             for domain in element_domains:
                 if self.__splits[domain].match(filename):
                     claimed_file = True
-                    if domain in include:
+                    if domain in include_domains:
                         include_file = True
-                    if domain in exclude:
+                    if domain in exclude_domains:
                         exclude_file = True
 
             if orphans and not claimed_file:

buildstream/plugins/elements/filter.py

@@ -47,6 +47,8 @@ from buildstream import Element, ElementError, Scope
 class FilterElement(Element):
     # pylint: disable=attribute-defined-outside-init
 
+    BST_ARTIFACT_VERSION = 1
+
     # The filter element's output is its dependencies, so
     # we must rebuild if the dependencies change even when
     # not in strict build plans.
@@ -102,7 +104,7 @@ class FilterElement(Element):
 
     def assemble(self, sandbox):
         with self.timed_activity("Staging artifact", silent_nested=True):
-            for dep in self.dependencies(Scope.BUILD):
+            for dep in self.dependencies(Scope.BUILD, recurse=False):
                 dep.stage_artifact(sandbox, include=self.include,
                                    exclude=self.exclude, orphans=self.include_orphans)
         return ""

buildstream/plugins/elements/junction.py

@@ -79,7 +79,7 @@ depend on a junction element itself.
 
    Therefore, if you require the most up-to-date version of a subproject,
    you must explicitly track the junction element by invoking:
-   `bst track JUNCTION_ELEMENT`.
+   `bst source track JUNCTION_ELEMENT`.
 
    Furthermore, elements within the subproject are also not tracked by default.
    For this, we must specify the `--track-cross-junctions` option. This option
@@ -93,7 +93,7 @@ cached yet. However, they can be fetched explicitly:
 
 .. code::
 
-   bst fetch junction.bst
+   bst source fetch junction.bst
 
 Other commands such as ``bst build`` implicitly fetch junction sources.
 
@@ -146,7 +146,7 @@ class JunctionElement(Element):
 
     def get_unique_key(self):
         # Junctions do not produce artifacts. get_unique_key() implementation
-        # is still required for `bst fetch`.
+        # is still required for `bst source fetch`.
         return 1
 
     def configure_sandbox(self, sandbox):

buildstream/plugins/sources/bzr.py

@@ -46,7 +46,7 @@ bzr - stage files from a bazaar repository
    # but revisions on a branch are of the form
    # <revision-branched-from>.<branch-number>.<revision-since-branching>
    # e.g. 6622.1.6.
-   # The ref must be specified to build, and 'bst track' will update the
+   # The ref must be specified to build, and 'bst source track' will update the
    # revision number to the one on the tip of the branch specified in 'track'.
    ref: 6622
 

buildstream/plugins/sources/deb.py

@@ -34,7 +34,7 @@ deb - stage files from .deb packages
    kind: deb
 
    # Specify the deb url. Using an alias defined in your project
-   # configuration is encouraged. 'bst track' will update the
+   # configuration is encouraged. 'bst source track' will update the
    # sha256sum in 'ref' to the downloaded file's sha256sum.
    url: upstream:foo.deb
 

buildstream/plugins/sources/git.py

@@ -112,7 +112,7 @@ git - stage files from a git repository
    #     o Enable `track-tags` feature
    #     o Set the `track` parameter to the desired commit sha which
    #       the current `ref` points to
-   #     o Run `bst track` for these elements, this will result in
+   #     o Run `bst source track` for these elements, this will result in
    #       populating the `tags` portion of the refs without changing
    #       the refs
    #     o Restore the `track` parameter to the branches which you have