NEWS

@@ -2,6 +2,18 @@
 buildstream 1.3.1
 =================
 
+  o Added `bst artifact log` subcommand for viewing build logs.
+
+  o BREAKING CHANGE: The bst source-bundle command has been removed. The
+    functionality it provided has been replaced by the `--include-build-scripts`
+    option of the `bst source-checkout` command. To produce a tarball containing
+    an element's sources and generated build scripts you can do the command
+    `bst source-checkout --include-build-scripts --tar foo.bst some-file.tar`
+
+  o BREAKING CHANGE: Default strip-commands have been removed as they are too
+    specific. Recommendation if you are building in Linux is to use the
+    ones being used in freedesktop-sdk project, for example
+
   o All elements must now be suffixed with `.bst`
     Attempting to use an element that does not have the `.bst` extension,
     will result in a warning.
@@ -79,6 +91,15 @@ buildstream 1.3.1
     plugin has now a tag tracking feature instead. This can be enabled
     by setting 'track-tags'.
 
+  o Opening a workspace now creates a .bstproject.yaml file that allows buildstream
+    commands to be run from a workspace that is not inside a project.
+
+  o Specifying an element is now optional for some commands when buildstream is run
+    from inside a workspace - the 'build', 'checkout', 'fetch', 'pull', 'push',
+    'shell', 'show', 'source-checkout', 'track', 'workspace close' and 'workspace reset'
+    commands are affected.
+
+
 =================
 buildstream 1.1.5
 =================

buildstream/_artifactcache/artifactcache.py

@@ -874,9 +874,7 @@ class ArtifactCache():
                             "\nValid values are, for example: 800M 10G 1T 50%\n"
                             .format(str(e))) from e
 
-        stat = os.statvfs(artifactdir_volume)
-        available_space = (stat.f_bsize * stat.f_bavail)
-
+        available_space, total_size = self._get_volume_space_info_for(artifactdir_volume)
         cache_size = self.get_cache_size()
 
         # Ensure system has enough storage for the cache_quota
@@ -893,7 +891,7 @@ class ArtifactCache():
                             "BuildStream requires a minimum cache quota of 2G.")
         elif cache_quota > cache_size + available_space:  # Check maximum
             if '%' in self.context.config_cache_quota:
-                available = (available_space / (stat.f_blocks * stat.f_bsize)) * 100
+                available = (available_space / total_size) * 100
                 available = '{}% of total disk space'.format(round(available, 1))
             else:
                 available = utils._pretty_size(available_space)
@@ -919,6 +917,20 @@ class ArtifactCache():
         self._cache_quota = cache_quota - headroom
         self._cache_lower_threshold = self._cache_quota / 2
 
+    # _get_volume_space_info_for
+    #
+    # Get the available space and total space for the given volume
+    #
+    # Args:
+    #     volume: volume for which to get the size
+    #
+    # Returns:
+    #     A tuple containing first the availabe number of bytes on the requested
+    #     volume, then the total number of bytes of the volume.
+    def _get_volume_space_info_for(self, volume):
+        stat = os.statvfs(volume)
+        return stat.f_bsize * stat.f_bavail, stat.f_bsize * stat.f_blocks
+
 
 # _configured_remote_artifact_cache_specs():
 #

buildstream/_context.py

@@ -32,7 +32,7 @@ from ._message import Message, MessageType
 from ._profile import Topics, profile_start, profile_end
 from ._artifactcache import ArtifactCache
 from ._artifactcache.cascache import CASCache
-from ._workspaces import Workspaces
+from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE
 from .plugin import _plugin_lookup
 
 
@@ -47,9 +47,12 @@ from .plugin import _plugin_lookup
 # verbosity levels and basically anything pertaining to the context
 # in which BuildStream was invoked.
 #
+# Args:
+#    directory (str): The directory that buildstream was invoked in
+#
 class Context():
 
-    def __init__(self):
+    def __init__(self, directory=None):
 
         # Filename indicating which configuration file was used, or None for the defaults
         self.config_origin = None
@@ -122,6 +125,10 @@ class Context():
         # remove a workspace directory.
         self.prompt_workspace_close_remove_dir = None
 
+        # Boolean, whether we double-check with the user that they meant to
+        # close the workspace when they're using it to access the project.
+        self.prompt_workspace_close_project_inaccessible = None
+
         # Boolean, whether we double-check with the user that they meant to do
         # a hard reset of a workspace, potentially losing changes.
         self.prompt_workspace_reset_hard = None
@@ -140,9 +147,11 @@ class Context():
         self._projects = []
         self._project_overrides = {}
         self._workspaces = None
+        self._workspace_project_cache = WorkspaceProjectCache()
         self._log_handle = None
         self._log_filename = None
         self._cascache = None
+        self._directory = directory
 
     # load()
     #
@@ -250,12 +259,15 @@ class Context():
             defaults, Mapping, 'prompt')
         _yaml.node_validate(prompt, [
             'auto-init', 'really-workspace-close-remove-dir',
+            'really-workspace-close-project-inaccessible',
             'really-workspace-reset-hard',
         ])
         self.prompt_auto_init = _node_get_option_str(
             prompt, 'auto-init', ['ask', 'no']) == 'ask'
         self.prompt_workspace_close_remove_dir = _node_get_option_str(
             prompt, 'really-workspace-close-remove-dir', ['ask', 'yes']) == 'ask'
+        self.prompt_workspace_close_project_inaccessible = _node_get_option_str(
+            prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask'
         self.prompt_workspace_reset_hard = _node_get_option_str(
             prompt, 'really-workspace-reset-hard', ['ask', 'yes']) == 'ask'
 
@@ -285,7 +297,7 @@ class Context():
     #
     def add_project(self, project):
         if not self._projects:
-            self._workspaces = Workspaces(project)
+            self._workspaces = Workspaces(project, self._workspace_project_cache)
         self._projects.append(project)
 
     # get_projects():
@@ -312,6 +324,16 @@ class Context():
     def get_workspaces(self):
         return self._workspaces
 
+    # get_workspace_project_cache():
+    #
+    # Return the WorkspaceProjectCache object used for this BuildStream invocation
+    #
+    # Returns:
+    #    (WorkspaceProjectCache): The WorkspaceProjectCache object
+    #
+    def get_workspace_project_cache(self):
+        return self._workspace_project_cache
+
     # get_overrides():
     #
     # Fetch the override dictionary for the active project. This returns
@@ -627,6 +649,20 @@ class Context():
             self._cascache = CASCache(self.artifactdir)
         return self._cascache
 
+    # guess_element()
+    #
+    # Attempts to interpret which element the user intended to run commands on
+    #
+    # Returns:
+    #    (str) The name of the element, or None if no element can be guessed
+    def guess_element(self):
+        workspace_project_dir, _ = utils._search_upward_for_files(self._directory, [WORKSPACE_PROJECT_FILE])
+        if workspace_project_dir:
+            workspace_project = self._workspace_project_cache.get(workspace_project_dir)
+            return workspace_project.get_default_element()
+        else:
+            return None
+
 
 # _node_get_option_str()
 #

buildstream/_frontend/app.py

@@ -164,7 +164,7 @@ class App():
         # Load the Context
         #
         try:
-            self.context = Context()
+            self.context = Context(directory)
             self.context.load(config)
         except BstError as e:
             self._error_exit(e, "Error loading user configuration")

buildstream/_frontend/cli.py

 import os
 import sys
+from contextlib import ExitStack
+from fnmatch import fnmatch
+from tempfile import TemporaryDirectory
 
 import click
 from .. import _yaml
@@ -59,18 +62,9 @@ def complete_target(args, incomplete):
     :return: all the possible user-specified completions for the param
     """
 
+    from .. import utils
     project_conf = 'project.conf'
 
-    def ensure_project_dir(directory):
-        directory = os.path.abspath(directory)
-        while not os.path.isfile(os.path.join(directory, project_conf)):
-            parent_dir = os.path.dirname(directory)
-            if directory == parent_dir:
-                break
-            directory = parent_dir
-
-        return directory
-
     # First resolve the directory, in case there is an
     # active --directory/-C option
     #
@@ -89,7 +83,7 @@ def complete_target(args, incomplete):
     else:
         # Check if this directory or any of its parent directories
         # contain a project config file
-        base_directory = ensure_project_dir(base_directory)
+        base_directory, _ = utils._search_upward_for_files(base_directory, [project_conf])
 
     # Now parse the project.conf just to find the element path,
     # this is unfortunately a bit heavy.
@@ -116,6 +110,23 @@ def complete_target(args, incomplete):
     return complete_list
 
 
+def complete_artifact(args, incomplete):
+    from .._context import Context
+    ctx = Context()
+
+    config = None
+    for i, arg in enumerate(args):
+        if arg in ('-c', '--config'):
+            config = args[i + 1]
+    ctx.load(config)
+
+    # element targets are valid artifact names
+    complete_list = complete_target(args, incomplete)
+    complete_list.extend(ref for ref in ctx.artifactcache.cas.list_refs() if ref.startswith(incomplete))
+
+    return complete_list
+
+
 def override_completions(cmd, cmd_param, args, incomplete):
     """
     :param cmd_param: command definition
@@ -130,13 +141,15 @@ def override_completions(cmd, cmd_param, args, incomplete):
     # We can't easily extend click's data structures without
     # modifying click itself, so just do some weak special casing
     # right here and select which parameters we want to handle specially.
-    if isinstance(cmd_param.type, click.Path) and \
-       (cmd_param.name == 'elements' or
+    if isinstance(cmd_param.type, click.Path):
+        if (cmd_param.name == 'elements' or
                 cmd_param.name == 'element' or
                 cmd_param.name == 'except_' or
                 cmd_param.opts == ['--track'] or
                 cmd_param.opts == ['--track-except']):
             return complete_target(args, incomplete)
+        if cmd_param.name == 'artifacts':
+            return complete_artifact(args, incomplete)
 
     raise CompleteUnhandled()
 
@@ -325,10 +338,15 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
     if track_save:
         click.echo("WARNING: --track-save is deprecated, saving is now unconditional", err=True)
 
+    with app.initialized(session_name="Build"):
+        if not all_ and not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
         if track_all:
             track_ = elements
 
-    with app.initialized(session_name="Build"):
         app.stream.build(elements,
                          track_targets=track_,
                          track_except=track_except,
@@ -380,6 +398,11 @@ def fetch(app, elements, deps, track_, except_, track_cross_junctions):
         deps = PipelineSelection.ALL
 
     with app.initialized(session_name="Fetch"):
+        if not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
         app.stream.fetch(elements,
                          selection=deps,
                          except_targets=except_,
@@ -416,6 +439,11 @@ def track(app, elements, deps, except_, cross_junctions):
         all:   All dependencies of all specified elements
     """
     with app.initialized(session_name="Track"):
+        if not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
         # Substitute 'none' for 'redirect' so that element redirections
         # will be done
         if deps == 'none':
@@ -451,7 +479,13 @@ def pull(app, elements, deps, remote):
         none:  No dependencies, just the element itself
         all:   All dependencies
     """
+
     with app.initialized(session_name="Pull"):
+        if not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
         app.stream.pull(elements, selection=deps, remote=remote)
 
 
@@ -484,6 +518,11 @@ def push(app, elements, deps, remote):
         all:   All dependencies
     """
     with app.initialized(session_name="Push"):
+        if not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
         app.stream.push(elements, selection=deps, remote=remote)
 
 
@@ -554,6 +593,11 @@ def show(app, elements, deps, except_, order, format_):
             $'---------- %{name} ----------\\n%{vars}'
     """
     with app.initialized():
+        if not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
         dependencies = app.stream.load_selection(elements,
                                                  selection=deps,
                                                  except_targets=except_)
@@ -582,7 +626,7 @@ def show(app, elements, deps, except_, order, format_):
               help="Mount a file or directory into the sandbox")
 @click.option('--isolate', is_flag=True, default=False,
               help='Create an isolated build sandbox')
-@click.argument('element',
+@click.argument('element', required=False,
                 type=click.Path(readable=False))
 @click.argument('command', type=click.STRING, nargs=-1)
 @click.pass_obj
@@ -613,6 +657,11 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
         scope = Scope.RUN
 
     with app.initialized():
+        if not element:
+            element = app.context.guess_element()
+            if not element:
+                raise AppError('Missing argument "ELEMENT".')
+
         dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
         element = dependencies[0]
         prompt = app.shell_prompt(element)
@@ -650,15 +699,24 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
               help="Create a tarball from the artifact contents instead "
                    "of a file tree. If LOCATION is '-', the tarball "
                    "will be dumped to the standard output.")
-@click.argument('element',
+@click.argument('element', required=False,
                 type=click.Path(readable=False))
-@click.argument('location', type=click.Path())
+@click.argument('location', type=click.Path(), required=False)
 @click.pass_obj
 def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
     """Checkout a built artifact to the specified location
     """
     from ..element import Scope
 
+    if not element and not location:
+        click.echo("ERROR: LOCATION is not specified", err=True)
+        sys.exit(-1)
+
+    if element and not location:
+        # Nasty hack to get around click's optional args
+        location = element
+        element = None
+
     if hardlinks and tar:
         click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
         sys.exit(-1)
@@ -671,6 +729,11 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
         scope = Scope.NONE
 
     with app.initialized():
+        if not element:
+            element = app.context.guess_element()
+            if not element:
+                raise AppError('Missing argument "ELEMENT".')
+
         app.stream.checkout(element,
                             location=location,
                             force=force,
@@ -684,6 +747,8 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
 #                  Source Checkout Command                      #
 ##################################################################
 @cli.command(name='source-checkout', short_help='Checkout sources for an element')
+@click.option('--force', '-f', default=False, is_flag=True,
+              help="Allow files to be overwritten")
 @click.option('--except', 'except_', multiple=True,
               type=click.Path(readable=False),
               help="Except certain dependencies")
@@ -692,19 +757,40 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
               help='The dependencies whose sources to checkout (default: none)')
 @click.option('--fetch', 'fetch_', default=False, is_flag=True,
               help='Fetch elements if they are not fetched')
-@click.argument('element',
-                type=click.Path(readable=False))
-@click.argument('location', type=click.Path())
+@click.option('--tar', 'tar', default=False, is_flag=True,
+              help='Create a tarball from the element\'s sources instead of a '
+                   'file tree.')
+@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
+@click.argument('element', required=False, type=click.Path(readable=False))
+@click.argument('location', type=click.Path(), required=False)
 @click.pass_obj
-def source_checkout(app, element, location, deps, fetch_, except_):
+def source_checkout(app, element, location, force, deps, fetch_, except_,
+                    tar, build_scripts):
     """Checkout sources of an element to the specified location
     """
+    if not element and not location:
+        click.echo("ERROR: LOCATION is not specified", err=True)
+        sys.exit(-1)
+
+    if element and not location:
+        # Nasty hack to get around click's optional args
+        location = element
+        element = None
+
     with app.initialized():
+        if not element:
+            element = app.context.guess_element()
+            if not element:
+                raise AppError('Missing argument "ELEMENT".')
+
         app.stream.source_checkout(element,
                                    location=location,
+                                   force=force,
                                    deps=deps,
                                    fetch=fetch_,
-                                   except_targets=except_)
+                                   except_targets=except_,
+                                   tar=tar,
+                                   include_build_scripts=build_scripts)
 
 
 ##################################################################
@@ -756,11 +842,15 @@ def workspace_open(app, no_checkout, force, track_, directory, elements):
 def workspace_close(app, remove_dir, all_, elements):
     """Close a workspace"""
 
-    if not (all_ or elements):
-        click.echo('ERROR: no elements specified', err=True)
-        sys.exit(-1)
-
     with app.initialized():
+        if not (all_ or elements):
+            # NOTE: I may need to revisit this when implementing multiple projects
+            # opening one workspace.
+            element = app.context.guess_element()
+            if element:
+                elements = (element,)
+            else:
+                raise AppError('No elements specified')
 
         # Early exit if we specified `all` and there are no workspaces
         if all_ and not app.stream.workspace_exists():
@@ -772,11 +862,19 @@ def workspace_close(app, remove_dir, all_, elements):
 
         elements = app.stream.redirect_element_names(elements)
 
-        # Check that the workspaces in question exist
+        # Check that the workspaces in question exist, and that it's safe to
+        # remove them.
         nonexisting = []
         for element_name in elements:
             if not app.stream.workspace_exists(element_name):
                 nonexisting.append(element_name)
+            if (app.stream.workspace_is_required(element_name) and app.interactive and
+                    app.context.prompt_workspace_close_project_inaccessible):
+                click.echo("Removing '{}' will prevent you from running "
+                           "BuildStream commands from the current directory".format(element_name))
+                if not click.confirm('Are you sure you want to close this workspace?'):
+                    click.echo('Aborting', err=True)
+                    sys.exit(-1)
         if nonexisting:
             raise AppError("Workspace does not exist", detail="\n".join(nonexisting))
 
@@ -809,6 +907,10 @@ def workspace_reset(app, soft, track_, all_, elements):
     with app.initialized():
 
         if not (all_ or elements):
+            element = app.context.guess_element()
+            if element:
+                elements = (element,)
+            else:
                 raise AppError('No elements specified to reset')
 
         if all_ and not app.stream.workspace_exists():
@@ -837,32 +939,99 @@ def workspace_list(app):
         app.stream.workspace_list()
 
 
-##################################################################
-#                     Source Bundle Command                      #
-##################################################################
-@cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed")
-@click.option('--except', 'except_', multiple=True,
-              type=click.Path(readable=False),
-              help="Elements to except from the tarball")
-@click.option('--compression', default='gz',
-              type=click.Choice(['none', 'gz', 'bz2', 'xz']),
-              help="Compress the tar file using the given algorithm.")
-@click.option('--track', 'track_', default=False, is_flag=True,
-              help="Track new source references before bundling")
-@click.option('--force', '-f', default=False, is_flag=True,
-              help="Overwrite an existing tarball")
-@click.option('--directory', default=os.getcwd(),
-              help="The directory to write the tarball to")
-@click.argument('element',
-                type=click.Path(readable=False))
+#############################################################
+#                     Artifact Commands                     #
+#############################################################
+def _classify_artifacts(names, cas, project_directory):
+    element_targets = []
+    artifact_refs = []
+    element_globs = []
+    artifact_globs = []
+
+    for name in names:
+        if name.endswith('.bst'):
+            if any(c in "*?[" for c in name):
+                element_globs.append(name)
+            else:
+                element_targets.append(name)
+        else:
+            if any(c in "*?[" for c in name):
+                artifact_globs.append(name)
+            else:
+                artifact_refs.append(name)
+
+    if element_globs:
+        for dirpath, _, filenames in os.walk(project_directory):
+            for filename in filenames:
+                element_path = os.path.join(dirpath, filename).lstrip(project_directory).lstrip('/')
+                if any(fnmatch(element_path, glob) for glob in element_globs):
+                    element_targets.append(element_path)
+
+    if artifact_globs:
+        artifact_refs.extend(ref for ref in cas.list_refs()
+                             if any(fnmatch(ref, glob) for glob in artifact_globs))
+
+    return element_targets, artifact_refs
+
+
+@cli.group(short_help="Manipulate cached artifacts")
+def artifact():
+    """Manipulate cached artifacts"""
+    pass
+
+
+################################################################
+#                     Artifact Log Command                     #
+################################################################
+@artifact.command(name='log', short_help="Show logs of an artifact")
+@click.argument('artifacts', type=click.Path(), nargs=-1)
 @click.pass_obj
-def source_bundle(app, element, force, directory,
-                  track_, compression, except_):
-    """Produce a source bundle to be manually executed
-    """
-    with app.initialized():
-        app.stream.source_bundle(element, directory,
-                                 track_first=track_,
-                                 force=force,
-                                 compression=compression,
-                                 except_targets=except_)
+def artifact_log(app, artifacts):
+    """Show logs of all artifacts"""
+    from .._exceptions import CASError
+    from .._message import MessageType
+    from .._pipeline import PipelineSelection
+    from ..storage._casbaseddirectory import CasBasedDirectory
+
+    with ExitStack() as stack:
+        stack.enter_context(app.initialized())
+        cache = app.context.artifactcache
+
+        elements, artifacts = _classify_artifacts(artifacts, cache.cas,
+                                                  app.project.directory)
+
+        vdirs = []
+        extractdirs = []
+        if artifacts:
+            for ref in artifacts:
+                try:
+                    cache_id = cache.cas.resolve_ref(ref, update_mtime=True)
+                    vdir = CasBasedDirectory(cache.cas, cache_id)
+                    vdirs.append(vdir)
+                except CASError as e:
+                    app._message(MessageType.WARN, "Artifact {} is not cached".format(ref), detail=str(e))
+                    continue
+        if elements:
+            elements = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
+            for element in elements:
+                if not element._cached():
+                    app._message(MessageType.WARN, "Element {} is not cached".format(element))
+                    continue
+                ref = cache.get_artifact_fullname(element, element._get_cache_key())
+                cache_id = cache.cas.resolve_ref(ref, update_mtime=True)
+                vdir = CasBasedDirectory(cache.cas, cache_id)
+                vdirs.append(vdir)
+
+        for vdir in vdirs:
+            # NOTE: If reading the logs feels unresponsive, here would be a good place to provide progress information.
+            logsdir = vdir.descend(["logs"])
+            td = stack.enter_context(TemporaryDirectory())
+            logsdir.export_files(td, can_link=True)
+            extractdirs.append(td)
+
+        for extractdir in extractdirs:
+            for log in (os.path.join(extractdir, log) for log in os.listdir(extractdir)):
+                # NOTE: Should click gain the ability to pass files to the pager this can be optimised.
+                with open(log) as f:
+                    data = f.read()
+                    click.echo_via_pager(data)

buildstream/_project.py

@@ -41,6 +41,7 @@ from .element import Element
 from ._message import Message, MessageType
 from ._includes import Includes
 from ._platform import Platform
+from ._workspaces import WORKSPACE_PROJECT_FILE
 
 
 # Project Configuration file
@@ -95,8 +96,10 @@ class Project():
         # The project name
         self.name = None
 
-        # The project directory
-        self.directory = self._ensure_project_dir(directory)
+        self._context = context  # The invocation Context, a private member
+
+        # The project directory, and whether the element whose workspace it was invoked from
+        self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory)
 
         # Absolute path to where elements are loaded from within the project
         self.element_path = None
@@ -117,7 +120,6 @@ class Project():
         #
         # Private Members
         #
-        self._context = context  # The invocation Context
 
         self._default_mirror = default_mirror    # The name of the preferred mirror.
 
@@ -371,6 +373,14 @@ class Project():
 
         self._load_second_pass()
 
+    # invoked_from_workspace_element()
+    #
+    # Returns the element whose workspace was used to invoke buildstream
+    # if buildstream was invoked from an external workspace
+    #
+    def invoked_from_workspace_element(self):
+        return self._invoked_from_workspace_element
+
     # cleanup()
     #
     # Cleans up resources used loading elements
@@ -650,7 +660,7 @@ class Project():
         # Source url aliases
         output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
 
-    # _ensure_project_dir()
+    # _find_project_dir()
     #
     # Returns path of the project directory, if a configuration file is found
     # in given directory or any of its parent directories.
@@ -661,18 +671,30 @@ class Project():
     # Raises:
     #    LoadError if project.conf is not found
     #
-    def _ensure_project_dir(self, directory):
-        directory = os.path.abspath(directory)
-        while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
-            parent_dir = os.path.dirname(directory)
-            if directory == parent_dir:
+    # Returns:
+    #    (str) - the directory that contains the project, and
+    #    (str) - the name of the element required to find the project, or None
+    #
+    def _find_project_dir(self, directory):
+        workspace_element = None
+        found_directory, filename = utils._search_upward_for_files(
+            directory, [_PROJECT_CONF_FILE, WORKSPACE_PROJECT_FILE]
+        )
+        if filename == _PROJECT_CONF_FILE:
+            project_directory = found_directory
+        elif filename == WORKSPACE_PROJECT_FILE:
+            workspace_project_cache = self._context.get_workspace_project_cache()
+            workspace_project = workspace_project_cache.get(found_directory)
+            if workspace_project:
+                project_directory = workspace_project.get_default_project_path()
+                workspace_element = workspace_project.get_default_element()
+        else:
             raise LoadError(
                 LoadErrorReason.MISSING_PROJECT_CONF,
                 '{} not found in current directory or any of its parent directories'
                 .format(_PROJECT_CONF_FILE))
-            directory = parent_dir
 
-        return directory
+        return project_directory, workspace_element
 
     def _load_plugin_factories(self, config, output):
         plugin_source_origins = []   # Origins of custom sources

buildstream/_stream.py

@@ -25,8 +25,8 @@ import stat
 import shlex
 import shutil
 import tarfile
-from contextlib import contextmanager
-from tempfile import TemporaryDirectory
+import tempfile
+from contextlib import contextmanager, suppress
 
 from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
 from ._message import Message, MessageType
@@ -449,11 +449,14 @@ class Stream():
     #
     def source_checkout(self, target, *,
                         location=None,
+                        force=False,
                         deps='none',
                         fetch=False,
-                        except_targets=()):
+                        except_targets=(),
+                        tar=False,
+                        include_build_scripts=False):
 
-        self._check_location_writable(location)
+        self._check_location_writable(location, force=force, tar=tar)
 
         elements, _ = self._load((target,), (),
                                  selection=deps,
@@ -467,7 +470,8 @@ class Stream():
 
         # Stage all sources determined by scope
         try:
-            self._write_element_sources(location, elements)
+            self._source_checkout(elements, location, force, deps,
+                                  fetch, tar, include_build_scripts)
         except BstError as e:
             raise StreamError("Error while writing sources"
                               ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
@@ -544,7 +548,8 @@ class Stream():
             if len(elements) != 1:
                 raise StreamError("Exactly one element can be given if --directory is used",
                                   reason='directory-with-multiple-elements')
-            expanded_directories = [custom_dir, ]
+            directory = os.path.abspath(custom_dir)
+            expanded_directories = [directory, ]
         else:
             # If this fails it is a bug in what ever calls this, usually cli.py and so can not be tested for via the
             # run bst test mechanism.
@@ -581,15 +586,7 @@ class Stream():
                     todo_elements = "\nDid not try to create workspaces for " + todo_elements
                 raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
 
-            workspaces.create_workspace(target._get_full_name(), directory)
-
-            if not no_checkout:
-                with target.timed_activity("Staging sources to {}".format(directory)):
-                    target._open_workspace()
-
-            # Saving the workspace once it is set up means that if the next workspace fails to be created before
-            # the configuration gets saved. The successfully created workspace still gets saved.
-            workspaces.save_config()
+            workspaces.create_workspace(target, directory, checkout=not no_checkout)
             self._message(MessageType.INFO, "Created a workspace for element: {}"
                           .format(target._get_full_name()))
 
@@ -672,10 +669,7 @@ class Stream():
                                       .format(workspace_path, e)) from e
 
             workspaces.delete_workspace(element._get_full_name())
-            workspaces.create_workspace(element._get_full_name(), workspace_path)
-
-            with element.timed_activity("Staging sources to {}".format(workspace_path)):
-                element._open_workspace()
+            workspaces.create_workspace(element, workspace_path, checkout=True)
 
             self._message(MessageType.INFO,
                           "Reset workspace for {} at: {}".format(element.name,
@@ -707,6 +701,20 @@ class Stream():
 
         return False
 
+    # workspace_is_required()
+    #
+    # Checks whether the workspace belonging to element_name is required to
+    # load the project
+    #
+    # Args:
+    #    element_name (str): The element whose workspace may be required
+    #
+    # Returns:
+    #    (bool): True if the workspace is required
+    def workspace_is_required(self, element_name):
+        invoked_elm = self._project.invoked_from_workspace_element()
+        return invoked_elm == element_name
+
     # workspace_list
     #
     # Serializes the workspaces and dumps them in YAML to stdout.
@@ -724,87 +732,6 @@ class Stream():
             'workspaces': workspaces
         })
 
-    # source_bundle()
-    #
-    # Create a host buildable tarball bundle for the given target.
-    #
-    # Args:
-    #    target (str): The target element to bundle
-    #    directory (str): The directory to output the tarball
-    #    track_first (bool): Track new source references before bundling
-    #    compression (str): The compression type to use
-    #    force (bool): Overwrite an existing tarball
-    #
-    def source_bundle(self, target, directory, *,
-                      track_first=False,
-                      force=False,
-                      compression="gz",
-                      except_targets=()):
-
-        if track_first:
-            track_targets = (target,)
-        else:
-            track_targets = ()
-
-        elements, track_elements = self._load((target,), track_targets,
-                                              selection=PipelineSelection.ALL,
-                                              except_targets=except_targets,
-                                              track_selection=PipelineSelection.ALL,
-                                              fetch_subprojects=True)
-
-        # source-bundle only supports one target
-        target = self.targets[0]
-
-        self._message(MessageType.INFO, "Bundling sources for target {}".format(target.name))
-
-        # Find the correct filename for the compression algorithm
-        tar_location = os.path.join(directory, target.normal_name + ".tar")
-        if compression != "none":
-            tar_location += "." + compression
-
-        # Attempt writing a file to generate a good error message
-        # early
-        #
-        # FIXME: A bit hackish
-        try:
-            open(tar_location, mode="x")
-            os.remove(tar_location)
-        except IOError as e:
-            raise StreamError("Cannot write to {0}: {1}"
-                              .format(tar_location, e)) from e
-
-        # Fetch and possibly track first
-        #
-        self._fetch(elements, track_elements=track_elements)
-
-        # We don't use the scheduler for this as it is almost entirely IO
-        # bound.
-
-        # Create a temporary directory to build the source tree in
-        builddir = self._context.builddir
-        os.makedirs(builddir, exist_ok=True)
-        prefix = "{}-".format(target.normal_name)
-
-        with TemporaryDirectory(prefix=prefix, dir=builddir) as tempdir:
-            source_directory = os.path.join(tempdir, 'source')
-            try:
-                os.makedirs(source_directory)
-            except OSError as e:
-                raise StreamError("Failed to create directory: {}"
-                                  .format(e)) from e
-
-            # Any elements that don't implement _write_script
-            # should not be included in the later stages.
-            elements = [
-                element for element in elements
-                if self._write_element_script(source_directory, element)
-            ]
-
-            self._write_element_sources(os.path.join(tempdir, "source"), elements)
-            self._write_build_script(tempdir, elements)
-            self._collect_sources(tempdir, tar_location,
-                                  target.normal_name, compression)
-
     # redirect_element_names()
     #
     # Takes a list of element names and returns a list where elements have been
@@ -1185,6 +1112,54 @@ class Stream():
 
         sandbox_vroot.export_files(directory, can_link=True, can_destroy=True)
 
+    # Helper function for source_checkout()
+    def _source_checkout(self, elements,
+                         location=None,
+                         force=False,
+                         deps='none',
+                         fetch=False,
+                         tar=False,
+                         include_build_scripts=False):
+        location = os.path.abspath(location)
+        location_parent = os.path.abspath(os.path.join(location, ".."))
+
+        # Stage all our sources in a temporary directory. The this
+        # directory can be used to either construct a tarball or moved
+        # to the final desired location.
+        temp_source_dir = tempfile.TemporaryDirectory(dir=location_parent)
+        try:
+            self._write_element_sources(temp_source_dir.name, elements)
+            if include_build_scripts:
+                self._write_build_scripts(temp_source_dir.name, elements)
+            if tar:
+                self._create_tarball(temp_source_dir.name, location)
+            else:
+                self._move_directory(temp_source_dir.name, location, force)
+        except OSError as e:
+            raise StreamError("Failed to checkout sources to {}: {}"
+                              .format(location, e)) from e
+        finally:
+            with suppress(FileNotFoundError):
+                temp_source_dir.cleanup()
+
+    # Move a directory src to dest. This will work across devices and
+    # may optionaly overwrite existing files.
+    def _move_directory(self, src, dest, force=False):
+        def is_empty_dir(path):
+            return os.path.isdir(dest) and not os.listdir(dest)
+
+        try:
+            os.rename(src, dest)
+            return
+        except OSError:
+            pass
+
+        if force or is_empty_dir(dest):
+            try:
+                utils.link_files(src, dest)
+            except utils.UtilError as e:
+                raise StreamError("Failed to move directory: {}".format(e)) from e
+
     # Write the element build script to the given directory
     def _write_element_script(self, directory, element):
         try:
@@ -1201,8 +1176,28 @@ class Stream():
                 os.makedirs(element_source_dir)
                 element._stage_sources_at(element_source_dir, mount_workspaces=False)
 
+    # Create a tarball from the content of directory
+    def _create_tarball(self, directory, tar_name):
+        try:
+            with utils.save_file_atomic(tar_name, mode='wb') as f:
+                # This TarFile does not need to be explicitly closed
+                # as the underlying file object will be closed be the
+                # save_file_atomic contect manager
+                tarball = tarfile.open(fileobj=f, mode='w')
+                for item in os.listdir(str(directory)):
+                    file_to_add = os.path.join(directory, item)
+                    tarball.add(file_to_add, arcname=item)
+        except OSError as e:
+            raise StreamError("Failed to create tar archive: {}".format(e)) from e
+
+    # Write all the build_scripts for elements in the directory location
+    def _write_build_scripts(self, location, elements):
+        for element in elements:
+            self._write_element_script(location, element)
+        self._write_master_build_script(location, elements)
+
     # Write a master build script to the sandbox
-    def _write_build_script(self, directory, elements):
+    def _write_master_build_script(self, directory, elements):
 
         module_string = ""
         for element in elements:

buildstream/_workspaces.py

@@ -25,6 +25,202 @@ from ._exceptions import LoadError, LoadErrorReason
 
 
 BST_WORKSPACE_FORMAT_VERSION = 3
+BST_WORKSPACE_PROJECT_FORMAT_VERSION = 1
+WORKSPACE_PROJECT_FILE = ".bstproject.yaml"
+
+
+# WorkspaceProject()
+#
+# An object to contain various helper functions and data required for
+# referring from a workspace back to buildstream.
+#
+# Args:
+#    directory (str): The directory that the workspace exists in.
+#
+class WorkspaceProject():
+    def __init__(self, directory):
+        self._projects = []
+        self._directory = directory
+
+    # get_default_project_path()
+    #
+    # Retrieves the default path to a project.
+    #
+    # Returns:
+    #    (str): The path to a project
+    #
+    def get_default_project_path(self):
+        return self._projects[0]['project-path']
+
+    # get_default_element()
+    #
+    # Retrieves the name of the element that owns this workspace.
+    #
+    # Returns:
+    #    (str): The name of an element
+    #
+    def get_default_element(self):
+        return self._projects[0]['element-name']
+
+    # to_dict()
+    #
+    # Turn the members data into a dict for serialization purposes
+    #
+    # Returns:
+    #    (dict): A dict representation of the WorkspaceProject
+    #
+    def to_dict(self):
+        ret = {
+            'projects': self._projects,
+            'format-version': BST_WORKSPACE_PROJECT_FORMAT_VERSION,
+        }
+        return ret
+
+    # from_dict()
+    #
+    # Loads a new WorkspaceProject from a simple dictionary
+    #
+    # Args:
+    #    directory (str): The directory that the workspace exists in
+    #    dictionary (dict): The dict to generate a WorkspaceProject from
+    #
+    # Returns:
+    #   (WorkspaceProject): A newly instantiated WorkspaceProject
+    #
+    @classmethod
+    def from_dict(cls, directory, dictionary):
+        # Only know how to handle one format-version at the moment.
+        format_version = int(dictionary['format-version'])
+        assert format_version == BST_WORKSPACE_PROJECT_FORMAT_VERSION, \
+            "Format version {} not found in {}".format(BST_WORKSPACE_PROJECT_FORMAT_VERSION, dictionary)
+
+        workspace_project = cls(directory)
+        for item in dictionary['projects']:
+            workspace_project.add_project(item['project-path'], item['element-name'])
+
+        return workspace_project
+
+    # load()
+    #
+    # Loads the WorkspaceProject for a given directory.
+    #
+    # Args:
+    #    directory (str): The directory
+    # Returns:
+    #    (WorkspaceProject): The created WorkspaceProject, if in a workspace, or
+    #    (NoneType): None, if the directory is not inside a workspace.
+    #
+    @classmethod
+    def load(cls, directory):
+        workspace_file = os.path.join(directory, WORKSPACE_PROJECT_FILE)
+        if os.path.exists(workspace_file):
+            data_dict = _yaml.load(workspace_file)
+            return cls.from_dict(directory, data_dict)
+        else:
+            return None
+
+    # write()
+    #
+    # Writes the WorkspaceProject to disk
+    #
+    def write(self):
+        os.makedirs(self._directory, exist_ok=True)
+        _yaml.dump(self.to_dict(), self.get_filename())
+
+    # get_filename()
+    #
+    # Returns the full path to the workspace local project file
+    #
+    def get_filename(self):
+        return os.path.join(self._directory, WORKSPACE_PROJECT_FILE)
+
+    # add_project()
+    #
+    # Adds an entry containing the project's path and element's name.
+    #
+    # Args:
+    #    project_path (str): The path to the project that opened the workspace.
+    #    element_name (str): The name of the element that the workspace belongs to.
+    #
+    def add_project(self, project_path, element_name):
+        assert (project_path and element_name)
+        self._projects.append({'project-path': project_path, 'element-name': element_name})
+
+
+# WorkspaceProjectCache()
+#
+# A class to manage workspace project data for multiple workspaces.
+#
+class WorkspaceProjectCache():
+    def __init__(self):
+        self._projects = {}  # Mapping of a workspace directory to its WorkspaceProject
+
+    # get()
+    #
+    # Returns a WorkspaceProject for a given directory, retrieving from the cache if
+    # present.
+    #
+    # Args:
+    #    directory (str): The directory to search for a WorkspaceProject.
+    #
+    # Returns:
+    #    (WorkspaceProject): The WorkspaceProject that was found for that directory.
+    #    or      (NoneType): None, if no WorkspaceProject can be found.
+    #
+    def get(self, directory):
+        try:
+            workspace_project = self._projects[directory]
+        except KeyError:
+            workspace_project = WorkspaceProject.load(directory)
+            if workspace_project:
+                self._projects[directory] = workspace_project
+
+        return workspace_project
+
+    # add()
+    #
+    # Adds the project path and element name to the WorkspaceProject that exists
+    # for that directory
+    #
+    # Args:
+    #    directory (str): The directory to search for a WorkspaceProject.
+    #    project_path (str): The path to the project that refers to this workspace
+    #    element_name (str): The element in the project that was refers to this workspace
+    #
+    # Returns:
+    #    (WorkspaceProject): The WorkspaceProject that was found for that directory.
+    #
+    def add(self, directory, project_path, element_name):
+        workspace_project = self.get(directory)
+        if not workspace_project:
+            workspace_project = WorkspaceProject(directory)
+            self._projects[directory] = workspace_project
+
+        workspace_project.add_project(project_path, element_name)
+        return workspace_project
+
+    # remove()
+    #
+    # Removes the project path and element name from the WorkspaceProject that exists
+    # for that directory.
+    #
+    # NOTE: This currently just deletes the file, but with support for multiple
+    # projects opening the same workspace, this will involve decreasing the count
+    # and deleting the file if there are no more projects.
+    #
+    # Args:
+    #    directory (str): The directory to search for a WorkspaceProject.
+    #
+    def remove(self, directory):
+        workspace_project = self.get(directory)
+        if not workspace_project:
+            raise LoadError(LoadErrorReason.MISSING_FILE,
+                            "Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE))
+        path = workspace_project.get_filename()
+        try:
+            os.unlink(path)
+        except FileNotFoundError:
+            pass
 
 
 # Workspace()
@@ -174,10 +370,15 @@ class Workspace():
         if recalculate or self._key is None:
             fullpath = self.get_absolute_path()
 
+            excluded_files = (WORKSPACE_PROJECT_FILE,)
+
             # Get a list of tuples of the the project relative paths and fullpaths
             if os.path.isdir(fullpath):
                 filelist = utils.list_relative_paths(fullpath)
-                filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
+                filelist = [
+                    (relpath, os.path.join(fullpath, relpath)) for relpath in filelist
+                    if relpath not in excluded_files
+                ]
             else:
                 filelist = [(self.get_absolute_path(), fullpath)]
 
@@ -199,12 +400,14 @@ class Workspace():
 #
 # Args:
 #    toplevel_project (Project): Top project used to resolve paths.
+#    workspace_project_cache (WorkspaceProjectCache): The cache of WorkspaceProjects
 #
 class Workspaces():
-    def __init__(self, toplevel_project):
+    def __init__(self, toplevel_project, workspace_project_cache):
         self._toplevel_project = toplevel_project
         self._bst_directory = os.path.join(toplevel_project.directory, ".bst")
         self._workspaces = self._load_config()
+        self._workspace_project_cache = workspace_project_cache
 
     # list()
     #
@@ -219,19 +422,36 @@ class Workspaces():
 
     # create_workspace()
     #
-    # Create a workspace in the given path for the given element.
+    # Create a workspace in the given path for the given element, and potentially
+    # checks-out the target into it.
     #
     # Args:
-    #    element_name (str) - The element name to create a workspace for
+    #    target (Element) - The element to create a workspace for
     #    path (str) - The path in which the workspace should be kept
+    #    checkout (bool): Whether to check-out the element's sources into the directory
     #
-    def create_workspace(self, element_name, path):
-        if path.startswith(self._toplevel_project.directory):
-            path = os.path.relpath(path, self._toplevel_project.directory)
+    def create_workspace(self, target, path, *, checkout):
+        element_name = target._get_full_name()
+        project_dir = self._toplevel_project.directory
+        if path.startswith(project_dir):
+            workspace_path = os.path.relpath(path, project_dir)
+        else:
+            workspace_path = path
 
-        self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
+        self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path)
 
-        return self._workspaces[element_name]
+        if checkout:
+            with target.timed_activity("Staging sources to {}".format(path)):
+                target._open_workspace()
+
+        workspace_project = self._workspace_project_cache.add(path, project_dir, element_name)
+        project_file_path = workspace_project.get_filename()
+
+        if os.path.exists(project_file_path):
+            target.warn("{} was staged from this element's sources".format(WORKSPACE_PROJECT_FILE))
+        workspace_project.write()
+
+        self.save_config()
 
     # get_workspace()
     #
@@ -280,8 +500,19 @@ class Workspaces():
     #    element_name (str) - The element name whose workspace to delete
     #
     def delete_workspace(self, element_name):
+        workspace = self.get_workspace(element_name)
         del self._workspaces[element_name]
 
+        # Remove from the cache if it exists
+        try:
+            self._workspace_project_cache.remove(workspace.get_absolute_path())
+        except LoadError as e:
+            # We might be closing a workspace with a deleted directory
+            if e.reason == LoadErrorReason.MISSING_FILE:
+                pass
+            else:
+                raise
+
     # save_config()
     #
     # Dump the current workspace element to the project configuration

buildstream/buildelement.py

@@ -35,6 +35,14 @@ This section will give a brief summary of how some of the common features work,
 some of them or the variables they use will be further detailed in the following
 sections.
 
+The `strip-binaries` variable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The `strip-binaries` variable is by default **empty**. You need to use the
+appropiate commands depending of the system you are building.
+If you are targetting Linux, ones known to work are the ones used by the
+`freedesktop-sdk <https://freedesktop-sdk.io/>`_, you can take a look to them in their
+`project.conf <https://gitlab.com/freedesktop-sdk/freedesktop-sdk/blob/freedesktop-sdk-18.08.21/project.conf#L74>`_
+
 Location for running commands
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 The ``command-subdir`` variable sets where the build commands will be executed,

buildstream/data/projectconfig.yaml

@@ -44,38 +44,8 @@ variables:
   # Indicates the build installation directory in the sandbox
   install-root: /buildstream-install
 
-  # Arguments for tooling used when stripping debug symbols
-  objcopy-link-args: --add-gnu-debuglink
-  objcopy-extract-args: |
-
-    --only-keep-debug --compress-debug-sections
-
-  strip-args: |
-
-    --remove-section=.comment --remove-section=.note --strip-unneeded
-
-  # Generic implementation for stripping debugging symbols
-  strip-binaries: |
-
-    cd "%{install-root}" && find -type f \
-      '(' -perm -111 -o -name '*.so*' \
-          -o -name '*.cmxs' -o -name '*.node' ')' \
-      -exec sh -ec \
-      'read -n4 hdr <"$1" # check for elf header
-       case "$1" in
-         %{install-root}%{debugdir}/*)
-           exit 0
-           ;;
-       esac
-       if [ "$hdr" != "$(printf \\x7fELF)" ]; then
-           exit 0
-       fi
-       debugfile="%{install-root}%{debugdir}/$1"
-       mkdir -p "$(dirname "$debugfile")"
-       objcopy %{objcopy-extract-args} "$1" "$debugfile"
-       chmod 644 "$debugfile"
-       strip %{strip-args} "$1"
-       objcopy %{objcopy-link-args} "$debugfile" "$1"' - {} ';'
+  # You need to override this with the commands specific for your system
+  strip-binaries: ""
 
   # Generic implementation for reproducible python builds
   fix-pyc-timestamps: |

buildstream/data/userconfig.yaml

@@ -128,6 +128,14 @@ prompt:
   #
   really-workspace-close-remove-dir: ask
 
+  # Whether to really proceed with 'bst workspace close' when doing so would
+  # stop them from running bst commands in this workspace.
+  #
+  #  ask - Ask the user if they are sure.
+  #  yes - Always close, without asking.
+  #
+  really-workspace-close-project-inaccessible: ask
+
   # Whether to really proceed with 'bst workspace reset' doing a hard reset of
   # a workspace, potentially losing changes.
   #

buildstream/plugins/sources/git.py

@@ -247,7 +247,7 @@ class GitMirror(SourceFetcher):
         else:
             remote_name = "origin"
 
-        self.source.call([self.source.host_git, 'fetch', remote_name, '--prune'],
+        self.source.call([self.source.host_git, 'fetch', remote_name, '--prune', '--force', '--tags'],
                          fail="Failed to fetch from remote git repository: {}".format(url),
                          fail_temporarily=True,
                          cwd=self.mirror)

buildstream/utils.py

@@ -1259,3 +1259,34 @@ def _message_digest(message_buffer):
     digest.hash = sha.hexdigest()
     digest.size_bytes = len(message_buffer)
     return digest
+
+
+# _search_upward_for_files()
+#
+# Searches upwards (from directory, then directory's parent directory...)
+# for any of the files listed in `filenames`.
+#
+# If multiple filenames are specified, and present in the same directory,
+# the first filename in the list will be returned.
+#
+# Args:
+#    directory (str): The directory to begin searching for files from
+#    filenames (list of str): The names of files to search for
+#
+# Returns:
+#    (str): The directory a file was found in, or None
+#    (str): The name of the first file that was found in that directory, or None
+#
+def _search_upward_for_files(directory, filenames):
+    directory = os.path.abspath(directory)
+    while True:
+        for filename in filenames:
+            file_path = os.path.join(directory, filename)
+            if os.path.isfile(file_path):
+                return directory, filename
+
+        parent_dir = os.path.dirname(directory)
+        if directory == parent_dir:
+            # i.e. we've reached the root of the filesystem
+            return None, None
+        directory = parent_dir

doc/sessions/developing.run

@@ -24,6 +24,11 @@ commands:
   output: ../source/sessions/developing-build-after-changes.html
   command: build hello.bst
 
+# Rebuild, from the workspace
+- directory: ../examples/developing/workspace_hello
+  output: ../source/sessions/developing-build-after-changes-workspace.html
+  command: build
+
 # Capture shell output with changes
 - directory: ../examples/developing/
   output: ../source/sessions/developing-shell-after-changes.html

doc/source/developing/workspaces.rst

@@ -50,11 +50,16 @@ We can open workspace_hello/hello.c and make the following change:
 .. literalinclude:: ../../examples/developing/update.patch
     :language: diff
 
-Now, rebuild the hello.bst element
+Now, rebuild the hello.bst element.
 
 .. raw:: html
    :file: ../sessions/developing-build-after-changes.html
 
+Note that if you run the command from inside the workspace, the element name is optional.
+
+.. raw:: html
+   :file: ../sessions/developing-build-after-changes-workspace.html
+
 Now running the hello command using bst shell:
 
 .. raw:: html

doc/source/sessions-stored/developing-build-after-changes-workspace.html

+<!--
+    WARNING: This file was generated with bst2html.py
+-->
+<div class="highlight" style="font-size:x-small"><pre>
+<span style="color:#C4A000;font-weight:bold">user@host</span>:<span style="color:#3456A4;font-weight:bold">~/workspace_hello</span>$ bst build
+
+<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#3465A4"><span style=""><span style="opacity:0.5">START  </span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Build
+<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#3465A4"><span style=""><span style="opacity:0.5">START  </span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Loading elements
+<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#4E9A06"><span style=""><span style="opacity:0.5">SUCCESS</span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Loading elements
+<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#3465A4"><span style=""><span style="opacity:0.5">START  </span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Resolving elements
+<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#4E9A06"><span style=""><span style="opacity:0.5">SUCCESS</span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Resolving elements
+<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#3465A4"><span style=""><span style="opacity:0.5">START  </span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Resolving cached state
+<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#4E9A06"><span style=""><span style="opacity:0.5">SUCCESS</span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Resolving cached state
+<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">--</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#3465A4"><span style=""><span style="opacity:0.5">START  </span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Checking sources
+<span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#4E9A06"><span style=""><span style="opacity:0.5">SUCCESS</span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Checking sources
+
+<span style="color:#C4A000"><span style="">BuildStream Version 1.3.0+1032.g41813d3a.dirty
+</span></span><span style="color:#06989A"><span style="opacity:0.5">  Session Start: </span></span><span style="color:#C4A000">Wednesday, 05-12-2018 at 16:47:38</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Project:       </span></span><span style="color:#C4A000">developing (/home/user/workspace/buildstream/buildstream/doc/examples/developing)</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Targets:       </span></span><span style="color:#C4A000">hello.bst</span>
+
+<span style="color:#C4A000"><span style="">User Configuration
+</span></span><span style="color:#06989A"><span style="opacity:0.5">  Configuration File:      </span></span><span style="color:#C4A000">/home/user/workspace/buildstream/buildstream/doc/run-bst-7ocq4_a7/buildstream.conf</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Log Files:               </span></span><span style="color:#C4A000">/home/user/workspace/buildstream/buildstream/doc/run-bst-7ocq4_a7/logs</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Source Mirrors:          </span></span><span style="color:#C4A000">/home/user/workspace/buildstream/buildstream/doc/run-bst-7ocq4_a7/sources</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Build Area:              </span></span><span style="color:#C4A000">/home/user/workspace/buildstream/buildstream/doc/run-bst-7ocq4_a7/build</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Artifact Cache:          </span></span><span style="color:#C4A000">/home/user/workspace/buildstream/buildstream/doc/run-bst-7ocq4_a7/artifacts</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Strict Build Plan:       </span></span><span style="color:#C4A000">Yes</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Maximum Fetch Tasks:     </span></span><span style="color:#C4A000">10</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Maximum Build Tasks:     </span></span><span style="color:#C4A000">4</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Maximum Push Tasks:      </span></span><span style="color:#C4A000">4</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Maximum Network Retries: </span></span><span style="color:#C4A000">2</span>
+
+<span style="color:#C4A000"><span style="">Pipeline
+</span></span><span style="color:#75507B">      cached</span> <span style="color:#C4A000">9afe69d645f0bee106749bc2101aae16ef437bb51e1b343ef1f16f04f0572efb</span> <span style="color:#3465A4"><span style="">base/alpine.bst</span></span> 
+<span style="color:#75507B">      cached</span> <span style="color:#C4A000">19f7c50c7a1db9ae4babe9d1f34f4cdbbf2428827d48673861fd1452d6c7e16b</span> <span style="color:#3465A4"><span style="">base.bst</span></span> 
+<span style="color:#75507B">      cached</span> <span style="color:#C4A000">faa419610e7309d36d15926a81a8d75bbc113443c23c8162e63843dd86b5f56a</span> <span style="color:#3465A4"><span style="">hello.bst</span></span> Workspace: /home/user/workspace/buildstream/buildstream/doc/examples/developing/workspace_hello
+<span style="color:#06989A"><span style="opacity:0.5">===============================================================================
+</span></span><span style="color:#06989A"><span style="opacity:0.5">[</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">:</span></span><span style="color:#C4A000">00</span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">][</span></span><span style="color:#06989A"><span style="opacity:0.5">] </span></span><span style="color:#4E9A06"><span style=""><span style="opacity:0.5">SUCCESS</span></span></span><span style="color:#06989A"><span style="opacity:0.5"> </span></span>Build
+
+<span style="color:#C4A000"><span style="">Pipeline Summary
+</span></span><span style="color:#06989A"><span style="opacity:0.5">  Total:       </span></span><span style="color:#C4A000">3</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Session:     </span></span><span style="color:#C4A000">0</span>
+<span style="color:#06989A"><span style="opacity:0.5">  Fetch Queue: </span></span><span style="color:#C4A000">processed </span><span style="color:#4E9A06">0</span><span style="color:#06989A"><span style="opacity:0.5">, </span></span><span style="color:#C4A000">skipped </span><span style="color:#C4A000">0</span><span style="color:#06989A"><span style="opacity:0.5">, </span></span><span style="color:#C4A000">failed </span><span style="color:#CC0000"><span style="opacity:0.5">0</span></span> 
+<span style="color:#06989A"><span style="opacity:0.5">  Build Queue: </span></span><span style="color:#C4A000">processed </span><span style="color:#4E9A06">0</span><span style="color:#06989A"><span style="opacity:0.5">, </span></span><span style="color:#C4A000">skipped </span><span style="color:#C4A000">0</span><span style="color:#06989A"><span style="opacity:0.5">, </span></span><span style="color:#C4A000">failed </span><span style="color:#CC0000"><span style="opacity:0.5">0</span></span>
+</pre></div>

doc/source/using_commands.rst

@@ -86,13 +86,6 @@ project's main directory.
 
 ----
 
-.. _invoking_source_bundle:
-
-.. click:: buildstream._frontend.cli:source_bundle
-   :prog: bst source bundle
-
-----
-
 .. _invoking_workspace:
 
 .. click:: buildstream._frontend.cli:workspace

tests/artifactcache/expiry.py

@@ -18,6 +18,7 @@
 #
 
 import os
+from unittest import mock
 
 import pytest
 
@@ -311,6 +312,8 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
     ("0", True),
     ("-1", False),
     ("pony", False),
+    ("7K", False),
+    ("70%", False),
     ("200%", False)
 ])
 @pytest.mark.datafiles(DATA_DIR)
@@ -324,7 +327,35 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success):
         }
     })
 
+    # We patch how we get space information
+    # Ideally we would instead create a FUSE device on which we control
+    # everything.
+    # If the value is a percentage, we fix the current values to take into
+    # account the block size, since this is important in how we compute the size
+
+    if quota.endswith("%"):  # We set the used space at 60% of total space
+        stats = os.statvfs(".")
+        free_space = 0.6 * stats.f_bsize * stats.f_blocks
+        total_space = stats.f_bsize * stats.f_blocks
+    else:
+        free_space = 6000
+        total_space = 10000
+
+    volume_space_patch = mock.patch(
+        "buildstream._artifactcache.artifactcache.ArtifactCache._get_volume_space_info_for",
+        autospec=True,
+        return_value=(free_space, total_space),
+    )
+
+    cache_size_patch = mock.patch(
+        "buildstream._artifactcache.artifactcache.ArtifactCache.get_cache_size",
+        autospec=True,
+        return_value=0,
+    )
+
+    with volume_space_patch, cache_size_patch:
         res = cli.run(project=project, args=['workspace', 'list'])
+
     if success:
         res.assert_success()
     else:

tests/cachekey/project/elements/build1.expected

-a0d000abc1dea8714cd27f348d0b798b35e7246c44e330c4b3f7912fabacc6db
\ No newline at end of file
+dadb8f86874f714b4f6d4c9025332934efb7e85c38f6a68b1267746ae8f43f24

tests/cachekey/project/elements/build2.expected

-79f546a78748d943a6958c99ab4ad03305f96fefd0b424b6b246b0c9816e00c6
\ No newline at end of file
+f81cefce283dd3581ba2fc865ff9c2763119274b114b12edb4e87196cfff8b2a