NEWS

@@ -2,6 +2,12 @@
 buildstream 1.3.1
 =================
 
+  o Added `bst artifact log` subcommand for viewing build logs.
+
+  o BREAKING CHANGE: Default strip-commands have been removed as they are too
+    specific. Recommendation if you are building in Linux is to use the
+    ones being used in freedesktop-sdk project, for example
+
   o All elements must now be suffixed with `.bst`
     Attempting to use an element that does not have the `.bst` extension,
     will result in a warning.
@@ -74,6 +80,19 @@ buildstream 1.3.1
   o Add sandbox API for command batching and use it for build, script, and
     compose elements.
 
+  o BREAKING CHANGE: The `git` plugin does not create a local `.git`
+    repository by default.  If `git describe` is required to work, the
+    plugin has now a tag tracking feature instead. This can be enabled
+    by setting 'track-tags'.
+
+  o Opening a workspace now creates a .bstproject.yaml file that allows buildstream
+    commands to be run from a workspace that is not inside a project.
+
+  o Specifying an element is now optional for some commands when buildstream is run
+    from inside a workspace - the 'build', 'checkout', 'fetch', 'pull', 'push',
+    'shell', 'show', 'source-checkout', 'track', 'workspace close' and 'workspace reset'
+    commands are affected.
+
 
 =================
 buildstream 1.1.5

buildstream/__init__.py

@@ -28,7 +28,7 @@ if "_BST_COMPLETION" not in os.environ:
 
     from .utils import UtilError, ProgramNotFoundError
     from .sandbox import Sandbox, SandboxFlags, SandboxCommandError
-    from .types import Scope, Consistency
+    from .types import Scope, Consistency, CoreWarnings
     from .plugin import Plugin
     from .source import Source, SourceError, SourceFetcher
     from .element import Element, ElementError

buildstream/_artifactcache/cascache.py

@@ -427,10 +427,7 @@ class CASCache():
     def push_message(self, remote, message):
 
         message_buffer = message.SerializeToString()
-        message_sha = hashlib.sha256(message_buffer)
-        message_digest = remote_execution_pb2.Digest()
-        message_digest.hash = message_sha.hexdigest()
-        message_digest.size_bytes = len(message_buffer)
+        message_digest = utils._message_digest(message_buffer)
 
         remote.init()
 

buildstream/_context.py

@@ -32,7 +32,7 @@ from ._message import Message, MessageType
 from ._profile import Topics, profile_start, profile_end
 from ._artifactcache import ArtifactCache
 from ._artifactcache.cascache import CASCache
-from ._workspaces import Workspaces
+from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE
 from .plugin import _plugin_lookup
 
 
@@ -47,9 +47,12 @@ from .plugin import _plugin_lookup
 # verbosity levels and basically anything pertaining to the context
 # in which BuildStream was invoked.
 #
+# Args:
+#    directory (str): The directory that buildstream was invoked in
+#
 class Context():
 
-    def __init__(self):
+    def __init__(self, directory=None):
 
         # Filename indicating which configuration file was used, or None for the defaults
         self.config_origin = None
@@ -122,6 +125,10 @@ class Context():
         # remove a workspace directory.
         self.prompt_workspace_close_remove_dir = None
 
+        # Boolean, whether we double-check with the user that they meant to
+        # close the workspace when they're using it to access the project.
+        self.prompt_workspace_close_project_inaccessible = None
+
         # Boolean, whether we double-check with the user that they meant to do
         # a hard reset of a workspace, potentially losing changes.
         self.prompt_workspace_reset_hard = None
@@ -140,9 +147,11 @@ class Context():
         self._projects = []
         self._project_overrides = {}
         self._workspaces = None
+        self._workspace_project_cache = WorkspaceProjectCache()
         self._log_handle = None
         self._log_filename = None
         self._cascache = None
+        self._directory = directory
 
     # load()
     #
@@ -250,12 +259,15 @@ class Context():
             defaults, Mapping, 'prompt')
         _yaml.node_validate(prompt, [
             'auto-init', 'really-workspace-close-remove-dir',
+            'really-workspace-close-project-inaccessible',
             'really-workspace-reset-hard',
         ])
         self.prompt_auto_init = _node_get_option_str(
             prompt, 'auto-init', ['ask', 'no']) == 'ask'
         self.prompt_workspace_close_remove_dir = _node_get_option_str(
             prompt, 'really-workspace-close-remove-dir', ['ask', 'yes']) == 'ask'
+        self.prompt_workspace_close_project_inaccessible = _node_get_option_str(
+            prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask'
         self.prompt_workspace_reset_hard = _node_get_option_str(
             prompt, 'really-workspace-reset-hard', ['ask', 'yes']) == 'ask'
 
@@ -285,7 +297,7 @@ class Context():
     #
     def add_project(self, project):
         if not self._projects:
-            self._workspaces = Workspaces(project)
+            self._workspaces = Workspaces(project, self._workspace_project_cache)
         self._projects.append(project)
 
     # get_projects():
@@ -312,6 +324,16 @@ class Context():
     def get_workspaces(self):
         return self._workspaces
 
+    # get_workspace_project_cache():
+    #
+    # Return the WorkspaceProjectCache object used for this BuildStream invocation
+    #
+    # Returns:
+    #    (WorkspaceProjectCache): The WorkspaceProjectCache object
+    #
+    def get_workspace_project_cache(self):
+        return self._workspace_project_cache
+
     # get_overrides():
     #
     # Fetch the override dictionary for the active project. This returns
@@ -627,6 +649,20 @@ class Context():
             self._cascache = CASCache(self.artifactdir)
         return self._cascache
 
+    # guess_element()
+    #
+    # Attempts to interpret which element the user intended to run commands on
+    #
+    # Returns:
+    #    (str) The name of the element, or None if no element can be guessed
+    def guess_element(self):
+        workspace_project_dir, _ = utils._search_upward_for_files(self._directory, [WORKSPACE_PROJECT_FILE])
+        if workspace_project_dir:
+            workspace_project = self._workspace_project_cache.get(workspace_project_dir)
+            return workspace_project.get_default_element()
+        else:
+            return None
+
 
 # _node_get_option_str()
 #

buildstream/_frontend/app.py

@@ -164,7 +164,7 @@ class App():
         # Load the Context
         #
         try:
-            self.context = Context()
+            self.context = Context(directory)
             self.context.load(config)
         except BstError as e:
             self._error_exit(e, "Error loading user configuration")

buildstream/_frontend/cli.py

 import os
 import sys
+from contextlib import ExitStack
+from fnmatch import fnmatch
+from tempfile import TemporaryDirectory
 
 import click
 from .. import _yaml
@@ -59,18 +62,9 @@ def complete_target(args, incomplete):
     :return: all the possible user-specified completions for the param
     """
 
+    from .. import utils
     project_conf = 'project.conf'
 
-    def ensure_project_dir(directory):
-        directory = os.path.abspath(directory)
-        while not os.path.isfile(os.path.join(directory, project_conf)):
-            parent_dir = os.path.dirname(directory)
-            if directory == parent_dir:
-                break
-            directory = parent_dir
-
-        return directory
-
     # First resolve the directory, in case there is an
     # active --directory/-C option
     #
@@ -89,7 +83,7 @@ def complete_target(args, incomplete):
     else:
         # Check if this directory or any of its parent directories
         # contain a project config file
-        base_directory = ensure_project_dir(base_directory)
+        base_directory, _ = utils._search_upward_for_files(base_directory, [project_conf])
 
     # Now parse the project.conf just to find the element path,
     # this is unfortunately a bit heavy.
@@ -116,6 +110,23 @@ def complete_target(args, incomplete):
     return complete_list
 
 
+def complete_artifact(args, incomplete):
+    from .._context import Context
+    ctx = Context()
+
+    config = None
+    for i, arg in enumerate(args):
+        if arg in ('-c', '--config'):
+            config = args[i + 1]
+    ctx.load(config)
+
+    # element targets are valid artifact names
+    complete_list = complete_target(args, incomplete)
+    complete_list.extend(ref for ref in ctx.artifactcache.cas.list_refs() if ref.startswith(incomplete))
+
+    return complete_list
+
+
 def override_completions(cmd, cmd_param, args, incomplete):
     """
     :param cmd_param: command definition
@@ -130,13 +141,15 @@ def override_completions(cmd, cmd_param, args, incomplete):
     # We can't easily extend click's data structures without
     # modifying click itself, so just do some weak special casing
     # right here and select which parameters we want to handle specially.
-    if isinstance(cmd_param.type, click.Path) and \
-       (cmd_param.name == 'elements' or
-        cmd_param.name == 'element' or
-        cmd_param.name == 'except_' or
-        cmd_param.opts == ['--track'] or
-        cmd_param.opts == ['--track-except']):
-        return complete_target(args, incomplete)
+    if isinstance(cmd_param.type, click.Path):
+        if (cmd_param.name == 'elements' or
+                cmd_param.name == 'element' or
+                cmd_param.name == 'except_' or
+                cmd_param.opts == ['--track'] or
+                cmd_param.opts == ['--track-except']):
+            return complete_target(args, incomplete)
+        if cmd_param.name == 'artifacts':
+            return complete_artifact(args, incomplete)
 
     raise CompleteUnhandled()
 
@@ -325,10 +338,15 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
     if track_save:
         click.echo("WARNING: --track-save is deprecated, saving is now unconditional", err=True)
 
-    if track_all:
-        track_ = elements
-
     with app.initialized(session_name="Build"):
+        if not all_ and not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
+        if track_all:
+            track_ = elements
+
         app.stream.build(elements,
                          track_targets=track_,
                          track_except=track_except,
@@ -380,6 +398,11 @@ def fetch(app, elements, deps, track_, except_, track_cross_junctions):
         deps = PipelineSelection.ALL
 
     with app.initialized(session_name="Fetch"):
+        if not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
         app.stream.fetch(elements,
                          selection=deps,
                          except_targets=except_,
@@ -416,6 +439,11 @@ def track(app, elements, deps, except_, cross_junctions):
         all:   All dependencies of all specified elements
     """
     with app.initialized(session_name="Track"):
+        if not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
         # Substitute 'none' for 'redirect' so that element redirections
         # will be done
         if deps == 'none':
@@ -451,7 +479,13 @@ def pull(app, elements, deps, remote):
         none:  No dependencies, just the element itself
         all:   All dependencies
     """
+
     with app.initialized(session_name="Pull"):
+        if not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
         app.stream.pull(elements, selection=deps, remote=remote)
 
 
@@ -484,6 +518,11 @@ def push(app, elements, deps, remote):
         all:   All dependencies
     """
     with app.initialized(session_name="Push"):
+        if not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
         app.stream.push(elements, selection=deps, remote=remote)
 
 
@@ -554,6 +593,11 @@ def show(app, elements, deps, except_, order, format_):
             $'---------- %{name} ----------\\n%{vars}'
     """
     with app.initialized():
+        if not elements:
+            guessed_target = app.context.guess_element()
+            if guessed_target:
+                elements = (guessed_target,)
+
         dependencies = app.stream.load_selection(elements,
                                                  selection=deps,
                                                  except_targets=except_)
@@ -582,7 +626,7 @@ def show(app, elements, deps, except_, order, format_):
               help="Mount a file or directory into the sandbox")
 @click.option('--isolate', is_flag=True, default=False,
               help='Create an isolated build sandbox')
-@click.argument('element',
+@click.argument('element', required=False,
                 type=click.Path(readable=False))
 @click.argument('command', type=click.STRING, nargs=-1)
 @click.pass_obj
@@ -613,6 +657,11 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
         scope = Scope.RUN
 
     with app.initialized():
+        if not element:
+            element = app.context.guess_element()
+            if not element:
+                raise AppError('Missing argument "ELEMENT".')
+
         dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
         element = dependencies[0]
         prompt = app.shell_prompt(element)
@@ -650,15 +699,24 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
               help="Create a tarball from the artifact contents instead "
                    "of a file tree. If LOCATION is '-', the tarball "
                    "will be dumped to the standard output.")
-@click.argument('element',
+@click.argument('element', required=False,
                 type=click.Path(readable=False))
-@click.argument('location', type=click.Path())
+@click.argument('location', type=click.Path(), required=False)
 @click.pass_obj
 def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
     """Checkout a built artifact to the specified location
     """
     from ..element import Scope
 
+    if not element and not location:
+        click.echo("ERROR: LOCATION is not specified", err=True)
+        sys.exit(-1)
+
+    if element and not location:
+        # Nasty hack to get around click's optional args
+        location = element
+        element = None
+
     if hardlinks and tar:
         click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
         sys.exit(-1)
@@ -671,6 +729,11 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
         scope = Scope.NONE
 
     with app.initialized():
+        if not element:
+            element = app.context.guess_element()
+            if not element:
+                raise AppError('Missing argument "ELEMENT".')
+
         app.stream.checkout(element,
                             location=location,
                             force=force,
@@ -692,14 +755,28 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
               help='The dependencies whose sources to checkout (default: none)')
 @click.option('--fetch', 'fetch_', default=False, is_flag=True,
               help='Fetch elements if they are not fetched')
-@click.argument('element',
+@click.argument('element', required=False,
                 type=click.Path(readable=False))
-@click.argument('location', type=click.Path())
+@click.argument('location', type=click.Path(), required=False)
 @click.pass_obj
 def source_checkout(app, element, location, deps, fetch_, except_):
     """Checkout sources of an element to the specified location
     """
+    if not element and not location:
+        click.echo("ERROR: LOCATION is not specified", err=True)
+        sys.exit(-1)
+
+    if element and not location:
+        # Nasty hack to get around click's optional args
+        location = element
+        element = None
+
     with app.initialized():
+        if not element:
+            element = app.context.guess_element()
+            if not element:
+                raise AppError('Missing argument "ELEMENT".')
+
         app.stream.source_checkout(element,
                                    location=location,
                                    deps=deps,
@@ -756,11 +833,15 @@ def workspace_open(app, no_checkout, force, track_, directory, elements):
 def workspace_close(app, remove_dir, all_, elements):
     """Close a workspace"""
 
-    if not (all_ or elements):
-        click.echo('ERROR: no elements specified', err=True)
-        sys.exit(-1)
-
     with app.initialized():
+        if not (all_ or elements):
+            # NOTE: I may need to revisit this when implementing multiple projects
+            # opening one workspace.
+            element = app.context.guess_element()
+            if element:
+                elements = (element,)
+            else:
+                raise AppError('No elements specified')
 
         # Early exit if we specified `all` and there are no workspaces
         if all_ and not app.stream.workspace_exists():
@@ -772,11 +853,19 @@ def workspace_close(app, remove_dir, all_, elements):
 
         elements = app.stream.redirect_element_names(elements)
 
-        # Check that the workspaces in question exist
+        # Check that the workspaces in question exist, and that it's safe to
+        # remove them.
         nonexisting = []
         for element_name in elements:
             if not app.stream.workspace_exists(element_name):
                 nonexisting.append(element_name)
+            if (app.stream.workspace_is_required(element_name) and app.interactive and
+                    app.context.prompt_workspace_close_project_inaccessible):
+                click.echo("Removing '{}' will prevent you from running "
+                           "BuildStream commands from the current directory".format(element_name))
+                if not click.confirm('Are you sure you want to close this workspace?'):
+                    click.echo('Aborting', err=True)
+                    sys.exit(-1)
         if nonexisting:
             raise AppError("Workspace does not exist", detail="\n".join(nonexisting))
 
@@ -809,7 +898,11 @@ def workspace_reset(app, soft, track_, all_, elements):
     with app.initialized():
 
         if not (all_ or elements):
-            raise AppError('No elements specified to reset')
+            element = app.context.guess_element()
+            if element:
+                elements = (element,)
+            else:
+                raise AppError('No elements specified to reset')
 
         if all_ and not app.stream.workspace_exists():
             raise AppError("No open workspaces to reset")
@@ -866,3 +959,101 @@ def source_bundle(app, element, force, directory,
                                  force=force,
                                  compression=compression,
                                  except_targets=except_)
+
+
+#############################################################
+#                     Artifact Commands                     #
+#############################################################
+def _classify_artifacts(names, cas, project_directory):
+    element_targets = []
+    artifact_refs = []
+    element_globs = []
+    artifact_globs = []
+
+    for name in names:
+        if name.endswith('.bst'):
+            if any(c in "*?[" for c in name):
+                element_globs.append(name)
+            else:
+                element_targets.append(name)
+        else:
+            if any(c in "*?[" for c in name):
+                artifact_globs.append(name)
+            else:
+                artifact_refs.append(name)
+
+    if element_globs:
+        for dirpath, _, filenames in os.walk(project_directory):
+            for filename in filenames:
+                element_path = os.path.join(dirpath, filename).lstrip(project_directory).lstrip('/')
+                if any(fnmatch(element_path, glob) for glob in element_globs):
+                    element_targets.append(element_path)
+
+    if artifact_globs:
+        artifact_refs.extend(ref for ref in cas.list_refs()
+                             if any(fnmatch(ref, glob) for glob in artifact_globs))
+
+    return element_targets, artifact_refs
+
+
+@cli.group(short_help="Manipulate cached artifacts")
+def artifact():
+    """Manipulate cached artifacts"""
+    pass
+
+
+################################################################
+#                     Artifact Log Command                     #
+################################################################
+@artifact.command(name='log', short_help="Show logs of an artifact")
+@click.argument('artifacts', type=click.Path(), nargs=-1)
+@click.pass_obj
+def artifact_log(app, artifacts):
+    """Show logs of all artifacts"""
+    from .._exceptions import CASError
+    from .._message import MessageType
+    from .._pipeline import PipelineSelection
+    from ..storage._casbaseddirectory import CasBasedDirectory
+
+    with ExitStack() as stack:
+        stack.enter_context(app.initialized())
+        cache = app.context.artifactcache
+
+        elements, artifacts = _classify_artifacts(artifacts, cache.cas,
+                                                  app.project.directory)
+
+        vdirs = []
+        extractdirs = []
+        if artifacts:
+            for ref in artifacts:
+                try:
+                    cache_id = cache.cas.resolve_ref(ref, update_mtime=True)
+                    vdir = CasBasedDirectory(cache.cas, cache_id)
+                    vdirs.append(vdir)
+                except CASError as e:
+                    app._message(MessageType.WARN, "Artifact {} is not cached".format(ref), detail=str(e))
+                    continue
+        if elements:
+            elements = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
+            for element in elements:
+                if not element._cached():
+                    app._message(MessageType.WARN, "Element {} is not cached".format(element))
+                    continue
+                ref = cache.get_artifact_fullname(element, element._get_cache_key())
+                cache_id = cache.cas.resolve_ref(ref, update_mtime=True)
+                vdir = CasBasedDirectory(cache.cas, cache_id)
+                vdirs.append(vdir)
+
+        for vdir in vdirs:
+            # NOTE: If reading the logs feels unresponsive, here would be a good place to provide progress information.
+            logsdir = vdir.descend(["logs"])
+            td = stack.enter_context(TemporaryDirectory())
+            logsdir.export_files(td, can_link=True)
+            extractdirs.append(td)
+
+        for extractdir in extractdirs:
+            for log in (os.path.join(extractdir, log) for log in os.listdir(extractdir)):
+                # NOTE: Should click gain the ability to pass files to the pager this can be optimised.
+                with open(log) as f:
+                    data = f.read()
+                    click.echo_via_pager(data)

buildstream/_loader/loader.py

@@ -36,7 +36,7 @@ from .types import Symbol, Dependency
 from .loadelement import LoadElement
 from . import MetaElement
 from . import MetaSource
-from ..plugin import CoreWarnings
+from ..types import CoreWarnings
 from .._message import Message, MessageType
 
 

buildstream/_options/optionarch.py

@@ -17,7 +17,7 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-import os
+from .._platform import Platform
 from .optionenum import OptionEnum
 
 
@@ -41,8 +41,7 @@ class OptionArch(OptionEnum):
         super(OptionArch, self).load(node, allow_default_definition=False)
 
     def load_default_value(self, node):
-        _, _, _, _, machine_arch = os.uname()
-        return machine_arch
+        return Platform.get_host_arch()
 
     def resolve(self):
 

buildstream/_options/optionos.py

+
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk>
+
+import os
+from .optionenum import OptionEnum
+
+
+# OptionOS
+#
+class OptionOS(OptionEnum):
+
+    OPTION_TYPE = 'os'
+
+    def load(self, node):
+        super(OptionOS, self).load(node, allow_default_definition=False)
+
+    def load_default_value(self, node):
+        return os.uname()[0]
+
+    def resolve(self):
+
+        # Validate that the default OS reported by uname() is explicitly
+        # supported by the project, if not overridden by user config or cli.
+        self.validate(self.value)

buildstream/_options/optionpool.py

@@ -28,6 +28,7 @@ from .optionenum import OptionEnum
 from .optionflags import OptionFlags
 from .optioneltmask import OptionEltMask
 from .optionarch import OptionArch
+from .optionos import OptionOS
 
 
 _OPTION_TYPES = {
@@ -36,6 +37,7 @@ _OPTION_TYPES = {
     OptionFlags.OPTION_TYPE: OptionFlags,
     OptionEltMask.OPTION_TYPE: OptionEltMask,
     OptionArch.OPTION_TYPE: OptionArch,
+    OptionOS.OPTION_TYPE: OptionOS,
 }
 
 

buildstream/_platform/linux.py

@@ -25,6 +25,7 @@ from .. import utils
 from ..sandbox import SandboxDummy
 
 from . import Platform
+from .._exceptions import PlatformError
 
 
 class Linux(Platform):
@@ -58,6 +59,9 @@ class Linux(Platform):
         else:
             self._user_ns_available = False
 
+        # Set linux32 option
+        self._linux32 = False
+
     def create_sandbox(self, *args, **kwargs):
         if not self._local_sandbox_available:
             return self._create_dummy_sandbox(*args, **kwargs)
@@ -71,11 +75,33 @@ class Linux(Platform):
 
         if self._user_ns_available:
             # User namespace support allows arbitrary build UID/GID settings.
-            return True
-        else:
+            pass
+        elif (config.build_uid != self._uid or config.build_gid != self._gid):
             # Without user namespace support, the UID/GID in the sandbox
             # will match the host UID/GID.
-            return config.build_uid == self._uid and config.build_gid == self._gid
+            return False
+
+        # We can't do builds for another host or architecture except x86-32 on
+        # x86-64
+        host_os = self.get_host_os()
+        host_arch = self.get_host_arch()
+        if config.build_os != host_os:
+            raise PlatformError("Configured and host OS don't match.")
+        elif config.build_arch != host_arch:
+            # We can use linux32 for building 32bit on 64bit machines
+            if (host_os == "Linux" and
+                    ((config.build_arch == "x86-32" and host_arch == "x86-64") or
+                     (config.build_arch == "aarch32" and host_arch == "aarch64"))):
+                # check linux32 is available
+                try:
+                    utils.get_host_tool('linux32')
+                    self._linux32 = True
+                except utils.ProgramNotFoundError:
+                    pass
+            else:
+                raise PlatformError("Configured architecture and host architecture don't match.")
+
+        return True
 
     ################################################
     #              Private Methods                 #
@@ -100,6 +126,7 @@ class Linux(Platform):
         kwargs['user_ns_available'] = self._user_ns_available
         kwargs['die_with_parent_available'] = self._die_with_parent_available
         kwargs['json_status_available'] = self._json_status_available
+        kwargs['linux32'] = self._linux32
         return SandboxBwrap(*args, **kwargs)
 
     def _check_user_ns_available(self):

buildstream/_platform/platform.py

@@ -73,6 +73,44 @@ class Platform():
         else:
             return min(cpu_count, cap)
 
+    @staticmethod
+    def get_host_os():
+        return os.uname()[0]
+
+    # get_host_arch():
+    #
+    # This returns the architecture of the host machine. The possible values
+    # map from uname -m in order to be a OS independent list.
+    #
+    # Returns:
+    #    (string): String representing the architecture
+    @staticmethod
+    def get_host_arch():
+        # get the hardware identifier from uname
+        uname_machine = os.uname()[4]
+        uname_to_arch = {
+            "aarch64": "aarch64",
+            "aarch64_be": "aarch64-be",
+            "amd64": "x86-64",
+            "arm": "aarch32",
+            "armv8l": "aarch64",
+            "armv8b": "aarch64-be",
+            "i386": "x86-32",
+            "i486": "x86-32",
+            "i586": "x86-32",
+            "i686": "x86-32",
+            "ppc64": "power-isa-be",
+            "ppc64le": "power-isa-le",
+            "sparc": "sparc-v9",
+            "sparc64": "sparc-v9",
+            "x86_64": "x86-64"
+        }
+        try:
+            return uname_to_arch[uname_machine]
+        except KeyError:
+            raise PlatformError("uname gave unsupported machine architecture: {}"
+                                .format(uname_machine))
+
     ##################################################################
     #                        Sandbox functions                       #
     ##################################################################

buildstream/_platform/unix.py

@@ -44,4 +44,13 @@ class Unix(Platform):
     def check_sandbox_config(self, config):
         # With the chroot sandbox, the UID/GID in the sandbox
         # will match the host UID/GID (typically 0/0).
-        return config.build_uid == self._uid and config.build_gid == self._gid
+        if config.build_uid != self._uid or config.build_gid != self._gid:
+            return False
+
+        # Check host os and architecture match
+        if config.build_os != self.get_host_os():
+            raise PlatformError("Configured and host OS don't match.")
+        elif config.build_arch != self.get_host_arch():
+            raise PlatformError("Configured and host architecture don't match.")
+
+        return True

buildstream/_project.py

@@ -33,7 +33,7 @@ from ._artifactcache import ArtifactCache
 from .sandbox import SandboxRemote
 from ._elementfactory import ElementFactory
 from ._sourcefactory import SourceFactory
-from .plugin import CoreWarnings
+from .types import CoreWarnings
 from ._projectrefs import ProjectRefs, ProjectRefStorage
 from ._versions import BST_FORMAT_VERSION
 from ._loader import Loader
@@ -41,6 +41,7 @@ from .element import Element
 from ._message import Message, MessageType
 from ._includes import Includes
 from ._platform import Platform
+from ._workspaces import WORKSPACE_PROJECT_FILE
 
 
 # Project Configuration file
@@ -95,8 +96,10 @@ class Project():
         # The project name
         self.name = None
 
-        # The project directory
-        self.directory = self._ensure_project_dir(directory)
+        self._context = context  # The invocation Context, a private member
+
+        # The project directory, and whether the element whose workspace it was invoked from
+        self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory)
 
         # Absolute path to where elements are loaded from within the project
         self.element_path = None
@@ -117,7 +120,6 @@ class Project():
         #
         # Private Members
         #
-        self._context = context  # The invocation Context
 
         self._default_mirror = default_mirror    # The name of the preferred mirror.
 
@@ -371,6 +373,14 @@ class Project():
 
         self._load_second_pass()
 
+    # invoked_from_workspace_element()
+    #
+    # Returns the element whose workspace was used to invoke buildstream
+    # if buildstream was invoked from an external workspace
+    #
+    def invoked_from_workspace_element(self):
+        return self._invoked_from_workspace_element
+
     # cleanup()
     #
     # Cleans up resources used loading elements
@@ -650,7 +660,7 @@ class Project():
         # Source url aliases
         output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
 
-    # _ensure_project_dir()
+    # _find_project_dir()
     #
     # Returns path of the project directory, if a configuration file is found
     # in given directory or any of its parent directories.
@@ -661,18 +671,30 @@ class Project():
     # Raises:
     #    LoadError if project.conf is not found
     #
-    def _ensure_project_dir(self, directory):
-        directory = os.path.abspath(directory)
-        while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
-            parent_dir = os.path.dirname(directory)
-            if directory == parent_dir:
-                raise LoadError(
-                    LoadErrorReason.MISSING_PROJECT_CONF,
-                    '{} not found in current directory or any of its parent directories'
-                    .format(_PROJECT_CONF_FILE))
-            directory = parent_dir
+    # Returns:
+    #    (str) - the directory that contains the project, and
+    #    (str) - the name of the element required to find the project, or None
+    #
+    def _find_project_dir(self, directory):
+        workspace_element = None
+        found_directory, filename = utils._search_upward_for_files(
+            directory, [_PROJECT_CONF_FILE, WORKSPACE_PROJECT_FILE]
+        )
+        if filename == _PROJECT_CONF_FILE:
+            project_directory = found_directory
+        elif filename == WORKSPACE_PROJECT_FILE:
+            workspace_project_cache = self._context.get_workspace_project_cache()
+            workspace_project = workspace_project_cache.get(found_directory)
+            if workspace_project:
+                project_directory = workspace_project.get_default_project_path()
+                workspace_element = workspace_project.get_default_element()
+        else:
+            raise LoadError(
+                LoadErrorReason.MISSING_PROJECT_CONF,
+                '{} not found in current directory or any of its parent directories'
+                .format(_PROJECT_CONF_FILE))
 
-        return directory
+        return project_directory, workspace_element
 
     def _load_plugin_factories(self, config, output):
         plugin_source_origins = []   # Origins of custom sources

buildstream/_scheduler/queues/buildqueue.py

@@ -106,10 +106,16 @@ class BuildQueue(Queue):
 
     def done(self, job, element, result, success):
 
-        if success:
-            # Inform element in main process that assembly is done
-            element._assemble_done()
+        # Inform element in main process that assembly is done
+        element._assemble_done()
 
-            # This has to be done after _assemble_done, such that the
-            # element may register its cache key as required
+        # This has to be done after _assemble_done, such that the
+        # element may register its cache key as required
+        #
+        # FIXME: Element._assemble() does not report both the failure state and the
+        #        size of the newly cached failed artifact, so we can only adjust the
+        #        artifact cache size for a successful build even though we know a
+        #        failed build also grows the artifact cache size.
+        #
+        if success:
             self._check_cache_size(job, element, result)

buildstream/_scheduler/queues/queue.py

@@ -292,7 +292,6 @@ class Queue():
     # See the Job object for an explanation of the call signature
     #
     def _job_done(self, job, element, success, result):
-        element._update_state()
 
         # Update values that need to be synchronized in the main task
         # before calling any queue implementation

buildstream/_stream.py

@@ -544,7 +544,8 @@ class Stream():
             if len(elements) != 1:
                 raise StreamError("Exactly one element can be given if --directory is used",
                                   reason='directory-with-multiple-elements')
-            expanded_directories = [custom_dir, ]
+            directory = os.path.abspath(custom_dir)
+            expanded_directories = [directory, ]
         else:
             # If this fails it is a bug in what ever calls this, usually cli.py and so can not be tested for via the
             # run bst test mechanism.
@@ -581,15 +582,7 @@ class Stream():
                     todo_elements = "\nDid not try to create workspaces for " + todo_elements
                 raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
 
-            workspaces.create_workspace(target._get_full_name(), directory)
-
-            if not no_checkout:
-                with target.timed_activity("Staging sources to {}".format(directory)):
-                    target._open_workspace()
-
-            # Saving the workspace once it is set up means that if the next workspace fails to be created before
-            # the configuration gets saved. The successfully created workspace still gets saved.
-            workspaces.save_config()
+            workspaces.create_workspace(target, directory, checkout=not no_checkout)
             self._message(MessageType.INFO, "Created a workspace for element: {}"
                           .format(target._get_full_name()))
 
@@ -672,10 +665,7 @@ class Stream():
                                       .format(workspace_path, e)) from e
 
             workspaces.delete_workspace(element._get_full_name())
-            workspaces.create_workspace(element._get_full_name(), workspace_path)
-
-            with element.timed_activity("Staging sources to {}".format(workspace_path)):
-                element._open_workspace()
+            workspaces.create_workspace(element, workspace_path, checkout=True)
 
             self._message(MessageType.INFO,
                           "Reset workspace for {} at: {}".format(element.name,
@@ -707,6 +697,20 @@ class Stream():
 
         return False
 
+    # workspace_is_required()
+    #
+    # Checks whether the workspace belonging to element_name is required to
+    # load the project
+    #
+    # Args:
+    #    element_name (str): The element whose workspace may be required
+    #
+    # Returns:
+    #    (bool): True if the workspace is required
+    def workspace_is_required(self, element_name):
+        invoked_elm = self._project.invoked_from_workspace_element()
+        return invoked_elm == element_name
+
     # workspace_list
     #
     # Serializes the workspaces and dumps them in YAML to stdout.
@@ -1199,7 +1203,7 @@ class Stream():
             element_source_dir = self._get_element_dirname(directory, element)
             if list(element.sources()):
                 os.makedirs(element_source_dir)
-                element._stage_sources_at(element_source_dir)
+                element._stage_sources_at(element_source_dir, mount_workspaces=False)
 
     # Write a master build script to the sandbox
     def _write_build_script(self, directory, elements):

buildstream/_versions.py

@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 18
+BST_FORMAT_VERSION = 20
 
 
 # The base BuildStream artifact version

buildstream/_workspaces.py

@@ -25,6 +25,202 @@ from ._exceptions import LoadError, LoadErrorReason
 
 
 BST_WORKSPACE_FORMAT_VERSION = 3
+BST_WORKSPACE_PROJECT_FORMAT_VERSION = 1
+WORKSPACE_PROJECT_FILE = ".bstproject.yaml"
+
+
+# WorkspaceProject()
+#
+# An object to contain various helper functions and data required for
+# referring from a workspace back to buildstream.
+#
+# Args:
+#    directory (str): The directory that the workspace exists in.
+#
+class WorkspaceProject():
+    def __init__(self, directory):
+        self._projects = []
+        self._directory = directory
+
+    # get_default_project_path()
+    #
+    # Retrieves the default path to a project.
+    #
+    # Returns:
+    #    (str): The path to a project
+    #
+    def get_default_project_path(self):
+        return self._projects[0]['project-path']
+
+    # get_default_element()
+    #
+    # Retrieves the name of the element that owns this workspace.
+    #
+    # Returns:
+    #    (str): The name of an element
+    #
+    def get_default_element(self):
+        return self._projects[0]['element-name']
+
+    # to_dict()
+    #
+    # Turn the members data into a dict for serialization purposes
+    #
+    # Returns:
+    #    (dict): A dict representation of the WorkspaceProject
+    #
+    def to_dict(self):
+        ret = {
+            'projects': self._projects,
+            'format-version': BST_WORKSPACE_PROJECT_FORMAT_VERSION,
+        }
+        return ret
+
+    # from_dict()
+    #
+    # Loads a new WorkspaceProject from a simple dictionary
+    #
+    # Args:
+    #    directory (str): The directory that the workspace exists in
+    #    dictionary (dict): The dict to generate a WorkspaceProject from
+    #
+    # Returns:
+    #   (WorkspaceProject): A newly instantiated WorkspaceProject
+    #
+    @classmethod
+    def from_dict(cls, directory, dictionary):
+        # Only know how to handle one format-version at the moment.
+        format_version = int(dictionary['format-version'])
+        assert format_version == BST_WORKSPACE_PROJECT_FORMAT_VERSION, \
+            "Format version {} not found in {}".format(BST_WORKSPACE_PROJECT_FORMAT_VERSION, dictionary)
+
+        workspace_project = cls(directory)
+        for item in dictionary['projects']:
+            workspace_project.add_project(item['project-path'], item['element-name'])
+
+        return workspace_project
+
+    # load()
+    #
+    # Loads the WorkspaceProject for a given directory.
+    #
+    # Args:
+    #    directory (str): The directory
+    # Returns:
+    #    (WorkspaceProject): The created WorkspaceProject, if in a workspace, or
+    #    (NoneType): None, if the directory is not inside a workspace.
+    #
+    @classmethod
+    def load(cls, directory):
+        workspace_file = os.path.join(directory, WORKSPACE_PROJECT_FILE)
+        if os.path.exists(workspace_file):
+            data_dict = _yaml.load(workspace_file)
+            return cls.from_dict(directory, data_dict)
+        else:
+            return None
+
+    # write()
+    #
+    # Writes the WorkspaceProject to disk
+    #
+    def write(self):
+        os.makedirs(self._directory, exist_ok=True)
+        _yaml.dump(self.to_dict(), self.get_filename())
+
+    # get_filename()
+    #
+    # Returns the full path to the workspace local project file
+    #
+    def get_filename(self):
+        return os.path.join(self._directory, WORKSPACE_PROJECT_FILE)
+
+    # add_project()
+    #
+    # Adds an entry containing the project's path and element's name.
+    #
+    # Args:
+    #    project_path (str): The path to the project that opened the workspace.
+    #    element_name (str): The name of the element that the workspace belongs to.
+    #
+    def add_project(self, project_path, element_name):
+        assert (project_path and element_name)
+        self._projects.append({'project-path': project_path, 'element-name': element_name})
+
+
+# WorkspaceProjectCache()
+#
+# A class to manage workspace project data for multiple workspaces.
+#
+class WorkspaceProjectCache():
+    def __init__(self):
+        self._projects = {}  # Mapping of a workspace directory to its WorkspaceProject
+
+    # get()
+    #
+    # Returns a WorkspaceProject for a given directory, retrieving from the cache if
+    # present.
+    #
+    # Args:
+    #    directory (str): The directory to search for a WorkspaceProject.
+    #
+    # Returns:
+    #    (WorkspaceProject): The WorkspaceProject that was found for that directory.
+    #    or      (NoneType): None, if no WorkspaceProject can be found.
+    #
+    def get(self, directory):
+        try:
+            workspace_project = self._projects[directory]
+        except KeyError:
+            workspace_project = WorkspaceProject.load(directory)
+            if workspace_project:
+                self._projects[directory] = workspace_project
+
+        return workspace_project
+
+    # add()
+    #
+    # Adds the project path and element name to the WorkspaceProject that exists
+    # for that directory
+    #
+    # Args:
+    #    directory (str): The directory to search for a WorkspaceProject.
+    #    project_path (str): The path to the project that refers to this workspace
+    #    element_name (str): The element in the project that was refers to this workspace
+    #
+    # Returns:
+    #    (WorkspaceProject): The WorkspaceProject that was found for that directory.
+    #
+    def add(self, directory, project_path, element_name):
+        workspace_project = self.get(directory)
+        if not workspace_project:
+            workspace_project = WorkspaceProject(directory)
+            self._projects[directory] = workspace_project
+
+        workspace_project.add_project(project_path, element_name)
+        return workspace_project
+
+    # remove()
+    #
+    # Removes the project path and element name from the WorkspaceProject that exists
+    # for that directory.
+    #
+    # NOTE: This currently just deletes the file, but with support for multiple
+    # projects opening the same workspace, this will involve decreasing the count
+    # and deleting the file if there are no more projects.
+    #
+    # Args:
+    #    directory (str): The directory to search for a WorkspaceProject.
+    #
+    def remove(self, directory):
+        workspace_project = self.get(directory)
+        if not workspace_project:
+            raise LoadError(LoadErrorReason.MISSING_FILE,
+                            "Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE))
+        path = workspace_project.get_filename()
+        try:
+            os.unlink(path)
+        except FileNotFoundError:
+            pass
 
 
 # Workspace()
@@ -174,10 +370,15 @@ class Workspace():
         if recalculate or self._key is None:
             fullpath = self.get_absolute_path()
 
+            excluded_files = (WORKSPACE_PROJECT_FILE,)
+
             # Get a list of tuples of the the project relative paths and fullpaths
             if os.path.isdir(fullpath):
                 filelist = utils.list_relative_paths(fullpath)
-                filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
+                filelist = [
+                    (relpath, os.path.join(fullpath, relpath)) for relpath in filelist
+                    if relpath not in excluded_files
+                ]
             else:
                 filelist = [(self.get_absolute_path(), fullpath)]
 
@@ -199,12 +400,14 @@ class Workspace():
 #
 # Args:
 #    toplevel_project (Project): Top project used to resolve paths.
+#    workspace_project_cache (WorkspaceProjectCache): The cache of WorkspaceProjects
 #
 class Workspaces():
-    def __init__(self, toplevel_project):
+    def __init__(self, toplevel_project, workspace_project_cache):
         self._toplevel_project = toplevel_project
         self._bst_directory = os.path.join(toplevel_project.directory, ".bst")
         self._workspaces = self._load_config()
+        self._workspace_project_cache = workspace_project_cache
 
     # list()
     #
@@ -219,19 +422,36 @@ class Workspaces():
 
     # create_workspace()
     #
-    # Create a workspace in the given path for the given element.
+    # Create a workspace in the given path for the given element, and potentially
+    # checks-out the target into it.
     #
     # Args:
-    #    element_name (str) - The element name to create a workspace for
+    #    target (Element) - The element to create a workspace for
     #    path (str) - The path in which the workspace should be kept
+    #    checkout (bool): Whether to check-out the element's sources into the directory
     #
-    def create_workspace(self, element_name, path):
-        if path.startswith(self._toplevel_project.directory):
-            path = os.path.relpath(path, self._toplevel_project.directory)
+    def create_workspace(self, target, path, *, checkout):
+        element_name = target._get_full_name()
+        project_dir = self._toplevel_project.directory
+        if path.startswith(project_dir):
+            workspace_path = os.path.relpath(path, project_dir)
+        else:
+            workspace_path = path
 
-        self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
+        self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path)
 
-        return self._workspaces[element_name]
+        if checkout:
+            with target.timed_activity("Staging sources to {}".format(path)):
+                target._open_workspace()
+
+        workspace_project = self._workspace_project_cache.add(path, project_dir, element_name)
+        project_file_path = workspace_project.get_filename()
+
+        if os.path.exists(project_file_path):
+            target.warn("{} was staged from this element's sources".format(WORKSPACE_PROJECT_FILE))
+        workspace_project.write()
+
+        self.save_config()
 
     # get_workspace()
     #
@@ -280,8 +500,19 @@ class Workspaces():
     #    element_name (str) - The element name whose workspace to delete
     #
     def delete_workspace(self, element_name):
+        workspace = self.get_workspace(element_name)
         del self._workspaces[element_name]
 
+        # Remove from the cache if it exists
+        try:
+            self._workspace_project_cache.remove(workspace.get_absolute_path())
+        except LoadError as e:
+            # We might be closing a workspace with a deleted directory
+            if e.reason == LoadErrorReason.MISSING_FILE:
+                pass
+            else:
+                raise
+
     # save_config()
     #
     # Dump the current workspace element to the project configuration

buildstream/_yaml.py

@@ -352,6 +352,7 @@ _sentinel = object()
 #    key (str): The key to get a value for in node
 #    indices (list of ints): Optionally decend into lists of lists
 #    default_value: Optionally return this value if the key is not found
+#    allow_none: (bool): Allow None to be a valid value
 #
 # Returns:
 #    The value if found in node, otherwise default_value is returned
@@ -362,7 +363,7 @@ _sentinel = object()
 # Note:
 #    Returned strings are stripped of leading and trailing whitespace
 #
-def node_get(node, expected_type, key, indices=None, default_value=_sentinel):
+def node_get(node, expected_type, key, indices=None, *, default_value=_sentinel, allow_none=False):
     value = node.get(key, default_value)
     provenance = node_get_provenance(node)
     if value is _sentinel:
@@ -377,8 +378,8 @@ def node_get(node, expected_type, key, indices=None, default_value=_sentinel):
             value = value[index]
             path += '[{:d}]'.format(index)
 
-    # We want to allow None as a valid value for any type
-    if value is None:
+    # Optionally allow None as a valid value for any type
+    if value is None and (allow_none or default_value is None):
         return None
 
     if not isinstance(value, expected_type):