NEWS

@@ -79,6 +79,10 @@ buildstream 1.3.1
     plugin has now a tag tracking feature instead. This can be enabled
     by setting 'track-tags'.
 
+  o Opening a workspace now creates a .bstproject.yaml file that allows buildstream
+    commands to be run from a workspace that is not inside a project.
+
+
 =================
 buildstream 1.1.5
 =================

buildstream/_context.py

@@ -122,6 +122,10 @@ class Context():
         # remove a workspace directory.
         self.prompt_workspace_close_remove_dir = None
 
+        # Boolean, whether we double-check with the user that they meant to
+        # close the workspace when they're using it to access the project.
+        self.prompt_workspace_close_project_inaccessible = None
+
         # Boolean, whether we double-check with the user that they meant to do
         # a hard reset of a workspace, potentially losing changes.
         self.prompt_workspace_reset_hard = None
@@ -251,12 +255,15 @@ class Context():
             defaults, Mapping, 'prompt')
         _yaml.node_validate(prompt, [
             'auto-init', 'really-workspace-close-remove-dir',
+            'really-workspace-close-project-inaccessible',
             'really-workspace-reset-hard',
         ])
         self.prompt_auto_init = _node_get_option_str(
             prompt, 'auto-init', ['ask', 'no']) == 'ask'
         self.prompt_workspace_close_remove_dir = _node_get_option_str(
             prompt, 'really-workspace-close-remove-dir', ['ask', 'yes']) == 'ask'
+        self.prompt_workspace_close_project_inaccessible = _node_get_option_str(
+            prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask'
         self.prompt_workspace_reset_hard = _node_get_option_str(
             prompt, 'really-workspace-reset-hard', ['ask', 'yes']) == 'ask'
 

buildstream/_frontend/cli.py

@@ -763,11 +763,18 @@ def workspace_close(app, remove_dir, all_, elements):
 
         elements = app.stream.redirect_element_names(elements)
 
-        # Check that the workspaces in question exist
+        # Check that the workspaces in question exist, and that it's safe to
+        # remove them.
         nonexisting = []
         for element_name in elements:
             if not app.stream.workspace_exists(element_name):
                 nonexisting.append(element_name)
+            if (app.stream.workspace_is_required(element_name) and app.interactive and
+                    app.context.prompt_workspace_close_project_inaccessible):
+                click.echo("Removing '{}' will prevent you from running buildstream commands".format(element_name))
+                if not click.confirm('Are you sure you want to close this workspace?'):
+                    click.echo('Aborting', err=True)
+                    sys.exit(-1)
         if nonexisting:
             raise AppError("Workspace does not exist", detail="\n".join(nonexisting))
 

buildstream/_project.py

@@ -95,8 +95,10 @@ class Project():
         # The project name
         self.name = None
 
-        # The project directory
-        self.directory = self._find_project_dir(directory)
+        self._context = context  # The invocation Context, a private member
+
+        # The project directory, and whether the project was found from an external workspace
+        self.directory, self._required_workspace_element = self._find_project_dir(directory)
 
         # Absolute path to where elements are loaded from within the project
         self.element_path = None
@@ -117,7 +119,6 @@ class Project():
         #
         # Private Members
         #
-        self._context = context  # The invocation Context
 
         self._default_mirror = default_mirror    # The name of the preferred mirror.
 
@@ -371,6 +372,14 @@ class Project():
 
         self._load_second_pass()
 
+    # required_workspace_element()
+    #
+    # Returns the element whose workspace is required to load this project,
+    # if any.
+    #
+    def required_workspace_element(self):
+        return self._required_workspace_element
+
     # cleanup()
     #
     # Cleans up resources used loading elements
@@ -661,18 +670,26 @@ class Project():
     # Raises:
     #    LoadError if project.conf is not found
     #
+    # Returns:
+    #    (str) - the directory that contains the project, and
+    #    (str) - the name of the element required to find the project, or an empty string
+    #
     def _find_project_dir(self, directory):
-        directory = os.path.abspath(directory)
-        while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
-            parent_dir = os.path.dirname(directory)
-            if directory == parent_dir:
+        workspace_element = ""
+        project_directory = utils._search_upward_for_file(directory, _PROJECT_CONF_FILE)
+        if not project_directory:
+            workspace_project_cache = self._context.get_workspace_project_cache()
+            workspace_project = workspace_project_cache.get(directory)
+            if workspace_project:
+                project_directory = workspace_project.get_default_project_path()
+                workspace_element = workspace_project.get_default_element()
+            else:
                 raise LoadError(
                     LoadErrorReason.MISSING_PROJECT_CONF,
                     '{} not found in current directory or any of its parent directories'
                     .format(_PROJECT_CONF_FILE))
-            directory = parent_dir
 
-        return directory
+        return project_directory, workspace_element
 
     def _load_plugin_factories(self, config, output):
         plugin_source_origins = []   # Origins of custom sources

buildstream/_stream.py

@@ -581,15 +581,7 @@ class Stream():
                     todo_elements = "\nDid not try to create workspaces for " + todo_elements
                 raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
 
-            workspaces.create_workspace(target._get_full_name(), directory)
-
-            if not no_checkout:
-                with target.timed_activity("Staging sources to {}".format(directory)):
-                    target._open_workspace()
-
-            # Saving the workspace once it is set up means that if the next workspace fails to be created before
-            # the configuration gets saved. The successfully created workspace still gets saved.
-            workspaces.save_config()
+            workspaces.create_workspace(target, directory, not no_checkout)
             self._message(MessageType.INFO, "Created a workspace for element: {}"
                           .format(target._get_full_name()))
 
@@ -672,10 +664,7 @@ class Stream():
                                       .format(workspace_path, e)) from e
 
             workspaces.delete_workspace(element._get_full_name())
-            workspaces.create_workspace(element._get_full_name(), workspace_path)
-
-            with element.timed_activity("Staging sources to {}".format(workspace_path)):
-                element._open_workspace()
+            workspaces.create_workspace(element, workspace_path, True)
 
             self._message(MessageType.INFO,
                           "Reset workspace for {} at: {}".format(element.name,
@@ -707,6 +696,20 @@ class Stream():
 
         return False
 
+    # workspace_is_required()
+    #
+    # Checks whether the workspace belonging to element_name is required to
+    # load the project
+    #
+    # Args:
+    #    element_name (str): The element whose workspace may be required
+    #
+    # Returns:
+    #    (bool): True if the workspace is required
+    def workspace_is_required(self, element_name):
+        required_elm = self._project.required_workspace_element()
+        return required_elm == element_name
+
     # workspace_list
     #
     # Serializes the workspaces and dumps them in YAML to stdout.

buildstream/_workspaces.py

@@ -387,10 +387,15 @@ class Workspace():
         if recalculate or self._key is None:
             fullpath = self.get_absolute_path()
 
+            excluded_files = (WORKSPACE_PROJECT_FILE,)
+
             # Get a list of tuples of the the project relative paths and fullpaths
             if os.path.isdir(fullpath):
                 filelist = utils.list_relative_paths(fullpath)
-                filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
+                filelist = [
+                    (relpath, os.path.join(fullpath, relpath)) for relpath in filelist
+                    if relpath not in excluded_files
+                ]
             else:
                 filelist = [(self.get_absolute_path(), fullpath)]
 

buildstream/data/userconfig.yaml

@@ -128,6 +128,14 @@ prompt:
   #
   really-workspace-close-remove-dir: ask
 
+  # Whether to really proceed with 'bst workspace close' when doing so would
+  # stop them from running bst commands in this workspace.
+  #
+  #  ask - Ask the user if they are sure.
+  #  yes - Always close, without asking.
+  #
+  really-workspace-close-project-inaccessible: ask
+
   # Whether to really proceed with 'bst workspace reset' doing a hard reset of
   # a workspace, potentially losing changes.
   #

tests/frontend/workspace.py

@@ -31,6 +31,7 @@ import shutil
 import subprocess
 from ruamel.yaml.comments import CommentedSet
 from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
+from tests.testutils import create_artifact_share
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
@@ -615,9 +616,12 @@ def test_list(cli, tmpdir, datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("kind", repo_kinds)
 @pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
-def test_build(cli, tmpdir, datafiles, kind, strict):
+@pytest.mark.parametrize("call_from", [("project"), ("workspace")])
+def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from):
+    tmpdir = tmpdir_factory.mktemp('')
     element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
     checkout = os.path.join(str(tmpdir), 'checkout')
+    args_pre = ['-C', workspace] if call_from == "workspace" else []
 
     # Modify workspace
     shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
@@ -640,15 +644,14 @@ def test_build(cli, tmpdir, datafiles, kind, strict):
     # Build modified workspace
     assert cli.get_element_state(project, element_name) == 'buildable'
     assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
-    result = cli.run(project=project, args=['build', element_name])
+    result = cli.run(project=project, args=args_pre + ['build', element_name])
     result.assert_success()
     assert cli.get_element_state(project, element_name) == 'cached'
     assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
 
     # Checkout the result
-    result = cli.run(project=project, args=[
-        'checkout', element_name, checkout
-    ])
+    result = cli.run(project=project,
+                     args=args_pre + ['checkout', element_name, checkout])
     result.assert_success()
 
     # Check that the pony.conf from the modified workspace exists
@@ -1055,3 +1058,131 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
         result = cli.run(project=project, args=["build", element_name])
         assert "BUG" not in result.stderr
         assert cli.get_element_state(project, element_name) != "cached"
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_fetch(cli, datafiles, tmpdir_factory):
+    # Fetching from a workspace outside a project doesn't fail horribly
+    tmpdir = tmpdir_factory.mktemp('')
+    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
+
+    result = cli.run(project=project, args=['-C', workspace, 'fetch', element_name])
+    result.assert_success()
+
+    # We already fetched it by opening the workspace, but we're also checking
+    # `bst show` works here
+    assert cli.get_element_state(project, element_name) == 'buildable'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_push_pull(cli, datafiles, tmpdir_factory):
+    # Pushing and pulling to/from an artifact cache works from an external workspace
+    tmpdir = tmpdir_factory.mktemp('')
+    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
+
+    with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+        result = cli.run(project=project, args=['-C', workspace, 'build', element_name])
+        result.assert_success()
+
+        cli.configure({
+            'artifacts': {'url': share.repo, 'push': True}
+        })
+
+        result = cli.run(project=project, args=['-C', workspace, 'push', element_name])
+        result.assert_success()
+
+        result = cli.run(project=project, args=['-C', workspace, 'pull', '--deps', 'all', element_name])
+        result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_track(cli, datafiles, tmpdir_factory):
+    # Tracking does not get horribly confused
+    tmpdir = tmpdir_factory.mktemp('')
+    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", True)
+
+    # The workspace is necessarily already tracked, so we only care that
+    # there's no weird errors.
+    result = cli.run(project=project, args=['-C', workspace, 'track', element_name])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_open_other(cli, datafiles, tmpdir_factory):
+    # From inside an external workspace, open another workspace
+    tmpdir1 = tmpdir_factory.mktemp('')
+    tmpdir2 = tmpdir_factory.mktemp('')
+    # Making use of the assumption that it's the same project in both invocations of open_workspace
+    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
+    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
+
+    # Closing the other element first, because I'm too lazy to create an
+    # element without opening it
+    result = cli.run(project=project, args=['workspace', 'close', beta_element])
+    result.assert_success()
+
+    result = cli.run(project=project, args=[
+        '-C', alpha_workspace, 'workspace', 'open', '--force', '--directory', beta_workspace, beta_element
+    ])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_close_other(cli, datafiles, tmpdir_factory):
+    # From inside an external workspace, close the other workspace
+    tmpdir1 = tmpdir_factory.mktemp('')
+    tmpdir2 = tmpdir_factory.mktemp('')
+    # Making use of the assumption that it's the same project in both invocations of open_workspace
+    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
+    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
+
+    result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_close_self(cli, datafiles, tmpdir_factory):
+    # From inside an external workspace, close it
+    tmpdir1 = tmpdir_factory.mktemp('')
+    tmpdir2 = tmpdir_factory.mktemp('')
+    # Making use of the assumption that it's the same project in both invocations of open_workspace
+    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
+    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
+
+    result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', alpha_element])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_reset_other(cli, datafiles, tmpdir_factory):
+    tmpdir1 = tmpdir_factory.mktemp('')
+    tmpdir2 = tmpdir_factory.mktemp('')
+    # Making use of the assumption that it's the same project in both invocations of open_workspace
+    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
+    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
+
+    result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'reset', beta_element])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_reset_self(cli, datafiles, tmpdir):
+    element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
+
+    # Command succeeds
+    result = cli.run(project=project, args=['-C', workspace, 'workspace', 'reset', element])
+    result.assert_success()
+
+    # Successive commands still work (i.e. .bstproject.yaml hasn't been deleted)
+    result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_list(cli, datafiles, tmpdir_factory):
+    tmpdir = tmpdir_factory.mktemp('')
+    # Making use of the assumption that it's the same project in both invocations of open_workspace
+    element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
+
+    result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
+    result.assert_success()

tests/integration/shell.py

@@ -353,3 +353,29 @@ def test_integration_devices(cli, tmpdir, datafiles):
 
     result = execute_shell(cli, project, ["true"], element=element_name)
     assert result.exit_code == 0
+
+
+# Test that a shell can be opened from an external workspace
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell):
+    tmpdir = tmpdir_factory.mktemp("")
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    element_name = 'autotools/amhello.bst'
+    workspace_dir = os.path.join(str(tmpdir), 'workspace')
+
+    result = cli.run(project=project, args=[
+        'workspace', 'open', '--directory', workspace_dir, element_name
+    ])
+    result.assert_success()
+
+    result = cli.run(project=project, args=['-C', workspace_dir, 'build', element_name])
+    result.assert_success()
+
+    command = ['shell']
+    if build_shell == 'build':
+        command.append('--build')
+    command.extend([element_name, '--', 'true'])
+    result = cli.run(project=project, cwd=workspace_dir, args=command)
+    result.assert_success()