Enhanced bst shell configuration and cli options

Some changes to the host-files configuration:

  o Dont require `host-files` to not be directories

    We need to specify directories to mount from `project.conf` after all.

  o Added possibility of specifying optional mounts, to avoid
    meaningless warnings where optional files don't exist on
    the host

Added --mount CLI option to `bst shell`

This allows users to explicitly mount whatever they want into the
sandbox environment for `bst shell`.

This closes issue #274

buildstream/_frontend/cli.py

@@ -462,13 +462,16 @@ def show(app, elements, deps, except_, order, format, downloadable):
 @click.option('--sysroot', '-s', default=None,
               type=click.Path(exists=True, file_okay=False, readable=True),
               help="An existing sysroot")
+@click.option('--mount', type=click.Tuple([click.Path(exists=True), str]), multiple=True,
+              metavar='HOSTPATH PATH',
+              help="Mount a file or directory into the sandbox")
 @click.option('--isolate', is_flag=True, default=False,
               help='Create an isolated build sandbox')
 @click.argument('element',
                 type=click.Path(dir_okay=False, readable=True))
 @click.argument('command', type=click.STRING, nargs=-1)
 @click.pass_obj
-def shell(app, element, sysroot, isolate, build, command):
+def shell(app, element, sysroot, mount, isolate, build, command):
     """Run a command in the target element's sandbox environment
 
     This will stage a temporary sysroot for running the target
@@ -486,6 +489,7 @@ def shell(app, element, sysroot, isolate, build, command):
     to run an interactive shell.
     """
     from ..element import Scope
+    from .._project import HostMount
     if build:
         scope = Scope.BUILD
     else:
@@ -509,9 +513,14 @@ def shell(app, element, sysroot, isolate, build, command):
         click.echo("Try building them first", err=True)
         sys.exit(-1)
 
+    mounts = [
+        HostMount(path, host_path)
+        for host_path, path in mount
+    ]
+
     try:
         element = app.pipeline.targets[0]
-        exitcode = app.shell(element, scope, sysroot, isolate=isolate, command=command)
+        exitcode = app.shell(element, scope, sysroot, mounts=mounts, isolate=isolate, command=command)
         sys.exit(exitcode)
     except BstError as e:
         click.echo("", err=True)

buildstream/_frontend/main.py

@@ -365,7 +365,7 @@ class App():
                     queue.failed_elements.remove(element)
                     queue.enqueue([element])
 
-    def shell(self, element, scope, directory, isolate=False, command=None):
+    def shell(self, element, scope, directory, *, mounts=None, isolate=False, command=None):
         _, key, dim = element._get_full_display_key()
         element_name = element._get_full_name()
 
@@ -380,7 +380,7 @@ class App():
         else:
             prompt = '[{}@{}:${{PWD}}]$ '.format(key, element_name)
 
-        return element._shell(scope, directory, isolate=isolate, prompt=prompt, command=command)
+        return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command)
 
     def tick(self, elapsed):
         self.maybe_render_status()

buildstream/_project.py

@@ -39,12 +39,29 @@ from ._sourcefactory import SourceFactory
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 2
+BST_FORMAT_VERSION = 3
 
 # The separator we use for user specified aliases
 _ALIAS_SEPARATOR = ':'
 
 
+# HostMount()
+#
+# A simple object describing the behavior of
+# a host mount.
+#
+class HostMount():
+
+    def __init__(self, path, host_path=None, optional=False):
+
+        self.path = path              # Path inside the sandbox
+        self.host_path = host_path    # Path on the host
+        self.optional = optional      # Optional mounts do not incur warnings or errors
+
+        if self.host_path is None:
+            self.host_path = self.path
+
+
 # Project()
 #
 # The Project Configuration
@@ -82,7 +99,7 @@ class Project():
         # Shell options
         self._shell_command = []      # The default interactive shell command
         self._shell_env_inherit = []  # Environment vars to inherit when non-isolated
-        self._shell_host_files = {}   # Mapping of sandbox paths to host paths
+        self._shell_host_files = []   # A list of HostMount objects
 
         profile_start(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
         self._load()
@@ -286,14 +303,20 @@ class Project():
         host_files = _yaml.node_get(shell_options, list, 'host-files', default_value=[])
         for host_file in host_files:
             if isinstance(host_file, str):
-                self._shell_host_files[host_file] = host_file
+                mount = HostMount(host_file)
             else:
+                # Some validation
                 index = host_files.index(host_file)
                 host_file_desc = _yaml.node_get(shell_options, Mapping, 'host-files', indices=[index])
-                _yaml.node_validate(host_file_desc, ['host', 'sandbox'])
-                host_path = _yaml.node_get(host_file_desc, str, 'host')
-                sandbox_path = _yaml.node_get(host_file_desc, str, 'sandbox')
-                self._shell_host_files[sandbox_path] = host_path
+                _yaml.node_validate(host_file_desc, ['path', 'host_path', 'optional'])
+
+                # Parse the host mount
+                path = _yaml.node_get(host_file_desc, str, 'path')
+                host_path = _yaml.node_get(host_file_desc, str, 'host_path', default_value='') or None
+                optional = _yaml.node_get(host_file_desc, bool, 'optional', default_value=False)
+                mount = HostMount(path, host_path, optional)
+
+            self._shell_host_files.append(mount)
 
     # _store_origin()
     #

buildstream/element.py

@@ -1404,6 +1404,7 @@ class Element(Plugin):
     # Args:
     #    scope (Scope): Either BUILD or RUN scopes are valid, or None
     #    directory (str): A directory to an existing sandbox, or None
+    #    mounts (list): A list of (str, str) tuples, representing host/target paths to mount
     #    isolate (bool): Whether to isolate the environment like we do in builds
     #    prompt (str): A suitable prompt string for PS1
     #    command (list): An argv to launch in the sandbox
@@ -1411,7 +1412,7 @@ class Element(Plugin):
     # Returns: Exit code
     #
     # If directory is not specified, one will be staged using scope
-    def _shell(self, scope=None, directory=None, isolate=False, prompt=None, command=None):
+    def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None):
 
         with self._prepare_sandbox(scope, directory) as sandbox:
             environment = self.get_environment()
@@ -1438,15 +1439,17 @@ class Element(Plugin):
                     if os.environ.get(inherit) is not None:
                         environment[inherit] = os.environ.get(inherit)
 
-                # Setup any project defined bind mounts
-                for target, source in _yaml.node_items(project._shell_host_files):
-                    if not os.path.exists(source):
-                        self.warn("Not mounting non-existing host file: {}".format(source))
-                    elif os.path.isdir(source):
-                        self.warn("Not mounting directory listed as host file: {}".format(source))
+                # Setup any requested bind mounts
+                if mounts is None:
+                    mounts = []
+
+                for mount in project._shell_host_files + mounts:
+                    if not os.path.exists(mount.host_path):
+                        if not mount.optional:
+                            self.warn("Not mounting non-existing host file: {}".format(mount.host_path))
                     else:
-                        sandbox.mark_directory(target)
-                        sandbox._set_mount_source(target, source)
+                        sandbox.mark_directory(mount.path)
+                        sandbox._set_mount_source(mount.path, mount.host_path)
 
             if command:
                 argv = [arg for arg in command]