buildgrid/_app/commands/cmd_execute.py

@@ -97,20 +97,6 @@ def request_dummy(context, number, wait_for_completion):
             context.logger.info(next(response))
 
 
-@cli.command('wait', short_help="Streams an operation until it is complete.")
-@click.argument('operation-name', nargs=1, type=click.STRING, required=True)
-@pass_context
-def wait_execution(context, operation_name):
-    stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
-    request = remote_execution_pb2.WaitExecutionRequest(instance_name=context.instance_name,
-                                                        name=operation_name)
-
-    response = stub.WaitExecution(request)
-
-    for stream in response:
-        context.logger.info(stream)
-
-
 @cli.command('command', short_help="Send a command to be executed.")
 @click.option('--output-file', nargs=2, type=(click.STRING, click.BOOL), multiple=True,
               help="Tuple of expected output file and is-executeable flag.")

buildgrid/_app/commands/cmd_operation.py

@@ -27,12 +27,13 @@ import sys
 import click
 import grpc
 
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
 from buildgrid._protos.google.longrunning import operations_pb2, operations_pb2_grpc
 
 from ..cli import pass_context
 
 
-@click.group(name='operation', short_help="Long running operations commands")
+@click.group(name='operation', short_help="Long running operations commands.")
 @click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
               help="Remote execution server's URL (port defaults to 50051 if no specified).")
 @click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
@@ -93,3 +94,17 @@ def lists(context):
 
     for op in response.operations:
         context.logger.info(op)
+
+
+@cli.command('wait', short_help="Streams an operation until it is complete.")
+@click.argument('operation-name', nargs=1, type=click.STRING, required=True)
+@pass_context
+def wait(context, operation_name):
+    stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
+    request = remote_execution_pb2.WaitExecutionRequest(instance_name=context.instance_name,
+                                                        name=operation_name)
+
+    response = stub.WaitExecution(request)
+
+    for stream in response:
+        context.logger.info(stream)

docs/source/data/buildstream-example-server.conf

+server:
+  port: 50051
+  insecure-mode: true
+
+instances:
+  - name: ""
+
+    storages:
+      - !lru-storage &main-storage
+        size: 512MB
+
+    services:
+      - !action-cache &main-action
+        storage: *main-storage
+        max_cached_refs: 256
+        allow_updates: true
+      - !execution
+        storage: *main-storage
+        action_cache: *main-action
+      - !cas
+        storage: *main-storage
+      - !reference-cache
+        storage: *main-storage
+        max_cached_refs: 128
+      - !bytestream
+        storage: *main-storage
\ No newline at end of file

docs/source/reference_cli.rst

 
-.. _cli_reference:
+.. _cli-reference:
 
 CLI reference
 =============
@@ -8,112 +8,118 @@ BuildGrid's Command Line Interface (CLI) reference documentation.
 
 ----
 
-.. _invoking_bgd:
+.. _invoking-bgd:
 
 .. click:: buildgrid._app:cli
    :prog: bgd
 
 ----
 
-.. _invoking_bgd_bot:
+.. _invoking-bgd-bot:
 
 .. click:: buildgrid._app.commands.cmd_bot:cli
    :prog: bgd bot
 
 ----
 
-.. _invoking_bgd_bot_buildbox:
+.. _invoking-bgd-bot-buildbox:
 
 .. click:: buildgrid._app.commands.cmd_bot:run_buildbox
    :prog: bgd bot buildbot
 
 ----
 
-.. _invoking_bgd_bot_dummy:
+.. _invoking-bgd-bot-dummy:
 
 .. click:: buildgrid._app.commands.cmd_bot:run_dummy
    :prog: bgd bot dummy
 
 ----
 
-.. _invoking_bgd_bot_temp_directory:
+.. _invoking-bgd-bot-temp-directory:
 
 .. click:: buildgrid._app.commands.cmd_bot:run_temp_directory
    :prog: bgd bot temp-directory
 
 ----
 
-.. _invoking_bgd_cas:
+.. _invoking-bgd-cas:
 
 .. click:: buildgrid._app.commands.cmd_cas:cli
    :prog: bgd cas
 
 ----
 
-.. _invoking_bgd_cas_upload_dir:
+.. _invoking-bgd-cas-upload-dir:
 
 .. click:: buildgrid._app.commands.cmd_cas:upload_dir
    :prog: bgd cas upload-dir
 
 ----
 
-.. _invoking_bgd_cas_upload_files:
+.. _invoking-bgd-cas-upload-files:
 
 .. click:: buildgrid._app.commands.cmd_cas:upload_files
    :prog: bgd cas upload-files
 
 ----
 
-.. _invoking_bgd_execute:
+.. _invoking-bgd-execute:
 
 .. click:: buildgrid._app.commands.cmd_execute:cli
    :prog: bgd execute
 
 ----
 
-.. _invoking_bgd_execute_command:
+.. _invoking-bgd-execute-command:
 
 .. click:: buildgrid._app.commands.cmd_execute:command
    :prog: bgd execute command
 
 ----
 
-.. _invoking_bgd_execute_list:
+.. _invoking-bgd-execute-request-dummy:
 
-.. click:: buildgrid._app.commands.cmd_execute:list_operations
-   :prog: bgd execute list
+.. click:: buildgrid._app.commands.cmd_execute:request_dummy
+   :prog: bgd execute request-dummy
 
 ----
 
-.. _invoking_bgd_execute_ request_dummy:
+.. _invoking-bgd-operation:
 
-.. click:: buildgrid._app.commands.cmd_execute:request_dummy
-   :prog: bgd execute request-dummy
+.. click:: buildgrid._app.commands.cmd_operation:cli
+   :prog: bgd operation
 
 ----
 
-.. _invoking_bgd_execute_status:
+.. _invoking-bgd-operation-list:
 
-.. click:: buildgrid._app.commands.cmd_execute:operation_status
-   :prog: bgd execute status
+.. click:: buildgrid._app.commands.cmd_operation:lists
+   :prog: bgd operation list
 
 ----
 
-.. _invoking_bgd_execute_wait:
+.. _invoking-bgd-operation-status:
 
-.. click:: buildgrid._app.commands.cmd_execute:wait_execution
-   :prog: bgd execute wait
+.. click:: buildgrid._app.commands.cmd_operation:status
+   :prog: bgd operation status
 
 ----
 
-.. _invoking_bgd_server:
+.. _invoking-bgd-operation-wait:
+
+.. click:: buildgrid._app.commands.cmd_operation:wait
+   :prog: bgd operation wait
+
+----
+.. _invoking-bgd-server:
 
 .. click:: buildgrid._app.commands.cmd_server:cli
    :prog: bgd server
 
 ----
 
-.. _invoking_bgd_server_start:
+.. _invoking-bgd-server-start:
 
 .. click:: buildgrid._app.commands.cmd_server:start
    :prog: bgd server start

docs/source/using.rst

@@ -7,7 +7,8 @@ Using
 This section covers how to run an use the BuildGrid build service.
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 3
 
    using_internal.rst
    using_bazel.rst
+   using_buildstream.rst

docs/source/using_bazel.rst

 
-.. _bazel-builds:
+.. _bazel-client:
 
-Bazel builds
+Bazel client
 ============
 
 `Bazel`_ is a *“fast, scalable, multi-language and extensible build system”*
@@ -18,7 +18,7 @@ Configuration
 -------------
 
 Bazel accepts many options that can either be specified as command line
-arguments when involking the ``bazel`` tool or stored in a `.bazelrc`_
+arguments when invoking the ``bazel`` tool or stored in a `.bazelrc`_
 configuration file. In order to activate remote build execution, the
 ``bazel build`` subcommand needs to be given specific `build options`_,
 including:
@@ -65,7 +65,7 @@ Next, change the current directory to the Bazel workspace root:
 
    cd bazel-examples/cpp-tutorial/stage3
 
-.. note::
+.. hint::
 
    All the commands in the instructions below are expected to be executed from
    that root directory (the stage3 example workspace's root directory).
@@ -80,7 +80,7 @@ below, paste it in a ``server.conf`` file in the root directory:
 This defines a single ``main`` server instance implementing a
 ``ContentAddressableStorage`` (CAS) + ``ByteStream`` service together with an
 ``Execution`` + ``ActionCache`` service, both using the same in-memory storage.
-You can then start the BuildGrid server deamon using that configuration by
+You can then start the BuildGrid server daemon using that configuration by
 running:
 
 .. code-block:: sh
@@ -97,8 +97,10 @@ has ``gcc`` installed, run:
    bgd bot --remote=http://localhost:50051 --parent=main temp-directory
 
 The ``--remote`` option is used to specify the server location (running on the
-same machine here, and listenning to port 50051). The ``--parent`` option is
-used to specifiy the server instance you except the bot to be attached to.
+same machine here, and listening to port 50051). The ``--parent`` option is used
+to specify the server instance you except the bot to be attached to. Refer to
+the :ref:`CLI reference section <invoking-bgd-bot-temp-directory>` for command
+line interface details.
 
 The BuildGrid server is now ready to accept jobs and execute them. Bazel needs
 some :ref:`configuration <bazel-configuration>` in order to run remote builds.
@@ -109,7 +111,7 @@ file in the root directory:
 
    build --spawn_strategy=remote --genrule_strategy=remote --remote_executor=localhost:50051 --remote_instance_name=main
 
-This ativates Bazel's remote execution mode and points to the ``main`` remote
+This activates Bazel's remote execution mode and points to the ``main`` remote
 execution server instance at ``localhost:50051``.
 
 You can finally have Bazel to build the example workspace by running:
@@ -118,7 +120,7 @@ You can finally have Bazel to build the example workspace by running:
 
    bazel build //main:hello-world
 
-You can verify that the example has been successfully build by runnig the
+You can verify that the example has been successfully build by running the
 generated executable. Simply invoke:
 
 .. code-block:: sh

docs/source/using_buildstream.rst

+
+.. _buildstream-client:
+
+BuildStream client
+==================
+
+`BuildStream`_ is a *“tool for integrating software stacks”*: it produces build
+outputs from multiple input sources, using the existing sources build systems.
+It supports remote build execution using the remote execution API (REAPI) v2.
+
+.. note::
+
+   No stable BuildStream version have been released with remote execution
+   support yet: you'll have to `install it from sources`_ in order use remote
+   build execution.
+
+.. _BuildStream: https://buildstream.build
+.. _install it from sources: https://buildstream.build/source_install.html
+
+
+.. _buildstream-configuration:
+
+Configuration
+-------------
+
+BuildStream uses `YAML`_ for build definition and configuration. It has two
+levels of configuration: user and project level. `User-level configuration`_ is
+stored in your ``buildstream.conf`` file while `project-level configuration`_
+is defined in each project's ``project.conf`` file.
+
+.. note::
+
+   At the moment, remote execution can only be configured at project-level.
+
+.. _YAML: http://yaml.org
+.. _User-level configuration: https://buildstream.gitlab.io/buildstream/using_config.html
+.. _project-level configuration: https://buildstream.gitlab.io/buildstream/format_project.html
+
+
+Project configuration
+~~~~~~~~~~~~~~~~~~~~~
+
+In order to activate remote build execution at project-level, the project's
+``project.conf`` file must declare two specific configuration nodes:
+
+- ``artifacts`` for `remote CAS endpoint details`_.
+- ``remote-execution`` for `remote execution endpoint details`_.
+
+.. important::
+
+   BuildStream does not support multi-instance remote execution servers and thus
+   will always submit remote execution request omitting the instance name
+   parameter.
+
+.. important::
+
+   If you are using BuildGrid's artifact server, the server instance **must**
+   accept pushes from your client for remote execution to be possible.
+
+
+.. _remote CAS endpoint details: https://buildstream.gitlab.io/buildstream/install_artifacts.html#user-configuration
+.. _remote execution endpoint details: https://buildstream.gitlab.io/buildstream/format_project.html#remote-execution
+
+
+BuildBox tool
+~~~~~~~~~~~~~
+
+BuildStream performs builds in a `sandbox`_ on top of a project-defined
+environment, not relying on any host-tools. BuildGrid supports this kind of
+builds using the ``buildbox`` bot, a specific kind of bot relying on
+`BuildBox`_ for build execution.
+
+BuildBox can execute build commands in a sandbox on top of an environment
+constructed from provided sources using `FUSE`_. It also uses `bubblewrap`_
+for sandboxing without requiring root privileges.
+
+BuildBox being a rather young project, it isn't packaged yet and you'll have to
+build it from sources. You may want follow the `manual instructions`_ or prefer
+to build it with BuildStream using the `dedicated integration project`_
+(recommanded).
+
+.. important::
+
+   Whatever the method you use to install BuildBox, you also have to install
+   bubblewrap along, minimum required version being `0.1.8`_.
+
+.. _sandbox: https://buildstream.gitlab.io/buildstream/additional_sandboxing.html
+.. _BuildBox: https://gitlab.com/BuildStream/buildbox
+.. _FUSE: https://en.wikipedia.org/wiki/Filesystem_in_Userspace
+.. _bubblewrap: https://github.com/projectatomic/bubblewrap
+.. _manual instructions: https://gitlab.com/BuildStream/buildbox/blob/master/INSTALL.rst
+.. _dedicated integration project: https://gitlab.com/BuildStream/buildbox-integration
+.. _0.1.8: https://github.com/projectatomic/bubblewrap/releases/tag/v0.1.8
+
+
+.. _buildstream-example:
+
+Example build
+-------------
+
+The BuildStream repository contains `example projects`_ used for testing purpose
+in the project's `usage documentation section`_. We'll focus here on
+instructions on how to build the `autotools example`_ running BuildStream and
+BuildGrid on your local machine, compiling the `GNU Automake`_ hello example
+program in a sandbox on top of a minimal `Alpine Linux`_ environment.
+
+First, you need to checkout the buildstream repository sources:
+
+.. code-block:: sh
+
+   git clone https://gitlab.com/BuildStream/buildstream.git
+
+Next, change the current directory to the BuildStream project root:
+
+.. code-block:: sh
+
+   cd buildstream/doc/examples/autotools
+
+.. hint::
+
+   All the commands in the instructions below are expected to be executed from
+   that root directory (the autotools example project's root directory).
+
+Before running BuildStream and building the example project, you'll have to setup
+and run a BuildGrid server and bot. A minimal server's configuration is given
+below, paste it in a ``server.conf`` file in the root directory:
+
+.. literalinclude:: ./data/buildstream-example-server.conf
+   :language: yaml
+
+This defines a single unnamed server instance implementing a
+``ContentAddressableStorage`` (CAS) + ``Reference`` + ``ByteStream`` service
+together with an ``Execution`` + ``ActionCache`` service, both using the
+same in-memory storage. You can then start the BuildGrid server daemon using
+that configuration by running:
+
+.. code-block:: sh
+
+   bgd server start server.conf
+
+In order to perform the actual build work, you need to attach a ``buildbox``
+worker bot to that server for that unnamed instance. Once you've make sure
+that the ``buildbox`` tool is functional on your machine (refer to
+:ref:`configuration <buildstream-configuration>`), run:
+
+.. code-block:: sh
+
+   bgd bot --remote=http://localhost:50051 --parent= buildbox --fuse-dir=fuse --local-cas=cache
+
+The ``--remote`` option is used to specify the server location (running on the
+same machine here, and listening to port 50051). The ``--parent`` option is
+used to specify the server instance you except the bot to be attached to (empty
+here). ``--fuse-dir`` and ``--local-cas`` are specific to the ``buildbox`` bot
+and respectively specify the sandbox mount point and local CAS cache locations.
+Refer to the :ref:`CLI reference section <invoking-bgd-bot-buildbox>` for
+for command line interface details.
+
+The BuildGrid server is now ready to accept jobs and execute them. The example
+project needs some :ref:`configuration <buildstream-configuration>` tweaks in
+order to be build remotely. Below is the configuration fragment you should
+append at the end of the ``project.conf`` file from the root directory:
+
+.. code-block:: yaml
+
+   artifacts:
+     url: http://localhost:50051
+     push: true
+
+   remote-execution:
+     url: http://localhost:50051
+
+This activates BuildGrid's remote execution mode and points to the unnamed
+remote execution server instance at ``localhost:50051``.
+
+You can finally have BuildStream to build the example project by running:
+
+.. code-block:: sh
+
+   bst build hello.bst
+
+You can verify that the example has been successfully build by running the
+generated executable. Simply invoke:
+
+.. code-block:: sh
+
+   bst shell hello.bst -- hello
+
+.. _example projects: https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples
+.. _usage documentation section: http://buildstream.gitlab.io/buildstream/main_using.html
+.. _autotools example: https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples/autotools
+.. _GNU Automake: https://www.gnu.org/software/automake
+.. _Alpine Linux: https://www.alpinelinux.org

docs/source/using_internal.rst

 
-.. _internal-builds:
+.. _internal-client:
 
-Internal builds
+Internal client
 ===============
 
+BuildGrid contains a minimal remote execution client that can be used through
+the ``bgd`` command line interface.
+
 
 .. _dummy-test: