.dockerignore

+# Ignore everything:
+*
+
+# Whitelist:
+!buildgrid
+!data/config
+!tests
+!LICENSE
+!README.rst
+!requirements.txt
+!requirements.auth.txt
+!setup.cfg
+!setup.py
+!_version.py
+!.coveragerc
+!.pylintrc

.gitlab-ci.yml

@@ -31,7 +31,7 @@ before_script:
 .run-dummy-job-template: &dummy-job
   stage: test
   script:
-    - ${BGD} server start buildgrid/_app/settings/default.yml &
+    - ${BGD} server start data/config/default.conf &
     - sleep 1 # Allow server to boot
     - ${BGD} bot dummy &
     - ${BGD} cas upload-dummy

BUILDSTREAM_README.rst

-Temp Demo Instructions
-======================
-
-A quick guide to getting remote execution working with BuildStream. Please change URL and certifcates / keys to your own.
-
-Downloaded and build::
-
-  https://gitlab.com/BuildStream/buildbox
-
-Copy build to bin/.
-
-Checkout branch::
-
-  https://gitlab.com/BuildStream/buildstream/tree/jmac/source_pushing_experiments
-
-Update to your URL::
-
-  https://gitlab.com/BuildStream/buildstream/blob/jmac/source_pushing_experiments/buildstream/sandbox/_sandboxremote.py#L73
-
-Start artifact server::
-
-  bst-artifact-server --port 11001 --server-key server.key --server-cert server.crt --client-certs client.crt --enable-push /home/user/
-
-Start bgd server::
-
-  bgd server start
-
-Run::
-
-  bgd bot buildbox
-
-Update project.conf in build area with::
-
-  artifacts:
-      url: https://localhost:11001
-      server-cert: server.crt
-
-      # Optional client key pair for authentication
-      client-key: client.key
-      client-cert: client.crt
-
-      push: true
-
-Run build with::
-
-  bst build --track something.bst

Dockerfile

+##
+# BuildGrid's Docker build manifest.
+#
+#  ¡FOR LOCAL DEVELOPMENT ONLY!
+#
+# Builds an image from local sources.
+#
+
 FROM python:3.5-stretch
 
-# Point the path to where buildgrid gets installed
-ENV PATH=$PATH:/root/.local/bin/
+# Use /app as working directory:
+WORKDIR /app
 
-# Upgrade python modules
-RUN python3 -m pip install --upgrade setuptools pip
+# Create a virtual environment:
+RUN [ \
+"python3", "-m", "venv", "/app/env" \
+]
 
-# Use /app as the current working directory
-WORKDIR /app
+# Upgrade Python core modules:
+RUN [ \
+"/app/env/bin/python", "-m", "pip", \
+"install", "--upgrade", \
+"setuptools", "pip", "wheel" \
+]
+
+# Install the main requirements:
+ADD requirements.txt /app
+RUN [ \
+"/app/env/bin/python", "-m", "pip", \
+"install", "--requirement", \
+"requirements.txt" \
+]
+
+# Install the auth. requirements:
+ADD requirements.auth.txt /app
+RUN [ \
+"/app/env/bin/python", "-m", "pip", \
+"install", "--requirement", \
+"requirements.auth.txt" \
+]
 
-# Copy the repo contents (source, config files, etc) in the WORKDIR
-COPY . .
+# Copy the repo. contents:
+COPY . /app
 
-# Install BuildGrid
-RUN pip install --user --editable .
+# Install BuildGrid:
+RUN [ \
+"/app/env/bin/python", "-m", "pip", \
+"install", "--editable", \
+".[auth,tests]" \
+]
 
-# Entry Point of the image (should get an additional argument from CMD, the path to the config file)
-ENTRYPOINT ["bgd", "server", "start", "-vv"]
+# Entry-point for the image:
+ENTRYPOINT [ \
+"/app/env/bin/bgd" \
+]
 
-# Default config file (used if no CMD specified when running)
-CMD ["buildgrid/_app/settings/default.yml"]
+# Default command (default config.):
+CMD [ \
+"server", "start", \
+"data/config/default.conf", \
+"-vvv" \
+]

buildgrid/_app/commands/cmd_server.py

@@ -119,6 +119,14 @@ def _create_server_from_config(configuration):
             click.echo("Error: Configuration, {}.".format(e), err=True)
             sys.exit(-1)
 
+    if 'thread-pool-size' in configuration:
+        try:
+            kargs['max_workers'] = int(configuration['thread-pool-size'])
+
+        except ValueError as e:
+            click.echo("Error: Configuration, {}.".format(e), err=True)
+            sys.exit(-1)
+
     server = BuildGridServer(**kargs)
 
     try:

buildgrid/_app/settings/default.yml

-server:
-  - !channel
-    port: 50051
-    insecure-mode: true
-
-description: |
-  A single default instance.
-
-instances:
-  - name: ''
-    description: |
-      The main server
-
-    storages:
-      - !disk-storage &main-storage
-        path: !expand-path $HOME/cas
-
-    services:
-      - !action-cache &main-action
-        storage: *main-storage
-        max-cached-refs: 256
-        allow-updates: true
-
-      - !execution
-        storage: *main-storage
-        action-cache: *main-action
-
-      - !cas
-        storage: *main-storage
-
-      - !bytestream
-        storage: *main-storage

buildgrid/_app/settings/reference.yml

@@ -136,3 +136,7 @@ monitoring:
   #  binary  - Protobuf binary format.
   #  json    - JSON format.
   serialization-format: binary
+
+##
+# Maximum number of gRPC threads.
+thread-pool-size: 20

buildgrid/_app/settings/remote-storage.yml

-server:
-  - !channel
-    port: 50051
-    insecure-mode: true
-
-description: |
-  A single default instance with remote storage.
-
-instances:
-  - name: ''
-    description: |
-      The main server
-
-    storages:
-      - !remote-storage &main-storage
-        url: http://localhost:50052
-        instance-name: main
-        # credentials:
-        #   tls-client-key: null
-        #   tls-client-cert: null
-        #   tls-server-cert: null
-
-    services:
-      - !action-cache &main-action
-        storage: *main-storage
-        max-cached-refs: 256
-        allow-updates: true
-
-      - !execution
-        storage: *main-storage
-        action-cache: *main-action
-
-      - !cas
-        storage: *main-storage
-
-      - !bytestream
-        storage: *main-storage

buildgrid/server/bots/instance.py

@@ -32,6 +32,8 @@ class BotsInterface:
 
     def __init__(self, scheduler):
         self.__logger = logging.getLogger(__name__)
+        # Turn on debug mode based on log verbosity level:
+        self.__debug = self.__logger.getEffectiveLevel() <= logging.DEBUG
 
         self._scheduler = scheduler
         self._instance_name = None
@@ -65,13 +67,11 @@ class BotsInterface:
         register with the service, the old one should be closed along
         with all its jobs.
         """
-        bot_id = bot_session.bot_id
-
-        if bot_id == "":
-            raise InvalidArgumentError("bot_id needs to be set by client")
+        if not bot_session.bot_id:
+            raise InvalidArgumentError("Bot's id must be set by client.")
 
         try:
-            self._check_bot_ids(bot_id)
+            self._check_bot_ids(bot_session.bot_id)
         except InvalidArgumentError:
             pass
 
@@ -79,21 +79,27 @@ class BotsInterface:
         name = "{}/{}".format(parent, str(uuid.uuid4()))
         bot_session.name = name
 
-        self._bot_ids[name] = bot_id
-
-        self.__logger.info("Created bot session name=[%s] with bot_id=[%s]", name, bot_id)
+        self._bot_ids[name] = bot_session.bot_id
 
         # We want to keep a copy of lease ids we have assigned
         self._assigned_leases[name] = set()
 
         self._request_leases(bot_session)
+
+        if self.__debug:
+            self.__logger.info("Opened session name=[%s] for bot=[%s], leases=[%s]",
+                               bot_session.name, bot_session.bot_id,
+                               ",".join([lease.id[:8] for lease in bot_session.leases]))
+        else:
+            self.__logger.info("Opened session, name=[%s] for bot=[%s]",
+                               bot_session.name, bot_session.bot_id)
+
         return bot_session
 
     def update_bot_session(self, name, bot_session):
         """ Client updates the server. Any changes in state to the Lease should be
         registered server side. Assigns available leases with work.
         """
-        self.__logger.debug("Updating bot session name=[%s]", name)
         self._check_bot_ids(bot_session.bot_id, name)
         self._check_assigned_leases(bot_session)
 
@@ -111,6 +117,15 @@ class BotsInterface:
                 bot_session.leases.remove(lease)
 
         self._request_leases(bot_session)
+
+        if self.__debug:
+            self.__logger.info("Sending session update, name=[%s], for bot=[%s], leases=[%s]",
+                               bot_session.name, bot_session.bot_id,
+                               ",".join([lease.id[:8] for lease in bot_session.leases]))
+        else:
+            self.__logger.info("Sending session update, name=[%s], for bot=[%s]",
+                               bot_session.name, bot_session.bot_id)
+
         return bot_session
 
     # --- Private API ---

buildgrid/server/instance.py

@@ -91,6 +91,8 @@ class BuildGridServer:
         self.__grpc_server = grpc.server(self.__grpc_executor,
                                          options=(('grpc.so_reuseport', 0),))
 
+        self.__logger.debug("Setting up gRPC server with thread-limit=[%s]", max_workers)
+
         self.__main_loop = asyncio.get_event_loop()
 
         self.__monitoring_bus = None

buildgrid/server/scheduler.py

@@ -150,14 +150,14 @@ class Scheduler:
             raise NotFoundError("Operation name does not exist: [{}]"
                                 .format(operation_name))
 
-        job.unregister_operation_peer(operation_name, peer)
-
         if not job.n_peers_for_operation(operation_name):
             del self.__jobs_by_operation[operation_name]
 
         if not job.n_peers and job.done and not job.lease:
             self._delete_job(job.name)
 
+        job.unregister_operation_peer(operation_name, peer)
+
     def queue_job_action(self, action, action_digest, platform_requirements=None,
                          priority=0, skip_cache_lookup=False):
         """Inserts a newly created job into the execution queue.

buildgrid/_app/settings/cas.yml → data/config/artifacts.conf

@@ -2,31 +2,34 @@ server:
   - !channel
     port: 50052
     insecure-mode: true
-    # credentials:
-    #   tls-server-key: null
-    #   tls-server-cert: null
-    #   tls-client-certs: null
 
-description: |
-  Just a CAS with some reference storage.
+description: >
+  Artifact server configuration:
+    - Unauthenticated plain HTTP at :50052
+    - Single instance: (empty-name)
+    - On-disk data stored in $HOME
+    - Hosted services:
+       - ReferenceStorage
+       - ContentAddressableStorage
+       - ByteStream
 
 instances:
   - name: ''
     description: |
-      The main server
+      The unique '' instance.
 
     storages:
-      - !disk-storage &main-storage
-        path: !expand-path $HOME/cas
+      - !disk-storage &data-store
+        path: !expand-path $HOME/.cache/buildgrid/store
 
     services:
       - !cas
-        storage: *main-storage
+        storage: *data-store
 
       - !bytestream
-        storage: *main-storage
+        storage: *data-store
 
       - !reference-cache
-        storage: *main-storage
-        max-cached-refs: 256
+        storage: *data-store
+        max-cached-refs: 512
         allow-updates: true

data/config/controller.conf

+server:
+  - !channel
+    port: 50051
+    insecure-mode: true
+
+description: >
+  Docker Compose controller configuration:
+    - Unauthenticated plain HTTP at :50051
+    - Single instance: local
+    - Expects a remote CAS at :50052
+    - Hosted services:
+       - ActionCache
+       - Execute
+
+authorization:
+  method: none
+
+monitoring:
+  enabled: false
+
+instances:
+  - name: local
+    description: |
+      The unique 'local' instance.
+
+    storages:
+      - !remote-storage &data-store
+        url: http://storage:50052
+        instance-name: local
+
+    services:
+      - !action-cache &build-cache
+        storage: *data-store
+        max-cached-refs: 256
+        cache-failed-actions: true
+        allow-updates: true
+
+      - !execution
+        storage: *data-store
+        action-cache: *build-cache
+        action-browser-url: http://localhost:8080
+
+thread-pool-size: 100

data/config/default.conf

+server:
+  - !channel
+    port: 50051
+    insecure-mode: true
+
+description: >
+  BuildGrid's default configuration:
+    - Unauthenticated plain HTTP at :50052
+    - Single instance: main
+    - In-memory data, max. 2Gio
+    - Hosted services:
+       - ActionCache
+       - Execute
+       - ContentAddressableStorage
+       - ByteStream
+
+authorization:
+  method: none
+
+monitoring:
+  enabled: false
+
+instances:
+  - name: ''
+    description: |
+      The unique '' instance.
+
+    storages:
+      - !lru-storage &data-store
+        size: 2048M
+
+    services:
+      - !action-cache &build-cache
+        storage: *data-store
+        max-cached-refs: 256
+        cache-failed-actions: true
+        allow-updates: true
+
+      - !execution
+        storage: *data-store
+        action-cache: *build-cache
+
+      - !cas
+        storage: *data-store
+
+      - !bytestream
+        storage: *data-store

data/config/storage.conf

+server:
+  - !channel
+    port: 50052
+    insecure-mode: true
+
+description: >
+  Docker Compose storage configuration:
+    - Unauthenticated plain HTTP at :50052
+    - Single instance: local
+    - On-disk data stored in /var
+    - Hosted services:
+       - ContentAddressableStorage
+       - ByteStream
+
+authorization:
+  method: none
+
+monitoring:
+  enabled: false
+
+instances:
+  - name: local
+    description: |
+      The unique 'local' instance.
+
+    storages:
+      - !disk-storage &data-store
+        path: /var/lib/buildgrid/store
+
+    services:
+      - !cas
+        storage: *data-store
+
+      - !bytestream
+        storage: *data-store
+
+thread-pool-size: 100

docker-compose.yml

+##
+# BuildGrid's Docker Compose manifest.
+#
+#   ¡FOR LOCAL DEVELOPMENT ONLY!
+#
+# Spins-up a 'local' grid instance:
+#   - Controller at http://localhost:50051
+#   - CAS + AC at: http://localhost:50052
+#
+# Basic usage:
+#  - docker-compose build
+#  - docker-compose up --scale bots=10
+#  - docker-compose down
+#  - docker volume inspect buildgrid_data
+#  - docker volume rm buildgrid_data
+#  - docker image rm buildgrid:local
+#
+version: "3.2"
+
+services:
+  storage:
+    build:
+      context: .
+    image: buildgrid:local
+    command: [
+      "server", "start", "-vvv",
+      "/app/config/storage.conf"]
+    volumes:
+      - type: volume
+        source: data
+        target: /var/lib/buildgrid/store
+        volume:
+          nocopy: true
+      - type: bind
+        source: ./data/config/storage.conf
+        target: /app/config/storage.conf
+    ports:
+      - "50052:50052"
+    networks:
+      - backend
+      - host
+
+  controller:
+    image: buildgrid:local
+    command: [
+      "server", "start", "-vvv",
+      "/app/config/controller.conf"]
+    volumes:
+      - type: bind
+        source: ./data/config/controller.conf
+        target: /app/config/controller.conf
+    ports:
+      - "50051:50051"
+    networks:
+      - backend
+      - host
+
+  bots:  # To be scaled horizontaly
+    image: buildgrid:local
+    command: [
+      "bot", "--parent=local",
+      "--remote=http://controller:50051",
+      "--remote-cas=http://storage:50052",
+      "host-tools"]
+    depends_on:
+      - controller
+    networks:
+      - backend
+
+networks:
+  backend:
+  host:
+
+volumes:
+  data:

docs/source/installation.rst

@@ -21,20 +21,24 @@ How to install BuildGrid directly onto your machine.
 Prerequisites
 ~~~~~~~~~~~~~
 
-BuildGrid only supports ``python3 >= 3.5`` but has no system requirements. Main
-Python dependencies, automatically handled during installation, include:
+BuildGrid only supports ``python3 >= 3.5.3`` but has no system requirements.
+Main Python dependencies, automatically handled during installation, include:
 
 - `boto3`_: the Amazon Web Services (AWS) SDK for Python.
 - `click`_: a Python composable command line library.
 - `grpcio`_: Google's `gRPC`_ Python interface.
+- `janus`_: a mixed sync-async Python queue.
 - `protobuf`_: Google's `protocol-buffers`_ Python interface.
+- `PyYAML`_: a YAML parser and emitter for Python.
 
 .. _boto3: https://pypi.org/project/boto3
 .. _click: https://pypi.org/project/click
 .. _grpcio: https://pypi.org/project/grpcio
 .. _gRPC: https://grpc.io
+.. _janus: https://pypi.org/project/janus
 .. _protobuf: https://pypi.org/project/protobuf
 .. _protocol-buffers: https://developers.google.com/protocol-buffers
+.. _PyYAML: https://pypi.org/project/PyYAML
 
 
 .. _install-host-source-install:
@@ -42,20 +46,30 @@ Python dependencies, automatically handled during installation, include:
 Install from sources
 ~~~~~~~~~~~~~~~~~~~~
 
-BuildGrid has ``setuptools`` support. In order to install it to your home
-directory, typically under ``~/.local``, simply run:
+BuildGrid has ``setuptools`` support. We recommend installing it in a dedicated
+`virtual environment`_. In order to do so in an environment named ``env``
+placed in the source tree, run:
 
 .. code-block:: sh
 
-   git clone https://gitlab.com/BuildGrid/buildgrid.git && cd buildgrid
-   pip3 install --user --editable .
+   git clone https://gitlab.com/BuildGrid/buildgrid.git
+   cd buildgrid
+   python3 -m venv env
+   env/bin/python -m pip install --upgrade setuptools pip wheel
+   env/bin/python -m pip install --editable .
 
-Additionally, and if your distribution does not already include it, you may
-have to adjust your ``PATH``, in ``~/.bashrc``, with:
+.. hint::
+
+   Once created, the virtual environment can be *activated* by sourcing the
+   ``env/bin/activate`` script. In an activated terminal session, simply run
+   ``deactivate`` to later *deactivate* it.
+
+Once completed, you can check that installation succeed by locally starting the
+BuildGrid server with default configuration. Simply run:
 
 .. code-block:: sh
 
-   export PATH="${PATH}:${HOME}/.local/bin"
+   env/bin/bgd server start data/config/default.conf -vvv
 
 .. note::
 
@@ -63,66 +77,112 @@ have to adjust your ``PATH``, in ``~/.bashrc``, with:
    ``tests``. They declare required dependency for, respectively, authentication
    and authorization management, generating documentation and running
    unit-tests. They can be use as helpers for setting up a development
-   environment. To use them simply run:
+   environment. To use them run:
 
    .. code-block:: sh
 
-      pip3 install --user --editable ".[auth,docs,tests]"
+      env/bin/python -m pip install --editable ".[auth,docs,tests]"
+
+.. _virtual environment: https://docs.python.org/3/library/venv.html
 
 
 .. install-docker:
 
-Installation through Docker
----------------------------
+Install through Docker
+----------------------
 
-How to build a Docker image that runs BuildGrid.
+BuildGrid comes with Docker support for local development use-cases.
 
-.. _install-docker-prerequisites:
+.. caution::
 
-Prerequisites
-~~~~~~~~~~~~~
+   The Docker manifests are intended to be use for **local development only**.
+   Do **not** use them in production.
 
-A working Docker installation. Please consult `Docker's Getting Started Guide`_ if you don't already have it installed.
+Please consult the `Get Started with Docker`_ guide if you are looking for
+instructions on how to setup Docker on your machine.
 
-.. _`Docker's Getting Started Guide`: https://www.docker.com/get-started
+.. _`Get Started with Docker`: https://www.docker.com/get-started
 
 
 .. _install-docker-build:
 
-Docker Container from Sources
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Docker build
+~~~~~~~~~~~~
 
-To clone the source code and build a Docker image, simply run:
+BuildGrid ships a ``Dockerfile`` manifest for building images from source using
+``docker build``. In order to produce a ``buildgrid:local`` base image, run:
 
 .. code-block:: sh
 
-   git clone https://gitlab.com/BuildGrid/buildgrid.git && cd buildgrid
-   docker build -t buildgrid_server .
+   git clone https://gitlab.com/BuildGrid/buildgrid.git
+   cd buildgrid
+   docker build --tag buildgrid:local .
 
 .. note::
 
-   The image built will contain the contents of the source code directory, including
-   configuration files.
-   
+   The image built will contain the Python sources, including example
+   configuration files. The main endpoint is the ``bgd`` CLI tools and the
+   default command shall run the BuildGrid server loading default configuration.
+
+Once completed, you can check that build succeed by locally starting in a
+container the BuildGrid server with default configuration. Simply run:
+
+.. code-block:: sh
+
+   docker run --interactive --publish 50051:50051 buildgrid:local
+
 .. hint::
 
-    Whenever the source code is updated or new configuration files are made, you need to re-build 
-    the image.
+   You can run any of the BuildGrid CLI tool using that image, simply pass extra
+   arguments to ``docker run`` the same way you would pass them to ``bgd``.
+
+    Bear in mind that whenever the source code or the configuration files are
+    updated, you **must** re-build the image.
+
+
+.. _install-docker-compose:
 
-After building the Docker image, to run BuildGrid using the default configuration file 
-(found in `buildgrid/_app/settings/default.yml`), simply run:
+Docker Compose
+~~~~~~~~~~~~~~
+
+BuildGrid ships a ``docker-compose.yml`` manifest for building and running a
+grid locally using ``docker-compose``. In order to produce a
+``buildgrid:local`` base image, run:
 
 .. code-block:: sh
 
-   docker run -i -p 50051:50051 buildgrid_server
+   git clone https://gitlab.com/BuildGrid/buildgrid.git
+   cd buildgrid
+   docker-compose build
+
+Once completed, you can start a minimal grid by running:
+
+.. code-block:: sh
+
+   docker-compose up
 
 .. note::
 
-    To run BuildGrid using a different configuration file, include the relative path to the
-    configuration file at the end of the command above. For example, to run the default 
-    standalone CAS server (without an execution service), simply run:
+   The grid is composed of three containers:
+
+   - An execution and action-cache service available at
+     ``http://localhost:50051``.
+   - An CAS service available at ``http://localhost:50052``.
+   - A single ``local`` instance with one host-tools based worker bot attached.
 
-       .. code-block:: sh
+.. hint::
+
+   You can spin up more bots by using ``docker-compose`` scaling capabilities:
+
+   .. code-block:: sh
+
+      docker-compose up --scale bots=12
+
+.. hint::
 
-            docker run -i -p 50052:50052 buildgrid_server buildgrid/_app/settings/cas.yml
+   The contained services configuration files are bind mounted into the
+   container, no need to rebuild the base image on configuration update.
+   Configuration files are read from:
 
+   - ``data/config/controller.conf`` for the execution service.
+   - ``data/config/storage.conf`` for the CAS and action-cache service.

docs/source/using_cas_server.rst

@@ -27,7 +27,7 @@ This defines a single ``main`` instance of the ``CAS``, ``Bytestream`` and ``Ref
 
 .. code-block:: sh
 
-   bgd server start example.conf
+   bgd server start data/config/artifacts.conf
 
 The server should now be available to use.
 

docs/source/using_internal.rst

@@ -17,7 +17,7 @@ In one terminal, start a server:
 
 .. code-block:: sh
 
-   bgd server start buildgrid/_app/settings/default.yml
+   bgd server start data/config/default.conf
 
 In another terminal, upload an action to CAS:
 
@@ -78,7 +78,7 @@ Now start a BuildGrid server, passing it a directory it can write a CAS to:
 
 .. code-block:: sh
 
-   bgd server start buildgrid/_app/settings/default.yml
+   bgd server start data/config/default.conf
 
 Start the following bot session:
 

requirements.auth.txt

+cryptography >= 1.8.0  # Required by pyjwt for RSA
+pyjwt >= 1.5.0