.gitlab-ci.yml

@@ -33,6 +33,7 @@ before_script:
     - ${BGD} server start buildgrid/_app/settings/default.yml &
     - sleep 1 # Allow server to boot
     - ${BGD} bot dummy &
+    - ${BGD} cas upload-dummy
     - ${BGD} execute request-dummy --wait-for-completion
 
 

buildgrid/_app/commands/cmd_cas.py

@@ -65,6 +65,23 @@ def cli(context, remote, instance_name, client_key, client_cert, server_cert):
     context.logger.debug("Starting for remote {}".format(context.remote))
 
 
+@cli.command('upload-dummy', short_help="Upload a dummy action. Should be used with `execute dummy-request`")
+@pass_context
+def upload_dummy(context):
+    context.logger.info("Uploading dummy action...")
+    action = remote_execution_pb2.Action(do_not_cache=True)
+    action_digest = create_digest(action.SerializeToString())
+
+    request = remote_execution_pb2.BatchUpdateBlobsRequest(instance_name=context.instance_name)
+    request.requests.add(digest=action_digest,
+                         data=action.SerializeToString())
+
+    stub = remote_execution_pb2_grpc.ContentAddressableStorageStub(context.channel)
+    response = stub.BatchUpdateBlobs(request)
+
+    context.logger.info(response)
+
+
 @cli.command('upload-files', short_help="Upload files to the CAS server.")
 @click.argument('files', nargs=-1, type=click.File('rb'), required=True)
 @pass_context

buildgrid/_app/commands/cmd_execute.py

@@ -76,9 +76,11 @@ def cli(context, remote, instance_name, client_key, client_cert, server_cert):
               help="Stream updates until jobs are completed.")
 @pass_context
 def request_dummy(context, number, wait_for_completion):
-    action_digest = remote_execution_pb2.Digest()
 
     context.logger.info("Sending execution request...")
+    action = remote_execution_pb2.Action(do_not_cache=True)
+    action_digest = create_digest(action.SerializeToString())
+
     stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
 
     request = remote_execution_pb2.ExecuteRequest(instance_name=context.instance_name,
@@ -90,9 +92,18 @@ def request_dummy(context, number, wait_for_completion):
         responses.append(stub.Execute(request))
 
     for response in responses:
+
         if wait_for_completion:
+            result = None
             for stream in response:
-                context.logger.info(stream)
+                result = stream
+                context.logger.info(result)
+
+            if not result.done:
+                click.echo("Result did not return True." +
+                           "Was the action uploaded to CAS?", err=True)
+                sys.exit(-1)
+
         else:
             context.logger.info(next(response))
 

buildgrid/server/_exceptions.py

@@ -46,3 +46,12 @@ class OutOfRangeError(BgdError):
 
     def __init__(self, message, detail=None, reason=None):
         super().__init__(message, detail=detail, domain=ErrorDomain.SERVER, reason=reason)
+
+
+class FailedPreconditionError(BgdError):
+    """ One or more errors occurred in setting up the action requested, such as a missing input
+    or command or no worker being available. The client may be able to fix the errors and retry.
+    """
+
+    def __init__(self, message, detail=None, reason=None):
+        super().__init__(message, detail=detail, domain=ErrorDomain.SERVER, reason=reason)

buildgrid/server/execution/instance.py

@@ -24,12 +24,12 @@ import logging
 from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Action
 
 from ..job import Job
-from .._exceptions import InvalidArgumentError
+from .._exceptions import InvalidArgumentError, FailedPreconditionError
 
 
 class ExecutionInstance:
 
-    def __init__(self, scheduler, storage=None):
+    def __init__(self, scheduler, storage):
         self.logger = logging.getLogger(__name__)
         self._storage = storage
         self._scheduler = scheduler
@@ -43,13 +43,12 @@ class ExecutionInstance:
         this action.
         """
 
-        do_not_cache = False
-        if self._storage is not None:
-            action = self._storage.get_message(action_digest, Action)
-            if action is not None:
-                do_not_cache = action.do_not_cache
+        action = self._storage.get_message(action_digest, Action)
 
-        job = Job(action_digest, do_not_cache, message_queue)
+        if not action:
+            raise FailedPreconditionError("Could not get action from storage.")
+
+        job = Job(action_digest, action.do_not_cache, message_queue)
         self.logger.info("Operation name: [{}]".format(job.name))
 
         self._scheduler.append_job(job, skip_cache_lookup)

buildgrid/server/execution/service.py

@@ -30,7 +30,7 @@ from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_p
 
 from buildgrid._protos.google.longrunning import operations_pb2
 
-from .._exceptions import InvalidArgumentError
+from .._exceptions import InvalidArgumentError, FailedPreconditionError
 
 
 class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
@@ -63,6 +63,12 @@ class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
             context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
             yield operations_pb2.Operation()
 
+        except FailedPreconditionError as e:
+            self.logger.error(e)
+            context.set_details(str(e))
+            context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
+            yield operations_pb2.Operation()
+
     def WaitExecution(self, request, context):
         try:
             names = request.name.split("/")

docs/source/using_internal.rst

-
 .. _internal-client:
 
 Internal client
@@ -19,7 +18,13 @@ In one terminal, start a server:
 
    bgd server start buildgrid/_app/settings/default.yml
 
-In another terminal, send a request for work:
+In another terminal, upload an action to CAS:
+
+.. code-block::sh
+
+   bgd cas upload-dummy
+
+Then send a request for work:
 
 .. code-block:: sh
 

tests/integration/execution_service.py

@@ -28,6 +28,7 @@ import pytest
 from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
 from buildgrid._protos.google.longrunning import operations_pb2
 
+from buildgrid.utils import create_digest
 from buildgrid.server import job
 from buildgrid.server.controller import ExecutionController
 from buildgrid.server.cas.storage import lru_memory_cache
@@ -37,6 +38,8 @@ from buildgrid.server.execution.service import ExecutionService
 
 
 server = mock.create_autospec(grpc.server)
+action = remote_execution_pb2.Action(do_not_cache=True)
+action_digest = create_digest(action.SerializeToString())
 
 
 @pytest.fixture
@@ -47,12 +50,16 @@ def context():
 
 @pytest.fixture(params=["action-cache", "no-action-cache"])
 def controller(request):
+    storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
+    write_session = storage.begin_write(action_digest)
+    storage.commit_write(action_digest, write_session)
+
     if request.param == "action-cache":
-        storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
         cache = ActionCache(storage, 50)
         yield ExecutionController(cache, storage)
+
     else:
-        yield ExecutionController()
+        yield ExecutionController(None, storage)
 
 
 # Instance to test
@@ -66,9 +73,6 @@ def instance(controller):
 
 @pytest.mark.parametrize("skip_cache_lookup", [True, False])
 def test_execute(skip_cache_lookup, instance, context):
-    action_digest = remote_execution_pb2.Digest()
-    action_digest.hash = 'zhora'
-
     request = remote_execution_pb2.ExecuteRequest(instance_name='',
                                                   action_digest=action_digest,
                                                   skip_cache_lookup=skip_cache_lookup)
@@ -91,10 +95,16 @@ def test_wrong_execute_instance(instance, context):
     context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)
 
 
-def test_wait_execution(instance, controller, context):
-    action_digest = remote_execution_pb2.Digest()
-    action_digest.hash = 'zhora'
+def test_no_action_digest_in_storage(instance, context):
+    request = remote_execution_pb2.ExecuteRequest(instance_name='',
+                                                  skip_cache_lookup=True)
+    response = instance.Execute(request, context)
+
+    next(response)
+    context.set_code.assert_called_once_with(grpc.StatusCode.FAILED_PRECONDITION)
 
+
+def test_wait_execution(instance, controller, context):
     j = job.Job(action_digest, None)
     j._operation.done = True
 

tests/integration/operations_service.py

@@ -24,18 +24,21 @@ import grpc
 from grpc._server import _Context
 import pytest
 
-from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
-from buildgrid._protos.google.longrunning import operations_pb2
-
+from buildgrid.utils import create_digest
 from buildgrid.server.controller import ExecutionController
-from buildgrid.server._exceptions import InvalidArgumentError
-
+from buildgrid.server.cas.storage import lru_memory_cache
 from buildgrid.server.operations import service
 from buildgrid.server.operations.service import OperationsService
+from buildgrid.server._exceptions import InvalidArgumentError
+
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from buildgrid._protos.google.longrunning import operations_pb2
 
 
 server = mock.create_autospec(grpc.server)
 instance_name = "blade"
+action = remote_execution_pb2.Action(do_not_cache=True)
+action_digest = create_digest(action.SerializeToString())
 
 
 # Can mock this
@@ -47,9 +50,6 @@ def context():
 # Requests to make
 @pytest.fixture
 def execute_request():
-    action_digest = remote_execution_pb2.Digest()
-    action_digest.hash = 'zhora'
-
     yield remote_execution_pb2.ExecuteRequest(instance_name='',
                                               action_digest=action_digest,
                                               skip_cache_lookup=True)
@@ -57,7 +57,11 @@ def execute_request():
 
 @pytest.fixture
 def controller():
-    yield ExecutionController()
+    storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
+    write_session = storage.begin_write(action_digest)
+    storage.commit_write(action_digest, write_session)
+
+    yield ExecutionController(None, storage)
 
 
 # Instance to test