.gitlab-ci.yml

@@ -33,6 +33,7 @@ before_script:
     - ${BGD} server start buildgrid/_app/settings/default.yml &
     - sleep 1 # Allow server to boot
     - ${BGD} bot dummy &
+    - ${BGD} cas upload-dummy
     - ${BGD} execute request-dummy --wait-for-completion
 
 

buildgrid/_app/bots/buildbox.py

@@ -19,7 +19,9 @@ import tempfile
 
 from google.protobuf import any_pb2
 
+from buildgrid.settings import HASH_LENGTH
 from buildgrid.client.cas import upload
+from buildgrid._exceptions import BotError
 from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
 from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
 from buildgrid.utils import read_file, write_file, parse_to_pb2_from_fetch
@@ -87,17 +89,30 @@ def work_buildbox(context, lease):
 
             command_line = subprocess.Popen(command_line,
                                             stdin=subprocess.PIPE,
-                                            stdout=subprocess.PIPE)
-            # TODO: Should return the stdout and stderr to the user.
-            command_line.communicate()
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.PIPE)
+            stdout, stderr = command_line.communicate()
+            action_result = remote_execution_pb2.ActionResult()
+            # TODO: Upload to CAS or output RAW
+            # For now, just pass raw
+            # https://gitlab.com/BuildGrid/buildgrid/issues/90
+            action_result.stdout_raw = stdout
+
+            if stderr:
+                # TODO: Upload to CAS or output RAW
+                # For now, just pass raw
+                # https://gitlab.com/BuildGrid/buildgrid/issues/90
+                logger.error("BuildBox error: [{}]".format(stderr))
+                raise BotError(stderr, detail=stdout, reason="Captured stderr")
 
             output_digest = remote_execution_pb2.Digest()
             output_digest.ParseFromString(read_file(output_digest_file.name))
 
             logger.debug("Output root digest: {}".format(output_digest))
 
-            if len(output_digest.hash) < 64:
-                logger.warning("Buildbox command failed - no output root digest present.")
+            if len(output_digest.hash) < HASH_LENGTH:
+                raise BotError("Output hash length too small",
+                               detail=stdout, reason="No output root digest present.")
 
             # TODO: Have BuildBox helping us creating the Tree instance here
             # See https://gitlab.com/BuildStream/buildbox/issues/7 for details
@@ -110,7 +125,6 @@ def work_buildbox(context, lease):
             output_directory.tree_digest.CopyFrom(output_tree_digest)
             output_directory.path = os.path.relpath(working_directory, start='/')
 
-            action_result = remote_execution_pb2.ActionResult()
             action_result.output_directories.extend([output_directory])
 
             action_result_any = any_pb2.Any()

buildgrid/_app/bots/temp_directory.py

@@ -77,11 +77,20 @@ def work_temp_directory(context, lease):
                                    universal_newlines=True,
                                    env=environment,
                                    stdin=subprocess.PIPE,
-                                   stdout=subprocess.PIPE)
-        # TODO: Should return the stdout and stderr in the ActionResult.
-        process.communicate()
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
+        stdout, stderr = process.communicate()
 
         action_result = remote_execution_pb2.ActionResult()
+        # TODO: Upload to CAS or output RAW
+        # For now, just pass raw
+        # https://gitlab.com/BuildGrid/buildgrid/issues/90
+        action_result.stdout_raw = stdout
+        action_result.stderr_raw = stderr
+
+        if stderr:
+            logger.error("Bot error: [{}]".format(stderr))
+            raise BotError(stderr, detail=stdout, reason="Captured stderr")
 
         with upload(context.cas_channel, instance=instance_name) as cas:
             for output_path in command.output_files:

buildgrid/_app/commands/cmd_cas.py

@@ -65,6 +65,23 @@ def cli(context, remote, instance_name, client_key, client_cert, server_cert):
     context.logger.debug("Starting for remote {}".format(context.remote))
 
 
+@cli.command('upload-dummy', short_help="Upload a dummy action. Should be used with `execute dummy-request`")
+@pass_context
+def upload_dummy(context):
+    context.logger.info("Uploading dummy action...")
+    action = remote_execution_pb2.Action(do_not_cache=True)
+    action_digest = create_digest(action.SerializeToString())
+
+    request = remote_execution_pb2.BatchUpdateBlobsRequest(instance_name=context.instance_name)
+    request.requests.add(digest=action_digest,
+                         data=action.SerializeToString())
+
+    stub = remote_execution_pb2_grpc.ContentAddressableStorageStub(context.channel)
+    response = stub.BatchUpdateBlobs(request)
+
+    context.logger.info(response)
+
+
 @cli.command('upload-files', short_help="Upload files to the CAS server.")
 @click.argument('files', nargs=-1, type=click.File('rb'), required=True)
 @pass_context

buildgrid/_app/commands/cmd_execute.py

@@ -76,9 +76,11 @@ def cli(context, remote, instance_name, client_key, client_cert, server_cert):
               help="Stream updates until jobs are completed.")
 @pass_context
 def request_dummy(context, number, wait_for_completion):
-    action_digest = remote_execution_pb2.Digest()
 
     context.logger.info("Sending execution request...")
+    action = remote_execution_pb2.Action(do_not_cache=True)
+    action_digest = create_digest(action.SerializeToString())
+
     stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
 
     request = remote_execution_pb2.ExecuteRequest(instance_name=context.instance_name,
@@ -90,9 +92,18 @@ def request_dummy(context, number, wait_for_completion):
         responses.append(stub.Execute(request))
 
     for response in responses:
+
         if wait_for_completion:
+            result = None
             for stream in response:
-                context.logger.info(stream)
+                result = stream
+                context.logger.info(result)
+
+            if not result.done:
+                click.echo("Result did not return True." +
+                           "Was the action uploaded to CAS?", err=True)
+                sys.exit(-1)
+
         else:
             context.logger.info(next(response))
 

buildgrid/bot/bot_session.py

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Disable broad exception catch
+# pylint: disable=broad-except
+
 
 """
 Bot Session
@@ -23,10 +26,14 @@ import asyncio
 import logging
 import platform
 import uuid
-
 from enum import Enum
 
+import grpc
+from google.protobuf import any_pb2
+
 from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2, worker_pb2
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from buildgrid._exceptions import BotError
 
 
 class BotStatus(Enum):
@@ -142,13 +149,35 @@ class BotSession:
 
     async def create_work(self, lease):
         self.logger.debug("Work created: [{}]".format(lease.id))
-
+        input_lease = lease
         loop = asyncio.get_event_loop()
-        lease = await loop.run_in_executor(None, self._work, self._context, lease)
+
+        try:
+            lease = await loop.run_in_executor(None, self._work, self._context, lease)
+
+        except BotError as e:
+            self.logger.error("Bot error thrown: [{}]".format(e))
+            lease = self._lease_error(input_lease, e)
+
+        except grpc.RpcError as e:
+            self.logger.error("Connection error thrown: [{}]".format(e))
+            lease = self._lease_error(input_lease, e)
+
+        except Exception as e:
+            self.logger.error("Connection error thrown: [{}]".format(e))
+            lease = self._lease_error(input_lease, e)
 
         self.logger.debug("Work complete: [{}]".format(lease.id))
         self.lease_completed(lease)
 
+    def _lease_error(self, lease, error):
+        action_result = remote_execution_pb2.ActionResult()
+        action_result.stderr_raw = str(error)
+        action_result_any = any_pb2.Any()
+        action_result_any.Pack(action_result)
+        lease.result.CopyFrom(action_result_any)
+        return lease
+
 
 class Worker:
     def __init__(self, properties=None, configs=None):

buildgrid/server/_exceptions.py

@@ -46,3 +46,12 @@ class OutOfRangeError(BgdError):
 
     def __init__(self, message, detail=None, reason=None):
         super().__init__(message, detail=detail, domain=ErrorDomain.SERVER, reason=reason)
+
+
+class FailedPreconditionError(BgdError):
+    """ One or more errors occurred in setting up the action requested, such as a missing input
+    or command or no worker being available. The client may be able to fix the errors and retry.
+    """
+
+    def __init__(self, message, detail=None, reason=None):
+        super().__init__(message, detail=detail, domain=ErrorDomain.SERVER, reason=reason)

buildgrid/server/execution/instance.py

@@ -24,12 +24,12 @@ import logging
 from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Action
 
 from ..job import Job
-from .._exceptions import InvalidArgumentError
+from .._exceptions import InvalidArgumentError, FailedPreconditionError
 
 
 class ExecutionInstance:
 
-    def __init__(self, scheduler, storage=None):
+    def __init__(self, scheduler, storage):
         self.logger = logging.getLogger(__name__)
         self._storage = storage
         self._scheduler = scheduler
@@ -43,13 +43,12 @@ class ExecutionInstance:
         this action.
         """
 
-        do_not_cache = False
-        if self._storage is not None:
-            action = self._storage.get_message(action_digest, Action)
-            if action is not None:
-                do_not_cache = action.do_not_cache
+        action = self._storage.get_message(action_digest, Action)
 
-        job = Job(action_digest, do_not_cache, message_queue)
+        if not action:
+            raise FailedPreconditionError("Could not get action from storage.")
+
+        job = Job(action_digest, action.do_not_cache, message_queue)
         self.logger.info("Operation name: [{}]".format(job.name))
 
         self._scheduler.append_job(job, skip_cache_lookup)

buildgrid/server/execution/service.py

@@ -30,7 +30,7 @@ from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_p
 
 from buildgrid._protos.google.longrunning import operations_pb2
 
-from .._exceptions import InvalidArgumentError
+from .._exceptions import InvalidArgumentError, FailedPreconditionError
 
 
 class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
@@ -63,6 +63,12 @@ class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
             context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
             yield operations_pb2.Operation()
 
+        except FailedPreconditionError as e:
+            self.logger.error(e)
+            context.set_details(str(e))
+            context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
+            yield operations_pb2.Operation()
+
     def WaitExecution(self, request, context):
         try:
             names = request.name.split("/")

buildgrid/server/job.py

@@ -21,6 +21,7 @@ from enum import Enum
 
 from google.protobuf import any_pb2
 
+from buildgrid._protos.google.rpc import code_pb2, status_pb2
 from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
 from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2
 from buildgrid._protos.google.longrunning import operations_pb2
@@ -121,10 +122,14 @@ class Job:
         self._operation.metadata.CopyFrom(self._pack_any(self.get_operation_meta()))
         if self.result is not None:
             self._operation.done = True
-            action_result = remote_execution_pb2.ActionResult()
-            self.result.Unpack(action_result)
-            response = remote_execution_pb2.ExecuteResponse(result=action_result,
-                                                            cached_result=self.result_cached)
+            status = status_pb2.Status()
+            status.code = code_pb2.OK
+            if self.result.stderr_raw or self.result.stderr_digest:
+                status.code = code_pb2.INTERNAL
+
+            response = remote_execution_pb2.ExecuteResponse(result=self.result,
+                                                            cached_result=self.result_cached,
+                                                            status=status)
             self._operation.response.CopyFrom(self._pack_any(response))
 
         return self._operation

buildgrid/server/scheduler.py

@@ -27,6 +27,7 @@ from google.protobuf import any_pb2
 
 
 from buildgrid.server._exceptions import NotFoundError
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
 from buildgrid._protos.google.longrunning import operations_pb2
 
 from .job import ExecuteStage, LeaseState
@@ -84,10 +85,13 @@ class Scheduler:
 
     def job_complete(self, name, result):
         job = self.jobs[name]
-        job.result = result
-        job.update_execute_stage(ExecuteStage.COMPLETED)
+        action_result = remote_execution_pb2.ActionResult()
+        result.Unpack(action_result)
+        job.result = action_result
         if not job.do_not_cache and self._action_cache is not None:
-            self._action_cache.update_action_result(job.action_digest, result)
+            if not (action_result.stderr_raw or action_result.stderr_digest):
+                self._action_cache.update_action_result(job.action_digest, result)
+        job.update_execute_stage(ExecuteStage.COMPLETED)
 
     def get_operations(self):
         response = operations_pb2.ListOperationsResponse()

buildgrid/settings.py

@@ -3,3 +3,4 @@ import hashlib
 
 # The hash function that CAS uses
 HASH = hashlib.sha256
+HASH_LENGTH = 64

docs/source/using_internal.rst

-
 .. _internal-client:
 
 Internal client
@@ -19,7 +18,13 @@ In one terminal, start a server:
 
    bgd server start buildgrid/_app/settings/default.yml
 
-In another terminal, send a request for work:
+In another terminal, upload an action to CAS:
+
+.. code-block::sh
+
+   bgd cas upload-dummy
+
+Then send a request for work:
 
 .. code-block:: sh
 

tests/integration/execution_service.py

@@ -28,6 +28,7 @@ import pytest
 from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
 from buildgrid._protos.google.longrunning import operations_pb2
 
+from buildgrid.utils import create_digest
 from buildgrid.server import job
 from buildgrid.server.controller import ExecutionController
 from buildgrid.server.cas.storage import lru_memory_cache
@@ -37,6 +38,8 @@ from buildgrid.server.execution.service import ExecutionService
 
 
 server = mock.create_autospec(grpc.server)
+action = remote_execution_pb2.Action(do_not_cache=True)
+action_digest = create_digest(action.SerializeToString())
 
 
 @pytest.fixture
@@ -47,12 +50,16 @@ def context():
 
 @pytest.fixture(params=["action-cache", "no-action-cache"])
 def controller(request):
+    storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
+    write_session = storage.begin_write(action_digest)
+    storage.commit_write(action_digest, write_session)
+
     if request.param == "action-cache":
-        storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
         cache = ActionCache(storage, 50)
         yield ExecutionController(cache, storage)
+
     else:
-        yield ExecutionController()
+        yield ExecutionController(None, storage)
 
 
 # Instance to test
@@ -66,9 +73,6 @@ def instance(controller):
 
 @pytest.mark.parametrize("skip_cache_lookup", [True, False])
 def test_execute(skip_cache_lookup, instance, context):
-    action_digest = remote_execution_pb2.Digest()
-    action_digest.hash = 'zhora'
-
     request = remote_execution_pb2.ExecuteRequest(instance_name='',
                                                   action_digest=action_digest,
                                                   skip_cache_lookup=skip_cache_lookup)
@@ -91,10 +95,16 @@ def test_wrong_execute_instance(instance, context):
     context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)
 
 
-def test_wait_execution(instance, controller, context):
-    action_digest = remote_execution_pb2.Digest()
-    action_digest.hash = 'zhora'
+def test_no_action_digest_in_storage(instance, context):
+    request = remote_execution_pb2.ExecuteRequest(instance_name='',
+                                                  skip_cache_lookup=True)
+    response = instance.Execute(request, context)
+
+    next(response)
+    context.set_code.assert_called_once_with(grpc.StatusCode.FAILED_PRECONDITION)
 
+
+def test_wait_execution(instance, controller, context):
     j = job.Job(action_digest, None)
     j._operation.done = True
 

tests/integration/operations_service.py

@@ -24,18 +24,21 @@ import grpc
 from grpc._server import _Context
 import pytest
 
-from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
-from buildgrid._protos.google.longrunning import operations_pb2
-
+from buildgrid.utils import create_digest
 from buildgrid.server.controller import ExecutionController
-from buildgrid.server._exceptions import InvalidArgumentError
-
+from buildgrid.server.cas.storage import lru_memory_cache
 from buildgrid.server.operations import service
 from buildgrid.server.operations.service import OperationsService
+from buildgrid.server._exceptions import InvalidArgumentError
+
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from buildgrid._protos.google.longrunning import operations_pb2
 
 
 server = mock.create_autospec(grpc.server)
 instance_name = "blade"
+action = remote_execution_pb2.Action(do_not_cache=True)
+action_digest = create_digest(action.SerializeToString())
 
 
 # Can mock this
@@ -47,9 +50,6 @@ def context():
 # Requests to make
 @pytest.fixture
 def execute_request():
-    action_digest = remote_execution_pb2.Digest()
-    action_digest.hash = 'zhora'
-
     yield remote_execution_pb2.ExecuteRequest(instance_name='',
                                               action_digest=action_digest,
                                               skip_cache_lookup=True)
@@ -57,7 +57,11 @@ def execute_request():
 
 @pytest.fixture
 def controller():
-    yield ExecutionController()
+    storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
+    write_session = storage.begin_write(action_digest)
+    storage.commit_write(action_digest, write_session)
+
+    yield ExecutionController(None, storage)
 
 
 # Instance to test