From 04fddfa508a17ca98a89e5938a84e482ebbcaec9 Mon Sep 17 00:00:00 2001
From: finn <finn.ball@codethink.com>
Date: Tue, 31 Jul 2018 11:39:19 +0100
Subject: [PATCH] Update BuildGrid to Remote Execution API v2
---
app/commands/cmd_bot.py | 2 +-
app/commands/cmd_execute.py | 35 ++++++-------------
buildgrid/server/build_grid_server.py | 2 +-
buildgrid/server/cas/bytestream_service.py | 2 +-
.../content_addressable_storage_service.py | 6 ++--
.../server/execution/action_cache_service.py | 2 +-
.../server/execution/execution_instance.py | 4 +--
.../server/execution/execution_service.py | 10 +++---
buildgrid/server/job.py | 4 +--
tests/cas/test_services.py | 18 +++++-----
tests/cas/test_storage.py | 2 +-
tests/integration/action_cache_service.py | 2 +-
tests/integration/bots_service.py | 2 +-
tests/integration/execution_service.py | 30 ++++++++--------
tests/integration/operations_service.py | 17 ++++-----
15 files changed, 62 insertions(+), 76 deletions(-)
diff --git a/app/commands/cmd_bot.py b/app/commands/cmd_bot.py
index 47176db30..c83d60ca2 100644
--- a/app/commands/cmd_bot.py
+++ b/app/commands/cmd_bot.py
@@ -39,7 +39,7 @@ from buildgrid._exceptions import BotError
from ..cli import pass_context
from buildgrid._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2, remote_execution_pb2_grpc
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
from google.protobuf import any_pb2
@click.group(short_help = 'Create a bot client')
diff --git a/app/commands/cmd_execute.py b/app/commands/cmd_execute.py
index a31eca77a..196ea9d46 100644
--- a/app/commands/cmd_execute.py
+++ b/app/commands/cmd_execute.py
@@ -30,8 +30,8 @@ import time
from ..cli import pass_context
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2, remote_execution_pb2_grpc
-from buildgrid._protos.google.devtools.remoteexecution.v1test.remote_execution_pb2 import ExecuteOperationMetadata
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
+from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import ExecuteOperationMetadata
from buildgrid._protos.google.longrunning import operations_pb2, operations_pb2_grpc
from google.protobuf import any_pb2
@@ -52,25 +52,19 @@ def cli(context, host, port):
@click.option('--wait-for-completion', is_flag=True)
@pass_context
def request(context, number, instance_name, wait_for_completion):
+ action_digest = remote_execution_pb2.Digest()
+ action_digest.hash = 'zhora'
+
context.logger.info("Sending execution request...\n")
stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
- action = remote_execution_pb2.Action(command_digest = None,
- input_root_digest = None,
- output_files = [],
- output_directories = None,
- platform = None,
- timeout = None,
- do_not_cache = True)
-
- action.command_digest.hash = 'foo'
-
request = remote_execution_pb2.ExecuteRequest(instance_name = instance_name,
- action = action,
+ action_digest = action_digest,
skip_cache_lookup = True)
for i in range(0, number):
response = stub.Execute(request)
- context.logger.info("Response name: {}".format(response.name))
+ for r in response:
+ context.logger.info(r)
try:
while wait_for_completion:
@@ -96,7 +90,7 @@ def operation_status(context, operation_name):
request = operations_pb2.GetOperationRequest(name=operation_name)
response = stub.GetOperation(request)
- _log_operation(context, response)
+ context.logger.info(response)
@cli.command('list', short_help='List operations')
@pass_context
@@ -113,13 +107,4 @@ def list_operations(context):
return
for op in response.operations:
- _log_operation(context, op)
-
-def _log_operation(context, operation):
- op_meta = ExecuteOperationMetadata()
- operation.metadata.Unpack(op_meta)
-
- context.logger.info("Name : {}".format(operation.name))
- context.logger.info("Done : {}".format(operation.done))
- context.logger.info("Stage : {}".format(ExecuteOperationMetadata.Stage.Name(op_meta.stage)))
- context.logger.info("Key : {}".format(operation.response))
+ context.logger.info(op)
diff --git a/buildgrid/server/build_grid_server.py b/buildgrid/server/build_grid_server.py
index 406fbb35d..73273778d 100644
--- a/buildgrid/server/build_grid_server.py
+++ b/buildgrid/server/build_grid_server.py
@@ -27,7 +27,7 @@ import grpc
from concurrent import futures
from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2_grpc
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc
from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2_grpc
from buildgrid._protos.google.longrunning import operations_pb2_grpc
diff --git a/buildgrid/server/cas/bytestream_service.py b/buildgrid/server/cas/bytestream_service.py
index 5b90bc4a5..adcd13545 100644
--- a/buildgrid/server/cas/bytestream_service.py
+++ b/buildgrid/server/cas/bytestream_service.py
@@ -25,7 +25,7 @@ CAS blobs.
import grpc
from buildgrid._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2 as re_pb2, remote_execution_pb2_grpc as re_pb2_grpc
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2, remote_execution_pb2_grpc as re_pb2_grpc
from ...settings import HASH
diff --git a/buildgrid/server/cas/content_addressable_storage_service.py b/buildgrid/server/cas/content_addressable_storage_service.py
index ae2bcc250..5c8cddf64 100644
--- a/buildgrid/server/cas/content_addressable_storage_service.py
+++ b/buildgrid/server/cas/content_addressable_storage_service.py
@@ -24,7 +24,7 @@ to check for missing CAS blobs and update them in bulk.
"""
import grpc
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2 as re_pb2, remote_execution_pb2_grpc as re_pb2_grpc
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2, remote_execution_pb2_grpc as re_pb2_grpc
class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServicer):
@@ -42,10 +42,10 @@ class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServ
storage = self._storage
requests = []
for request_proto in request.requests:
- requests.append((request_proto.content_digest, request_proto.data))
+ requests.append((request_proto.digest, request_proto.data))
response = re_pb2.BatchUpdateBlobsResponse()
for (digest, _), status in zip(requests, storage.bulk_update_blobs(requests)):
response_proto = response.responses.add()
- response_proto.blob_digest.CopyFrom(digest)
+ response_proto.digest.CopyFrom(digest)
response_proto.status.CopyFrom(status)
return response
diff --git a/buildgrid/server/execution/action_cache_service.py b/buildgrid/server/execution/action_cache_service.py
index 0f6f897fa..64fddbe9a 100644
--- a/buildgrid/server/execution/action_cache_service.py
+++ b/buildgrid/server/execution/action_cache_service.py
@@ -25,7 +25,7 @@ Action Cache currently not implemented.
import logging
import grpc
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2, remote_execution_pb2_grpc
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
class ActionCacheService(remote_execution_pb2_grpc.ActionCacheServicer):
diff --git a/buildgrid/server/execution/execution_instance.py b/buildgrid/server/execution/execution_instance.py
index 0203e9af1..5dc486e4d 100644
--- a/buildgrid/server/execution/execution_instance.py
+++ b/buildgrid/server/execution/execution_instance.py
@@ -34,12 +34,12 @@ class ExecutionInstance():
self.logger = logging.getLogger(__name__)
self._scheduler = scheduler
- def execute(self, action, skip_cache_lookup):
+ def execute(self, action_digest, skip_cache_lookup):
""" Sends a job for execution.
Queues an action and creates an Operation instance to be associated with
this action.
"""
- job = Job(action)
+ job = Job(action_digest)
self.logger.info("Operation name: {}".format(job.name))
if not skip_cache_lookup:
diff --git a/buildgrid/server/execution/execution_service.py b/buildgrid/server/execution/execution_service.py
index 3e72787f7..af00a72bb 100644
--- a/buildgrid/server/execution/execution_service.py
+++ b/buildgrid/server/execution/execution_service.py
@@ -25,7 +25,7 @@ Serves remote execution requests.
import grpc
import logging
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2, remote_execution_pb2_grpc
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
from buildgrid._protos.google.longrunning import operations_pb2_grpc, operations_pb2
from ._exceptions import InvalidArgumentError
@@ -40,17 +40,17 @@ class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
# Ignore request.instance_name for now
# Have only one instance
try:
- return self._instance.execute(request.action,
- request.skip_cache_lookup)
+ yield self._instance.execute(request.action_digest,
+ request.skip_cache_lookup)
except InvalidArgumentError as e:
self.logger.error(e)
context.set_details(str(e))
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
- return operations_pb2.Operation()
+ yield operations_pb2.Operation()
except NotImplementedError as e:
self.logger.error(e)
context.set_details(str(e))
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
- return operations_pb2.Operation()
+ yield operations_pb2.Operation()
diff --git a/buildgrid/server/job.py b/buildgrid/server/job.py
index 2a94baf79..8b5319cba 100644
--- a/buildgrid/server/job.py
+++ b/buildgrid/server/job.py
@@ -18,11 +18,11 @@
import logging
import uuid
-import buildgrid._protos.google.devtools.remoteexecution.v1test.remote_execution_pb2
+import buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2
from enum import Enum
-from buildgrid._protos.google.devtools.remoteexecution.v1test.remote_execution_pb2 import ExecuteOperationMetadata, ExecuteResponse
+from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import ExecuteOperationMetadata, ExecuteResponse
from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2, worker_pb2
from buildgrid._protos.google.longrunning import operations_pb2
from google.protobuf import any_pb2
diff --git a/tests/cas/test_services.py b/tests/cas/test_services.py
index f28bb4afb..ad42d5b5c 100644
--- a/tests/cas/test_services.py
+++ b/tests/cas/test_services.py
@@ -18,7 +18,7 @@
import io
from buildgrid._protos.google.bytestream import bytestream_pb2
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2 as re_pb2
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
import pytest
from buildgrid.server.cas.storage.storage_abc import StorageABC
@@ -170,22 +170,22 @@ def test_cas_batch_update_blobs(instance):
storage = SimpleStorage()
servicer = ContentAddressableStorageService(storage)
update_requests = [
- re_pb2.UpdateBlobRequest(
- content_digest=re_pb2.Digest(hash=HASH(b'abc').hexdigest(), size_bytes=3), data=b'abc'),
- re_pb2.UpdateBlobRequest(
- content_digest=re_pb2.Digest(hash="invalid digest!", size_bytes=1000),
+ re_pb2.BatchUpdateBlobsRequest.Request(
+ digest=re_pb2.Digest(hash=HASH(b'abc').hexdigest(), size_bytes=3), data=b'abc'),
+ re_pb2.BatchUpdateBlobsRequest.Request(
+ digest=re_pb2.Digest(hash="invalid digest!", size_bytes=1000),
data=b'wrong data')
]
request = re_pb2.BatchUpdateBlobsRequest(instance_name=instance, requests=update_requests)
response = servicer.BatchUpdateBlobs(request, None)
assert len(response.responses) == 2
for blob_response in response.responses:
- if blob_response.blob_digest == update_requests[0].content_digest:
+ if blob_response.digest == update_requests[0].digest:
assert blob_response.status.code == 0
- elif blob_response.blob_digest == update_requests[1].content_digest:
+ elif blob_response.digest == update_requests[1].digest:
assert blob_response.status.code != 0
else:
raise Exception("Unexpected blob response")
assert len(storage.data) == 1
- assert (update_requests[0].content_digest.hash, 3) in storage.data
- assert storage.data[(update_requests[0].content_digest.hash, 3)] == b'abc'
+ assert (update_requests[0].digest.hash, 3) in storage.data
+ assert storage.data[(update_requests[0].digest.hash, 3)] == b'abc'
diff --git a/tests/cas/test_storage.py b/tests/cas/test_storage.py
index 576a6072b..3847d35a9 100644
--- a/tests/cas/test_storage.py
+++ b/tests/cas/test_storage.py
@@ -18,7 +18,7 @@
import tempfile
import boto3
-from buildgrid._protos.google.devtools.remoteexecution.v1test.remote_execution_pb2 import Digest
+from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Digest
from moto import mock_s3
import pytest
diff --git a/tests/integration/action_cache_service.py b/tests/integration/action_cache_service.py
index afe8892a1..6ddd2f29b 100644
--- a/tests/integration/action_cache_service.py
+++ b/tests/integration/action_cache_service.py
@@ -21,7 +21,7 @@ import pytest
from unittest import mock
from grpc._server import _Context
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from buildgrid.server import scheduler
from buildgrid.server.execution import execution_instance, action_cache_service
diff --git a/tests/integration/bots_service.py b/tests/integration/bots_service.py
index 33655364f..836912395 100644
--- a/tests/integration/bots_service.py
+++ b/tests/integration/bots_service.py
@@ -23,7 +23,7 @@ import uuid
from unittest import mock
from grpc._server import _Context
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2, worker_pb2
from google.protobuf import any_pb2
diff --git a/tests/integration/execution_service.py b/tests/integration/execution_service.py
index 3672d244d..08d41231a 100644
--- a/tests/integration/execution_service.py
+++ b/tests/integration/execution_service.py
@@ -22,7 +22,7 @@ import uuid
from unittest import mock
from grpc._server import _Context
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from buildgrid._protos.google.longrunning import operations_pb2
from google.protobuf import any_pb2
@@ -49,22 +49,22 @@ def instance(execution):
@pytest.mark.parametrize("skip_cache_lookup", [True, False])
def test_execute(skip_cache_lookup, instance, context):
- action = remote_execution_pb2.Action()
- action.command_digest.hash = 'zhora'
+ action_digest = remote_execution_pb2.Digest()
+ action_digest.hash = 'zhora'
request = remote_execution_pb2.ExecuteRequest(instance_name = '',
- action = action,
+ action_digest = action_digest,
skip_cache_lookup = skip_cache_lookup)
+ response = instance.Execute(request, context)
- result = instance.Execute(request, context)
+ for result in response:
+ assert isinstance(result, operations_pb2.Operation)
- assert isinstance(result, operations_pb2.Operation)
-
- if skip_cache_lookup is False:
- context.set_code.assert_called_once_with(grpc.StatusCode.UNIMPLEMENTED)
- else:
- metadata = remote_execution_pb2.ExecuteOperationMetadata()
- result.metadata.Unpack(metadata)
- assert metadata.stage == job.ExecuteStage.QUEUED.value
- assert uuid.UUID(result.name, version=4)
- assert result.done is False
+ if skip_cache_lookup is False:
+ context.set_code.assert_called_once_with(grpc.StatusCode.UNIMPLEMENTED)
+ else:
+ metadata = remote_execution_pb2.ExecuteOperationMetadata()
+ result.metadata.Unpack(metadata)
+ assert metadata.stage == job.ExecuteStage.QUEUED.value
+ assert uuid.UUID(result.name, version=4)
+ assert result.done is False
diff --git a/tests/integration/operations_service.py b/tests/integration/operations_service.py
index aa8674fc1..246e354ed 100644
--- a/tests/integration/operations_service.py
+++ b/tests/integration/operations_service.py
@@ -21,7 +21,7 @@ import pytest
from unittest import mock
from grpc._server import _Context
-from buildgrid._protos.google.devtools.remoteexecution.v1test import remote_execution_pb2
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from buildgrid._protos.google.longrunning import operations_pb2
from buildgrid.server import scheduler, job
@@ -37,11 +37,11 @@ def context():
# Requests to make
@pytest.fixture
def execute_request():
- action = remote_execution_pb2.Action()
- action.command_digest.hash = 'zhora'
+ action_digest = remote_execution_pb2.Digest()
+ action_digest.hash = 'zhora'
yield remote_execution_pb2.ExecuteRequest(instance_name = '',
- action = action,
+ action_digest = action_digest,
skip_cache_lookup = True)
@pytest.fixture
@@ -59,10 +59,11 @@ def instance(execution):
# Queue an execution, get operation corresponding to that request
def test_get_operation(instance, execute_request, context):
- response_execute = instance._instance.execute(execute_request.action,
+ response_execute = instance._instance.execute(execute_request.action_digest,
execute_request.skip_cache_lookup)
request = operations_pb2.GetOperationRequest()
+
request.name = response_execute.name
response = instance.GetOperation(request, context)
@@ -75,7 +76,7 @@ def test_get_operation_fail(instance, context):
context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)
def test_list_operations(instance, execute_request, context):
- response_execute = instance._instance.execute(execute_request.action,
+ response_execute = instance._instance.execute(execute_request.action_digest,
execute_request.skip_cache_lookup)
request = operations_pb2.ListOperationsRequest()
@@ -84,7 +85,7 @@ def test_list_operations(instance, execute_request, context):
assert response.operations[0].name == response_execute.name
def test_list_operations_with_result(instance, execute_request, context):
- response_execute = instance._instance.execute(execute_request.action,
+ response_execute = instance._instance.execute(execute_request.action_digest,
execute_request.skip_cache_lookup)
action_result = remote_execution_pb2.ActionResult()
@@ -109,7 +110,7 @@ def test_list_operations_empty(instance, context):
# Send execution off, delete, try to find operation should fail
def test_delete_operation(instance, execute_request, context):
- response_execute = instance._instance.execute(execute_request.action,
+ response_execute = instance._instance.execute(execute_request.action_digest,
execute_request.skip_cache_lookup)
request = operations_pb2.DeleteOperationRequest()
request.name = response_execute.name
--
GitLab