.gitlab-ci.yml

-image: buildstream/buildstream-fedora:master-81-06ae434
+# Use a debian image with python 3.5 (and pip3) pre-installed
+image: python:3.5-stretch
 
 variables:
   BGD: bgd --verbose
@@ -8,6 +9,7 @@ stages:
   - post
 
 before_script:
+  - pip3 install setuptools
   - export PATH=~/.local/bin:${PATH}
   - pip3 install --user -e .
 
@@ -28,33 +30,22 @@ before_script:
   script:
     - ${BGD} server start &
     - sleep 1 # Allow server to boot
-    - ${BGD} bot --host=0.0.0.0 dummy &
+    - ${BGD} bot --host=0.0.0.0 --continuous dummy &
     - ${BGD} execute --host=0.0.0.0 request --wait-for-completion
 
-tests-debian:
-  image: buildstream/buildstream-debian
-  <<: *linux-tests
-
-# Need to yum install until we have our own image
-tests-fedora:
+tests-debian-stretch:
   <<: *linux-tests
-  script:
-    - yum -y install clang libffi-devel openssl-devel python3-devel
-    - python3 setup.py test
-    - mkdir -p coverage/
-    - cp .coverage.* coverage/coverage."${CI_JOB_NAME}"
 
 run-dummy-job-debian:
   image: buildstream/buildstream-debian
   <<: *dummy-job
 
-run-dummy-job-fedora:
-  <<: *dummy-job
 
 coverage:
   stage: post
   coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
   script:
+    - pip3 install coverage==4.4.0
     - mkdir report
     - cd report
     - cp ../coverage/coverage.* .
@@ -62,14 +53,14 @@ coverage:
     - coverage combine --rcfile=../.coveragerc -a coverage.*
     - coverage report --rcfile=../.coveragerc -m
   dependencies:
-  - tests-fedora
+  - tests-debian-stretch
 
 # Deploy, only for merges which land on master branch.
 #
 pages:
   stage: post
   dependencies:
-  - tests-fedora
+  - tests-debian-stretch
   script:
   - mv coverage/ public/
   artifacts:

CONTRIBUTING.rst

@@ -9,8 +9,23 @@ We welcome contributions in the form of bug fixes or feature additions / enhance
 
 Any major feature additions should be raised as a proposal on the `Mailing List <https://lists.buildgrid.build/cgi-bin/mailman/listinfo/buildgrid/>`_ to be discussed, and then eventually followed up with an issue here on gitlab. We recommend that you propose the feature in advance of commencing work. We are also on irc, but do not have our own dedicated channel - you can find us on #buildstream on GIMPNet and #bazel on freenode.
 
-The author of any patch is expected to take ownership of that code and is to support it for a reasonable
-time-frame. This means addressing any unforeseen side effects and quirks the feature may have introduced.
+The author of any patch is expected to take ownership of that code and is to support it for a reasonable time-frame. This means addressing any unforeseen side effects and quirks the feature may have introduced. More on this below in 'Granting Committer Access'.
+
+Granting Committer Access
+-------------------------
+
+We'll hand out commit access to anyone who has successfully landed a single patch to the code base. Please request this via irc or the mailing list.
+
+This of course relies on contributors being responsive and show willingness to address problems after landing branches there should not be any problems here.
+
+What we are expecting of committers here in general is basically to
+escalate the review in cases of uncertainty:
+
+* If the patch/branch is very trivial (obvious few line changes or typos etc), and you are confident of the change, there is no need for review.
+
+* If the patch/branch is non trivial, please obtain a review from another committer who is familiar with the area which the branch effects. An approval from someone who is not the patch author will be needed before any merge. 
+
+We don't have any detailed policy for "bad actors", but will of course handle things on a case by case basis - commit access should not result in commit wars or be used as a tool to subvert the project when disagreements arise, such incidents (if any) would surely lead to temporary suspension of commit rights.
 
 Patch Submissions
 -----------------

app/commands/cmd_bot.py

@@ -30,32 +30,45 @@ import os
 import random
 import subprocess
 import tempfile
+import time
 
 from pathlib import Path, PurePath
 
 from buildgrid.bot import bot
+from buildgrid.bot.bot_session import BotSession, Device, Lease, Worker
 from buildgrid._exceptions import BotError
 
 from ..cli import pass_context
 
-from google.bytestream import bytestream_pb2, bytestream_pb2_grpc
-from google.devtools.remoteexecution.v1test import remote_execution_pb2, remote_execution_pb2_grpc
+from buildgrid._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
 from google.protobuf import any_pb2
 
 @click.group(short_help = 'Create a bot client')
+@click.option('--continuous', is_flag=True)
 @click.option('--parent', default='bgd_test')
 @click.option('--number-of-leases', default=1)
 @click.option('--port', default='50051')
 @click.option('--host', default='localhost')
 @pass_context
-def cli(context, host, port, number_of_leases, parent):
+def cli(context, host, port, number_of_leases, parent, continuous):
     context.logger = logging.getLogger(__name__)
     context.logger.info("Starting on port {}".format(port))
 
+    context.continuous = continuous
     context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
-    context.number_of_leases = number_of_leases
     context.parent = parent
 
+    worker = Worker()
+    worker.add_device(Device().get())
+
+    bot_session = BotSession(parent)
+    bot_session.add_worker(worker.get())
+    for i in range(0, number_of_leases):
+        bot_session.add_lease(Lease().get())
+
+    context.bot_session = bot_session.get()
+
 @cli.command('dummy', short_help='Create a dummy bot session')
 @pass_context
 def dummy(context):
@@ -63,15 +76,13 @@ def dummy(context):
     Simple dummy client. Creates a session, accepts leases, does fake work and
     updates the server.
     """
-
-    context.logger.info("Creating a bot session")
-
     try:
-        bot.Bot(work=_work_dummy,
-                context=context,
-                channel=context.channel,
-                parent=context.parent,
-                number_of_leases=context.number_of_leases)
+        b = bot.Bot(bot_session=context.bot_session,
+                    channel=context.channel)
+        b.session(context.parent,
+                  _work_dummy,
+                  context,
+                  context.continuous)
 
     except KeyboardInterrupt:
         pass
@@ -85,7 +96,7 @@ def dummy(context):
 @click.option('--port', show_default = True, default=11001)
 @click.option('--remote', show_default = True, default='localhost')
 @pass_context
-def _work_buildbox(context, remote, port, server_cert, client_key, client_cert, local_cas, fuse_dir):
+def work_buildbox(context, remote, port, server_cert, client_key, client_cert, local_cas, fuse_dir):
     """
     Uses BuildBox to run commands.
     """
@@ -101,11 +112,14 @@ def _work_buildbox(context, remote, port, server_cert, client_key, client_cert,
     context.fuse_dir = fuse_dir
 
     try:
-        bot.Bot(work=_work_buildbox,
-                context=context,
-                channel=context.channel,
-                parent=context.parent,
-                number_of_leases=context.number_of_leases)
+        b = bot.Bot(work=_work_buildbox,
+                    bot_session=context.bot_session,
+                    channel=context.channel,
+                    parent=context.parent)
+
+        b.session(context.parent,
+                  _work_buildbox,
+                  context)
 
     except KeyboardInterrupt:
         pass
@@ -117,7 +131,7 @@ async def _work_dummy(context, lease):
 async def _work_buildbox(context, lease):
     logger = context.logger
 
-    action_any = lease.inline_assignment
+    action_any = lease.payload
     action = remote_execution_pb2.Action()
     action_any.Unpack(action)
 
@@ -173,7 +187,7 @@ async def _work_buildbox(context, lease):
     action_result_any = any_pb2.Any()
     action_result_any.Pack(action_result)
 
-    lease.inline_assignment.CopyFrom(action_result_any)
+    lease.result.CopyFrom(action_result_any)
 
     return lease
 

app/commands/cmd_execute.py

@@ -30,9 +30,9 @@ import time
 
 from ..cli import pass_context
 
-from google.devtools.remoteexecution.v1test import remote_execution_pb2, remote_execution_pb2_grpc
-from google.devtools.remoteexecution.v1test.remote_execution_pb2 import ExecuteOperationMetadata
-from google.longrunning import operations_pb2, operations_pb2_grpc
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
+from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import ExecuteOperationMetadata
+from buildgrid._protos.google.longrunning import operations_pb2, operations_pb2_grpc
 from google.protobuf import any_pb2
 
 @click.group(short_help = "Simple execute client")
@@ -52,39 +52,24 @@ def cli(context, host, port):
 @click.option('--wait-for-completion', is_flag=True)
 @pass_context
 def request(context, number, instance_name, wait_for_completion):
+    action_digest = remote_execution_pb2.Digest()
+
     context.logger.info("Sending execution request...\n")
     stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
 
-    action = remote_execution_pb2.Action(command_digest = None,
-                                         input_root_digest = None,
-                                         output_files = [],
-                                         output_directories = None,
-                                         platform = None,
-                                         timeout = None,
-                                         do_not_cache = True)
-
-    action.command_digest.hash = 'foo'
-
     request = remote_execution_pb2.ExecuteRequest(instance_name = instance_name,
-                                                  action = action,
+                                                  action_digest = action_digest,
                                                   skip_cache_lookup = True)
+    responses = []
     for i in range(0, number):
-        response = stub.Execute(request)
-        context.logger.info("Response name: {}".format(response.name))
-
-    try:
-        while wait_for_completion:
-            request = operations_pb2.ListOperationsRequest()
-            context.logger.debug('Querying to see if jobs are complete.')
-            stub = operations_pb2_grpc.OperationsStub(context.channel)
-            response = stub.ListOperations(request)
-            if all(operation.done for operation in response.operations):
-                context.logger.info('Jobs complete')
-                break
-            time.sleep(1)
-
-    except KeyboardInterrupt:
-        pass
+        responses.append(stub.Execute(request))
+
+    for response in responses:
+        if wait_for_completion:
+            for stream in response:
+                context.logger.info(stream)
+        else:
+            context.logger.info(next(response))
 
 @cli.command('status', short_help='Get the status of an operation')
 @click.argument('operation-name')
@@ -96,7 +81,7 @@ def operation_status(context, operation_name):
     request = operations_pb2.GetOperationRequest(name=operation_name)
 
     response = stub.GetOperation(request)
-    _log_operation(context, response)
+    context.logger.info(response)
 
 @cli.command('list', short_help='List operations')
 @pass_context
@@ -113,13 +98,16 @@ def list_operations(context):
         return
 
     for op in response.operations:
-        _log_operation(context, op)
+        context.logger.info(op)
+
+@cli.command('wait', short_help='Streams an operation until it is complete')
+@click.argument('operation-name')
+@pass_context
+def wait_execution(context, operation_name):
+    stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
+    request = remote_execution_pb2.WaitExecutionRequest(name=operation_name)
 
-def _log_operation(context, operation):
-    op_meta = ExecuteOperationMetadata()
-    operation.metadata.Unpack(op_meta)
+    response = stub.WaitExecution(request)
 
-    context.logger.info("Name  : {}".format(operation.name))
-    context.logger.info("Done  : {}".format(operation.done))
-    context.logger.info("Stage : {}".format(ExecuteOperationMetadata.Stage.Name(op_meta.stage)))
-    context.logger.info("Key   : {}".format(operation.response))
+    for stream in response:
+        context.logger.info(stream)

buildgrid/_protos/build/bazel/semver/semver.proto

+// Copyright 2018 The Bazel Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package build.bazel.semver;
+
+message SemVer {
+  int32 major = 1;
+  int32 minor = 2;
+  int32 patch = 3;
+  string prerelease = 4;
+}

buildgrid/_protos/build/bazel/semver/semver_pb2.py

+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: build/bazel/semver/semver.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='build/bazel/semver/semver.proto',
+  package='build.bazel.semver',
+  syntax='proto3',
+  serialized_pb=_b('\n\x1f\x62uild/bazel/semver/semver.proto\x12\x12\x62uild.bazel.semver\"I\n\x06SemVer\x12\r\n\x05major\x18\x01 \x01(\x05\x12\r\n\x05minor\x18\x02 \x01(\x05\x12\r\n\x05patch\x18\x03 \x01(\x05\x12\x12\n\nprerelease\x18\x04 \x01(\tb\x06proto3')
+)
+
+
+
+
+_SEMVER = _descriptor.Descriptor(
+  name='SemVer',
+  full_name='build.bazel.semver.SemVer',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='major', full_name='build.bazel.semver.SemVer.major', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='minor', full_name='build.bazel.semver.SemVer.minor', index=1,
+      number=2, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='patch', full_name='build.bazel.semver.SemVer.patch', index=2,
+      number=3, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='prerelease', full_name='build.bazel.semver.SemVer.prerelease', index=3,
+      number=4, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=55,
+  serialized_end=128,
+)
+
+DESCRIPTOR.message_types_by_name['SemVer'] = _SEMVER
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+SemVer = _reflection.GeneratedProtocolMessageType('SemVer', (_message.Message,), dict(
+  DESCRIPTOR = _SEMVER,
+  __module__ = 'build.bazel.semver.semver_pb2'
+  # @@protoc_insertion_point(class_scope:build.bazel.semver.SemVer)
+  ))
+_sym_db.RegisterMessage(SemVer)
+
+
+# @@protoc_insertion_point(module_scope)

google/api/annotations_pb2.py → buildgrid/_protos/google/api/annotations_pb2.py

@@ -13,7 +13,7 @@ from google.protobuf import descriptor_pb2
 _sym_db = _symbol_database.Default()
 
 
-from google.api import http_pb2 as google_dot_api_dot_http__pb2
+from buildgrid._protos.google.api import http_pb2 as google_dot_api_dot_http__pb2
 from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2