buildgrid/_app/commands/cmd_server.py

@@ -26,14 +26,10 @@ import sys
 
 import click
 
-from buildgrid.server.controller import ExecutionController
-from buildgrid.server.actioncache.storage import ActionCache
-from buildgrid.server.cas.instance import ByteStreamInstance, ContentAddressableStorageInstance
-from buildgrid.server.referencestorage.storage import ReferenceCache
+from buildgrid.server.instance import BuildGridServer
 
 from ..cli import pass_context
 from ..settings import parser
-from ..server import BuildGridServer
 
 
 @click.group(name='server', short_help="Start a local server instance.")
@@ -50,58 +46,12 @@ def start(context, config):
         settings = parser.get_parser().safe_load(f)
 
     try:
-        server_settings = settings['server']
-        insecure_mode = server_settings['insecure-mode']
-
-        credentials = None
-        if not insecure_mode:
-            credential_settings = server_settings['credentials']
-            server_key = credential_settings['tls-server-key']
-            server_cert = credential_settings['tls-server-cert']
-            client_certs = credential_settings['tls-client-certs']
-            credentials = context.load_server_credentials(server_key, server_cert, client_certs)
-
-            if not credentials:
-                click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
-                           "Set `insecure-mode: false` in order to deactivate TLS encryption.\n", err=True)
-                sys.exit(-1)
-
-        port = server_settings['port']
-        instances = settings['instances']
-
-        execution_controllers = _instance_maker(instances, ExecutionController)
-
-        execution_instances = {}
-        bots_interfaces = {}
-        operations_instances = {}
-
-        # TODO: map properly in parser
-        # Issue 82
-        for k, v in execution_controllers.items():
-            execution_instances[k] = v.execution_instance
-            bots_interfaces[k] = v.bots_interface
-            operations_instances[k] = v.operations_instance
-
-        reference_caches = _instance_maker(instances, ReferenceCache)
-        action_caches = _instance_maker(instances, ActionCache)
-        cas = _instance_maker(instances, ContentAddressableStorageInstance)
-        bytestreams = _instance_maker(instances, ByteStreamInstance)
+        server = _create_server_from_config(settings)
 
     except KeyError as e:
         click.echo("ERROR: Could not parse config: {}.\n".format(str(e)), err=True)
         sys.exit(-1)
 
-    server = BuildGridServer(port=port,
-                             credentials=credentials,
-                             execution_instances=execution_instances,
-                             bots_interfaces=bots_interfaces,
-                             operations_instances=operations_instances,
-                             reference_storage_instances=reference_caches,
-                             action_cache_instances=action_caches,
-                             cas_instances=cas,
-                             bytestream_instances=bytestreams)
-
-    context.logger.info("Starting server on port {}".format(port))
     loop = asyncio.get_event_loop()
     try:
         server.start()
@@ -116,15 +66,23 @@ def start(context, config):
         loop.close()
 
 
-# Turn away now if you want to keep your eyes
-def _instance_maker(instances, service_type):
-    # TODO get this mapped in parser
-    made = {}
+def _create_server_from_config(config):
+    server_settings = config['server']
+    if len(server_settings) < 1:
+        click.echo("ERROR: Need at least one channel in config: {}.\n".format(config), err=True)
+        sys.exit(-1)
+
+    server = BuildGridServer()
 
+    for channel in server_settings:
+        server.add_port(channel.address, channel.credentials)
+
+    instances = config['instances']
     for instance in instances:
-        services = instance['services']
         instance_name = instance['name']
+        services = instance['services']
+
         for service in services:
-            if isinstance(service, service_type):
-                made[instance_name] = service
-    return made
+            service.register_instance_with_server(instance_name, server)
+
+    return server

buildgrid/_app/server.py

-# Copyright (C) 2018 Bloomberg LP
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#  <http://www.apache.org/licenses/LICENSE-2.0>
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-"""
-BuildGridServer
-==============
-
-Creates a BuildGrid server, binding all the requisite service instances together.
-"""
-
-import logging
-from concurrent import futures
-
-import grpc
-
-from buildgrid.server.cas.service import ByteStreamService, ContentAddressableStorageService
-from buildgrid.server.actioncache.service import ActionCacheService
-from buildgrid.server.execution.service import ExecutionService
-from buildgrid.server.operations.service import OperationsService
-from buildgrid.server.bots.service import BotsService
-from buildgrid.server.referencestorage.service import ReferenceStorageService
-
-
-class BuildGridServer:
-
-    def __init__(self, port=50051, max_workers=10, credentials=None,
-                 execution_instances=None, bots_interfaces=None, operations_instances=None,
-                 operations_service_instances=None, reference_storage_instances=None,
-                 action_cache_instances=None, cas_instances=None, bytestream_instances=None):
-
-        self.logger = logging.getLogger(__name__)
-        address = '[::]:{0}'.format(port)
-
-        server = grpc.server(futures.ThreadPoolExecutor(max_workers))
-
-        if credentials is not None:
-            self.logger.info("Secure connection")
-            server.add_secure_port(address, credentials)
-
-        else:
-            self.logger.info("Insecure connection")
-            server.add_insecure_port(address)
-
-        if execution_instances:
-            self.logger.debug("Adding execution instances {}".format(
-                execution_instances.keys()))
-            ExecutionService(server, execution_instances)
-
-        if bots_interfaces:
-            self.logger.debug("Adding bots interfaces {}".format(
-                bots_interfaces.keys()))
-            BotsService(server, bots_interfaces)
-
-        if operations_instances:
-            self.logger.debug("Adding operations instances {}".format(
-                operations_instances.keys()))
-            OperationsService(server, operations_instances)
-
-        if reference_storage_instances:
-            self.logger.debug("Adding reference storages {}".format(
-                reference_storage_instances.keys()))
-            ReferenceStorageService(server, reference_storage_instances)
-
-        if action_cache_instances:
-            self.logger.debug("Adding action cache instances {}".format(
-                action_cache_instances.keys()))
-            ActionCacheService(server, action_cache_instances)
-
-        if cas_instances:
-            self.logger.debug("Adding cas instances {}".format(
-                cas_instances.keys()))
-            ContentAddressableStorageService(server, cas_instances)
-
-        if bytestream_instances:
-            self.logger.debug("Adding bytestream instances {}".format(
-                bytestream_instances.keys()))
-            ByteStreamService(server, bytestream_instances)
-
-        self._server = server
-
-    def start(self):
-        self._server.start()
-
-    def stop(self):
-        self._server.stop(grace=0)

buildgrid/_app/settings/cas.yml

 server:
-  port: 50052
-  insecure-mode: true
-  credentials:
-    tls-server-key: null
-    tls-server-cert: null
-    tls-client-certs: null
+  - !channel
+    port: 50051
+    insecure_mode: true
+#    credentials:
+#      tls-server-key: null
+#      tls-server-cert: null
+#      tls-client-certs: null
 
 description: |
   Just a CAS with some reference storage.

buildgrid/_app/settings/default.yml

 server:
-  port: 50051
-  insecure-mode: true
-  credentials:
-    tls-server-key: null
-    tls-server-cert: null
-    tls-client-certs: null
+  - !channel
+    port: 50051
+    insecure_mode: true
+#    credentials:
+#      tls-server-key: null
+#      tls-server-cert: null
+#      tls-client-certs: null
 
 description: |
   A single default instance

buildgrid/_app/settings/parser.py

@@ -37,8 +37,35 @@ from ..cli import Context
 class YamlFactory(yaml.YAMLObject):
     @classmethod
     def from_yaml(cls, loader, node):
-        values = loader.construct_mapping(node, deep=True)
-        return cls(**values)
+        if isinstance(node, yaml.ScalarNode):
+            value = loader.construct_scalar(node)
+            return cls(value)
+
+        else:
+            values = loader.construct_mapping(node, deep=True)
+            return cls(**values)
+
+
+class Channel(YamlFactory):
+
+    yaml_tag = u'!channel'
+
+    def __init__(self, port, insecure_mode, credentials=None):
+        self.address = '[::]:{0}'.format(port)
+        self.credentials = None
+
+        context = Context()
+
+        if not insecure_mode:
+            server_key = credentials['tls-server-key']
+            server_cert = credentials['tls-server-cert']
+            client_certs = credentials['tls-client-certs']
+            self.credentials = context.load_server_credentials(server_key, server_cert, client_certs)
+
+            if not credentials:
+                click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
+                           "Set `insecure-mode: false` in order to deactivate TLS encryption.\n", err=True)
+                sys.exit(-1)
 
 
 class Disk(YamlFactory):
@@ -169,6 +196,7 @@ def _parse_size(size):
 
 def get_parser():
 
+    yaml.SafeLoader.add_constructor(Channel.yaml_tag, Channel.from_yaml)
     yaml.SafeLoader.add_constructor(Execution.yaml_tag, Execution.from_yaml)
     yaml.SafeLoader.add_constructor(Action.yaml_tag, Action.from_yaml)
     yaml.SafeLoader.add_constructor(Reference.yaml_tag, Reference.from_yaml)

buildgrid/_app/settings/remote-storage.yml

 server:
-  port: 50051
-  insecure-mode: true
-  credentials:
-    tls-server-key: null
-    tls-server-cert: null
-    tls-client-certs: null
-
+  - !channel
+    port: 50051
+    insecure_mode: true
+#    credentials:
+#      tls-server-key: null
+#      tls-server-cert: null
+#      tls-client-certs: null
 
 description: |
   A single default instance with remote storage.

buildgrid/server/actioncache/service.py

@@ -32,13 +32,16 @@ from .._exceptions import InvalidArgumentError, NotFoundError
 
 class ActionCacheService(remote_execution_pb2_grpc.ActionCacheServicer):
 
-    def __init__(self, server, instances):
-        self._instances = instances
-
+    def __init__(self, server):
         self.logger = logging.getLogger(__name__)
 
+        self._instances = {}
+
         remote_execution_pb2_grpc.add_ActionCacheServicer_to_server(self, server)
 
+    def add_instance(self, name, instance):
+        self._instances[name] = instance
+
     def GetActionResult(self, request, context):
         try:
             instance = self._get_instance(request.instance_name)

buildgrid/server/actioncache/storage.py

@@ -26,6 +26,9 @@ from ..referencestorage.storage import ReferenceCache
 
 class ActionCache(ReferenceCache):
 
+    def register_instance_with_server(self, instance_name, server):
+        server.add_action_cache_instance(self, instance_name)
+
     def get_action_result(self, action_digest):
         key = self._get_key(action_digest)
         return self.get_action_reference(key)

buildgrid/server/bots/instance.py

@@ -36,6 +36,9 @@ class BotsInterface:
         self._bot_sessions = {}
         self._scheduler = scheduler
 
+    def register_instance_with_server(self, instance_name, server):
+        server.add_bots_interface(self, instance_name)
+
     def create_bot_session(self, parent, bot_session):
         """ Creates a new bot session. Server should assign a unique
         name to the session. If a bot with the same bot id tries to

buildgrid/server/bots/service.py

@@ -33,12 +33,16 @@ from .._exceptions import InvalidArgumentError, OutofSyncError
 
 class BotsService(bots_pb2_grpc.BotsServicer):
 
-    def __init__(self, server, instances):
-        self._instances = instances
+    def __init__(self, server):
         self.logger = logging.getLogger(__name__)
 
+        self._instances = {}
+
         bots_pb2_grpc.add_BotsServicer_to_server(self, server)
 
+    def add_instance(self, name, instance):
+        self._instances[name] = instance
+
     def CreateBotSession(self, request, context):
         try:
             parent = request.parent

buildgrid/server/cas/instance.py

@@ -31,6 +31,9 @@ class ContentAddressableStorageInstance:
     def __init__(self, storage):
         self._storage = storage
 
+    def register_instance_with_server(self, instance_name, server):
+        server.add_cas_instance(self, instance_name)
+
     def find_missing_blobs(self, blob_digests):
         storage = self._storage
         return re_pb2.FindMissingBlobsResponse(
@@ -60,6 +63,9 @@ class ByteStreamInstance:
     def __init__(self, storage):
         self._storage = storage
 
+    def register_instance_with_server(self, instance_name, server):
+        server.add_bytestream_instance(self, instance_name)
+
     def read(self, path, read_offset, read_limit):
         storage = self._storage
 

buildgrid/server/cas/service.py

@@ -35,12 +35,16 @@ from .._exceptions import InvalidArgumentError, NotFoundError, OutOfRangeError
 
 class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressableStorageServicer):
 
-    def __init__(self, server, instances):
+    def __init__(self, server):
         self.logger = logging.getLogger(__name__)
-        self._instances = instances
+
+        self._instances = {}
 
         remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(self, server)
 
+    def add_instance(self, name, instance):
+        self._instances[name] = instance
+
     def FindMissingBlobs(self, request, context):
         try:
             instance = self._get_instance(request.instance_name)
@@ -75,12 +79,16 @@ class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressa
 
 class ByteStreamService(bytestream_pb2_grpc.ByteStreamServicer):
 
-    def __init__(self, server, instances):
+    def __init__(self, server):
         self.logger = logging.getLogger(__name__)
-        self._instances = instances
+
+        self._instances = {}
 
         bytestream_pb2_grpc.add_ByteStreamServicer_to_server(self, server)
 
+    def add_instance(self, name, instance):
+        self._instances[name] = instance
+
     def Read(self, request, context):
         try:
             path = request.resource_name.split("/")

buildgrid/server/controller.py

@@ -45,6 +45,11 @@ class ExecutionController:
         self._bots_interface = BotsInterface(scheduler)
         self._operations_instance = OperationsInstance(scheduler)
 
+    def register_instance_with_server(self, instance_name, server):
+        server.add_execution_instance(self._execution_instance, instance_name)
+        server.add_bots_interface(self._bots_interface, instance_name)
+        server.add_operations_instance(self._operations_instance, instance_name)
+
     def stream_operation_updates(self, message_queue, operation_name):
         operation = message_queue.get()
         while not operation.done:

buildgrid/server/execution/instance.py

@@ -34,6 +34,9 @@ class ExecutionInstance:
         self._storage = storage
         self._scheduler = scheduler
 
+    def register_instance_with_server(self, instance_name, server):
+        server.add_execution_instance(self, instance_name)
+
     def execute(self, action_digest, skip_cache_lookup, message_queue=None):
         """ Sends a job for execution.
         Queues an action and creates an Operation instance to be associated with

buildgrid/server/execution/service.py

@@ -35,12 +35,14 @@ from .._exceptions import InvalidArgumentError
 
 class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
 
-    def __init__(self, server, instances):
+    def __init__(self, server):
         self.logger = logging.getLogger(__name__)
-        self._instances = instances
-
+        self._instances = {}
         remote_execution_pb2_grpc.add_ExecutionServicer_to_server(self, server)
 
+    def add_instance(self, name, instance):
+        self._instances[name] = instance
+
     def Execute(self, request, context):
         try:
             message_queue = queue.Queue()

buildgrid/server/instance.py

+# Copyright (C) 2018 Bloomberg LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  <http://www.apache.org/licenses/LICENSE-2.0>
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import os
+from concurrent import futures
+
+import grpc
+
+from .cas.service import ByteStreamService, ContentAddressableStorageService
+from .actioncache.service import ActionCacheService
+from .execution.service import ExecutionService
+from .operations.service import OperationsService
+from .bots.service import BotsService
+from .referencestorage.service import ReferenceStorageService
+
+
+class BuildGridServer:
+    """Creates a BuildGrid server.
+
+    The :class:`BuildGridServer` class binds together all the
+    requisite services.
+    """
+
+    def __init__(self, max_workers=None):
+        """Initializes a new :class:`BuildGridServer` instance.
+
+        Args:
+            max_workers (int, optional): A pool of max worker threads.
+        """
+
+        self.logger = logging.getLogger(__name__)
+
+        if max_workers is None:
+            # Use max_workers default from Python 3.5+
+            max_workers = (os.cpu_count() or 1) * 5
+
+        server = grpc.server(futures.ThreadPoolExecutor(max_workers))
+
+        self._server = server
+
+        self._execution_service = None
+        self._bots_service = None
+        self._operations_service = None
+        self._reference_storage_service = None
+        self._action_cache_service = None
+        self._cas_service = None
+        self._bytestream_service = None
+
+    def start(self):
+        """Starts the server.
+        """
+        self._server.start()
+
+    def stop(self, grace=0):
+        """Stops the server.
+        """
+        self._server.stop(grace)
+
+    def add_port(self, address, credentials):
+        """Adds a port to the server.
+
+        Must be called before the server starts. If a credentials object exists,
+        it will make a secure port.
+
+        Args:
+            address (str): The address with port number.
+            credentials (:obj:`grpc.ChannelCredentials`): Credentials object.
+        """
+        if credentials is not None:
+            self.logger.info("Adding secure connection on: [{}]".format(address))
+            self._server.add_secure_port(address, credentials)
+
+        else:
+            self.logger.info("Adding insecure connection on [{}]".format(address))
+            self._server.add_insecure_port(address)
+
+    def add_execution_instance(self, instance, instance_name):
+        """Adds an :obj:`ExecutionInstance` to the service.
+
+        If no service exists, it creates one.
+
+        Args:
+            instance (:obj:`ExecutionInstance`): Instance to add.
+            instance_name (str): Instance name.
+        """
+        if self._execution_service is None:
+            self._execution_service = ExecutionService(self._server)
+
+        self._execution_service.add_instance(instance_name, instance)
+
+    def add_bots_interface(self, instance, instance_name):
+        """Adds a :obj:`BotsInterface` to the service.
+
+        If no service exists, it creates one.
+
+        Args:
+            instance (:obj:`BotsInterface`): Instance to add.
+            instance_name (str): Instance name.
+        """
+        if self._bots_service is None:
+            self._bots_service = BotsService(self._server)
+
+        self._bots_service.add_instance(instance_name, instance)
+
+    def add_operations_instance(self, instance, instance_name):
+        """Adds an :obj:`OperationsInstance` to the service.
+
+        If no service exists, it creates one.
+
+        Args:
+            instance (:obj:`OperationsInstance`): Instance to add.
+            instance_name (str): Instance name.
+        """
+        if self._operations_service is None:
+            self._operations_service = OperationsService(self._server)
+
+        self._operations_service.add_instance(instance_name, instance)
+
+    def add_reference_storage_instance(self, instance, instance_name):
+        """Adds a :obj:`ReferenceCache` to the service.
+
+        If no service exists, it creates one.
+
+        Args:
+            instance (:obj:`ReferenceCache`): Instance to add.
+            instance_name (str): Instance name.
+        """
+        if self._reference_storage_service is None:
+            self._reference_storage_service = ReferenceStorageService(self._server)
+
+        self._reference_storage_service.add_instance(instance_name, instance)
+
+    def add_action_cache_instance(self, instance, instance_name):
+        """Adds a :obj:`ReferenceCache` to the service.
+
+        If no service exists, it creates one.
+
+        Args:
+            instance (:obj:`ReferenceCache`): Instance to add.
+            instance_name (str): Instance name.
+        """
+        if self._action_cache_service is None:
+            self._action_cache_service = ActionCacheService(self._server)
+
+        self._action_cache_service.add_instance(instance_name, instance)
+
+    def add_cas_instance(self, instance, instance_name):
+        """Stores a :obj:`ContentAddressableStorageInstance` to the service.
+
+        If no service exists, it creates one.
+
+        Args:
+            instance (:obj:`ReferenceCache`): Instance to add.
+            instance_name (str): Instance name.
+        """
+        if self._cas_service is None:
+            self._cas_service = ContentAddressableStorageService(self._server)
+
+        self._cas_service.add_instance(instance_name, instance)
+
+    def add_bytestream_instance(self, instance, instance_name):
+        """Stores a :obj:`ByteStreamInstance` to the service.
+
+        If no service exists, it creates one.
+
+        Args:
+            instance (:obj:`ByteStreamInstance`): Instance to add.
+            instance_name (str): Instance name.
+        """
+        if self._bytestream_service is None:
+            self._bytestream_service = ByteStreamService(self._server)
+
+        self._bytestream_service.add_instance(instance_name, instance)

buildgrid/server/operations/instance.py

@@ -30,6 +30,9 @@ class OperationsInstance:
         self.logger = logging.getLogger(__name__)
         self._scheduler = scheduler
 
+    def register_instance_with_server(self, instance_name, server):
+        server.add_operations_instance(self, instance_name)
+
     def get_operation(self, name):
         operation = self._scheduler.jobs.get(name)
 

buildgrid/server/operations/service.py

@@ -32,12 +32,16 @@ from .._exceptions import InvalidArgumentError
 
 class OperationsService(operations_pb2_grpc.OperationsServicer):
 
-    def __init__(self, server, instances):
-        self._instances = instances
+    def __init__(self, server):
         self.logger = logging.getLogger(__name__)
 
+        self._instances = {}
+
         operations_pb2_grpc.add_OperationsServicer_to_server(self, server)
 
+    def add_instance(self, name, instance):
+        self._instances[name] = instance
+
     def GetOperation(self, request, context):
         try:
             name = request.name

buildgrid/server/referencestorage/service.py

@@ -25,13 +25,16 @@ from .._exceptions import InvalidArgumentError, NotFoundError
 
 class ReferenceStorageService(buildstream_pb2_grpc.ReferenceStorageServicer):
 
-    def __init__(self, server, instances):
+    def __init__(self, server):
         self.logger = logging.getLogger(__name__)
 
-        self._instances = instances
+        self._instances = {}
 
         buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server(self, server)
 
+    def add_instance(self, name, instance):
+        self._instances[name] = instance
+
     def GetReference(self, request, context):
         try:
             instance = self._get_instance(request.instance_name)

buildgrid/server/referencestorage/storage.py

@@ -44,6 +44,9 @@ class ReferenceCache:
         self._max_cached_refs = max_cached_refs
         self._digest_map = collections.OrderedDict()
 
+    def register_instance_with_server(self, instance_name, server):
+        server.add_reference_storage_instance(self, instance_name)
+
     @property
     def allow_updates(self):
         return self._allow_updates