Add server-side gRPC TLS encryption support

#63

buildgrid/_app/cli.py

@@ -25,6 +25,10 @@ import os
 import logging
 
 import click
+import grpc
+from xdg import XDG_CACHE_HOME, XDG_CONFIG_HOME, XDG_DATA_HOME
+
+from buildgrid.utils import read_file
 
 from . import _logging
 
@@ -35,7 +39,57 @@ class Context:
 
     def __init__(self):
         self.verbose = False
-        self.home = os.getcwd()
+
+        self.user_home = os.getcwd()
+
+        self.cache_home = os.path.join(XDG_CACHE_HOME, 'buildgrid')
+        self.config_home = os.path.join(XDG_CONFIG_HOME, 'buildgrid')
+        self.data_home = os.path.join(XDG_DATA_HOME, 'buildgrid')
+
+    def load_server_credentials(self, server_key=None, server_cert=None,
+                                client_certs=None, use_default_client_certs=False):
+        """Looks-up and loads TLS server gRPC credentials.
+
+        Every private and public keys are expected to be PEM-encoded.
+
+        Args:
+            server_key(str): private server key file path.
+            server_cert(str): public server certificate file path.
+            client_certs(str): public client certificates file path.
+            use_default_client_certs(bool, optional): whether or not to try
+                loading public client certificates from default location.
+                Defaults to False.
+
+        Returns:
+            :obj:`ServerCredentials`: The credentials for use for a
+            TLS-encrypted gRPC server channel.
+        """
+        if not server_key or not os.path.exists(server_key):
+            server_key = os.path.join(self.config_home, 'server.key')
+            if not os.path.exists(server_key):
+                return None
+
+        if not server_cert or not os.path.exists(server_cert):
+            server_cert = os.path.join(self.config_home, 'server.crt')
+            if not os.path.exists(server_cert):
+                return None
+
+        if not client_certs or not os.path.exists(client_certs):
+            if use_default_client_certs:
+                client_certs = os.path.join(self.config_home, 'client.crt')
+            else:
+                client_certs = None
+
+        server_key_pem = read_file(server_key)
+        server_cert_pem = read_file(server_cert)
+        if client_certs and os.path.exists(client_certs):
+            client_certs_pem = read_file(client_certs)
+        else:
+            client_certs_pem = None
+
+        return grpc.ssl_server_credentials([(server_key_pem, server_cert_pem)],
+                                           root_certificates=client_certs_pem,
+                                           require_client_auth=bool(client_certs))
 
 
 pass_context = click.make_pass_decorator(Context, ensure=True)

buildgrid/_app/commands/cmd_server.py

@@ -22,6 +22,7 @@ Create a BuildGrid server.
 
 import asyncio
 import logging
+import sys
 
 import click
 
@@ -41,18 +42,25 @@ _SIZE_PREFIXES = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40}
 @pass_context
 def cli(context):
     context.logger = logging.getLogger(__name__)
-    context.logger.info("BuildGrid server booting up")
 
 
 @cli.command('start', short_help="Setup a new server instance.")
 @click.argument('instances', nargs=-1, type=click.STRING)
 @click.option('--port', type=click.INT, default='50051', show_default=True,
               help="The port number to be listened.")
-@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
-              help="Maximum number of actions to keep in the ActionCache.")
+@click.option('--server-key', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Private server key for TLS (PEM-encoded)")
+@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Public server certificate for TLS (PEM-encoded)")
+@click.option('--client-certs', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Public client certificates for TLS (PEM-encoded)")
+@click.option('--allow-insecure', type=click.BOOL, is_flag=True,
+              help="Whether or not to allow unencrypted connections.")
 @click.option('--allow-update-action-result/--forbid-update-action-result',
               'allow_uar', default=True, show_default=True,
               help="Whether or not to allow clients to manually edit the action cache.")
+@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
+              help="Maximum number of actions to keep in the ActionCache.")
 @click.option('--cas', type=click.Choice(('lru', 's3', 'disk', 'with-cache')),
               help="The CAS storage type to use.")
 @click.option('--cas-cache', type=click.Choice(('lru', 's3', 'disk')),
@@ -68,9 +76,21 @@ def cli(context):
 @click.option('--cas-disk-directory', type=click.Path(file_okay=False, dir_okay=True, writable=True),
               help="For --cas=disk, the folder to store CAS blobs in.")
 @pass_context
-def start(context, instances, port, max_cached_actions, allow_uar, cas, **cas_args):
-    """ Starts a BuildGrid server.
-    """
+def start(context, port, allow_insecure, server_key, server_cert, client_certs,
+          instances, max_cached_actions, allow_uar, cas, **cas_args):
+    """Setups a new server instance."""
+    credentials = None
+    if not allow_insecure:
+        credentials = context.load_server_credentials(server_key, server_cert, client_certs)
+    if not credentials and not allow_insecure:
+        click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
+                   "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
+        sys.exit(-1)
+
+    context.credentials = credentials
+    context.port = port
+
+    context.logger.info("BuildGrid server booting up")
     context.logger.info("Starting on port {}".format(port))
 
     cas_storage = _make_cas_storage(context, cas, cas_args)
@@ -85,8 +105,9 @@ def start(context, instances, port, max_cached_actions, allow_uar, cas, **cas_ar
     if instances is None:
         instances = ['main']
 
-    server = buildgrid_server.BuildGridServer(port,
-                                              instances,
+    server = buildgrid_server.BuildGridServer(port=context.port,
+                                              credentials=context.credentials,
+                                              instances=instances,
                                               cas_storage=cas_storage,
                                               action_cache=action_cache)
     loop = asyncio.get_event_loop()

buildgrid/server/buildgrid_server.py

@@ -40,11 +40,16 @@ from .worker.bots_service import BotsService
 
 class BuildGridServer:
 
-    def __init__(self, port='50051', instances=None, max_workers=10, action_cache=None, cas_storage=None):
-        port = '[::]:{0}'.format(port)
+    def __init__(self, port=50051, credentials=None, instances=None,
+                 max_workers=10, action_cache=None, cas_storage=None):
+        address = '[::]:{0}'.format(port)
 
         self._server = grpc.server(futures.ThreadPoolExecutor(max_workers))
-        self._server.add_insecure_port(port)
+
+        if credentials is not None:
+            self._server.add_secure_port(address, credentials)
+        else:
+            self._server.add_insecure_port(address)
 
         if cas_storage is not None:
             cas_service = ContentAddressableStorageService(cas_storage)

docs/source/using_dummy_build.rst

@@ -8,7 +8,7 @@ In one terminal, start a server:
 
 .. code-block:: sh
 
-   bgd server start
+   bgd server start --allow-insecure
 
 In another terminal, send a request for work:
 

docs/source/using_simple_build.rst

@@ -27,7 +27,7 @@ Now start a BuildGrid server, passing it a directory it can write a CAS to:
 
 .. code-block:: sh
 
-   bgd server start --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
+   bgd server start --allow-insecure --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
 
 Start the following bot session:
 

setup.py

@@ -116,6 +116,7 @@ setup(
         'Click',
         'boto3 < 1.8.0',
         'botocore < 1.11.0',
+        'xdg',
     ],
     entry_points={
         'console_scripts': [