buildgrid/_app/cli.py

@@ -25,6 +25,9 @@ import os
 import logging
 
 import click
+import grpc
+
+from buildgrid.utils import read_file
 
 from . import _logging
 
@@ -35,7 +38,114 @@ class Context:
 
     def __init__(self):
         self.verbose = False
-        self.home = os.getcwd()
+
+        self.user_home = os.getcwd()
+
+        self.user_cache_home = os.environ.get('XDG_CACHE_HOME')
+        if not self.user_cache_home:
+            self.user_cache_home = os.path.expanduser('~/.cache')
+        self.cache_home = os.path.join(self.user_cache_home, 'buildgrid')
+
+        self.user_config_home = os.environ.get('XDG_CONFIG_HOME')
+        if not self.user_config_home:
+            self.user_config_home = os.path.expanduser('~/.config')
+        self.config_home = os.path.join(self.user_config_home, 'buildgrid')
+
+        self.user_data_home = os.environ.get('XDG_DATA_HOME')
+        if not self.user_data_home:
+            self.user_data_home = os.path.expanduser('~/.local/share')
+        self.data_home = os.path.join(self.user_data_home, 'buildgrid')
+
+    def load_client_credentials(self, client_key=None, client_cert=None,
+                                server_cert=None, use_default_client_keys=False):
+        """Looks-up and loads TLS client gRPC credentials.
+
+        Args:
+            client_key(str): root certificate file path.
+            client_cert(str): private key file path.
+            server_cert(str): certificate chain file path.
+            use_default_client_keys(bool, optional): whether or not to try
+                loading client keys from default location. Defaults to False.
+
+        Returns:
+            :obj:`ChannelCredentials`: The credentials for use for a
+            TLS-encrypted gRPC client channel.
+        """
+        if not client_key or not os.path.exists(client_key):
+            if use_default_client_keys:
+                client_key = os.path.join(self.config_home, 'client.key')
+            else:
+                client_key = None
+
+        if not client_cert or not os.path.exists(client_cert):
+            if use_default_client_keys:
+                client_cert = os.path.join(self.config_home, 'client.crt')
+            else:
+                client_cert = None
+
+        if not server_cert or not os.path.exists(server_cert):
+            server_cert = os.path.join(self.config_home, 'server.crt')
+            if not os.path.exists(server_cert):
+                return None
+
+        server_cert_pem = read_file(server_cert)
+        if client_key and os.path.exists(client_key):
+            client_key_pem = read_file(client_key)
+        else:
+            client_key_pem = None
+        if client_key_pem and client_cert and os.path.exists(client_cert):
+            client_cert_pem = read_file(client_cert)
+        else:
+            client_cert_pem = None
+
+        return grpc.ssl_channel_credentials(root_certificates=server_cert_pem,
+                                            private_key=client_key_pem,
+                                            certificate_chain=client_cert_pem)
+
+    def load_server_credentials(self, server_key=None, server_cert=None,
+                                client_certs=None, use_default_client_certs=False):
+        """Looks-up and loads TLS server gRPC credentials.
+
+        Every private and publick keys are expected to be PEM-encoded.
+
+        Args:
+            server_key(str): private server key file path.
+            server_cert(str): public server certificate file path.
+            client_certs(str): public client certificates file path.
+            use_default_client_certs(bool, optional): whether or not to try
+                loading public client certificates from default location.
+                Defaults to False.
+
+        Returns:
+            :obj:`ServerCredentials`: The credentials for use for a
+            TLS-encrypted gRPC server channel.
+        """
+        if not server_key or not os.path.exists(server_key):
+            server_key = os.path.join(self.config_home, 'server.key')
+            if not os.path.exists(server_key):
+                return None
+
+        if not server_cert or not os.path.exists(server_cert):
+            server_cert = os.path.join(self.config_home, 'server.crt')
+            if not os.path.exists(server_cert):
+                return None
+
+        if not client_certs or not os.path.exists(client_certs):
+            if use_default_client_certs:
+                client_certs = os.path.join(self.config_home, 'client.crt')
+            else:
+                client_certs = None
+
+        server_key_pem = read_file(server_key)
+        server_cert_pem = read_file(server_cert)
+        if client_certs and os.path.exists(client_certs):
+            client_certs_pem = read_file(client_certs)
+        else:
+            client_certs_pem = None
+
+        return grpc.ssl_server_credentials([(server_key_pem, server_cert_pem)],
+                                           root_certificates=client_certs_pem,
+                                           require_client_auth=bool(client_certs))
 
 
 pass_context = click.make_pass_decorator(Context, ensure=True)

buildgrid/_app/commands/cmd_bot.py

@@ -21,8 +21,9 @@ Create a bot interface and request work
 """
 
 import logging
-
 from pathlib import Path, PurePath
+import sys
+from urllib.parse import urlparse
 
 import click
 import grpc
@@ -35,20 +36,37 @@ from ..cli import pass_context
 
 
 @click.group(name='bot', short_help="Create and register bot clients.")
+@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
+              help="Remote execution server's URL (port defaults to 50051 if no specified).")
+@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Private client key for TLS (PEM-encoded)")
+@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Public client certificate for TLS (PEM-encoded)")
+@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Public server certificate for TLS (PEM-encoded)")
 @click.option('--parent', type=click.STRING, default='bgd_test', show_default=True,
               help="Targeted farm resource.")
-@click.option('--port', type=click.INT, default='50051', show_default=True,
-              help="Remote server's port number.")
-@click.option('--host', type=click.STRING, default='localhost', show_default=True,
-              help="Renote server's hostname.")
 @pass_context
-def cli(context, host, port, parent):
-    channel = grpc.insecure_channel('{}:{}'.format(host, port))
-    interface = bot_interface.BotInterface(channel)
+def cli(context, remote, client_key, client_cert, server_cert, parent):
+    url = urlparse(remote)
+
+    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
+
+    if url.scheme == 'http':
+        context.channel = grpc.insecure_channel(context.remote)
+    else:
+        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
+        if not credentials:
+            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
+                       "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
+            sys.exit(-1)
+
+        context.channel = grpc.secure_channel(context.remote, credentials)
 
     context.logger = logging.getLogger(__name__)
-    context.logger.info("Starting on port {}".format(port))
-    context.channel = channel
+    context.logger.debug("Starting for remote {}".format(context.remote))
+
+    interface = bot_interface.BotInterface(context.channel)
 
     worker = Worker()
     worker.add_device(Device())

buildgrid/_app/commands/cmd_cas.py

@@ -21,6 +21,9 @@ Request work to be executed and monitor status of jobs.
 """
 
 import logging
+import sys
+from urllib.parse import urlparse
+
 import click
 import grpc
 
@@ -31,17 +34,33 @@ from ..cli import pass_context
 
 
 @click.group(name='cas', short_help="Interact with the CAS server.")
-@click.option('--port', type=click.INT, default='50051', show_default=True,
-              help="Remote server's port number.")
-@click.option('--host', type=click.STRING, default='localhost', show_default=True,
-              help="Remote server's hostname.")
+@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
+              help="Remote execution server's URL (port defaults to 50051 if no specified).")
+@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Private client key for TLS (PEM-encoded)")
+@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Public client certificate for TLS (PEM-encoded)")
+@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Public server certificate for TLS (PEM-encoded)")
 @pass_context
-def cli(context, host, port):
-    context.logger = logging.getLogger(__name__)
-    context.logger.info("Starting on port {}".format(port))
+def cli(context, remote, client_key, client_cert, server_cert):
+    url = urlparse(remote)
+
+    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
 
-    context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
-    context.port = port
+    if url.scheme == 'http':
+        context.channel = grpc.insecure_channel(context.remote)
+    else:
+        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
+        if not credentials:
+            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
+                       "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
+            sys.exit(-1)
+
+        context.channel = grpc.secure_channel(context.remote, credentials)
+
+    context.logger = logging.getLogger(__name__)
+    context.logger.debug("Starting for remote {}".format(context.remote))
 
 
 @cli.command('upload-files', short_help="Upload files to the CAS server.")

buildgrid/_app/commands/cmd_execute.py

@@ -22,8 +22,11 @@ Request work to be executed and monitor status of jobs.
 
 import errno
 import logging
-import stat
 import os
+import stat
+import sys
+from urllib.parse import urlparse
+
 import click
 import grpc
 
@@ -36,17 +39,33 @@ from ..cli import pass_context
 
 
 @click.group(name='execute', short_help="Execute simple operations.")
-@click.option('--port', type=click.INT, default='50051', show_default=True,
-              help="Remote server's port number.")
-@click.option('--host', type=click.STRING, default='localhost', show_default=True,
-              help="Remote server's hostname.")
+@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
+              help="Remote execution server's URL (port defaults to 50051 if no specified).")
+@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Private client key for TLS (PEM-encoded)")
+@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Public client certificate for TLS (PEM-encoded)")
+@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Public server certificate for TLS (PEM-encoded)")
 @pass_context
-def cli(context, host, port):
-    context.logger = logging.getLogger(__name__)
-    context.logger.info("Starting on port {}".format(port))
+def cli(context, remote, client_key, client_cert, server_cert):
+    url = urlparse(remote)
 
-    context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
-    context.port = port
+    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
+
+    if url.scheme == 'http':
+        context.channel = grpc.insecure_channel(context.remote)
+    else:
+        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
+        if not credentials:
+            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
+                       "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
+            sys.exit(-1)
+
+        context.channel = grpc.secure_channel(context.remote, credentials)
+
+    context.logger = logging.getLogger(__name__)
+    context.logger.debug("Starting for remote {}".format(context.remote))
 
 
 @cli.command('request-dummy', short_help="Send a dummy action.")

buildgrid/_app/commands/cmd_server.py

@@ -22,6 +22,7 @@ Create a BuildGrid server.
 
 import asyncio
 import logging
+import sys
 
 import click
 
@@ -41,17 +42,24 @@ _SIZE_PREFIXES = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40}
 @pass_context
 def cli(context):
     context.logger = logging.getLogger(__name__)
-    context.logger.info("BuildGrid server booting up")
 
 
 @cli.command('start', short_help="Setup a new server instance.")
-@click.option('--port', type=click.INT, default='50051', show_default=True,
+@click.option('--port', '-p', type=click.INT, default='50051', show_default=True,
               help="The port number to be listened.")
-@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
-              help="Maximum number of actions to keep in the ActionCache.")
+@click.option('--server-key', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Private server key for TLS (PEM-encoded)")
+@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Public server certificate for TLS (PEM-encoded)")
+@click.option('--client-certs', type=click.Path(exists=True, dir_okay=False), default=None,
+              help="Public client certificates for TLS (PEM-encoded)")
+@click.option('--allow-insecure', type=click.BOOL, is_flag=True,
+              help="Whether or not to allow unencrypted connections.")
 @click.option('--allow-update-action-result/--forbid-update-action-result',
               'allow_uar', default=True, show_default=True,
               help="Whether or not to allow clients to manually edit the action cache.")
+@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
+              help="Maximum number of actions to keep in the ActionCache.")
 @click.option('--cas', type=click.Choice(('lru', 's3', 'disk', 'with-cache')),
               help="The CAS storage type to use.")
 @click.option('--cas-cache', type=click.Choice(('lru', 's3', 'disk')),
@@ -67,7 +75,21 @@ def cli(context):
 @click.option('--cas-disk-directory', type=click.Path(file_okay=False, dir_okay=True, writable=True),
               help="For --cas=disk, the folder to store CAS blobs in.")
 @pass_context
-def start(context, port, max_cached_actions, allow_uar, cas, **cas_args):
+def start(context, port, allow_insecure, server_key, server_cert, client_certs,
+          max_cached_actions, allow_uar, cas, **cas_args):
+    """Setups a new server instance."""
+    credentials = None
+    if not allow_insecure:
+        credentials = context.load_server_credentials(server_key, server_cert, client_certs)
+    if not credentials and not allow_insecure:
+        click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
+                   "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
+        sys.exit(-1)
+
+    context.credentials = credentials
+    context.port = port
+
+    context.logger.info("BuildGrid server booting up")
     context.logger.info("Starting on port {}".format(port))
 
     cas_storage = _make_cas_storage(context, cas, cas_args)
@@ -79,7 +101,8 @@ def start(context, port, max_cached_actions, allow_uar, cas, **cas_args):
     else:
         action_cache = ActionCache(cas_storage, max_cached_actions, allow_uar)
 
-    server = build_grid_server.BuildGridServer(port,
+    server = build_grid_server.BuildGridServer(port=context.port,
+                                               credentials=context.credentials,
                                                cas_storage=cas_storage,
                                                action_cache=action_cache)
     loop = asyncio.get_event_loop()

buildgrid/server/build_grid_server.py

@@ -42,14 +42,18 @@ from .worker.bots_interface import BotsInterface
 
 class BuildGridServer:
 
-    def __init__(self, port='50051', max_workers=10, cas_storage=None, action_cache=None):
+    def __init__(self, port='50051', credentials=None, max_workers=10, cas_storage=None, action_cache=None):
         port = '[::]:{0}'.format(port)
         scheduler = Scheduler(action_cache)
         bots_interface = BotsInterface(scheduler)
         execution_instance = ExecutionInstance(scheduler, cas_storage)
 
         self._server = grpc.server(futures.ThreadPoolExecutor(max_workers))
-        self._server.add_insecure_port(port)
+
+        if credentials is not None:
+            self._server.add_secure_port(port, credentials)
+        else:
+            self._server.add_insecure_port(port)
 
         bots_pb2_grpc.add_BotsServicer_to_server(BotsService(bots_interface),
                                                  self._server)

docs/source/using_dummy_build.rst

@@ -8,7 +8,7 @@ In one terminal, start a server:
 
 .. code-block:: sh
 
-   bgd server start
+   bgd server start --allow-insecure
 
 In another terminal, send a request for work:
 

docs/source/using_simple_build.rst

@@ -27,7 +27,7 @@ Now start a BuildGrid server, passing it a directory it can write a CAS to:
 
 .. code-block:: sh
 
-   bgd server start --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
+   bgd server start --allow-insecure --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
 
 Start the following bot session: