Split remote execution from artifact cache

Description

Currently remote execution can only specify one URL, which is used for the REAPI server (usually BuildGrid), and remote storage has to be the same as the artifact cache. This MR splits those up, so you can specify another CAS remote, and separately specify the endpoint for the execution service.

You can still specify the same server for your artifact cache and remote execution store if you want.

At the moment, local RE CAS storage is still in the same location as artifact cache storage, it's just the remotes that differ.

Proposed changes

Changes proposed in this merge request:

• Change config for remote execution so it can accept separate configuration for the execution server and storage service.
• Create our own CASCache objects in SandboxRemote instead of using the artifact cache's objects.
• Add documentation for the new config options
• Add tests for new config options
• Remove the methods in ArtifactCache which are only necessary for remote execution.

I'm not proposing to add a full CI remote execution test, but I will add some tests which check the new config option behaves sensibly.

This merge request, when approved, will close: #750 (closed)

---

buildstream/_artifactcache/artifactcache.py

@@ -21,7 +21,6 @@ import multiprocessing
 import os
 import signal
 import string
-from collections import namedtuple
 from collections.abc import Mapping
 
 from ..types import _KeyStrength
@@ -31,7 +30,7 @@ from .. import _signals
 from .. import utils
 from .. import _yaml
 
-from .cascache import CASCache, CASRemote
+from .cascache import CASCache, CASRemote, CASRemoteSpec
 
 
 CACHE_SIZE_FILE = "cache_size"
@@ -45,48 +44,8 @@ CACHE_SIZE_FILE = "cache_size"
 #     push (bool): Whether we should attempt to push artifacts to this cache,
 #                  in addition to pulling from it.
 #
-class ArtifactCacheSpec(namedtuple('ArtifactCacheSpec', 'url push server_cert client_key client_cert')):
-
-    # _new_from_config_node
-    #
-    # Creates an ArtifactCacheSpec() from a YAML loaded node
-    #
-    @staticmethod
-    def _new_from_config_node(spec_node, basedir=None):
-        _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert'])
-        url = _yaml.node_get(spec_node, str, 'url')
-        push = _yaml.node_get(spec_node, bool, 'push', default_value=False)
-        if not url:
-            provenance = _yaml.node_get_provenance(spec_node, 'url')
-            raise LoadError(LoadErrorReason.INVALID_DATA,
-                            "{}: empty artifact cache URL".format(provenance))
-
-        server_cert = _yaml.node_get(spec_node, str, 'server-cert', default_value=None)
-        if server_cert and basedir:
-            server_cert = os.path.join(basedir, server_cert)
-
-        client_key = _yaml.node_get(spec_node, str, 'client-key', default_value=None)
-        if client_key and basedir:
-            client_key = os.path.join(basedir, client_key)
-
-        client_cert = _yaml.node_get(spec_node, str, 'client-cert', default_value=None)
-        if client_cert and basedir:
-            client_cert = os.path.join(basedir, client_cert)
-
-        if client_key and not client_cert:
-            provenance = _yaml.node_get_provenance(spec_node, 'client-key')
-            raise LoadError(LoadErrorReason.INVALID_DATA,
-                            "{}: 'client-key' was specified without 'client-cert'".format(provenance))
-
-        if client_cert and not client_key:
-            provenance = _yaml.node_get_provenance(spec_node, 'client-cert')
-            raise LoadError(LoadErrorReason.INVALID_DATA,
-                            "{}: 'client-cert' was specified without 'client-key'".format(provenance))
-
-        return ArtifactCacheSpec(url, push, server_cert, client_key, client_cert)
-
-
-ArtifactCacheSpec.__new__.__defaults__ = (None, None, None)
+class ArtifactCacheSpec(CASRemoteSpec):
+    pass
 
 
 # An ArtifactCache manages artifacts.