Add support for user remote execution configuration (890e6ba3) · Commits · BuildStream / buildstream

buildstream/_context.py

+9 −2

Original line number	Diff line number	Diff line
		@@ -34,6 +34,7 @@ from ._artifactcache import ArtifactCache
		from ._artifactcache.cascache import CASCache
		from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE
		from .plugin import _plugin_lookup
		from .sandbox import SandboxRemote


		# Context()
		@@ -72,6 +73,9 @@ class Context():
		# The locations from which to push and pull prebuilt artifacts
		self.artifact_cache_specs = None

		# The global remote execution configuration
		self.remote_execution_specs = None

		# The directory to store build logs
		self.logdir = None

		@@ -187,7 +191,7 @@ class Context():
		_yaml.node_validate(defaults, [
		'sourcedir', 'builddir', 'artifactdir', 'logdir',
		'scheduler', 'artifacts', 'logging', 'projects',
		'cache', 'prompt', 'workspacedir',
		'cache', 'prompt', 'workspacedir', 'remote-execution'
		])

		for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
		@@ -212,6 +216,8 @@ class Context():
		# Load artifact share configuration
		self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)

		self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults)

		# Load pull build trees configuration
		self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')

		@@ -271,7 +277,8 @@ class Context():
		# Shallow validation of overrides, parts of buildstream which rely
		# on the overrides are expected to validate elsewhere.
		for _, overrides in _yaml.node_items(self._project_overrides):
		_yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror'])
		_yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror',
		'remote-execution'])

		profile_end(Topics.LOAD_CONTEXT, 'load')

buildstream/_project.py

+10 −1

Original line number	Diff line number	Diff line
		@@ -507,7 +507,16 @@ class Project():
		self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)

		# Load remote-execution configuration for this project
		self.remote_execution_specs = SandboxRemote.specs_from_config_node(config, self.directory)
		project_specs = SandboxRemote.specs_from_config_node(config, self.directory)
		override_specs = SandboxRemote.specs_from_config_node(
		self._context.get_overrides(self.name), self.directory)

		if override_specs is not None:
		self.remote_execution_specs = override_specs
		elif project_specs is not None:
		self.remote_execution_specs = project_specs
		else:
		self.remote_execution_specs = self._context.remote_execution_specs

		# Load sandbox environment variables
		self.base_environment = _yaml.node_get(config, Mapping, 'environment')

buildstream/sandbox/_sandboxremote.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -81,7 +81,7 @@ class SandboxRemote(Sandbox):
		self._get_context().message(Message(None, MessageType.INFO, msg))

		@staticmethod
		def specs_from_config_node(config_node, basedir):
		def specs_from_config_node(config_node, basedir=None):

		def require_node(config, keyname):
		val = config.get(keyname)

doc/source/format_project.rst

+4 −0

Original line number	Diff line number	Diff line
		@@ -218,6 +218,7 @@ The use of ports are required to distinguish between pull only access and
		push/pull access. For information regarding the server/client certificates
		and keys, please see: :ref:`Key pair for the server <server_authentication>`.

		.. _project_remote_execution:

		Remote execution
		~~~~~~~~~~~~~~~~
		@@ -268,6 +269,9 @@ instance names.

		The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.

		Remote execution configuration can be also provided in the `user
		configuration <user_config_remote_execution>`.

		.. _project_essentials_mirrors:

		Mirrors

doc/source/using_config.rst

+48 −0

Original line number	Diff line number	Diff line
		@@ -100,6 +100,54 @@ pull only access and push/pull access. For information regarding this and the
		server/client certificates and keys, please see:
		:ref:`Key pair for the server <server_authentication>`.

		.. _user_config_remote_execution:

		Remote execution
		~~~~~~~~~~~~~~~~

		The same configuration for :ref:`remote execution <project_remote_execution>`
		in ``project.conf`` can be provided in the user configuation.

		There is only one remote execution configuration used per project.

		The project overrides will be taken in priority. The global
		configuration will be used as fallback.

		1. Global remote execution fallback:

		.. code:: yaml

		remote-execution:
		execution-service:
		url: http://execution.fallback.example.com:50051
		instance-name: main
		storage-service:
		url: https://storage.fallback.example.com:11002/
		server-cert: /keys/server.crt
		client-cert: /keys/client.crt
		client-key: /keys/client.key
		instance-name: main
		action-cache-service:
		url: http://action.flalback.example.com:50052

		2. Project override:

		.. code:: yaml

		projects:
		some_project:
		remote-execution:
		execution-service:
		url: http://execution.some_project.example.com:50051
		instance-name: main
		storage-service:
		url: https://storage.some_project.example.com:11002/
		server-cert: /some_project_keys/server.crt
		client-cert: /some_project_keys/client.crt
		client-key: /some_project_keys/client.key
		instance-name: main
		action-cache-service:
		url: http://action.some_project.example.com:50052


		Strict build plan