WIP: Opening a workspace with a cached build (98172166) · Commits · BuildStream / buildstream

buildstream/_artifactcache/artifactcache.py

+14 −0

Original line number	Diff line number	Diff line
		@@ -603,6 +603,20 @@ class ArtifactCache():
		raise ImplError("Cache '{kind}' does not implement calculate_cache_size()"
		.format(kind=type(self).__name__))

		# checkout_artifact_subdir()
		#
		# Checkout given artifact subdir into provided directory
		#
		# Args:
		# element (Element): The Element
		# key (str): The cache key to use
		# subdir (str): The subdir to checkout
		# tmpdir (str): The dir to place the subdir content
		#
		def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
		raise ImplError("Cache '{kind}' does not implement checkout_artifact_subdir()"
		.format(kind=type(self).__name__))

		################################################
		# Local Private Methods #
		################################################

buildstream/_artifactcache/cascache.py

+7 −0

Original line number	Diff line number	Diff line
		@@ -452,6 +452,13 @@ class CASCache(ArtifactCache):

		return pushed

		def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
		tree = self.resolve_ref(self.get_artifact_fullname(element, key))

		# This assumes that the subdir digest is present in the element tree
		subdirdigest = self._get_subdir(tree, subdir)
		self._checkout(tmpdir, subdirdigest)

		################################################
		# API Private Methods #
		################################################

buildstream/_context.py

+7 −1

Original line number	Diff line number	Diff line
		@@ -114,6 +114,9 @@ class Context():
		# Whether or not to attempt to pull buildtrees globally
		self.pullbuildtrees = False

		# Whether to not include artifact buildtrees in workspaces if available
		self.workspacebuildtrees = True

		# Private variables
		self._cache_key = None
		self._message_handler = None
		@@ -164,7 +167,7 @@ class Context():
		_yaml.node_validate(defaults, [
		'sourcedir', 'builddir', 'artifactdir', 'logdir',
		'scheduler', 'artifacts', 'logging', 'projects',
		'cache', 'pullbuildtrees'
		'cache', 'pullbuildtrees', 'workspacebuildtrees'
		])

		for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
		@@ -192,6 +195,9 @@ class Context():
		# Load pull buildtrees configuration
		self.pullbuildtrees = _yaml.node_get(defaults, bool, 'pullbuildtrees', default_value='False')

		# Load workspace buildtrees configuration
		self.workspacebuildtrees = _yaml.node_get(defaults, bool, 'workspacebuildtrees', default_value='True')

		# Load logging config
		logging = _yaml.node_get(defaults, Mapping, 'logging')
		_yaml.node_validate(logging, [

buildstream/_frontend/cli.py

+11 −3

Original line number	Diff line number	Diff line
		@@ -686,12 +686,16 @@ def workspace():
		help="Overwrite files existing in checkout directory")
		@click.option('--track', 'track_', default=False, is_flag=True,
		help="Track and fetch new source references before checking out the workspace")
		@click.option('--no-cache', default=False, is_flag=True,
		help="Do not checkout the cached buildtree")
		@click.argument('element',
		type=click.Path(readable=False))
		@click.argument('directory', type=click.Path(file_okay=False))
		@click.pass_obj
		def workspace_open(app, no_checkout, force, track_, element, directory):
		"""Open a workspace for manual source modification"""
		def workspace_open(app, no_checkout, force, track_, no_cache, element, directory):
		"""Open a workspace for manual source modification, the elements buildtree
		will be provided if available in the local artifact cache.
		"""

		if os.path.exists(directory):

		@@ -703,11 +707,15 @@ def workspace_open(app, no_checkout, force, track_, element, directory):
		click.echo("Checkout directory is not empty: {}".format(directory), err=True)
		sys.exit(-1)

		if not no_cache:
		click.echo("WARNING: Workspace will be opened without the cached buildtree if not cached locally")

		with app.initialized():
		app.stream.workspace_open(element, directory,
		no_checkout=no_checkout,
		track_first=track_,
		force=force)
		force=force,
		no_cache=no_cache)


		##################################################################

buildstream/_stream.py

+27 −2

Original line number	Diff line number	Diff line
		@@ -456,11 +456,17 @@ class Stream():
		# no_checkout (bool): Whether to skip checking out the source
		# track_first (bool): Whether to track and fetch first
		# force (bool): Whether to ignore contents in an existing directory
		# no_cache (bool): Whether to not include the cached buildtree
		#
		def workspace_open(self, target, directory, *,
		no_checkout,
		track_first,
		force):
		force,
		no_cache):

		# Override no_cache if the global user conf workspacebuildtrees is false
		if not self._context.workspacebuildtrees:
		no_cache = True

		if track_first:
		track_targets = (target,)
		@@ -473,6 +479,20 @@ class Stream():
		target = elements[0]
		directory = os.path.abspath(directory)

		# Check if given target has a buildtree artifact cached locally
		buildtree = None
		if target._cached():
		buildtree = self._artifacts.contains_subdir_artifact(target, target._get_cache_key(), 'buildtree')

		# If we're running in the default state, make the user aware of buildtree usage
		if not no_cache:
		if buildtree:
		self._message(MessageType.INFO, "{} buildtree artifact is available,"
		" workspace will be opened with it".format(target.name))
		else:
		self._message(MessageType.WARN, "{} buildtree artifact not available,"
		" workspace will be opened with source checkout".format(target.name))

		if not list(target.sources()):
		build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
		if not build_depends:
		@@ -504,6 +524,7 @@ class Stream():
		"fetch the latest version of the " +
		"source.")

		# Presume workspace to be forced if previous StreamError not raised
		if workspace:
		workspaces.delete_workspace(target._get_full_name())
		workspaces.save_config()
		@@ -515,9 +536,13 @@ class Stream():

		workspaces.create_workspace(target._get_full_name(), directory)

		if not no_checkout:
		if (not buildtree or no_cache) and not no_checkout:
		with target.timed_activity("Staging sources to {}".format(directory)):
		target._open_workspace()
		# Handle opening workspace with buildtree included
		elif buildtree and not no_cache:
		with target.timed_activity("Staging buildtree to {}".format(directory)):
		target._open_workspace(buildtree=buildtree)

		workspaces.save_config()
		self._message(MessageType.INFO, "Saved workspace configuration")