Move KeyStrength enum out to element_enums.py (f9aa09d3) · Commits · BuildStream / buildstream

buildstream/_artifactcache/artifactcache.py

+3 −3

Original line number	Diff line number	Diff line
		@@ -21,7 +21,7 @@ import os
		import string
		from collections import Mapping, namedtuple

		from ..element import _KeyStrength
		from ..element_enums import KeyStrength
		from .._exceptions import ArtifactError, ImplError, LoadError, LoadErrorReason
		from .._message import Message, MessageType
		from .. import utils
		@@ -201,8 +201,8 @@ class ArtifactCache():
		# user inconvenience.

		for element in elements:
		strong_key = element._get_cache_key(strength=_KeyStrength.STRONG)
		weak_key = element._get_cache_key(strength=_KeyStrength.WEAK)
		strong_key = element._get_cache_key(strength=KeyStrength.STRONG)
		weak_key = element._get_cache_key(strength=KeyStrength.WEAK)

		for key in (strong_key, weak_key):
		if key and key not in self.required_artifacts:

buildstream/element.py

+20 −53

Original line number	Diff line number	Diff line
		@@ -78,7 +78,6 @@ import stat
		import copy
		from collections import Mapping, OrderedDict
		from contextlib import contextmanager
		from enum import Enum
		import tempfile
		import shutil

		@@ -97,41 +96,9 @@ from ._platform import Platform
		from .sandbox._config import SandboxConfig

		from .storage.directory import Directory
		from .storage._filebaseddirectory import FileBasedDirectory, VirtualDirectoryError


		# _KeyStrength():
		#
		# Strength of cache key
		#
		class _KeyStrength(Enum):

		# Includes strong cache keys of all build dependencies and their
		# runtime dependencies.
		STRONG = 1

		# Includes names of direct build dependencies but does not include
		# cache keys of dependencies.
		WEAK = 2


		class Scope(Enum):
		"""Types of scope for a given element"""

		ALL = 1
		"""All elements which the given element depends on, following
		all elements required for building. Including the element itself.
		"""

		BUILD = 2
		"""All elements required for building the element, including their
		respective run dependencies. Not including the given element itself.
		"""

		RUN = 3
		"""All elements required for running the element. Including the element
		itself.
		"""
		from .storage._filebaseddirectory import FileBasedDirectory
		from .storage.directory import VirtualDirectoryError
		from .element_enums import KeyStrength, Scope


		class ElementError(BstError):
		@@ -1026,7 +993,7 @@ class Element(Plugin):
		# if the pull job is still pending as the remote cache may have an artifact
		# that matches the strict cache key, which is preferred over a locally
		# cached artifact with a weak cache key match.
		if not dependency._cached_success() or not dependency._get_cache_key(strength=_KeyStrength.STRONG):
		if not dependency._cached_success() or not dependency._get_cache_key(strength=KeyStrength.STRONG):
		return False

		if not self.__assemble_scheduled:
		@@ -1039,15 +1006,15 @@ class Element(Plugin):
		# Returns the cache key
		#
		# Args:
		# strength (_KeyStrength): Either STRONG or WEAK key strength
		# strength (KeyStrength): Either STRONG or WEAK key strength
		#
		# Returns:
		# (str): A hex digest cache key for this Element, or None
		#
		# None is returned if information for the cache key is missing.
		#
		def _get_cache_key(self, strength=_KeyStrength.STRONG):
		if strength == _KeyStrength.STRONG:
		def _get_cache_key(self, strength=KeyStrength.STRONG):
		if strength == KeyStrength.STRONG:
		return self.__cache_key
		else:
		return self.__weak_cache_key
		@@ -1107,7 +1074,7 @@ class Element(Plugin):
		# but does not include keys of dependencies.
		if self.BST_STRICT_REBUILD:
		dependencies = [
		e._get_cache_key(strength=_KeyStrength.WEAK)
		e._get_cache_key(strength=KeyStrength.WEAK)
		for e in self.dependencies(Scope.BUILD)
		]
		else:
		@@ -1132,7 +1099,7 @@ class Element(Plugin):
		# are sufficient. However, don't update the `cached` attributes
		# until the full cache query below.
		if (not self.__assemble_scheduled and not self.__assemble_done and
		not self.__cached_success(keystrength=_KeyStrength.WEAK) and
		not self.__cached_success(keystrength=KeyStrength.WEAK) and
		not self._pull_pending() and self._is_required()):
		self._schedule_assemble()
		return
		@@ -1624,7 +1591,7 @@ class Element(Plugin):
		# Store keys.yaml
		_yaml.dump(_yaml.node_sanitize({
		'strong': self._get_cache_key(),
		'weak': self._get_cache_key(_KeyStrength.WEAK),
		'weak': self._get_cache_key(KeyStrength.WEAK),
		}), os.path.join(metadir, 'keys.yaml'))

		# Store dependencies.yaml
		@@ -1693,7 +1660,7 @@ class Element(Plugin):
		self._update_state()

		def _pull_strong(self, *, progress=None):
		weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
		weak_key = self._get_cache_key(strength=KeyStrength.WEAK)

		key = self.__strict_cache_key
		if not self.__artifacts.pull(self, key, progress=progress):
		@@ -1705,7 +1672,7 @@ class Element(Plugin):
		return True

		def _pull_weak(self, *, progress=None):
		weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
		weak_key = self._get_cache_key(strength=KeyStrength.WEAK)

		if not self.__artifacts.pull(self, weak_key, progress=progress):
		return False
		@@ -1714,7 +1681,7 @@ class Element(Plugin):
		self._pull_done()

		# create tag for strong cache key
		key = self._get_cache_key(strength=_KeyStrength.STRONG)
		key = self._get_cache_key(strength=KeyStrength.STRONG)
		self.__artifacts.link_key(self, weak_key, key)

		return True
		@@ -2085,13 +2052,13 @@ class Element(Plugin):
		if keystrength is None:
		return self.__cached

		return self.__strong_cached if keystrength == _KeyStrength.STRONG else self.__weak_cached
		return self.__strong_cached if keystrength == KeyStrength.STRONG else self.__weak_cached

		# __assert_cached()
		#
		# Raises an error if the artifact is not cached.
		#
		def __assert_cached(self, keystrength=_KeyStrength.STRONG):
		def __assert_cached(self, keystrength=KeyStrength.STRONG):
		assert self.__is_cached(keystrength=keystrength), "{}: Missing artifact {}".format(
		self, self._get_brief_display_key())

		@@ -2415,7 +2382,7 @@ class Element(Plugin):
		# Use weak cache key, if artifact is missing for strong cache key
		# and the context allows use of weak cache keys
		if not context.get_strict() and not self.__artifacts.contains(self, key):
		key = self._get_cache_key(strength=_KeyStrength.WEAK)
		key = self._get_cache_key(strength=KeyStrength.WEAK)

		return (self.__artifacts.extract(self, key), key)

		@@ -2558,7 +2525,7 @@ class Element(Plugin):
		self.__assert_cached(keystrength=keystrength)
		assert self.__build_result is None

		artifact_base, _ = self.__extract(key=self.__weak_cache_key if keystrength is _KeyStrength.WEAK
		artifact_base, _ = self.__extract(key=self.__weak_cache_key if keystrength is KeyStrength.WEAK
		else self.__strict_cache_key)

		metadir = os.path.join(artifact_base, 'meta')
		@@ -2572,7 +2539,7 @@ class Element(Plugin):

		def __get_build_result(self, keystrength):
		if keystrength is None:
		keystrength = _KeyStrength.STRONG if self._get_context().get_strict() else _KeyStrength.WEAK
		keystrength = KeyStrength.STRONG if self._get_context().get_strict() else KeyStrength.WEAK

		if self.__build_result is None:
		self.__load_build_result(keystrength)
		@@ -2590,10 +2557,10 @@ class Element(Plugin):
		keys = []

		# tag with strong cache key based on dependency versions used for the build
		keys.append(self._get_cache_key(strength=_KeyStrength.STRONG))
		keys.append(self._get_cache_key(strength=KeyStrength.STRONG))

		# also store under weak cache key
		keys.append(self._get_cache_key(strength=_KeyStrength.WEAK))
		keys.append(self._get_cache_key(strength=KeyStrength.WEAK))

		return utils._deduplicate(keys)

buildstream/element_enums.py

0 → 100644

+61 −0

Original line number	Diff line number	Diff line
		#
		# Copyright (C) 2018 Bloomberg LLC
		#
		# This program is free software; you can redistribute it and/or
		# modify it under the terms of the GNU Lesser General Public
		# License as published by the Free Software Foundation; either
		# version 2 of the License, or (at your option) any later version.
		#
		# This library is distributed in the hope that it will be useful,
		# but WITHOUT ANY WARRANTY; without even the implied warranty of
		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
		# Lesser General Public License for more details.
		#
		# You should have received a copy of the GNU Lesser General Public
		# License along with this library. If not, see <http://www.gnu.org/licenses/>.
		#
		# Authors:
		# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
		# Jim MacArthur <jim.macarthur@codethink.co.uk>

		"""
		Element - Globally visible enumerations
		=======================================

		"""

		from enum import Enum


		# KeyStrength():
		#
		# Strength of cache key
		#
		class KeyStrength(Enum):

		# Includes strong cache keys of all build dependencies and their
		# runtime dependencies.
		STRONG = 1

		# Includes names of direct build dependencies but does not include
		# cache keys of dependencies.
		WEAK = 2


		class Scope(Enum):
		"""Types of scope for a given element"""

		ALL = 1
		"""All elements which the given element depends on, following
		all elements required for building. Including the element itself.
		"""

		BUILD = 2
		"""All elements required for building the element, including their
		respective run dependencies. Not including the given element itself.
		"""

		RUN = 3
		"""All elements required for running the element. Including the element
		itself.
		"""