Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • willsalmon/buildstream
  • CumHoleZH/buildstream
  • tchaik/buildstream
  • DCotyPortfolio/buildstream
  • jesusoctavioas/buildstream
  • patrickmmartin/buildstream
  • franred/buildstream
  • tintou/buildstream
  • alatiera/buildstream
  • martinblanchard/buildstream
  • neverdie22042524/buildstream
  • Mattlk13/buildstream
  • PServers/buildstream
  • phamnghia610909/buildstream
  • chiaratolentino/buildstream
  • eysz7-x-x/buildstream
  • kerrick1/buildstream
  • matthew-yates/buildstream
  • twofeathers/buildstream
  • mhadjimichael/buildstream
  • pointswaves/buildstream
  • Mr.JackWilson/buildstream
  • Tw3akG33k/buildstream
  • AlexFazakas/buildstream
  • eruidfkiy/buildstream
  • clamotion2/buildstream
  • nanonyme/buildstream
  • wickyjaaa/buildstream
  • nmanchev/buildstream
  • bojorquez.ja/buildstream
  • mostynb/buildstream
  • highpit74/buildstream
  • Demo112/buildstream
  • ba2014sheer/buildstream
  • tonimadrino/buildstream
  • usuario2o/buildstream
  • Angelika123456/buildstream
  • neo355/buildstream
  • corentin-ferlay/buildstream
  • coldtom/buildstream
  • wifitvbox81/buildstream
  • 358253885/buildstream
  • seanborg/buildstream
  • SotK/buildstream
  • DouglasWinship/buildstream
  • karansthr97/buildstream
  • louib/buildstream
  • bwh-ct/buildstream
  • robjh/buildstream
  • we88c0de/buildstream
  • zhengxian5555/buildstream
51 results
Show changes
Commits on Source (6)
Hide whitespace changes
Inline Side-by-side
Showing
with 437 additions and 362 deletions
.gitignore
View file @ 5e10e2e8
......@@ -2,6 +2,9 @@
buildstream/**/*.pyc
tests/**/*.pyc
# Build output directory
build
# Setuptools distribution folder.
/dist/
......
buildstream/_artifactcache.py
View file @ 5e10e2e8
......@@ -22,12 +22,12 @@ import os
from collections.abc import Mapping
from .types import _KeyStrength
from ._exceptions import ArtifactError, CASError, LoadError, LoadErrorReason
from ._exceptions import ArtifactError, CASError
from ._message import Message, MessageType
from . import utils
from . import _yaml
from ._cas import CASRemote, CASRemoteSpec
from ._cas import CASRemote, CASRemoteSpec, CASCacheUsage
from .storage._casbaseddirectory import CasBasedDirectory
......@@ -46,39 +46,6 @@ class ArtifactCacheSpec(CASRemoteSpec):
pass
# ArtifactCacheUsage
#
# A simple object to report the current artifact cache
# usage details.
#
# Note that this uses the user configured cache quota
# rather than the internal quota with protective headroom
# removed, to provide a more sensible value to display to
# the user.
#
# Args:
# artifacts (ArtifactCache): The artifact cache to get the status of
#
class ArtifactCacheUsage():
def __init__(self, artifacts):
context = artifacts.context
self.quota_config = context.config_cache_quota # Configured quota
self.quota_size = artifacts._cache_quota_original # Resolved cache quota in bytes
self.used_size = artifacts.get_cache_size() # Size used by artifacts in bytes
self.used_percent = 0 # Percentage of the quota used
if self.quota_size is not None:
self.used_percent = int(self.used_size * 100 / self.quota_size)
# Formattable into a human readable string
#
def __str__(self):
return "{} / {} ({}%)" \
.format(utils._pretty_size(self.used_size, dec_places=1),
self.quota_config,
self.used_percent)
# An ArtifactCache manages artifacts.
#
# Args:
......@@ -87,19 +54,17 @@ class ArtifactCacheUsage():
class ArtifactCache():
def __init__(self, context):
self.context = context
self.extractdir = os.path.join(context.artifactdir, 'extract')
self.extractdir = context.extractdir
self.cas = context.get_cascache()
self.casquota = context.get_casquota()
self.casquota._calculate_cache_quota()
self.global_remote_specs = []
self.project_remote_specs = {}
self._required_elements = set() # The elements required for this session
self._cache_size = None # The current cache size, sometimes it's an estimate
self._cache_quota = None # The cache quota
self._cache_quota_original = None # The cache quota as specified by the user, in bytes
self._cache_quota_headroom = None # The headroom in bytes before reaching the quota or full disk
self._cache_lower_threshold = None # The target cache size for a cleanup
self._remotes_setup = False # Check to prevent double-setup of remotes
# Per-project list of _CASRemote instances.
......@@ -110,8 +75,6 @@ class ArtifactCache():
os.makedirs(self.extractdir, exist_ok=True)
self._calculate_cache_quota()
# setup_remotes():
#
# Sets up which remotes to use
......@@ -235,7 +198,7 @@ class ArtifactCache():
space_saved = 0
# Start off with an announcement with as much info as possible
volume_size, volume_avail = self._get_cache_volume_size()
volume_size, volume_avail = self.casquota._get_cache_volume_size()
self._message(MessageType.STATUS, "Starting cache cleanup",
detail=("Elements required by the current build plan: {}\n" +
"User specified quota: {} ({})\n" +
......@@ -243,8 +206,8 @@ class ArtifactCache():
"Cache volume: {} total, {} available")
.format(len(self._required_elements),
context.config_cache_quota,
utils._pretty_size(self._cache_quota_original, dec_places=2),
utils._pretty_size(self.get_cache_size(), dec_places=2),
utils._pretty_size(self.casquota._cache_quota, dec_places=2),
utils._pretty_size(self.casquota.get_cache_size(), dec_places=2),
utils._pretty_size(volume_size, dec_places=2),
utils._pretty_size(volume_avail, dec_places=2)))
......@@ -261,9 +224,11 @@ class ArtifactCache():
])
# Do a real computation of the cache size once, just in case
self.compute_cache_size()
self.casquota.compute_cache_size()
usage = CASCacheUsage(self.casquota)
self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
while self.get_cache_size() >= self._cache_lower_threshold:
while self.casquota.get_cache_size() >= self.casquota._cache_lower_threshold:
try:
to_remove = artifacts.pop(0)
except IndexError:
......@@ -280,7 +245,7 @@ class ArtifactCache():
"Please increase the cache-quota in {} and/or make more disk space."
.format(removed_ref_count,
utils._pretty_size(space_saved, dec_places=2),
utils._pretty_size(self.get_cache_size(), dec_places=2),
utils._pretty_size(self.casquota.get_cache_size(), dec_places=2),
len(self._required_elements),
(context.config_origin or default_conf)))
......@@ -306,7 +271,7 @@ class ArtifactCache():
to_remove))
# Remove the size from the removed size
self.set_cache_size(self._cache_size - size)
self.casquota.set_cache_size(self.casquota._cache_size - size)
# User callback
#
......@@ -322,29 +287,12 @@ class ArtifactCache():
"Cache usage is now: {}")
.format(removed_ref_count,
utils._pretty_size(space_saved, dec_places=2),
utils._pretty_size(self.get_cache_size(), dec_places=2)))
return self.get_cache_size()
# compute_cache_size()
#
# Computes the real artifact cache size by calling
# the abstract calculate_cache_size() method.
#
# Returns:
# (int): The size of the artifact cache.
#
def compute_cache_size(self):
old_cache_size = self._cache_size
new_cache_size = self.cas.calculate_cache_size()
if old_cache_size != new_cache_size:
self._cache_size = new_cache_size
utils._pretty_size(self.casquota.get_cache_size(), dec_places=2)))
usage = ArtifactCacheUsage(self)
self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
return self.casquota.get_cache_size()
return self._cache_size
def full(self):
return self.casquota.full()
# add_artifact_size()
#
......@@ -355,71 +303,10 @@ class ArtifactCache():
# artifact_size (int): The size to add.
#
def add_artifact_size(self, artifact_size):
cache_size = self.get_cache_size()
cache_size = self.casquota.get_cache_size()
cache_size += artifact_size
self.set_cache_size(cache_size)
# get_cache_size()
#
# Fetches the cached size of the cache, this is sometimes
# an estimate and periodically adjusted to the real size
# when a cache size calculation job runs.
#
# When it is an estimate, the value is either correct, or
# it is greater than the actual cache size.
#
# Returns:
# (int) An approximation of the artifact cache size, in bytes.
#
def get_cache_size(self):
# If we don't currently have an estimate, figure out the real cache size.
if self._cache_size is None:
stored_size = self._read_cache_size()
if stored_size is not None:
self._cache_size = stored_size
else:
self.compute_cache_size()
return self._cache_size
# set_cache_size()
#
# Forcefully set the overall cache size.
#
# This is used to update the size in the main process after
# having calculated in a cleanup or a cache size calculation job.
#
# Args:
# cache_size (int): The size to set.
#
def set_cache_size(self, cache_size):
assert cache_size is not None
self._cache_size = cache_size
self._write_cache_size(self._cache_size)
# full()
#
# Checks if the artifact cache is full, either
# because the user configured quota has been exceeded
# or because the underlying disk is almost full.
#
# Returns:
# (bool): True if the artifact cache is full
#
def full(self):
if self.get_cache_size() > self._cache_quota:
return True
_, volume_avail = self._get_cache_volume_size()
if volume_avail < self._cache_quota_headroom:
return True
return False
self.casquota.set_cache_size(cache_size)
# preflight():
#
......@@ -885,142 +772,6 @@ class ArtifactCache():
with self.context.timed_activity("Initializing remote caches", silent_nested=True):
self.initialize_remotes(on_failure=remote_failed)
# _write_cache_size()
#
# Writes the given size of the artifact to the cache's size file
#
# Args:
# size (int): The size of the artifact cache to record
#
def _write_cache_size(self, size):
assert isinstance(size, int)
size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
with utils.save_file_atomic(size_file_path, "w") as f:
f.write(str(size))
# _read_cache_size()
#
# Reads and returns the size of the artifact cache that's stored in the
# cache's size file
#
# Returns:
# (int): The size of the artifact cache, as recorded in the file
#
def _read_cache_size(self):
size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
if not os.path.exists(size_file_path):
return None
with open(size_file_path, "r") as f:
size = f.read()
try:
num_size = int(size)
except ValueError as e:
raise ArtifactError("Size '{}' parsed from '{}' was not an integer".format(
size, size_file_path)) from e
return num_size
# _calculate_cache_quota()
#
# Calculates and sets the cache quota and lower threshold based on the
# quota set in Context.
# It checks that the quota is both a valid expression, and that there is
# enough disk space to satisfy that quota
#
def _calculate_cache_quota(self):
# Headroom intended to give BuildStream a bit of leeway.
# This acts as the minimum size of cache_quota and also
# is taken from the user requested cache_quota.
#
if 'BST_TEST_SUITE' in os.environ:
self._cache_quota_headroom = 0
else:
self._cache_quota_headroom = 2e9
try:
cache_quota = utils._parse_size(self.context.config_cache_quota,
self.context.artifactdir)
except utils.UtilError as e:
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}\nPlease specify the value in bytes or as a % of full disk space.\n"
"\nValid values are, for example: 800M 10G 1T 50%\n"
.format(str(e))) from e
total_size, available_space = self._get_cache_volume_size()
cache_size = self.get_cache_size()
# Ensure system has enough storage for the cache_quota
#
# If cache_quota is none, set it to the maximum it could possibly be.
#
# Also check that cache_quota is at least as large as our headroom.
#
if cache_quota is None: # Infinity, set to max system storage
cache_quota = cache_size + available_space
if cache_quota < self._cache_quota_headroom: # Check minimum
raise LoadError(LoadErrorReason.INVALID_DATA,
"Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
"BuildStream requires a minimum cache quota of 2G.")
elif cache_quota > total_size:
# A quota greater than the total disk size is certianly an error
raise ArtifactError("Your system does not have enough available " +
"space to support the cache quota specified.",
detail=("You have specified a quota of {quota} total disk space.\n" +
"The filesystem containing {local_cache_path} only " +
"has {total_size} total disk space.")
.format(
quota=self.context.config_cache_quota,
local_cache_path=self.context.artifactdir,
total_size=utils._pretty_size(total_size)),
reason='insufficient-storage-for-quota')
elif cache_quota > cache_size + available_space:
# The quota does not fit in the available space, this is a warning
if '%' in self.context.config_cache_quota:
available = (available_space / total_size) * 100
available = '{}% of total disk space'.format(round(available, 1))
else:
available = utils._pretty_size(available_space)
self._message(MessageType.WARN,
"Your system does not have enough available " +
"space to support the cache quota specified.",
detail=("You have specified a quota of {quota} total disk space.\n" +
"The filesystem containing {local_cache_path} only " +
"has {available_size} available.")
.format(quota=self.context.config_cache_quota,
local_cache_path=self.context.artifactdir,
available_size=available))
# Place a slight headroom (2e9 (2GB) on the cache_quota) into
# cache_quota to try and avoid exceptions.
#
# Of course, we might still end up running out during a build
# if we end up writing more than 2G, but hey, this stuff is
# already really fuzzy.
#
self._cache_quota_original = cache_quota
self._cache_quota = cache_quota - self._cache_quota_headroom
self._cache_lower_threshold = self._cache_quota / 2
# _get_cache_volume_size()
#
# Get the available space and total space for the volume on
# which the artifact cache is located.
#
# Returns:
# (int): The total number of bytes on the volume
# (int): The number of available bytes on the volume
#
# NOTE: We use this stub to allow the test cases
# to override what an artifact cache thinks
# about it's disk size and available bytes.
#
def _get_cache_volume_size(self):
return utils._get_volume_size(self.context.artifactdir)
# _configured_remote_artifact_cache_specs():
#
......
buildstream/_cas/__init__.py
View file @ 5e10e2e8
......@@ -17,5 +17,5 @@
# Authors:
# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
from .cascache import CASCache
from .cascache import CASCache, CASQuota, CASCacheUsage
from .casremote import CASRemote, CASRemoteSpec
buildstream/_cas/cascache.py
View file @ 5e10e2e8
......@@ -32,17 +32,53 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from .._protos.buildstream.v2 import buildstream_pb2
from .. import utils
from .._exceptions import CASCacheError
from .._exceptions import CASCacheError, LoadError, LoadErrorReason
from .._message import Message, MessageType
from .casremote import BlobNotFound, _CASBatchRead, _CASBatchUpdate
_BUFFER_SIZE = 65536
CACHE_SIZE_FILE = "cache_size"
# CASCacheUsage
#
# A simple object to report the current CAS cache usage details.
#
# Note that this uses the user configured cache quota
# rather than the internal quota with protective headroom
# removed, to provide a more sensible value to display to
# the user.
#
# Args:
# cas (CASQuota): The CAS cache to get the status of
#
class CASCacheUsage():
def __init__(self, casquota):
self.quota_config = casquota._config_cache_quota # Configured quota
self.quota_size = casquota._cache_quota_original # Resolved cache quota in bytes
self.used_size = casquota.get_cache_size() # Size used by artifacts in bytes
self.used_percent = 0 # Percentage of the quota used
if self.quota_size is not None:
self.used_percent = int(self.used_size * 100 / self.quota_size)
# Formattable into a human readable string
#
def __str__(self):
return "{} / {} ({}%)" \
.format(utils._pretty_size(self.used_size, dec_places=1),
self.quota_config,
self.used_percent)
# A CASCache manages a CAS repository as specified in the Remote Execution API.
#
# Args:
# path (str): The root directory for the CAS repository
# cache_quota (int): User configured cache quota
#
class CASCache():
......@@ -459,16 +495,6 @@ class CASCache():
except FileNotFoundError as e:
raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
# calculate_cache_size()
#
# Return the real disk usage of the CAS cache.
#
# Returns:
# (int): The size of the cache.
#
def calculate_cache_size(self):
return utils._get_dir_size(self.casdir)
# list_refs():
#
# List refs in Least Recently Modified (LRM) order.
......@@ -1043,6 +1069,248 @@ class CASCache():
batch.send()
class CASQuota:
def __init__(self, context):
self.cas = context.get_cascache()
self.casdir = self.cas.casdir
self._config_cache_quota = context.config_cache_quota
self._config_cache_quota_string = context.config_cache_quota_string
self._cache_size = None # The current cache size, sometimes it's an estimate
self._cache_quota = None # The cache quota
self._cache_quota_original = None # The cache quota as specified by the user, in bytes
self._cache_quota_headroom = None # The headroom in bytes before reaching the quota or full disk
self._cache_lower_threshold = None # The target cache size for a cleanup
self.available_space = None
self._message = context.message
self._calculate_cache_quota()
# compute_cache_size()
#
# Computes the real artifact cache size by calling
# the abstract calculate_cache_size() method.
#
# Returns:
# (int): The size of the artifact cache.
#
def compute_cache_size(self):
old_cache_size = self._cache_size
new_cache_size = self.calculate_cache_size()
if old_cache_size != new_cache_size:
self._cache_size = new_cache_size
return self._cache_size
# calculate_cache_size()
#
# Return the real disk usage of the CAS cache.
#
# Returns:
# (int): The size of the cache.
#
def calculate_cache_size(self):
return utils._get_dir_size(self.casdir)
# get_cache_size()
#
# Fetches the cached size of the cache, this is sometimes
# an estimate and periodically adjusted to the real size
# when a cache size calculation job runs.
#
# When it is an estimate, the value is either correct, or
# it is greater than the actual cache size.
#
# Returns:
# (int) An approximation of the artifact cache size, in bytes.
#
def get_cache_size(self):
# If we don't currently have an estimate, figure out the real cache size.
if self._cache_size is None:
stored_size = self._read_cache_size()
if stored_size is not None:
self._cache_size = stored_size
else:
self._cache_size = self.compute_cache_size()
return self._cache_size
# set_cache_size()
#
# Forcefully set the overall cache size.
#
# This is used to update the size in the main process after
# having calculated in a cleanup or a cache size calculation job.
#
# Args:
# cache_size (int): The size to set.
#
def set_cache_size(self, cache_size):
assert cache_size is not None
self._cache_size = cache_size
self._write_cache_size(self._cache_size)
# full()
#
# Checks if the artifact cache is full, either
# because the user configured quota has been exceeded
# or because the underlying disk is almost full.
#
# Returns:
# (bool): True if the artifact cache is full
#
def full(self):
if self.get_cache_size() > self._cache_quota:
return True
_, volume_avail = self._get_cache_volume_size()
if volume_avail < self._cache_quota_headroom:
return True
return False
################################################
# Local Private Methods #
################################################
# _read_cache_size()
#
# Reads and returns the size of the artifact cache that's stored in the
# cache's size file
#
# Returns:
# (int): The size of the artifact cache, as recorded in the file
#
def _read_cache_size(self):
size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
if not os.path.exists(size_file_path):
return None
with open(size_file_path, "r") as f:
size = f.read()
try:
num_size = int(size)
except ValueError as e:
raise CASCacheError("Size '{}' parsed from '{}' was not an integer".format(
size, size_file_path)) from e
return num_size
# _write_cache_size()
#
# Writes the given size of the artifact to the cache's size file
#
# Args:
# size (int): The size of the artifact cache to record
#
def _write_cache_size(self, size):
assert isinstance(size, int)
size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
with utils.save_file_atomic(size_file_path, "w") as f:
f.write(str(size))
# _get_cache_volume_size()
#
# Get the available space and total space for the volume on
# which the artifact cache is located.
#
# Returns:
# (int): The total number of bytes on the volume
# (int): The number of available bytes on the volume
#
# NOTE: We use this stub to allow the test cases
# to override what an artifact cache thinks
# about it's disk size and available bytes.
#
def _get_cache_volume_size(self):
return utils._get_volume_size(self.casdir)
# _calculate_cache_quota()
#
# Calculates and sets the cache quota and lower threshold based on the
# quota set in Context.
# It checks that the quota is both a valid expression, and that there is
# enough disk space to satisfy that quota
#
def _calculate_cache_quota(self):
# Headroom intended to give BuildStream a bit of leeway.
# This acts as the minimum size of cache_quota and also
# is taken from the user requested cache_quota.
#
if 'BST_TEST_SUITE' in os.environ:
self._cache_quota_headroom = 0
else:
self._cache_quota_headroom = 2e9
total_size, available_space = self._get_cache_volume_size()
cache_size = self.get_cache_size()
self.available_space = available_space
# Ensure system has enough storage for the cache_quota
#
# If cache_quota is none, set it to the maximum it could possibly be.
#
# Also check that cache_quota is at least as large as our headroom.
#
cache_quota = self._config_cache_quota
if cache_quota is None: # Infinity, set to max system storage
cache_quota = cache_size + available_space
if cache_quota < self._cache_quota_headroom: # Check minimum
raise LoadError(LoadErrorReason.INVALID_DATA,
"Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
"BuildStream requires a minimum cache quota of 2G.")
elif cache_quota > total_size:
# A quota greater than the total disk size is certianly an error
raise CASCacheError("Your system does not have enough available " +
"space to support the cache quota specified.",
detail=("You have specified a quota of {quota} total disk space.\n" +
"The filesystem containing {local_cache_path} only " +
"has {total_size} total disk space.")
.format(
quota=self._config_cache_quota,
local_cache_path=self.casdir,
total_size=utils._pretty_size(total_size)),
reason='insufficient-storage-for-quota')
elif cache_quota > cache_size + available_space:
# The quota does not fit in the available space, this is a warning
if '%' in self._config_cache_quota_string:
available = (available_space / total_size) * 100
available = '{}% of total disk space'.format(round(available, 1))
else:
available = utils._pretty_size(available_space)
self._message(Message(
None,
MessageType.WARN,
"Your system does not have enough available " +
"space to support the cache quota specified.",
detail=("You have specified a quota of {quota} total disk space.\n" +
"The filesystem containing {local_cache_path} only " +
"has {available_size} available.")
.format(quota=self._config_cache_quota,
local_cache_path=self.casdir,
available_size=available)))
# Place a slight headroom (2e9 (2GB) on the cache_quota) into
# cache_quota to try and avoid exceptions.
#
# Of course, we might still end up running out during a build
# if we end up writing more than 2G, but hey, this stuff is
# already really fuzzy.
#
self._cache_quota_original = cache_quota
self._cache_quota = cache_quota - self._cache_quota_headroom
self._cache_lower_threshold = self._cache_quota / 2
def _grouper(iterable, n):
while True:
try:
......
buildstream/_context.py
View file @ 5e10e2e8
......@@ -30,8 +30,8 @@ from . import _yaml
from ._exceptions import LoadError, LoadErrorReason, BstError
from ._message import Message, MessageType
from ._profile import Topics, profile_start, profile_end
from ._artifactcache import ArtifactCache, ArtifactCacheUsage
from ._cas import CASCache
from ._artifactcache import ArtifactCache
from ._cas import CASCache, CASQuota, CASCacheUsage
from ._workspaces import Workspaces, WorkspaceProjectCache
from .plugin import _plugin_lookup
from .sandbox import SandboxRemote
......@@ -58,18 +58,27 @@ class Context():
# Filename indicating which configuration file was used, or None for the defaults
self.config_origin = None
# The directory under which other directories are based
self.cachedir = None
# The directory where various sources are stored
self.sourcedir = None
# The directory where build sandboxes will be created
self.builddir = None
# The directory for CAS
self.casdir = None
# Extract directory
self.extractdir = None
# The directory for temporary files
self.tmpdir = None
# Default root location for workspaces
self.workspacedir = None
# The local binary artifact cache directory
self.artifactdir = None
# The locations from which to push and pull prebuilt artifacts
self.artifact_cache_specs = None
......@@ -118,6 +127,9 @@ class Context():
# Size of the artifact cache in bytes
self.config_cache_quota = None
# User specified cache quota, used for display messages
self.config_cache_quota_string = None
# Whether or not to attempt to pull build trees globally
self.pull_buildtrees = None
......@@ -142,6 +154,7 @@ class Context():
self._log_handle = None
self._log_filename = None
self._cascache = None
self._casquota = None
self._directory = directory
# load()
......@@ -179,13 +192,22 @@ class Context():
user_config = _yaml.load(config)
_yaml.composite(defaults, user_config)
# Give obsoletion warnings
if defaults.get('builddir'):
raise LoadError(LoadErrorReason.INVALID_DATA,
"builddir is obsolete, use cachedir")
if defaults.get('artifactdir'):
raise LoadError(LoadErrorReason.INVALID_DATA,
"artifactdir is obsolete")
_yaml.node_validate(defaults, [
'sourcedir', 'builddir', 'artifactdir', 'logdir',
'scheduler', 'artifacts', 'logging', 'projects',
'cache', 'prompt', 'workspacedir', 'remote-execution'
'cachedir', 'sourcedir', 'builddir', 'logdir', 'scheduler',
'artifacts', 'logging', 'projects', 'cache', 'prompt',
'workspacedir', 'remote-execution',
])
for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
for directory in ['cachedir', 'sourcedir', 'logdir', 'workspacedir']:
# Allow the ~ tilde expansion and any environment variables in
# path specification in the config files.
#
......@@ -195,14 +217,34 @@ class Context():
path = os.path.normpath(path)
setattr(self, directory, path)
# add directories not set by users
self.extractdir = os.path.join(self.cachedir, 'extract')
self.tmpdir = os.path.join(self.cachedir, 'tmp')
self.casdir = os.path.join(self.cachedir, 'cas')
self.builddir = os.path.join(self.cachedir, 'build')
# Move old artifact cas to cas if it exists and create symlink
old_casdir = os.path.join(self.cachedir, 'artifacts', 'cas')
if (os.path.exists(old_casdir) and not os.path.islink(old_casdir) and
not os.path.exists(self.casdir)):
os.rename(old_casdir, self.casdir)
os.symlink(self.casdir, old_casdir)
# Load quota configuration
# We need to find the first existing directory in the path of
# our artifactdir - the artifactdir may not have been created
# yet.
# We need to find the first existing directory in the path of our
# cachedir - the cachedir may not have been created yet.
cache = _yaml.node_get(defaults, Mapping, 'cache')
_yaml.node_validate(cache, ['quota', 'pull-buildtrees', 'cache-buildtrees'])
self.config_cache_quota = _yaml.node_get(cache, str, 'quota')
self.config_cache_quota_string = _yaml.node_get(cache, str, 'quota')
try:
self.config_cache_quota = utils._parse_size(self.config_cache_quota_string,
self.casdir)
except utils.UtilError as e:
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}\nPlease specify the value in bytes or as a % of full disk space.\n"
"\nValid values are, for example: 800M 10G 1T 50%\n"
.format(str(e))) from e
# Load artifact share configuration
self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
......@@ -262,15 +304,15 @@ class Context():
return self._artifactcache
# get_artifact_cache_usage()
# get_cache_usage()
#
# Fetches the current usage of the artifact cache
#
# Returns:
# (ArtifactCacheUsage): The current status
# (CASCacheUsage): The current status
#
def get_artifact_cache_usage(self):
return ArtifactCacheUsage(self.artifactcache)
def get_cache_usage(self):
return CASCacheUsage(self.get_casquota())
# add_project():
#
......@@ -640,9 +682,14 @@ class Context():
def get_cascache(self):
if self._cascache is None:
self._cascache = CASCache(self.artifactdir)
self._cascache = CASCache(self.cachedir)
return self._cascache
def get_casquota(self):
if self._casquota is None:
self._casquota = CASQuota(self)
return self._casquota
# _node_get_option_str()
#
......
buildstream/_frontend/status.py
View file @ 5e10e2e8
......@@ -404,7 +404,7 @@ class _StatusHeader():
#
# ~~~~~~ cache: 69% ~~~~~~
#
usage = self._context.get_artifact_cache_usage()
usage = self._context.get_cache_usage()
usage_percent = '{}%'.format(usage.used_percent)
size = 21
......
buildstream/_frontend/widget.py
View file @ 5e10e2e8
......@@ -486,7 +486,7 @@ class LogLine(Widget):
values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S')
values["Project"] = "{} ({})".format(project.name, project.directory)
values["Targets"] = ", ".join([t.name for t in stream.targets])
values["Cache Usage"] = "{}".format(context.get_artifact_cache_usage())
values["Cache Usage"] = "{}".format(context.get_cache_usage())
text += self._format_values(values)
# User configurations
......@@ -495,10 +495,10 @@ class LogLine(Widget):
values = OrderedDict()
values["Configuration File"] = \
"Default Configuration" if not context.config_origin else context.config_origin
values["Cache Directory"] = context.cachedir
values["Log Files"] = context.logdir
values["Source Mirrors"] = context.sourcedir
values["Build Area"] = context.builddir
values["Artifact Cache"] = context.artifactdir
values["Strict Build Plan"] = "Yes" if context.get_strict() else "No"
values["Maximum Fetch Tasks"] = context.sched_fetchers
values["Maximum Build Tasks"] = context.sched_builders
......
buildstream/_scheduler/jobs/cachesizejob.py
View file @ 5e10e2e8
......@@ -25,14 +25,14 @@ class CacheSizeJob(Job):
self._complete_cb = complete_cb
context = self._scheduler.context
self._artifacts = context.artifactcache
self._casquota = context.get_casquota()
def child_process(self):
return self._artifacts.compute_cache_size()
return self._casquota.compute_cache_size()
def parent_complete(self, status, result):
if status == JobStatus.OK:
self._artifacts.set_cache_size(result)
self._casquota.set_cache_size(result)
if self._complete_cb:
self._complete_cb(status, result)
......
buildstream/_scheduler/jobs/cleanupjob.py
View file @ 5e10e2e8
......@@ -25,27 +25,27 @@ class CleanupJob(Job):
self._complete_cb = complete_cb
context = self._scheduler.context
self._casquota = context.get_casquota()
self._artifacts = context.artifactcache
def child_process(self):
def progress():
self.send_message('update-cache-size',
self._artifacts.get_cache_size())
self._casquota.get_cache_size())
return self._artifacts.clean(progress)
def handle_message(self, message_type, message):
# Update the cache size in the main process as we go,
# this provides better feedback in the UI.
if message_type == 'update-cache-size':
self._artifacts.set_cache_size(message)
self._casquota.set_cache_size(message)
return True
return False
def parent_complete(self, status, result):
if status == JobStatus.OK:
self._artifacts.set_cache_size(result)
self._casquota.set_cache_size(result)
if self._complete_cb:
self._complete_cb(status, result)
buildstream/data/userconfig.yaml
View file @ 5e10e2e8
......@@ -13,11 +13,8 @@
# Location to store sources
sourcedir: ${XDG_CACHE_HOME}/buildstream/sources
# Location to perform builds
builddir: ${XDG_CACHE_HOME}/buildstream/build
# Location to store local binary artifacts
artifactdir: ${XDG_CACHE_HOME}/buildstream/artifacts
# Root location for other directories in the cache
cachedir: ${XDG_CACHE_HOME}/buildstream
# Location to store build logs
logdir: ${XDG_CACHE_HOME}/buildstream/logs
......
buildstream/element.py
View file @ 5e10e2e8
......@@ -1448,7 +1448,7 @@ class Element(Plugin):
# It's advantageous to have this temporary directory on
# the same file system as the rest of our cache.
with self.timed_activity("Staging sources", silent_nested=True), \
utils._tempdir(dir=context.artifactdir, prefix='staging-temp') as temp_staging_directory:
utils._tempdir(dir=context.tmpdir, prefix='staging-temp') as temp_staging_directory:
import_dir = temp_staging_directory
......
buildstream/plugintestutils/runcli.py
View file @ 5e10e2e8
......@@ -277,10 +277,10 @@ class Cli():
*, cache_dir=None):
# Read configuration to figure out where artifacts are stored
if not cache_dir:
default = os.path.join(project, 'cache', 'artifacts')
default = os.path.join(project, 'cache')
if self.config is not None:
cache_dir = self.config.get('artifactdir', default)
cache_dir = self.config.get('cachedir', default)
else:
cache_dir = default
......@@ -582,11 +582,21 @@ def cli_integration(tmpdir, integration_cache):
# We want to cache sources for integration tests more permanently,
# to avoid downloading the huge base-sdk repeatedly
fixture.configure({
'cachedir': integration_cache.cachedir,
'sourcedir': integration_cache.sources,
'artifactdir': integration_cache.artifacts
})
return fixture
yield fixture
# remove following folders if necessary
try:
shutil.rmtree(os.path.join(integration_cache.cachedir, 'build'))
except FileNotFoundError:
pass
try:
shutil.rmtree(os.path.join(integration_cache.cachedir, 'tmp'))
except FileNotFoundError:
pass
@contextmanager
......@@ -626,10 +636,8 @@ def configured(directory, config=None):
if not config.get('sourcedir', False):
config['sourcedir'] = os.path.join(directory, 'sources')
if not config.get('builddir', False):
config['builddir'] = os.path.join(directory, 'build')
if not config.get('artifactdir', False):
config['artifactdir'] = os.path.join(directory, 'artifacts')
if not config.get('cachedir', False):
config['cachedir'] = directory
if not config.get('logdir', False):
config['logdir'] = os.path.join(directory, 'logs')
......
conftest.py
View file @ 5e10e2e8
......@@ -53,16 +53,16 @@ def pytest_runtest_setup(item):
class IntegrationCache():
def __init__(self, cache):
cache = os.path.abspath(cache)
self.root = os.path.abspath(cache)
os.makedirs(cache, exist_ok=True)
# Use the same sources every time
self.sources = os.path.join(cache, 'sources')
self.sources = os.path.join(self.root, 'sources')
# Create a temp directory for the duration of the test for
# the artifacts directory
try:
self.artifacts = tempfile.mkdtemp(dir=cache, prefix='artifacts-')
self.cachedir = tempfile.mkdtemp(dir=self.root, prefix='cache-')
except OSError as e:
raise AssertionError("Unable to create test directory !") from e
......@@ -84,7 +84,11 @@ def integration_cache(request):
# Clean up the artifacts after each test run - we only want to
# cache sources between runs
try:
shutil.rmtree(cache.artifacts)
shutil.rmtree(cache.cachedir)
except FileNotFoundError:
pass
try:
shutil.rmtree(os.path.join(cache.root, 'cas'))
except FileNotFoundError:
pass
......
doc/bst2html.py
View file @ 5e10e2e8
......@@ -194,10 +194,9 @@ def workdir(source_cache=None):
bst_config_file = os.path.join(tempdir, 'buildstream.conf')
config = {
'cachedir': tempdir,
'sourcedir': source_cache,
'artifactdir': os.path.join(tempdir, 'artifacts'),
'logdir': os.path.join(tempdir, 'logs'),
'builddir': os.path.join(tempdir, 'build'),
}
_yaml.dump(config, bst_config_file)
......@@ -411,12 +410,10 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
# Encode and save the output if that was asked for
output = _yaml.node_get(command, str, 'output', default_value=None)
if output is not None:
# Convert / Generate a nice <div>
converted = generate_html(command_out, directory, config_file,
source_cache, tempdir, palette,
command_str, command_fake_output is not None)
# Save it
filename = os.path.join(desc_dir, output)
filename = os.path.realpath(filename)
......
doc/sessions/running-commands.run
View file @ 5e10e2e8
......@@ -2,7 +2,7 @@
commands:
# Make it fetch first
- directory: ../examples/running-commands
command: fetch hello.bst
command: source fetch hello.bst
# Capture a show output
- directory: ../examples/running-commands
......
setup.cfg
View file @ 5e10e2e8
......@@ -20,4 +20,4 @@ env =
[pycodestyle]
max-line-length = 119
ignore = E129,E125,W504,W605
exclude = .git/**,.tox/**,doc/source/conf.py,buildstream/_fuse/fuse.py,buildstream/_protos/**/*py
exclude = .git/**,.tox/**,.eggs/**,build/**,doc/source/conf.py,buildstream/_fuse/fuse.py,buildstream/_protos/**/*py,tmp/**
tests/artifactcache/cache_size.py
View file @ 5e10e2e8
......@@ -50,15 +50,15 @@ def test_cache_size_write(cli, tmpdir):
create_project(project_dir)
# Artifact cache must be in a known place
artifactdir = os.path.join(project_dir, "artifacts")
cli.configure({"artifactdir": artifactdir})
casdir = os.path.join(project_dir, "cas")
cli.configure({"cachedir": project_dir})
# Build, to populate the cache
res = cli.run(project=project_dir, args=["build", "test.bst"])
res.assert_success()
# Inspect the artifact cache
sizefile = os.path.join(artifactdir, CACHE_SIZE_FILE)
sizefile = os.path.join(casdir, CACHE_SIZE_FILE)
assert os.path.isfile(sizefile)
with open(sizefile, "r") as f:
size_data = f.read()
......@@ -81,11 +81,11 @@ def test_quota_over_1024T(cli, tmpdir):
_yaml.dump({'name': 'main'}, str(project.join("project.conf")))
volume_space_patch = mock.patch(
"buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
"buildstream._cas.CASQuota._get_cache_volume_size",
autospec=True,
return_value=(1025 * TiB, 1025 * TiB)
)
with volume_space_patch:
result = cli.run(project, args=["build", "file.bst"])
result.assert_main_error(ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota')
result.assert_main_error(ErrorDomain.CAS, 'insufficient-storage-for-quota')
tests/artifactcache/expiry.py
View file @ 5e10e2e8
......@@ -341,7 +341,7 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
# Not enough space on disk even if you cleaned up
("11K", ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota'),
("11K", ErrorDomain.CAS, 'insufficient-storage-for-quota'),
# Not enough space for these caches
("7K", 'warning', 'Your system does not have enough available'),
......@@ -355,7 +355,7 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
cli.configure({
'cache': {
'quota': quota,
}
},
})
# We patch how we get space information
......@@ -373,13 +373,13 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
total_space = 10000
volume_space_patch = mock.patch(
"buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
"buildstream.utils._get_volume_size",
autospec=True,
return_value=(total_space, free_space),
)
cache_size_patch = mock.patch(
"buildstream._artifactcache.ArtifactCache.get_cache_size",
"buildstream._cas.CASQuota.get_cache_size",
autospec=True,
return_value=0,
)
......@@ -417,7 +417,7 @@ def test_extract_expiry(cli, datafiles, tmpdir):
res.assert_success()
# Get a snapshot of the extracts in advance
extractdir = os.path.join(project, 'cache', 'artifacts', 'extract', 'test', 'target')
extractdir = os.path.join(project, 'cache', 'extract', 'test', 'target')
extracts = os.listdir(extractdir)
assert(len(extracts) == 1)
extract = os.path.join(extractdir, extracts[0])
......@@ -436,7 +436,7 @@ def test_extract_expiry(cli, datafiles, tmpdir):
# Now we should have a directory for the cached target2.bst, which
# replaced target.bst in the cache, we should not have a directory
# for the target.bst
refsdir = os.path.join(project, 'cache', 'artifacts', 'cas', 'refs', 'heads')
refsdir = os.path.join(project, 'cache', 'cas', 'refs', 'heads')
refsdirtest = os.path.join(refsdir, 'test')
refsdirtarget = os.path.join(refsdirtest, 'target')
refsdirtarget2 = os.path.join(refsdirtest, 'target2')
......
tests/artifactcache/junctions.py
View file @ 5e10e2e8
......@@ -70,8 +70,8 @@ def test_push_pull(cli, tmpdir, datafiles):
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
artifacts = os.path.join(cli.directory, 'artifacts')
shutil.rmtree(artifacts)
cas = os.path.join(cli.directory, 'cas')
shutil.rmtree(cas)
# Assert that nothing is cached locally anymore
state = cli.get_element_state(project, 'target.bst')
......
tests/artifactcache/pull.py
View file @ 5e10e2e8
......@@ -57,7 +57,7 @@ def test_pull(cli, tmpdir, datafiles):
# Set up an artifact cache.
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
# Configure artifact share
artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
cache_dir = os.path.join(str(tmpdir), 'cache')
user_config_file = str(tmpdir.join('buildstream.conf'))
user_config = {
'scheduler': {
......@@ -66,7 +66,8 @@ def test_pull(cli, tmpdir, datafiles):
'artifacts': {
'url': share.repo,
'push': True,
}
},
'cachedir': cache_dir
}
# Write down the user configuration file
......@@ -93,7 +94,6 @@ def test_pull(cli, tmpdir, datafiles):
# Fake minimal context
context = Context()
context.load(config=user_config_file)
context.artifactdir = os.path.join(str(tmpdir), 'cache', 'artifacts')
context.set_message_handler(message_handler)
# Load the project and CAS cache
......@@ -111,7 +111,7 @@ def test_pull(cli, tmpdir, datafiles):
# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
process = multiprocessing.Process(target=_queue_wrapper,
args=(_test_pull, queue, user_config_file, project_dir,
artifact_dir, 'target.bst', element_key))
cache_dir, 'target.bst', element_key))
try:
# Keep SIGINT blocked in the child process
......@@ -128,12 +128,14 @@ def test_pull(cli, tmpdir, datafiles):
assert cas.contains(element, element_key)
def _test_pull(user_config_file, project_dir, artifact_dir,
def _test_pull(user_config_file, project_dir, cache_dir,
element_name, element_key, queue):
# Fake minimal context
context = Context()
context.load(config=user_config_file)
context.artifactdir = artifact_dir
context.cachedir = cache_dir
context.casdir = os.path.join(cache_dir, 'cas')
context.tmpdir = os.path.join(cache_dir, 'tmp')
context.set_message_handler(message_handler)
# Load the project manually
......@@ -166,7 +168,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
# Set up an artifact cache.
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
# Configure artifact share
artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
rootcache_dir = os.path.join(str(tmpdir), 'cache')
user_config_file = str(tmpdir.join('buildstream.conf'))
user_config = {
'scheduler': {
......@@ -175,7 +177,8 @@ def test_pull_tree(cli, tmpdir, datafiles):
'artifacts': {
'url': share.repo,
'push': True,
}
},
'cachedir': rootcache_dir
}
# Write down the user configuration file
......@@ -196,7 +199,6 @@ def test_pull_tree(cli, tmpdir, datafiles):
# Fake minimal context
context = Context()
context.load(config=user_config_file)
context.artifactdir = os.path.join(str(tmpdir), 'cache', 'artifacts')
context.set_message_handler(message_handler)
# Load the project and CAS cache
......@@ -219,7 +221,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
process = multiprocessing.Process(target=_queue_wrapper,
args=(_test_push_tree, queue, user_config_file, project_dir,
artifact_dir, artifact_digest))
artifact_digest))
try:
# Keep SIGINT blocked in the child process
......@@ -247,7 +249,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
# Use subprocess to avoid creation of gRPC threads in main BuildStream process
process = multiprocessing.Process(target=_queue_wrapper,
args=(_test_pull_tree, queue, user_config_file, project_dir,
artifact_dir, tree_digest))
tree_digest))
try:
# Keep SIGINT blocked in the child process
......@@ -269,11 +271,10 @@ def test_pull_tree(cli, tmpdir, datafiles):
assert os.path.exists(cas.objpath(directory_digest))
def _test_push_tree(user_config_file, project_dir, artifact_dir, artifact_digest, queue):
def _test_push_tree(user_config_file, project_dir, artifact_digest, queue):
# Fake minimal context
context = Context()
context.load(config=user_config_file)
context.artifactdir = artifact_dir
context.set_message_handler(message_handler)
# Load the project manually
......@@ -305,11 +306,10 @@ def _test_push_tree(user_config_file, project_dir, artifact_dir, artifact_digest
queue.put("No remote configured")
def _test_pull_tree(user_config_file, project_dir, artifact_dir, artifact_digest, queue):
def _test_pull_tree(user_config_file, project_dir, artifact_digest, queue):
# Fake minimal context
context = Context()
context.load(config=user_config_file)
context.artifactdir = artifact_dir
context.set_message_handler(message_handler)
# Load the project manually
......