Commit 5e10e2e8 authored by Raoul Hidalgo Charman's avatar Raoul Hidalgo Charman Committed by Jürg Billeter

CASQuota: Move cache check methods into new Class

This sits in Context allowing artifact cache to check the cas quota
while not being used for CASServer.

A lot of code that checks cache quota has been touched.

Part of #870
parent 4b62b936
Pipeline #48235129 passed with stages
in 97 minutes and 33 seconds
This diff is collapsed.
......@@ -17,5 +17,5 @@
# Authors:
# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
from .cascache import CASCache
from .cascache import CASCache, CASQuota, CASCacheUsage
from .casremote import CASRemote, CASRemoteSpec
This diff is collapsed.
......@@ -30,8 +30,8 @@ from . import _yaml
from ._exceptions import LoadError, LoadErrorReason, BstError
from ._message import Message, MessageType
from ._profile import Topics, profile_start, profile_end
from ._artifactcache import ArtifactCache, ArtifactCacheUsage
from ._cas import CASCache
from ._artifactcache import ArtifactCache
from ._cas import CASCache, CASQuota, CASCacheUsage
from ._workspaces import Workspaces, WorkspaceProjectCache
from .plugin import _plugin_lookup
from .sandbox import SandboxRemote
......@@ -127,6 +127,9 @@ class Context():
# Size of the artifact cache in bytes
self.config_cache_quota = None
# User specified cache quota, used for display messages
self.config_cache_quota_string = None
# Whether or not to attempt to pull build trees globally
self.pull_buildtrees = None
......@@ -151,6 +154,7 @@ class Context():
self._log_handle = None
self._log_filename = None
self._cascache = None
self._casquota = None
self._directory = directory
# load()
......@@ -232,7 +236,15 @@ class Context():
cache = _yaml.node_get(defaults, Mapping, 'cache')
_yaml.node_validate(cache, ['quota', 'pull-buildtrees', 'cache-buildtrees'])
self.config_cache_quota = _yaml.node_get(cache, str, 'quota')
self.config_cache_quota_string = _yaml.node_get(cache, str, 'quota')
try:
self.config_cache_quota = utils._parse_size(self.config_cache_quota_string,
self.casdir)
except utils.UtilError as e:
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}\nPlease specify the value in bytes or as a % of full disk space.\n"
"\nValid values are, for example: 800M 10G 1T 50%\n"
.format(str(e))) from e
# Load artifact share configuration
self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
......@@ -292,15 +304,15 @@ class Context():
return self._artifactcache
# get_artifact_cache_usage()
# get_cache_usage()
#
# Fetches the current usage of the artifact cache
#
# Returns:
# (ArtifactCacheUsage): The current status
# (CASCacheUsage): The current status
#
def get_artifact_cache_usage(self):
return ArtifactCacheUsage(self.artifactcache)
def get_cache_usage(self):
return CASCacheUsage(self.get_casquota())
# add_project():
#
......@@ -673,6 +685,11 @@ class Context():
self._cascache = CASCache(self.cachedir)
return self._cascache
def get_casquota(self):
if self._casquota is None:
self._casquota = CASQuota(self)
return self._casquota
# _node_get_option_str()
#
......
......@@ -404,7 +404,7 @@ class _StatusHeader():
#
# ~~~~~~ cache: 69% ~~~~~~
#
usage = self._context.get_artifact_cache_usage()
usage = self._context.get_cache_usage()
usage_percent = '{}%'.format(usage.used_percent)
size = 21
......
......@@ -486,7 +486,7 @@ class LogLine(Widget):
values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S')
values["Project"] = "{} ({})".format(project.name, project.directory)
values["Targets"] = ", ".join([t.name for t in stream.targets])
values["Cache Usage"] = "{}".format(context.get_artifact_cache_usage())
values["Cache Usage"] = "{}".format(context.get_cache_usage())
text += self._format_values(values)
# User configurations
......
......@@ -25,14 +25,14 @@ class CacheSizeJob(Job):
self._complete_cb = complete_cb
context = self._scheduler.context
self._artifacts = context.artifactcache
self._casquota = context.get_casquota()
def child_process(self):
return self._artifacts.compute_cache_size()
return self._casquota.compute_cache_size()
def parent_complete(self, status, result):
if status == JobStatus.OK:
self._artifacts.set_cache_size(result)
self._casquota.set_cache_size(result)
if self._complete_cb:
self._complete_cb(status, result)
......
......@@ -25,27 +25,27 @@ class CleanupJob(Job):
self._complete_cb = complete_cb
context = self._scheduler.context
self._casquota = context.get_casquota()
self._artifacts = context.artifactcache
def child_process(self):
def progress():
self.send_message('update-cache-size',
self._artifacts.get_cache_size())
self._casquota.get_cache_size())
return self._artifacts.clean(progress)
def handle_message(self, message_type, message):
# Update the cache size in the main process as we go,
# this provides better feedback in the UI.
if message_type == 'update-cache-size':
self._artifacts.set_cache_size(message)
self._casquota.set_cache_size(message)
return True
return False
def parent_complete(self, status, result):
if status == JobStatus.OK:
self._artifacts.set_cache_size(result)
self._casquota.set_cache_size(result)
if self._complete_cb:
self._complete_cb(status, result)
......@@ -81,11 +81,11 @@ def test_quota_over_1024T(cli, tmpdir):
_yaml.dump({'name': 'main'}, str(project.join("project.conf")))
volume_space_patch = mock.patch(
"buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
"buildstream._cas.CASQuota._get_cache_volume_size",
autospec=True,
return_value=(1025 * TiB, 1025 * TiB)
)
with volume_space_patch:
result = cli.run(project, args=["build", "file.bst"])
result.assert_main_error(ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota')
result.assert_main_error(ErrorDomain.CAS, 'insufficient-storage-for-quota')
......@@ -341,7 +341,7 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
# Not enough space on disk even if you cleaned up
("11K", ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota'),
("11K", ErrorDomain.CAS, 'insufficient-storage-for-quota'),
# Not enough space for these caches
("7K", 'warning', 'Your system does not have enough available'),
......@@ -355,7 +355,7 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
cli.configure({
'cache': {
'quota': quota,
}
},
})
# We patch how we get space information
......@@ -373,13 +373,13 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
total_space = 10000
volume_space_patch = mock.patch(
"buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
"buildstream.utils._get_volume_size",
autospec=True,
return_value=(total_space, free_space),
)
cache_size_patch = mock.patch(
"buildstream._artifactcache.ArtifactCache.get_cache_size",
"buildstream._cas.CASQuota.get_cache_size",
autospec=True,
return_value=0,
)
......
......@@ -46,7 +46,6 @@ class ArtifactShare():
# in tests as a remote artifact push/pull configuration
#
self.repodir = os.path.join(self.directory, 'repo')
os.makedirs(self.repodir)
self.cas = CASCache(self.repodir)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment