buildstream/_artifactcache.py

@@ -22,7 +22,7 @@ import os
 from collections.abc import Mapping
 
 from .types import _KeyStrength
-from ._exceptions import ArtifactError, CASError, LoadError, LoadErrorReason
+from ._exceptions import ArtifactError, CASError
 from ._message import Message, MessageType
 from . import utils
 from . import _yaml
@@ -46,39 +46,6 @@ class ArtifactCacheSpec(CASRemoteSpec):
     pass
 
 
-# ArtifactCacheUsage
-#
-# A simple object to report the current artifact cache
-# usage details.
-#
-# Note that this uses the user configured cache quota
-# rather than the internal quota with protective headroom
-# removed, to provide a more sensible value to display to
-# the user.
-#
-# Args:
-#    artifacts (ArtifactCache): The artifact cache to get the status of
-#
-class ArtifactCacheUsage():
-
-    def __init__(self, artifacts):
-        context = artifacts.context
-        self.quota_config = context.config_cache_quota       # Configured quota
-        self.quota_size = artifacts._cache_quota_original    # Resolved cache quota in bytes
-        self.used_size = artifacts.get_cache_size()          # Size used by artifacts in bytes
-        self.used_percent = 0                                # Percentage of the quota used
-        if self.quota_size is not None:
-            self.used_percent = int(self.used_size * 100 / self.quota_size)
-
-    # Formattable into a human readable string
-    #
-    def __str__(self):
-        return "{} / {} ({}%)" \
-            .format(utils._pretty_size(self.used_size, dec_places=1),
-                    self.quota_config,
-                    self.used_percent)
-
-
 # An ArtifactCache manages artifacts.
 #
 # Args:
@@ -87,19 +54,17 @@ class ArtifactCacheUsage():
 class ArtifactCache():
     def __init__(self, context):
         self.context = context
-        self.extractdir = os.path.join(context.artifactdir, 'extract')
+        self.extractdir = context.extractdir
 
         self.cas = context.get_cascache()
+        self.casquota = context.get_casquota()
+        self.casquota._calculate_cache_quota()
 
         self.global_remote_specs = []
         self.project_remote_specs = {}
 
         self._required_elements = set()       # The elements required for this session
-        self._cache_size = None               # The current cache size, sometimes it's an estimate
-        self._cache_quota = None              # The cache quota
-        self._cache_quota_original = None     # The cache quota as specified by the user, in bytes
-        self._cache_quota_headroom = None     # The headroom in bytes before reaching the quota or full disk
-        self._cache_lower_threshold = None    # The target cache size for a cleanup
+
         self._remotes_setup = False           # Check to prevent double-setup of remotes
 
         # Per-project list of _CASRemote instances.
@@ -110,8 +75,6 @@ class ArtifactCache():
 
         os.makedirs(self.extractdir, exist_ok=True)
 
-        self._calculate_cache_quota()
-
     # setup_remotes():
     #
     # Sets up which remotes to use
@@ -235,7 +198,7 @@ class ArtifactCache():
         space_saved = 0
 
         # Start off with an announcement with as much info as possible
-        volume_size, volume_avail = self._get_cache_volume_size()
+        volume_size, volume_avail = self.casquota._get_cache_volume_size()
         self._message(MessageType.STATUS, "Starting cache cleanup",
                       detail=("Elements required by the current build plan: {}\n" +
                               "User specified quota: {} ({})\n" +
@@ -243,8 +206,8 @@ class ArtifactCache():
                               "Cache volume: {} total, {} available")
                       .format(len(self._required_elements),
                               context.config_cache_quota,
-                              utils._pretty_size(self._cache_quota_original, dec_places=2),
-                              utils._pretty_size(self.get_cache_size(), dec_places=2),
+                              utils._pretty_size(self.casquota._cache_quota, dec_places=2),
+                              utils._pretty_size(self.casquota.get_cache_size(), dec_places=2),
                               utils._pretty_size(volume_size, dec_places=2),
                               utils._pretty_size(volume_avail, dec_places=2)))
 
@@ -261,9 +224,10 @@ class ArtifactCache():
             ])
 
         # Do a real computation of the cache size once, just in case
-        self.compute_cache_size()
+        usage = self.casquota.compute_cache_size()
+        self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
 
-        while self.get_cache_size() >= self._cache_lower_threshold:
+        while self.casquota.get_cache_size() >= self.casquota._cache_lower_threshold:
             try:
                 to_remove = artifacts.pop(0)
             except IndexError:
@@ -280,7 +244,7 @@ class ArtifactCache():
                           "Please increase the cache-quota in {} and/or make more disk space."
                           .format(removed_ref_count,
                                   utils._pretty_size(space_saved, dec_places=2),
-                                  utils._pretty_size(self.get_cache_size(), dec_places=2),
+                                  utils._pretty_size(self.casquota.get_cache_size(), dec_places=2),
                                   len(self._required_elements),
                                   (context.config_origin or default_conf)))
 
@@ -306,7 +270,7 @@ class ArtifactCache():
                                   to_remove))
 
                 # Remove the size from the removed size
-                self.set_cache_size(self._cache_size - size)
+                self.casquota.set_cache_size(self.casquota._cache_size - size)
 
                 # User callback
                 #
@@ -322,29 +286,12 @@ class ArtifactCache():
                               "Cache usage is now: {}")
                       .format(removed_ref_count,
                               utils._pretty_size(space_saved, dec_places=2),
-                              utils._pretty_size(self.get_cache_size(), dec_places=2)))
-
-        return self.get_cache_size()
-
-    # compute_cache_size()
-    #
-    # Computes the real artifact cache size by calling
-    # the abstract calculate_cache_size() method.
-    #
-    # Returns:
-    #    (int): The size of the artifact cache.
-    #
-    def compute_cache_size(self):
-        old_cache_size = self._cache_size
-        new_cache_size = self.cas.calculate_cache_size()
-
-        if old_cache_size != new_cache_size:
-            self._cache_size = new_cache_size
+                              utils._pretty_size(self.casquota.get_cache_size(), dec_places=2)))
 
-            usage = ArtifactCacheUsage(self)
-            self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
+        return self.casquota.get_cache_size()
 
-        return self._cache_size
+    def full(self):
+        return self.casquota.full()
 
     # add_artifact_size()
     #
@@ -355,71 +302,10 @@ class ArtifactCache():
     #     artifact_size (int): The size to add.
     #
     def add_artifact_size(self, artifact_size):
-        cache_size = self.get_cache_size()
+        cache_size = self.casquota.get_cache_size()
         cache_size += artifact_size
 
-        self.set_cache_size(cache_size)
-
-    # get_cache_size()
-    #
-    # Fetches the cached size of the cache, this is sometimes
-    # an estimate and periodically adjusted to the real size
-    # when a cache size calculation job runs.
-    #
-    # When it is an estimate, the value is either correct, or
-    # it is greater than the actual cache size.
-    #
-    # Returns:
-    #     (int) An approximation of the artifact cache size, in bytes.
-    #
-    def get_cache_size(self):
-
-        # If we don't currently have an estimate, figure out the real cache size.
-        if self._cache_size is None:
-            stored_size = self._read_cache_size()
-            if stored_size is not None:
-                self._cache_size = stored_size
-            else:
-                self.compute_cache_size()
-
-        return self._cache_size
-
-    # set_cache_size()
-    #
-    # Forcefully set the overall cache size.
-    #
-    # This is used to update the size in the main process after
-    # having calculated in a cleanup or a cache size calculation job.
-    #
-    # Args:
-    #     cache_size (int): The size to set.
-    #
-    def set_cache_size(self, cache_size):
-
-        assert cache_size is not None
-
-        self._cache_size = cache_size
-        self._write_cache_size(self._cache_size)
-
-    # full()
-    #
-    # Checks if the artifact cache is full, either
-    # because the user configured quota has been exceeded
-    # or because the underlying disk is almost full.
-    #
-    # Returns:
-    #    (bool): True if the artifact cache is full
-    #
-    def full(self):
-
-        if self.get_cache_size() > self._cache_quota:
-            return True
-
-        _, volume_avail = self._get_cache_volume_size()
-        if volume_avail < self._cache_quota_headroom:
-            return True
-
-        return False
+        self.casquota.set_cache_size(cache_size)
 
     # preflight():
     #
@@ -882,142 +768,6 @@ class ArtifactCache():
         with self.context.timed_activity("Initializing remote caches", silent_nested=True):
             self.initialize_remotes(on_failure=remote_failed)
 
-    # _write_cache_size()
-    #
-    # Writes the given size of the artifact to the cache's size file
-    #
-    # Args:
-    #    size (int): The size of the artifact cache to record
-    #
-    def _write_cache_size(self, size):
-        assert isinstance(size, int)
-        size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
-        with utils.save_file_atomic(size_file_path, "w") as f:
-            f.write(str(size))
-
-    # _read_cache_size()
-    #
-    # Reads and returns the size of the artifact cache that's stored in the
-    # cache's size file
-    #
-    # Returns:
-    #    (int): The size of the artifact cache, as recorded in the file
-    #
-    def _read_cache_size(self):
-        size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
-
-        if not os.path.exists(size_file_path):
-            return None
-
-        with open(size_file_path, "r") as f:
-            size = f.read()
-
-        try:
-            num_size = int(size)
-        except ValueError as e:
-            raise ArtifactError("Size '{}' parsed from '{}' was not an integer".format(
-                size, size_file_path)) from e
-
-        return num_size
-
-    # _calculate_cache_quota()
-    #
-    # Calculates and sets the cache quota and lower threshold based on the
-    # quota set in Context.
-    # It checks that the quota is both a valid expression, and that there is
-    # enough disk space to satisfy that quota
-    #
-    def _calculate_cache_quota(self):
-        # Headroom intended to give BuildStream a bit of leeway.
-        # This acts as the minimum size of cache_quota and also
-        # is taken from the user requested cache_quota.
-        #
-        if 'BST_TEST_SUITE' in os.environ:
-            self._cache_quota_headroom = 0
-        else:
-            self._cache_quota_headroom = 2e9
-
-        try:
-            cache_quota = utils._parse_size(self.context.config_cache_quota,
-                                            self.context.artifactdir)
-        except utils.UtilError as e:
-            raise LoadError(LoadErrorReason.INVALID_DATA,
-                            "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
-                            "\nValid values are, for example: 800M 10G 1T 50%\n"
-                            .format(str(e))) from e
-
-        total_size, available_space = self._get_cache_volume_size()
-        cache_size = self.get_cache_size()
-
-        # Ensure system has enough storage for the cache_quota
-        #
-        # If cache_quota is none, set it to the maximum it could possibly be.
-        #
-        # Also check that cache_quota is at least as large as our headroom.
-        #
-        if cache_quota is None:  # Infinity, set to max system storage
-            cache_quota = cache_size + available_space
-        if cache_quota < self._cache_quota_headroom:  # Check minimum
-            raise LoadError(LoadErrorReason.INVALID_DATA,
-                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
-                            "BuildStream requires a minimum cache quota of 2G.")
-        elif cache_quota > total_size:
-            # A quota greater than the total disk size is certianly an error
-            raise ArtifactError("Your system does not have enough available " +
-                                "space to support the cache quota specified.",
-                                detail=("You have specified a quota of {quota} total disk space.\n" +
-                                        "The filesystem containing {local_cache_path} only " +
-                                        "has {total_size} total disk space.")
-                                .format(
-                                    quota=self.context.config_cache_quota,
-                                    local_cache_path=self.context.artifactdir,
-                                    total_size=utils._pretty_size(total_size)),
-                                reason='insufficient-storage-for-quota')
-        elif cache_quota > cache_size + available_space:
-            # The quota does not fit in the available space, this is a warning
-            if '%' in self.context.config_cache_quota:
-                available = (available_space / total_size) * 100
-                available = '{}% of total disk space'.format(round(available, 1))
-            else:
-                available = utils._pretty_size(available_space)
-
-            self._message(MessageType.WARN,
-                          "Your system does not have enough available " +
-                          "space to support the cache quota specified.",
-                          detail=("You have specified a quota of {quota} total disk space.\n" +
-                                  "The filesystem containing {local_cache_path} only " +
-                                  "has {available_size} available.")
-                          .format(quota=self.context.config_cache_quota,
-                                  local_cache_path=self.context.artifactdir,
-                                  available_size=available))
-
-        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
-        # cache_quota to try and avoid exceptions.
-        #
-        # Of course, we might still end up running out during a build
-        # if we end up writing more than 2G, but hey, this stuff is
-        # already really fuzzy.
-        #
-        self._cache_quota_original = cache_quota
-        self._cache_quota = cache_quota - self._cache_quota_headroom
-        self._cache_lower_threshold = self._cache_quota / 2
-
-    # _get_cache_volume_size()
-    #
-    # Get the available space and total space for the volume on
-    # which the artifact cache is located.
-    #
-    # Returns:
-    #    (int): The total number of bytes on the volume
-    #    (int): The number of available bytes on the volume
-    #
-    # NOTE: We use this stub to allow the test cases
-    #       to override what an artifact cache thinks
-    #       about it's disk size and available bytes.
-    #
-    def _get_cache_volume_size(self):
-        return utils._get_volume_size(self.context.artifactdir)
-
 
 # _configured_remote_artifact_cache_specs():
 #

buildstream/_cas/__init__.py

@@ -17,5 +17,5 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-from .cascache import CASCache
+from .cascache import CASCache, CASQuota, CASCacheUsage
 from .casremote import CASRemote, CASRemoteSpec

buildstream/_cas/cascache.py

@@ -32,17 +32,53 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
 from .._protos.buildstream.v2 import buildstream_pb2
 
 from .. import utils
-from .._exceptions import CASCacheError
+from .._exceptions import CASCacheError, LoadError, LoadErrorReason
+from .._message import Message, MessageType
 
 from .casremote import BlobNotFound, _CASBatchRead, _CASBatchUpdate
 
 _BUFFER_SIZE = 65536
 
 
+CACHE_SIZE_FILE = "cache_size"
+
+
+# CASCacheUsage
+#
+# A simple object to report the current CAS cache usage details.
+#
+# Note that this uses the user configured cache quota
+# rather than the internal quota with protective headroom
+# removed, to provide a more sensible value to display to
+# the user.
+#
+# Args:
+#    cas (CASQuota): The CAS cache to get the status of
+#
+class CASCacheUsage():
+
+    def __init__(self, casquota):
+        self.quota_config = casquota._config_cache_quota          # Configured quota
+        self.quota_size = casquota._cache_quota_original          # Resolved cache quota in bytes
+        self.used_size = casquota.get_cache_size()                # Size used by artifacts in bytes
+        self.used_percent = 0                                # Percentage of the quota used
+        if self.quota_size is not None:
+            self.used_percent = int(self.used_size * 100 / self.quota_size)
+
+    # Formattable into a human readable string
+    #
+    def __str__(self):
+        return "{} / {} ({}%)" \
+            .format(utils._pretty_size(self.used_size, dec_places=1),
+                    self.quota_config,
+                    self.used_percent)
+
+
 # A CASCache manages a CAS repository as specified in the Remote Execution API.
 #
 # Args:
 #     path (str): The root directory for the CAS repository
+#     cache_quota (int): User configured cache quota
 #
 class CASCache():
 
@@ -459,16 +495,6 @@ class CASCache():
         except FileNotFoundError as e:
             raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
 
-    # calculate_cache_size()
-    #
-    # Return the real disk usage of the CAS cache.
-    #
-    # Returns:
-    #    (int): The size of the cache.
-    #
-    def calculate_cache_size(self):
-        return utils._get_dir_size(self.casdir)
-
     # list_refs():
     #
     # List refs in Least Recently Modified (LRM) order.
@@ -1043,6 +1069,240 @@ class CASCache():
         batch.send()
 
 
+class CASQuota:
+    def __init__(self, context):
+        self.cas = context.get_cascache()
+        self.casdir = self.cas.casdir
+        self._config_cache_quota = context.config_cache_quota
+        self._cache_size = None               # The current cache size, sometimes it's an estimate
+        self._cache_quota = None              # The cache quota
+        self._cache_quota_original = None     # The cache quota as specified by the user, in bytes
+        self._cache_quota_headroom = None
+        self._cache_lower_threshold = None    # The target cache size for a cleanup
+        self.available_space = None
+
+        self._message = context.message
+
+        self._calculate_cache_quota()
+
+    # compute_cache_size()
+    #
+    # Computes the real artifact cache size by calling
+    # the abstract calculate_cache_size() method.
+    #
+    # Returns:
+    #    (int): The size of the artifact cache.
+    #
+    def compute_cache_size(self):
+        old_cache_size = self._cache_size
+        new_cache_size = self.calculate_cache_size()
+
+        if old_cache_size != new_cache_size:
+            self._cache_size = new_cache_size
+
+        return self._cache_size
+
+    # calculate_cache_size()
+    #
+    # Return the real disk usage of the CAS cache.
+    #
+    # Returns:
+    #    (int): The size of the cache.
+    #
+    def calculate_cache_size(self):
+        return utils._get_dir_size(self.casdir)
+
+    # get_cache_size()
+    #
+    # Fetches the cached size of the cache, this is sometimes
+    # an estimate and periodically adjusted to the real size
+    # when a cache size calculation job runs.
+    #
+    # When it is an estimate, the value is either correct, or
+    # it is greater than the actual cache size.
+    #
+    # Returns:
+    #     (int) An approximation of the artifact cache size, in bytes.
+    #
+    def get_cache_size(self):
+
+        # If we don't currently have an estimate, figure out the real cache size.
+        if self._cache_size is None:
+            stored_size = self._read_cache_size()
+            if stored_size is not None:
+                self._cache_size = stored_size
+            else:
+                self._cache_size = self.compute_cache_size()
+
+        return self._cache_size
+
+    # set_cache_size()
+    #
+    # Forcefully set the overall cache size.
+    #
+    # This is used to update the size in the main process after
+    # having calculated in a cleanup or a cache size calculation job.
+    #
+    # Args:
+    #     cache_size (int): The size to set.
+    #
+    def set_cache_size(self, cache_size):
+
+        assert cache_size is not None
+
+        self._cache_size = cache_size
+        self._write_cache_size(self._cache_size)
+
+    # full()
+    #
+    # Checks if the artifact cache is full, either
+    # because the user configured quota has been exceeded
+    # or because the underlying disk is almost full.
+    #
+    # Returns:
+    #    (bool): True if the artifact cache is full
+    #
+    def full(self):
+
+        if self.get_cache_size() > self._cache_quota:
+            return True
+
+        _, volume_avail = self._get_cache_volume_size()
+        if volume_avail < self._cache_quota_headroom:
+            return True
+
+        return False
+
+    ################################################
+    #             Local Private Methods            #
+    ################################################
+
+    # _read_cache_size()
+    #
+    # Reads and returns the size of the artifact cache that's stored in the
+    # cache's size file
+    #
+    # Returns:
+    #    (int): The size of the artifact cache, as recorded in the file
+    #
+    def _read_cache_size(self):
+        size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
+
+        if not os.path.exists(size_file_path):
+            return None
+
+        with open(size_file_path, "r") as f:
+            size = f.read()
+
+        try:
+            num_size = int(size)
+        except ValueError as e:
+            raise CASCacheError("Size '{}' parsed from '{}' was not an integer".format(
+                size, size_file_path)) from e
+
+        return num_size
+
+    # _write_cache_size()
+    #
+    # Writes the given size of the artifact to the cache's size file
+    #
+    # Args:
+    #    size (int): The size of the artifact cache to record
+    #
+    def _write_cache_size(self, size):
+        assert isinstance(size, int)
+        size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
+        with utils.save_file_atomic(size_file_path, "w") as f:
+            f.write(str(size))
+
+    # _get_cache_volume_size()
+    #
+    # Get the available space and total space for the volume on
+    # which the artifact cache is located.
+    #
+    # Returns:
+    #    (int): The total number of bytes on the volume
+    #    (int): The number of available bytes on the volume
+    #
+    # NOTE: We use this stub to allow the test cases
+    #       to override what an artifact cache thinks
+    #       about it's disk size and available bytes.
+    #
+    def _get_cache_volume_size(self):
+        return utils._get_volume_size(self.casdir)
+
+    # _calculate_cache_quota()
+    #
+    # Calculates and sets the cache quota and lower threshold based on the
+    # quota set in Context.
+    # It checks that the quota is both a valid expression, and that there is
+    # enough disk space to satisfy that quota
+    #
+    def _calculate_cache_quota(self):
+        # Headroom intended to give BuildStream a bit of leeway.
+        # This acts as the minimum size of cache_quota and also
+        # is taken from the user requested cache_quota.
+        #
+        if 'BST_TEST_SUITE' in os.environ:
+            self._cache_quota_headroom = 0
+        else:
+            self._cache_quota_headroom = 2e9
+
+        total_size, available_space = self._get_cache_volume_size()
+        cache_size = self.get_cache_size()
+        self.available_space = available_space
+
+        # Ensure system has enough storage for the cache_quota
+        #
+        # If cache_quota is none, set it to the maximum it could possibly be.
+        #
+        # Also check that cache_quota is at least as large as our headroom.
+        #
+        cache_quota = self._config_cache_quota
+        if cache_quota is None:  # Infinity, set to max system storage
+            cache_quota = cache_size + available_space
+        if cache_quota < self._cache_quota_headroom:  # Check minimum
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
+                            "BuildStream requires a minimum cache quota of 2G.")
+        elif cache_quota > total_size:
+            # A quota greater than the total disk size is certianly an error
+            raise CASCacheError("Your system does not have enough available " +
+                                "space to support the cache quota specified.",
+                                detail=("You have specified a quota of {quota} total disk space.\n" +
+                                        "The filesystem containing {local_cache_path} only " +
+                                        "has {total_size} total disk space.")
+                                .format(
+                                    quota=self._config_cache_quota,
+                                    local_cache_path=self.casdir,
+                                    total_size=utils._pretty_size(total_size)),
+                                reason='insufficient-storage-for-quota')
+
+        elif cache_quota > cache_size + available_space:
+            self._message(Message(
+                None,
+                MessageType.WARN,
+                "Your system does not have enough available " +
+                "space to support the cache quota specified.",
+                detail=("You have specified a quota of {quota} total disk space.\n" +
+                        "The filesystem containing {local_cache_path} only " +
+                        "has {available_size} available.")
+                .format(quota=self._config_cache_quota,
+                        local_cache_path=self.casdir,
+                        available_size=self.available_space)))
+
+        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
+        # cache_quota to try and avoid exceptions.
+        #
+        # Of course, we might still end up running out during a build
+        # if we end up writing more than 2G, but hey, this stuff is
+        # already really fuzzy.
+        #
+        self._cache_quota_original = cache_quota
+        self._cache_quota = cache_quota - self._cache_quota_headroom
+        self._cache_lower_threshold = self._cache_quota / 2
+
+
 def _grouper(iterable, n):
     while True:
         try:

buildstream/_context.py

@@ -30,8 +30,8 @@ from . import _yaml
 from ._exceptions import LoadError, LoadErrorReason, BstError
 from ._message import Message, MessageType
 from ._profile import Topics, profile_start, profile_end
-from ._artifactcache import ArtifactCache, ArtifactCacheUsage
-from ._cas import CASCache
+from ._artifactcache import ArtifactCache
+from ._cas import CASCache, CASQuota, CASCacheUsage
 from ._workspaces import Workspaces, WorkspaceProjectCache
 from .plugin import _plugin_lookup
 from .sandbox import SandboxRemote
@@ -58,18 +58,27 @@ class Context():
         # Filename indicating which configuration file was used, or None for the defaults
         self.config_origin = None
 
+        # The directory under which other directories are based
+        self.cachedir = None
+
         # The directory where various sources are stored
         self.sourcedir = None
 
         # The directory where build sandboxes will be created
         self.builddir = None
 
+        # The directory for CAS
+        self.casdir = None
+
+        # Extract directory
+        self.extractdir = None
+
+        # The directory for temporary files
+        self.tmpdir = None
+
         # Default root location for workspaces
         self.workspacedir = None
 
-        # The local binary artifact cache directory
-        self.artifactdir = None
-
         # The locations from which to push and pull prebuilt artifacts
         self.artifact_cache_specs = None
 
@@ -143,6 +152,7 @@ class Context():
         self._log_handle = None
         self._log_filename = None
         self._cascache = None
+        self._casquota = None
         self._directory = directory
 
     # load()
@@ -180,13 +190,22 @@ class Context():
             user_config = _yaml.load(config)
             _yaml.composite(defaults, user_config)
 
+        # Give obsoletion warnings
+        if defaults.get('builddir'):
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "builddir is obsolete, use cachedir")
+
+        if defaults.get('artifactdir'):
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "artifactdir is obsolete")
+
         _yaml.node_validate(defaults, [
-            'sourcedir', 'builddir', 'artifactdir', 'logdir',
-            'scheduler', 'artifacts', 'logging', 'projects',
-            'cache', 'prompt', 'workspacedir', 'remote-execution'
+            'cachedir', 'sourcedir', 'builddir', 'logdir', 'scheduler',
+            'artifacts', 'logging', 'projects', 'cache', 'prompt',
+            'workspacedir', 'remote-execution',
         ])
 
-        for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
+        for directory in ['cachedir', 'sourcedir', 'logdir', 'workspacedir']:
             # Allow the ~ tilde expansion and any environment variables in
             # path specification in the config files.
             #
@@ -196,14 +215,33 @@ class Context():
             path = os.path.normpath(path)
             setattr(self, directory, path)
 
+        # add directories not set by users
+        self.extractdir = os.path.join(self.cachedir, 'extract')
+        self.tmpdir = os.path.join(self.cachedir, 'tmp')
+        self.casdir = os.path.join(self.cachedir, 'cas')
+        self.builddir = os.path.join(self.cachedir, 'build')
+
+        # Move old artifact cas to cas if it exists and create symlink
+        old_casdir = os.path.join(self.cachedir, 'artifacts', 'cas')
+        if os.path.exists(old_casdir) and not os.path.islink(old_casdir):
+            os.rename(old_casdir, self.casdir)
+            os.symlink(self.casdir, old_casdir)
+
         # Load quota configuration
-        # We need to find the first existing directory in the path of
-        # our artifactdir - the artifactdir may not have been created
-        # yet.
+        # We need to find the first existing directory in the path of our
+        # cachedir - the cachedir may not have been created yet.
         cache = _yaml.node_get(defaults, Mapping, 'cache')
         _yaml.node_validate(cache, ['quota', 'pull-buildtrees'])
 
-        self.config_cache_quota = _yaml.node_get(cache, str, 'quota')
+        config_cache_quota = _yaml.node_get(cache, str, 'quota')
+        try:
+            self.config_cache_quota = utils._parse_size(config_cache_quota,
+                                                        self.casdir)
+        except utils.UtilError as e:
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
+                            "\nValid values are, for example: 800M 10G 1T 50%\n"
+                            .format(str(e))) from e
 
         # Load artifact share configuration
         self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
@@ -275,15 +313,15 @@ class Context():
 
         return self._artifactcache
 
-    # get_artifact_cache_usage()
+    # get_cache_usage()
     #
     # Fetches the current usage of the artifact cache
     #
     # Returns:
-    #     (ArtifactCacheUsage): The current status
+    #     (CASCacheUsage): The current status
     #
-    def get_artifact_cache_usage(self):
-        return ArtifactCacheUsage(self.artifactcache)
+    def get_cache_usage(self):
+        return CASCacheUsage(self.get_casquota())
 
     # add_project():
     #
@@ -653,9 +691,14 @@ class Context():
 
     def get_cascache(self):
         if self._cascache is None:
-            self._cascache = CASCache(self.artifactdir)
+            self._cascache = CASCache(self.cachedir)
         return self._cascache
 
+    def get_casquota(self):
+        if self._casquota is None:
+            self._casquota = CASQuota(self)
+        return self._casquota
+
 
 # _node_get_option_str()
 #

buildstream/_frontend/status.py

@@ -404,7 +404,7 @@ class _StatusHeader():
         #
         #  ~~~~~~ cache: 69% ~~~~~~
         #
-        usage = self._context.get_artifact_cache_usage()
+        usage = self._context.get_cache_usage()
         usage_percent = '{}%'.format(usage.used_percent)
 
         size = 21

buildstream/_frontend/widget.py

@@ -486,7 +486,7 @@ class LogLine(Widget):
         values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S')
         values["Project"] = "{} ({})".format(project.name, project.directory)
         values["Targets"] = ", ".join([t.name for t in stream.targets])
-        values["Cache Usage"] = "{}".format(context.get_artifact_cache_usage())
+        values["Cache Usage"] = "{}".format(context.get_cache_usage())
         text += self._format_values(values)
 
         # User configurations
@@ -495,10 +495,10 @@ class LogLine(Widget):
         values = OrderedDict()
         values["Configuration File"] = \
             "Default Configuration" if not context.config_origin else context.config_origin
+        values["Cache directory"] = context.cachedir
         values["Log Files"] = context.logdir
         values["Source Mirrors"] = context.sourcedir
         values["Build Area"] = context.builddir
-        values["Artifact Cache"] = context.artifactdir
         values["Strict Build Plan"] = "Yes" if context.get_strict() else "No"
         values["Maximum Fetch Tasks"] = context.sched_fetchers
         values["Maximum Build Tasks"] = context.sched_builders

buildstream/_scheduler/jobs/cachesizejob.py

@@ -25,14 +25,14 @@ class CacheSizeJob(Job):
         self._complete_cb = complete_cb
 
         context = self._scheduler.context
-        self._artifacts = context.artifactcache
+        self._casquota = context.get_casquota()
 
     def child_process(self):
-        return self._artifacts.compute_cache_size()
+        return self._casquota.compute_cache_size()
 
     def parent_complete(self, status, result):
         if status == JobStatus.OK:
-            self._artifacts.set_cache_size(result)
+            self._casquota.set_cache_size(result)
 
         if self._complete_cb:
             self._complete_cb(status, result)

buildstream/_scheduler/jobs/cleanupjob.py

@@ -25,27 +25,27 @@ class CleanupJob(Job):
         self._complete_cb = complete_cb
 
         context = self._scheduler.context
+        self._casquota = context.get_casquota()
         self._artifacts = context.artifactcache
 
     def child_process(self):
         def progress():
             self.send_message('update-cache-size',
-                              self._artifacts.get_cache_size())
+                              self._casquota.get_cache_size())
         return self._artifacts.clean(progress)
 
     def handle_message(self, message_type, message):
-
         # Update the cache size in the main process as we go,
         # this provides better feedback in the UI.
         if message_type == 'update-cache-size':
-            self._artifacts.set_cache_size(message)
+            self._casquota.set_cache_size(message)
             return True
 
         return False
 
     def parent_complete(self, status, result):
         if status == JobStatus.OK:
-            self._artifacts.set_cache_size(result)
+            self._casquota.set_cache_size(result)
 
         if self._complete_cb:
             self._complete_cb(status, result)

buildstream/data/projectconfig.yaml

@@ -90,10 +90,16 @@ split-rules:
   # built element to run, this includes stripped executables
   # and shared libraries by default.
   runtime:
+  - |
+    %{bindir}
   - |
     %{bindir}/*
+  - |
+    %{sbindir}
   - |
     %{sbindir}/*
+  - |
+    %{libexecdir}
   - |
     %{libexecdir}/*
   - |

buildstream/data/userconfig.yaml

@@ -13,11 +13,8 @@
 # Location to store sources
 sourcedir: ${XDG_CACHE_HOME}/buildstream/sources
 
-# Location to perform builds
-builddir: ${XDG_CACHE_HOME}/buildstream/build
-
-# Location to store local binary artifacts
-artifactdir: ${XDG_CACHE_HOME}/buildstream/artifacts
+# Root location for other directories in the cache
+cachedir: ${XDG_CACHE_HOME}/buildstream
 
 # Location to store build logs
 logdir: ${XDG_CACHE_HOME}/buildstream/logs

buildstream/element.py

@@ -1435,7 +1435,7 @@ class Element(Plugin):
         # It's advantageous to have this temporary directory on
         # the same file system as the rest of our cache.
         with self.timed_activity("Staging sources", silent_nested=True), \
-            utils._tempdir(dir=context.artifactdir, prefix='staging-temp') as temp_staging_directory:
+            utils._tempdir(dir=context.tmpdir, prefix='staging-temp') as temp_staging_directory:
 
             import_dir = temp_staging_directory
 

buildstream/plugintestutils/runcli.py

@@ -277,15 +277,13 @@ class Cli():
                                    *, cache_dir=None):
         # Read configuration to figure out where artifacts are stored
         if not cache_dir:
-            default = os.path.join(project, 'cache', 'artifacts')
-
-            if self.config is not None:
-                cache_dir = self.config.get('artifactdir', default)
-            else:
-                cache_dir = default
+            cache_dir = os.path.join(project, 'cache')
 
         cache_dir = os.path.join(cache_dir, 'cas', 'refs', 'heads')
 
+        # replace forward slashes
+        element_name = element_name.replace('/', '-')
+
         cache_dir = os.path.splitext(os.path.join(cache_dir, 'test', element_name))[0]
         shutil.rmtree(cache_dir)
 
@@ -582,11 +580,21 @@ def cli_integration(tmpdir, integration_cache):
     # We want to cache sources for integration tests more permanently,
     # to avoid downloading the huge base-sdk repeatedly
     fixture.configure({
+        'cachedir': integration_cache.root,
         'sourcedir': integration_cache.sources,
-        'artifactdir': integration_cache.artifacts
     })
 
-    return fixture
+    yield fixture
+
+    # remove following folders if necessary
+    try:
+        shutil.rmtree(os.path.join(integration_cache.root, 'build'))
+    except FileNotFoundError:
+        pass
+    try:
+        shutil.rmtree(os.path.join(integration_cache.root, 'tmp'))
+    except FileNotFoundError:
+        pass
 
 
 @contextmanager
@@ -626,10 +634,8 @@ def configured(directory, config=None):
 
     if not config.get('sourcedir', False):
         config['sourcedir'] = os.path.join(directory, 'sources')
-    if not config.get('builddir', False):
-        config['builddir'] = os.path.join(directory, 'build')
-    if not config.get('artifactdir', False):
-        config['artifactdir'] = os.path.join(directory, 'artifacts')
+    if not config.get('cachedir', False):
+        config['cachedir'] = directory
     if not config.get('logdir', False):
         config['logdir'] = os.path.join(directory, 'logs')
 

buildstream/storage/_casbaseddirectory.py

@@ -800,11 +800,11 @@ class CasBasedDirectory(Directory):
         directory_list = filter(lambda i: isinstance(i[1].buildstream_object, CasBasedDirectory),
                                 self.index.items())
 
-        if file_list == [] and relpath != "":
+        if relpath != "":
             yield relpath
-        else:
-            for (k, v) in sorted(file_list):
-                yield os.path.join(relpath, k)
+
+        for (k, v) in sorted(file_list):
+            yield os.path.join(relpath, k)
 
         for (k, v) in sorted(directory_list):
             yield from v.buildstream_object.list_relative_paths(relpath=os.path.join(relpath, k))

buildstream/utils.py

@@ -117,9 +117,6 @@ def list_relative_paths(directory):
     This generator is useful for checking the full manifest of
     a directory.
 
-    Note that directories will be yielded only if they are
-    empty.
-
     Symbolic links will not be followed, but will be included
     in the manifest.
 
@@ -156,11 +153,9 @@ def list_relative_paths(directory):
         # `directory`, prefer to have no prefix in that case.
         basepath = relpath if relpath != '.' and dirpath != directory else ''
 
-        # We've decended into an empty directory, in this case we
-        # want to include the directory itself, but not in any other
-        # case.
-        if not filenames:
-            yield relpath
+        # First yield the walked directory itself, except for the root
+        if basepath != '':
+            yield basepath
 
         # List the filenames in the walked directory
         for f in filenames:

conftest.py

@@ -53,16 +53,16 @@ def pytest_runtest_setup(item):
 class IntegrationCache():
 
     def __init__(self, cache):
-        cache = os.path.abspath(cache)
+        self.root = os.path.abspath(cache)
         os.makedirs(cache, exist_ok=True)
 
         # Use the same sources every time
-        self.sources = os.path.join(cache, 'sources')
+        self.sources = os.path.join(self.root, 'sources')
 
         # Create a temp directory for the duration of the test for
         # the artifacts directory
         try:
-            self.artifacts = tempfile.mkdtemp(dir=cache, prefix='artifacts-')
+            self.artifacts = tempfile.mkdtemp(dir=self.root, prefix='artifacts-')
         except OSError as e:
             raise AssertionError("Unable to create test directory !") from e
 
@@ -87,6 +87,10 @@ def integration_cache(request):
         shutil.rmtree(cache.artifacts)
     except FileNotFoundError:
         pass
+    try:
+        shutil.rmtree(os.path.join(cache.root, 'cas'))
+    except FileNotFoundError:
+        pass
 
 
 #################################################

doc/bst2html.py

@@ -194,10 +194,9 @@ def workdir(source_cache=None):
 
         bst_config_file = os.path.join(tempdir, 'buildstream.conf')
         config = {
+            'cachedir': tempdir,
             'sourcedir': source_cache,
-            'artifactdir': os.path.join(tempdir, 'artifacts'),
             'logdir': os.path.join(tempdir, 'logs'),
-            'builddir': os.path.join(tempdir, 'build'),
         }
         _yaml.dump(config, bst_config_file)
 
@@ -411,12 +410,10 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
         # Encode and save the output if that was asked for
         output = _yaml.node_get(command, str, 'output', default_value=None)
         if output is not None:
-
             # Convert / Generate a nice <div>
             converted = generate_html(command_out, directory, config_file,
                                       source_cache, tempdir, palette,
                                       command_str, command_fake_output is not None)
-
             # Save it
             filename = os.path.join(desc_dir, output)
             filename = os.path.realpath(filename)

doc/sessions/running-commands.run

@@ -2,7 +2,7 @@
 commands:
 # Make it fetch first
 - directory: ../examples/running-commands
-  command: fetch hello.bst
+  command: source fetch hello.bst
 
 # Capture a show output
 - directory: ../examples/running-commands

tests/artifactcache/cache_size.py

@@ -50,15 +50,15 @@ def test_cache_size_write(cli, tmpdir):
     create_project(project_dir)
 
     # Artifact cache must be in a known place
-    artifactdir = os.path.join(project_dir, "artifacts")
-    cli.configure({"artifactdir": artifactdir})
+    casdir = os.path.join(project_dir, "cas")
+    cli.configure({"cachedir": project_dir})
 
     # Build, to populate the cache
     res = cli.run(project=project_dir, args=["build", "test.bst"])
     res.assert_success()
 
     # Inspect the artifact cache
-    sizefile = os.path.join(artifactdir, CACHE_SIZE_FILE)
+    sizefile = os.path.join(casdir, CACHE_SIZE_FILE)
     assert os.path.isfile(sizefile)
     with open(sizefile, "r") as f:
         size_data = f.read()
@@ -81,11 +81,11 @@ def test_quota_over_1024T(cli, tmpdir):
     _yaml.dump({'name': 'main'}, str(project.join("project.conf")))
 
     volume_space_patch = mock.patch(
-        "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
+        "buildstream._cas.CASQuota._get_cache_volume_size",
         autospec=True,
         return_value=(1025 * TiB, 1025 * TiB)
     )
 
     with volume_space_patch:
         result = cli.run(project, args=["build", "file.bst"])
-        result.assert_main_error(ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota')
+        result.assert_main_error(ErrorDomain.CAS, 'insufficient-storage-for-quota')

tests/artifactcache/expiry.py

@@ -341,7 +341,7 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
     ("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
 
     # Not enough space on disk even if you cleaned up
-    ("11K", ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota'),
+    ("11K", ErrorDomain.CAS, 'insufficient-storage-for-quota'),
 
     # Not enough space for these caches
     ("7K", 'warning', 'Your system does not have enough available'),
@@ -355,7 +355,7 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
     cli.configure({
         'cache': {
             'quota': quota,
-        }
+        },
     })
 
     # We patch how we get space information
@@ -373,13 +373,13 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
         total_space = 10000
 
     volume_space_patch = mock.patch(
-        "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
+        "buildstream.utils._get_volume_size",
         autospec=True,
         return_value=(total_space, free_space),
     )
 
     cache_size_patch = mock.patch(
-        "buildstream._artifactcache.ArtifactCache.get_cache_size",
+        "buildstream._cas.CASQuota.get_cache_size",
         autospec=True,
         return_value=0,
     )
@@ -417,7 +417,7 @@ def test_extract_expiry(cli, datafiles, tmpdir):
     res.assert_success()
 
     # Get a snapshot of the extracts in advance
-    extractdir = os.path.join(project, 'cache', 'artifacts', 'extract', 'test', 'target')
+    extractdir = os.path.join(project, 'cache', 'extract', 'test', 'target')
     extracts = os.listdir(extractdir)
     assert(len(extracts) == 1)
     extract = os.path.join(extractdir, extracts[0])
@@ -436,7 +436,7 @@ def test_extract_expiry(cli, datafiles, tmpdir):
     # Now we should have a directory for the cached target2.bst, which
     # replaced target.bst in the cache, we should not have a directory
     # for the target.bst
-    refsdir = os.path.join(project, 'cache', 'artifacts', 'cas', 'refs', 'heads')
+    refsdir = os.path.join(project, 'cache', 'cas', 'refs', 'heads')
     refsdirtest = os.path.join(refsdir, 'test')
     refsdirtarget = os.path.join(refsdirtest, 'target')
     refsdirtarget2 = os.path.join(refsdirtest, 'target2')

tests/artifactcache/junctions.py

@@ -70,8 +70,8 @@ def test_push_pull(cli, tmpdir, datafiles):
         # Now we've pushed, delete the user's local artifact cache
         # directory and try to redownload it from the share
         #
-        artifacts = os.path.join(cli.directory, 'artifacts')
-        shutil.rmtree(artifacts)
+        cas = os.path.join(cli.directory, 'cas')
+        shutil.rmtree(cas)
 
         # Assert that nothing is cached locally anymore
         state = cli.get_element_state(project, 'target.bst')