buildstream/_artifactcache/artifactcache.py

@@ -937,15 +937,22 @@ class ArtifactCache():
                             "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
                             "BuildStream requires a minimum cache quota of 2G.")
         elif cache_quota > cache_size + available_space:  # Check maximum
+            if '%' in self.context.config_cache_quota:
+                available = (available_space / (stat.f_blocks * stat.f_bsize)) * 100
+                available = '{}% of total disk space'.format(round(available, 1))
+            else:
+                available = utils._pretty_size(available_space)
+
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             ("Your system does not have enough available " +
                              "space to support the cache quota specified.\n" +
-                             "You currently have:\n" +
-                             "- {used} of cache in use at {local_cache_path}\n" +
-                             "- {available} of available system storage").format(
-                                 used=utils._pretty_size(cache_size),
-                                 local_cache_path=self.context.artifactdir,
-                                 available=utils._pretty_size(available_space)))
+                             "\nYou have specified a quota of {quota} total disk space.\n" +
+                             "- The filesystem containing {local_cache_path} only " +
+                             "has: {available_size} available.")
+                            .format(
+                                quota=self.context.config_cache_quota,
+                                local_cache_path=self.context.artifactdir,
+                                available_size=available))
 
         # Place a slight headroom (2e9 (2GB) on the cache_quota) into
         # cache_quota to try and avoid exceptions.

buildstream/plugin.py

@@ -111,6 +111,7 @@ Class Reference
 
 import os
 import subprocess
+import sys
 from contextlib import contextmanager
 from weakref import WeakValueDictionary
 
@@ -190,7 +191,7 @@ class Plugin():
         # Dont send anything through the Message() pipeline at destruction time,
         # any subsequent lookup of plugin by unique id would raise KeyError.
         if self.__context.log_debug:
-            print("DEBUG: Destroyed: {}".format(self))
+            sys.stderr.write("DEBUG: Destroyed: {}\n".format(self))
 
     def __str__(self):
         return "{kind} {typetag} at {provenance}".format(

buildstream/plugins/sources/pip.py

@@ -80,6 +80,7 @@ _PYPI_INDEX_URL = 'https://pypi.org/simple/'
 
 # Used only for finding pip command
 _PYTHON_VERSIONS = [
+    'python',  # when running in a venv, we might not have the exact version
     'python2.7',
     'python3.0',
     'python3.1',

doc/source/using_config.rst

@@ -147,6 +147,44 @@ The default mirror is defined by its name, e.g.
    ``--default-mirror`` command-line option.
 
 
+Local cache expiry
+~~~~~~~~~~~~~~~~~~
+BuildStream locally caches artifacts, build trees, log files and sources within a
+cache located at ``~/.cache/buildstream`` (unless a $XDG_CACHE_HOME environment
+variable exists). When building large projects, this cache can get very large,
+thus BuildStream will attempt to clean up the cache automatically by expiring the least
+recently *used* artifacts.
+
+By default, cache expiry will begin once the file system which contains the cache
+approaches maximum usage. However, it is also possible to impose a quota on the local
+cache in the user configuration. This can be done in two ways:
+
+1. By restricting the maximum size of the cache directory itself.
+
+For example, to ensure that BuildStream's cache does not grow beyond 100 GB,
+simply declare the following in your user configuration (``~/.config/buildstream.conf``):
+
+.. code:: yaml
+
+  cache:
+    quota: 100G
+
+This quota defines the maximum size of the artifact cache in bytes.
+Other accepted values are: K, M, G or T (or you can simply declare the value in bytes, without the suffix).
+This uses the same format as systemd's
+`resource-control <https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html>`_.
+
+2. By expiring artifacts once the file system which contains the cache exceeds a specified usage.
+
+To ensure that we start cleaning the cache once we've used 80% of local disk space (on the file system
+which mounts the cache):
+
+.. code:: yaml
+
+  cache:
+    quota: 80%
+
+
 Default configuration
 ---------------------
 The default BuildStream configuration is specified here for reference:

tests/utils/misc.py

@@ -27,4 +27,5 @@ def test_parse_size_over_1024T(cli, tmpdir):
     patched_statvfs = mock_os.mock_statvfs(f_bavail=bavail, f_bsize=BLOCK_SIZE)
     with mock_os.monkey_patch("statvfs", patched_statvfs):
         result = cli.run(project, args=["build", "file.bst"])
-        assert "1025T of available system storage" in result.stderr
+        failure_msg = 'Your system does not have enough available space to support the cache quota specified.'
+        assert failure_msg in result.stderr