CONTRIBUTING.rst

@@ -97,7 +97,13 @@ a new merge request. You can also `create a merge request for an existing branch
 You may open merge requests for the branches you create before you are ready
 to have them reviewed and considered for inclusion if you like. Until your merge
 request is ready for review, the merge request title must be prefixed with the
-``WIP:`` identifier.
+``WIP:`` identifier. GitLab `treats this specially
+<https://docs.gitlab.com/ee/user/project/merge_requests/work_in_progress_merge_requests.html>`_,
+which helps reviewers.
+
+Consider marking a merge request as WIP again if you are taking a while to
+address a review point. This signals that the next action is on you, and it
+won't appear in a reviewer's search for non-WIP merge requests to review.
 
 
 Organized commits
@@ -122,6 +128,12 @@ If a commit in your branch modifies behavior such that a test must also
 be changed to match the new behavior, then the tests should be updated
 with the same commit, so that every commit passes its own tests.
 
+These principles apply whenever a branch is non-WIP. So for example, don't push
+'fixup!' commits when addressing review comments, instead amend the commits
+directly before pushing. GitLab has `good support
+<https://docs.gitlab.com/ee/user/project/merge_requests/versions.html>`_ for
+diffing between pushes, so 'fixup!' commits are not necessary for reviewers.
+
 
 Commit messages
 ~~~~~~~~~~~~~~~
@@ -144,6 +156,16 @@ number must be referenced in the commit message.
 
   Fixes #123
 
+Note that the 'why' of a change is as important as the 'what'.
+
+When reviewing this, folks can suggest better alternatives when they know the
+'why'. Perhaps there are other ways to avoid an error when things are not
+frobnicated.
+
+When folks modify this code, there may be uncertainty around whether the foos
+should always be frobnicated. The comments, the commit message, and issue #123
+should shed some light on that.
+
 In the case that you have a commit which necessarily modifies multiple
 components, then the summary line should still mention generally what
 changed (if possible), followed by a colon and a brief summary.

buildstream/_artifactcache/artifactcache.py

@@ -937,15 +937,22 @@ class ArtifactCache():
                             "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
                             "BuildStream requires a minimum cache quota of 2G.")
         elif cache_quota > cache_size + available_space:  # Check maximum
+            if '%' in self.context.config_cache_quota:
+                available = (available_space / (stat.f_blocks * stat.f_bsize)) * 100
+                available = '{}% of total disk space'.format(round(available, 1))
+            else:
+                available = utils._pretty_size(available_space)
+
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             ("Your system does not have enough available " +
                              "space to support the cache quota specified.\n" +
-                             "You currently have:\n" +
-                             "- {used} of cache in use at {local_cache_path}\n" +
-                             "- {available} of available system storage").format(
-                                 used=utils._pretty_size(cache_size),
-                                 local_cache_path=self.context.artifactdir,
-                                 available=utils._pretty_size(available_space)))
+                             "\nYou have specified a quota of {quota} total disk space.\n" +
+                             "- The filesystem containing {local_cache_path} only " +
+                             "has: {available_size} available.")
+                            .format(
+                                quota=self.context.config_cache_quota,
+                                local_cache_path=self.context.artifactdir,
+                                available_size=available))
 
         # Place a slight headroom (2e9 (2GB) on the cache_quota) into
         # cache_quota to try and avoid exceptions.

buildstream/scriptelement.py

@@ -202,7 +202,7 @@ class ScriptElement(Element):
         sandbox.set_environment(self.get_environment())
 
         # Tell the sandbox to mount the install root
-        directories = {'/': False}
+        directories = {self.__install_root: False}
 
         # Mark the artifact directories in the layout
         for item in self.__layout:
@@ -211,7 +211,10 @@ class ScriptElement(Element):
             directories[destination] = item['element'] or was_artifact
 
         for directory, artifact in directories.items():
-            sandbox.mark_directory(directory, artifact=artifact)
+            # Root does not need to be marked as it is always mounted
+            # with artifact (unless explicitly marked non-artifact)
+            if directory != '/':
+                sandbox.mark_directory(directory, artifact=artifact)
 
     def stage(self, sandbox):
 

doc/source/using_config.rst

@@ -147,6 +147,44 @@ The default mirror is defined by its name, e.g.
    ``--default-mirror`` command-line option.
 
 
+Local cache expiry
+~~~~~~~~~~~~~~~~~~
+BuildStream locally caches artifacts, build trees, log files and sources within a
+cache located at ``~/.cache/buildstream`` (unless a $XDG_CACHE_HOME environment
+variable exists). When building large projects, this cache can get very large,
+thus BuildStream will attempt to clean up the cache automatically by expiring the least
+recently *used* artifacts.
+
+By default, cache expiry will begin once the file system which contains the cache
+approaches maximum usage. However, it is also possible to impose a quota on the local
+cache in the user configuration. This can be done in two ways:
+
+1. By restricting the maximum size of the cache directory itself.
+
+For example, to ensure that BuildStream's cache does not grow beyond 100 GB,
+simply declare the following in your user configuration (``~/.config/buildstream.conf``):
+
+.. code:: yaml
+
+  cache:
+    quota: 100G
+
+This quota defines the maximum size of the artifact cache in bytes.
+Other accepted values are: K, M, G or T (or you can simply declare the value in bytes, without the suffix).
+This uses the same format as systemd's
+`resource-control <https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html>`_.
+
+2. By expiring artifacts once the file system which contains the cache exceeds a specified usage.
+
+To ensure that we start cleaning the cache once we've used 80% of local disk space (on the file system
+which mounts the cache):
+
+.. code:: yaml
+
+  cache:
+    quota: 80%
+
+
 Default configuration
 ---------------------
 The default BuildStream configuration is specified here for reference:

tests/integration/project/elements/script/corruption-2.bst

+kind: script
+
+depends:
+- filename: base.bst
+  type: build
+- filename: script/corruption-image.bst
+  type: build
+
+config:
+  commands:
+  - echo smashed >>/canary

tests/integration/project/elements/script/marked-tmpdir.bst

+kind: compose
+
+depends:
+- filename: base.bst
+  type: build
+
+public:
+  bst:
+    split-rules:
+      remove:
+        - "/tmp/**"
+        - "/tmp"

tests/integration/project/elements/script/no-tmpdir.bst

+kind: filter
+
+depends:
+- filename: script/marked-tmpdir.bst
+  type: build
+
+config:
+  exclude:
+  - remove
+  include-orphans: True
+
+

tests/integration/project/elements/script/tmpdir.bst

+kind: script
+
+depends:
+- filename: script/no-tmpdir.bst
+  type: build
+
+config:
+  commands:
+  - |
+    mkdir -p /tmp/blah

tests/integration/script.py

@@ -184,3 +184,41 @@ def test_regression_cache_corruption(cli, tmpdir, datafiles):
 
     with open(os.path.join(checkout_after, 'canary')) as f:
         assert f.read() == 'alive\n'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_regression_tmpdir(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    element_name = 'script/tmpdir.bst'
+
+    res = cli.run(project=project, args=['build', element_name])
+    assert res.exit_code == 0
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_regression_cache_corruption_2(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    checkout_original = os.path.join(cli.directory, 'checkout-original')
+    checkout_after = os.path.join(cli.directory, 'checkout-after')
+    element_name = 'script/corruption-2.bst'
+    canary_element_name = 'script/corruption-image.bst'
+
+    res = cli.run(project=project, args=['build', canary_element_name])
+    assert res.exit_code == 0
+
+    res = cli.run(project=project, args=['checkout', canary_element_name,
+                                         checkout_original])
+    assert res.exit_code == 0
+
+    with open(os.path.join(checkout_original, 'canary')) as f:
+        assert f.read() == 'alive\n'
+
+    res = cli.run(project=project, args=['build', element_name])
+    assert res.exit_code == 0
+
+    res = cli.run(project=project, args=['checkout', canary_element_name,
+                                         checkout_after])
+    assert res.exit_code == 0
+
+    with open(os.path.join(checkout_after, 'canary')) as f:
+        assert f.read() == 'alive\n'

tests/utils/misc.py

@@ -27,4 +27,5 @@ def test_parse_size_over_1024T(cli, tmpdir):
     patched_statvfs = mock_os.mock_statvfs(f_bavail=bavail, f_bsize=BLOCK_SIZE)
     with mock_os.monkey_patch("statvfs", patched_statvfs):
         result = cli.run(project, args=["build", "file.bst"])
-        assert "1025T of available system storage" in result.stderr
+        failure_msg = 'Your system does not have enough available space to support the cache quota specified.'
+        assert failure_msg in result.stderr