Merge branch 'valentindavid/faster_startup-1.2' into 'bst-1.2'

Use ArtifactCache's get_cache_size when calculating the quota

See merge request !701

buildstream/_context.py

@@ -124,6 +124,8 @@ class Context():
         self._workspaces = None
         self._log_handle = None
         self._log_filename = None
+        self._config_cache_quota = None
+        self._artifactdir_volume = None
 
     # load()
     #
@@ -187,67 +189,19 @@ class Context():
         while not os.path.exists(artifactdir_volume):
             artifactdir_volume = os.path.dirname(artifactdir_volume)
 
+        self._artifactdir_volume = artifactdir_volume
+
         # We read and parse the cache quota as specified by the user
         cache_quota = _yaml.node_get(cache, str, 'quota', default_value='infinity')
         try:
-            cache_quota = utils._parse_size(cache_quota, artifactdir_volume)
+            cache_quota = utils._parse_size(cache_quota, self._artifactdir_volume)
         except utils.UtilError as e:
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
                             "\nValid values are, for example: 800M 10G 1T 50%\n"
                             .format(str(e))) from e
 
-        # Headroom intended to give BuildStream a bit of leeway.
-        # This acts as the minimum size of cache_quota and also
-        # is taken from the user requested cache_quota.
-        #
-        if 'BST_TEST_SUITE' in os.environ:
-            headroom = 0
-        else:
-            headroom = 2e9
-
-        stat = os.statvfs(artifactdir_volume)
-        available_space = (stat.f_bsize * stat.f_bavail)
-
-        # Again, the artifact directory may not yet have been created yet
-        #
-        if not os.path.exists(self.artifactdir):
-            cache_size = 0
-        else:
-            cache_size = utils._get_dir_size(self.artifactdir)
-
-        # Ensure system has enough storage for the cache_quota
-        #
-        # If cache_quota is none, set it to the maximum it could possibly be.
-        #
-        # Also check that cache_quota is atleast as large as our headroom.
-        #
-        if cache_quota is None:  # Infinity, set to max system storage
-            cache_quota = cache_size + available_space
-        if cache_quota < headroom:  # Check minimum
-            raise LoadError(LoadErrorReason.INVALID_DATA,
-                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
-                            "BuildStream requires a minimum cache quota of 2G.")
-        elif cache_quota > cache_size + available_space:  # Check maximum
-            raise LoadError(LoadErrorReason.INVALID_DATA,
-                            ("Your system does not have enough available " +
-                             "space to support the cache quota specified.\n" +
-                             "You currently have:\n" +
-                             "- {used} of cache in use at {local_cache_path}\n" +
-                             "- {available} of available system storage").format(
-                                 used=utils._pretty_size(cache_size),
-                                 local_cache_path=self.artifactdir,
-                                 available=utils._pretty_size(available_space)))
-
-        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
-        # cache_quota to try and avoid exceptions.
-        #
-        # Of course, we might still end up running out during a build
-        # if we end up writing more than 2G, but hey, this stuff is
-        # already really fuzzy.
-        #
-        self.cache_quota = cache_quota - headroom
-        self.cache_lower_threshold = self.cache_quota / 2
+        self._config_cache_quota = cache_quota
 
         # Load artifact share configuration
         self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
@@ -571,6 +525,53 @@ class Context():
     def get_log_filename(self):
         return self._log_filename
 
+    def set_cache_quota(self, cache_size):
+        # Headroom intended to give BuildStream a bit of leeway.
+        # This acts as the minimum size of cache_quota and also
+        # is taken from the user requested cache_quota.
+        #
+        if 'BST_TEST_SUITE' in os.environ:
+            headroom = 0
+        else:
+            headroom = 2e9
+
+        stat = os.statvfs(self._artifactdir_volume)
+        available_space = (stat.f_bsize * stat.f_bavail)
+
+        # Ensure system has enough storage for the cache_quota
+        #
+        # If cache_quota is none, set it to the maximum it could possibly be.
+        #
+        # Also check that cache_quota is atleast as large as our headroom.
+        #
+        cache_quota = self._config_cache_quota
+        if cache_quota is None:  # Infinity, set to max system storage
+            cache_quota = cache_size + available_space
+        if cache_quota < headroom:  # Check minimum
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
+                            "BuildStream requires a minimum cache quota of 2G.")
+        elif cache_quota > cache_size + available_space:  # Check maximum
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            ("Your system does not have enough available " +
+                             "space to support the cache quota specified.\n" +
+                             "You currently have:\n" +
+                             "- {used} of cache in use at {local_cache_path}\n" +
+                             "- {available} of available system storage").format(
+                                 used=utils._pretty_size(cache_size),
+                                 local_cache_path=self.artifactdir,
+                                 available=utils._pretty_size(available_space)))
+
+        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
+        # cache_quota to try and avoid exceptions.
+        #
+        # Of course, we might still end up running out during a build
+        # if we end up writing more than 2G, but hey, this stuff is
+        # already really fuzzy.
+        #
+        self.cache_quota = cache_quota - headroom
+        self.cache_lower_threshold = self.cache_quota / 2
+
     # _record_message()
     #
     # Records the message if recording is enabled

buildstream/_frontend/app.py

@@ -202,6 +202,10 @@ class App():
 
         Platform.create_instance(self.context)
 
+        platform = Platform.get_platform()
+        cache_size = platform._artifact_cache.calculate_cache_size()
+        self.context.set_cache_quota(cache_size)
+
         # Create the logger right before setting the message handler
         self.logger = LogLine(self.context,
                               self._content_profile,