Commit 270da060 authored by Jürg Billeter's avatar Jürg Billeter

Do not pull/fetch/build elements that are not required

For `bst build --deps plan`, do not process elements in pull/fetch/build
queues until they are requested by a reverse dependency.
parent b78df1e2
Pipeline #21851276 passed with stages
in 20 minutes and 57 seconds
......@@ -38,6 +38,11 @@ class BuildQueue(Queue):
# state of dependencies may have changed, recalculate element state
element._update_state()
if not element._is_required():
# Artifact is not currently required but it may be requested later.
# Keep it in the queue.
return QueueStatus.WAIT
if element._cached():
return QueueStatus.SKIP
......
......@@ -47,6 +47,11 @@ class FetchQueue(Queue):
# state of dependencies may have changed, recalculate element state
element._update_state()
if not element._is_required():
# Artifact is not currently required but it may be requested later.
# Keep it in the queue.
return QueueStatus.WAIT
# Optionally skip elements that are already in the artifact cache
if self._skip_cached:
if not element._can_query_cache():
......
......@@ -39,6 +39,11 @@ class PullQueue(Queue):
# state of dependencies may have changed, recalculate element state
element._update_state()
if not element._is_required():
# Artifact is not currently required but it may be requested later.
# Keep it in the queue.
return QueueStatus.WAIT
if not element._can_query_cache():
return QueueStatus.WAIT
......
......@@ -181,7 +181,8 @@ class Stream():
track_except_targets=track_except,
track_cross_junctions=track_cross_junctions,
use_artifact_config=True,
fetch_subprojects=True)
fetch_subprojects=True,
dynamic_plan=True)
# Remove the tracking elements from the main targets
elements = self._pipeline.subtract_elements(elements, track_elements)
......@@ -755,7 +756,8 @@ class Stream():
track_cross_junctions=False,
use_artifact_config=False,
artifact_remote_url=None,
fetch_subprojects=False):
fetch_subprojects=False,
dynamic_plan=False):
# Load rewritable if we have any tracking selection to make
rewritable = False
......@@ -806,6 +808,16 @@ class Stream():
selected,
except_elements)
if selection == PipelineSelection.PLAN and dynamic_plan:
# We use a dynamic build plan, only request artifacts of top-level targets,
# others are requested dynamically as needed.
# This avoids pulling, fetching, or building unneeded build-only dependencies.
for element in elements:
element._set_required()
else:
for element in selected:
element._set_required()
return selected, track_selected
# _message()
......
......@@ -225,6 +225,7 @@ class Element(Plugin):
self.__whitelist_regex = None # Resolved regex object to check if file is allowed to overlap
self.__staged_sources_directory = None # Location where Element.stage_sources() was called
self.__tainted = None # Whether the artifact is tainted and should not be shared
self.__required = False # Whether the artifact is required in the current session
# hash tables of loaded artifact metadata, hashed by key
self.__metadata_keys = {} # Strong and weak keys for this key
......@@ -1069,7 +1070,7 @@ class Element(Plugin):
# until the full cache query below.
cached = self.__artifacts.contains(self, self.__weak_cache_key)
if (not self.__assemble_scheduled and not self.__assemble_done and
not cached and not self._pull_pending()):
not cached and not self._pull_pending() and self._is_required()):
self._schedule_assemble()
return
......@@ -1091,7 +1092,7 @@ class Element(Plugin):
self.__strong_cached = self.__artifacts.contains(self, self.__strict_cache_key)
if (not self.__assemble_scheduled and not self.__assemble_done and
not self.__cached and not self._pull_pending()):
not self.__cached and not self._pull_pending() and self._is_required()):
# Workspaced sources are considered unstable if a build is pending
# as the build will modify the contents of the workspace.
# Determine as early as possible if a build is pending to discard
......@@ -1322,15 +1323,45 @@ class Element(Plugin):
# Ensure deterministic owners of sources at build time
utils._set_deterministic_user(directory)
# _set_required():
#
# Mark this element and its runtime dependencies as required.
# This unblocks pull/fetch/build.
#
def _set_required(self):
if self.__required:
# Already done
return
self.__required = True
# Request artifacts of runtime dependencies
for dep in self.dependencies(Scope.RUN, recurse=False):
dep._set_required()
self._update_state()
# _is_required():
#
# Returns whether this element has been marked as required.
#
def _is_required(self):
return self.__required
# _schedule_assemble():
#
# This is called in the main process before the element is assembled
# in a subprocess.
#
def _schedule_assemble(self):
assert self._is_required()
assert not self.__assemble_scheduled
self.__assemble_scheduled = True
# Requests artifacts of build dependencies
for dep in self.dependencies(Scope.BUILD, recurse=False):
dep._set_required()
# Invalidate workspace key as the build modifies the workspace directory
workspace = self._get_workspace()
if workspace:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment