buildstream/__init__.py

@@ -34,3 +34,8 @@ if "_BST_COMPLETION" not in os.environ:
     from .element import Element, ElementError
     from .buildelement import BuildElement
     from .scriptelement import ScriptElement
+
+    # XXX We are exposing a private member here as we expect it to move to a
+    # separate package soon. See the following discussion for more details:
+    # https://gitlab.com/BuildStream/buildstream/issues/739#note_124819869
+    from ._gitsourcebase import _GitSourceBase

buildstream/_scheduler/queues/fetchqueue.py

@@ -40,10 +40,7 @@ class FetchQueue(Queue):
         self._skip_cached = skip_cached
 
     def process(self, element):
-        previous_sources = []
-        for source in element.sources():
-            source._fetch(previous_sources)
-            previous_sources.append(source)
+        element._fetch()
 
     def status(self, element):
         # state of dependencies may have changed, recalculate element state

buildstream/element.py

@@ -2022,6 +2022,20 @@ class Element(Plugin):
 
         return True
 
+    # _fetch()
+    #
+    # Fetch the element's sources.
+    #
+    # Raises:
+    #    SourceError: If one of the element sources has an error
+    #
+    def _fetch(self):
+        previous_sources = []
+        for source in self.sources():
+            if source._get_consistency() < Consistency.CACHED:
+                source._fetch(previous_sources)
+            previous_sources.append(source)
+
     #############################################################
     #                   Private Local Methods                   #
     #############################################################

tests/sources/no-fetch-cached/files/file

+Hello World!

tests/sources/no-fetch-cached/plugins/sources/always_cached.py

+"""
+always_cached
+=============
+
+This is a test source plugin that is always cached.
+Used to test that BuildStream core does not call fetch() for cached sources.
+
+"""
+
+from buildstream import Consistency, Source
+
+
+class AlwaysCachedSource(Source):
+
+    def configure(self, node):
+        pass
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        return None
+
+    def get_consistency(self):
+        return Consistency.CACHED
+
+    def load_ref(self, node):
+        pass
+
+    def get_ref(self):
+        return None
+
+    def set_ref(self, ref, node):
+        pass
+
+    def fetch(self):
+        # Source is always cached, so fetch() should never be called
+        assert False
+
+    def stage(self, directory):
+        pass
+
+
+def setup():
+    return AlwaysCachedSource

tests/sources/no-fetch-cached/project.conf

+# Project with local source plugins
+name: no-fetch-cached
+
+plugins:
+- origin: local
+  path: plugins/sources
+  sources:
+    always_cached: 0

tests/sources/no_fetch_cached.py

+import os
+import pytest
+
+from buildstream import _yaml
+
+from tests.testutils import cli, create_repo
+from tests.testutils.site import HAVE_GIT
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'no-fetch-cached'
+)
+
+
+##################################################################
+#                              Tests                             #
+##################################################################
+# Test that fetch() is not called for cached sources
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(DATA_DIR)
+def test_no_fetch_cached(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    # Create the repo from 'files' subdir
+    repo = create_repo('git', str(tmpdir))
+    ref = repo.create(os.path.join(project, 'files'))
+
+    # Write out test target with a cached and a non-cached source
+    element = {
+        'kind': 'import',
+        'sources': [
+            repo.source_config(ref=ref),
+            {
+                'kind': 'always_cached'
+            }
+        ]
+    }
+    _yaml.dump(element, os.path.join(project, 'target.bst'))
+
+    # Test fetch of target with a cached and a non-cached source
+    result = cli.run(project=project, args=[
+        'source', 'fetch', 'target.bst'
+    ])
+    result.assert_success()