diff --git a/buildstream/element.py b/buildstream/element.py
index dddba27edc200b9fbf7f5150c61d277cfad5aa5f..24ea89664ed1aba0336f0e9c9bc4672802c59d3c 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1738,8 +1738,8 @@ class Element(Plugin):
                 metadir = os.path.join(self.__extract(), 'meta')
                 meta = _yaml.load(os.path.join(metadir, 'artifact.yaml'))
                 self.__cache_key = meta['keys']['strong']
-            elif self._buildable():
-                # Artifact will be built, not downloaded
+            elif self.__assemble_scheduled or self.__assemble_done:
+                # Artifact will or has been built, not downloaded
                 dependencies = [
                     e._get_cache_key() for e in self.dependencies(Scope.BUILD)
                 ]