From f57a9da9a79aa2dcb769f3599ec25d1c567216fd Mon Sep 17 00:00:00 2001
From: Tristan Maat <tristan.maat@codethink.com>
Date: Thu, 13 Jul 2017 17:36:30 +0100
Subject: [PATCH] WIP: Add workspace artifact cache metadata
---
buildstream/_pipeline.py | 4 +---
buildstream/element.py | 51 ++++++++++++++++++++++++++++++++++++----
2 files changed, 47 insertions(+), 8 deletions(-)
mode change 100644 => 100755 buildstream/element.py
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index c87eb9ecbe..cf5938384b 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -164,9 +164,7 @@ class Pipeline():
self.unused_workspaces.append((element_name, source, workspace))
continue
- # Usually few sources, performance should be fine
- sources = list(element.sources())
- sources[source]._set_workspace(workspace)
+ element._set_source_workspace(source, workspace)
for element in self.dependencies(Scope.ALL):
if cache_ticker:
diff --git a/buildstream/element.py b/buildstream/element.py
old mode 100644
new mode 100755
index 964e082d80..17c874a66f
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -120,6 +120,9 @@ class Element(Plugin):
self.__config = self.__extract_config(meta)
self.configure(self.__config)
+ # Generic artifact data for this element
+ self.__artifact_meta = self._load_artifact_meta()
+
def __lt__(self, other):
return self.name < other.name
@@ -692,6 +695,17 @@ class Element(Plugin):
def _set_cached(self):
self.__cached = True
+ # _tainted():
+ #
+ # Returns:
+ # (bool) Whether this element should be excluded from pushing.
+ #
+ def _tainted(self):
+ workspaced = _yaml.node_get(self.__artifact_meta, bool, 'workspaced', default_value=False)
+
+ # Other conditions should be or-ed
+ return workspaced
+
# _set_built():
#
# Forcefully set the built state on the element.
@@ -880,8 +894,18 @@ class Element(Plugin):
if self.__log_path:
shutil.copyfile(self.__log_path, os.path.join(logsdir, 'build.log'))
+ # Update workspace status
+ #
+ # FIXME: Does this work in the case that a source
+ # is not cached but does have a workspace?
+ #
+ self.__artifact_meta['workspaced'] = any(source._has_workspace()
+ for source in self.sources())
+
# Store public data
_yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
+ # Store artifact metadata
+ _yaml.dump(_yaml.node_sanitize(self.__artifact_meta), os.path.join(metadir, 'artifact.yaml'))
with self.timed_activity("Caching Artifact"):
self.__artifacts.commit(self, assembledir)
@@ -943,6 +967,12 @@ class Element(Plugin):
def _push(self):
self._assert_cached()
+ if self._tainted():
+ self.warn("Not pushing tainted artifact.",
+ detail=("The artifact was built with a workspaced source"
+ if self.__artifact_meta['workspaced'] else ""))
+ return False
+
with self.timed_activity("Pushing Artifact"):
return self.__artifacts.push(self)
@@ -983,14 +1013,16 @@ class Element(Plugin):
os.makedirs(directory, exist_ok=True)
return os.path.join(directory, logfile)
+ # Set a source's workspace
+ #
+ def _set_source_workspace(self, source_index, path):
+ self.__sources[source_index]._set_workspace(path)
+ self.__artifact_meta['workspaced'] = True
+
# Whether this element has a source that is workspaced.
#
def _workspaced(self):
- for source in self.sources():
- if source._has_workspace():
- return True
-
- return False
+ return self.__artifact_meta['workspaced']
# Get all source workspace directories.
#
@@ -1344,3 +1376,12 @@ class Element(Plugin):
# Load the public data from the artifact
metadir = os.path.join(self.__artifacts.extract(self), 'meta')
self.__dynamic_public = _yaml.load(os.path.join(metadir, 'public.yaml'))
+
+ def _load_artifact_meta(self):
+ if not self._cached():
+ return {}
+
+ assert(self.__artifact_meta is None)
+
+ metadir = os.path.join(self.__artifacts.extract(self), 'meta')
+ self.__artifact_meta = _yaml.load(os.path.join(metadir, 'artifact.yaml'))
--
GitLab