buildstream/source.py

@@ -76,6 +76,39 @@ these methods are mandatory to implement.
   :ref:`SourceFetcher <core_source_fetcher>`.
 
 
+Accessing previous sources
+--------------------------
+*Since: 1.4*
+
+In the general case, all sources are fetched and tracked independently of one
+another. In situations where a source needs to access previous source(s) in
+order to perform its own track and/or fetch, following attributes can be set to
+request access to previous sources:
+
+* :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_TRACK`
+
+  Indicate that access to previous sources is required during track
+
+* :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH`
+
+  Indicate that access to previous sources is required during fetch
+
+The intended use of such plugins is to fetch external dependencies of other
+sources, typically using some kind of package manager, such that all the
+dependencies of the original source(s) are available at build time.
+
+When implementing such a plugin, implementors should adhere to the following
+guidelines:
+
+* Implementations must be able to store the obtained artifacts in a
+  subdirectory.
+
+* Implementations must be able to deterministically generate a unique ref, such
+  that two refs are different if and only if they produce different outputs.
+
+* Implementations must not introduce host contamination.
+
+
 .. _core_source_fetcher:
 
 SourceFetcher - Object for fetching individual URLs
@@ -92,6 +125,8 @@ mentioned, these methods are mandatory to implement.
   Fetches the URL associated with this SourceFetcher, optionally taking an
   alias override.
 
+Class Reference
+---------------
 """
 
 import os
@@ -156,7 +191,7 @@ class SourceFetcher():
     #############################################################
     #                      Abstract Methods                     #
     #############################################################
-    def fetch(self, alias_override=None):
+    def fetch(self, alias_override=None, **kwargs):
         """Fetch remote sources and mirror them locally, ensuring at least
         that the specific reference is cached locally.
 
@@ -209,6 +244,32 @@ class Source(Plugin):
     __defaults = {}          # The defaults from the project
     __defaults_set = False   # Flag, in case there are not defaults at all
 
+    BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False
+    """Whether access to previous sources is required during track
+
+    When set to True:
+      * all sources listed before this source in the given element will be
+        fetched before this source is tracked
+      * Source.track() will be called with an additional keyword argument
+        `previous_sources_dir` where previous sources will be staged
+      * this source can not be the first source for an element
+
+    *Since: 1.4*
+    """
+
+    BST_REQUIRES_PREVIOUS_SOURCES_FETCH = False
+    """Whether access to previous sources is required during fetch
+
+    When set to True:
+      * all sources listed before this source in the given element will be
+        fetched before this source is fetched
+      * Source.fetch() will be called with an additional keyword argument
+        `previous_sources_dir` where previous sources will be staged
+      * this source can not be the first source for an element
+
+    *Since: 1.4*
+    """
+
     def __init__(self, context, project, meta, *, alias_override=None):
         provenance = _yaml.node_get_provenance(meta.config)
         super().__init__("{}-{}".format(meta.element_name, meta.element_index),
@@ -305,9 +366,15 @@ class Source(Plugin):
         """
         raise ImplError("Source plugin '{}' does not implement set_ref()".format(self.get_kind()))
 
-    def track(self):
+    def track(self, **kwargs):
         """Resolve a new ref from the plugin's track option
 
+        Args:
+           previous_sources_dir (str): directory where previous sources are staged.
+                                       Note that this keyword argument is available only when
+                                       :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_TRACK`
+                                       is set to True.
+
         Returns:
            (simple object): A new internal source reference, or None
 
@@ -326,10 +393,16 @@ class Source(Plugin):
         # Allow a non implementation
         return None
 
-    def fetch(self):
+    def fetch(self, **kwargs):
         """Fetch remote sources and mirror them locally, ensuring at least
         that the specific reference is cached locally.
 
+        Args:
+           previous_sources_dir (str): directory where previous sources are staged.
+                                       Note that this keyword argument is available only when
+                                       :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH`
+                                       is set to True.
+
         Raises:
            :class:`.SourceError`
 
@@ -519,50 +592,19 @@ class Source(Plugin):
 
     # Wrapper function around plugin provided fetch method
     #
-    def _fetch(self):
-        project = self._get_project()
-        source_fetchers = self.get_source_fetchers()
-        if source_fetchers:
-            for fetcher in source_fetchers:
-                alias = fetcher._get_alias()
-                success = False
-                for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
-                    try:
-                        fetcher.fetch(uri)
-                    # FIXME: Need to consider temporary vs. permanent failures,
-                    #        and how this works with retries.
-                    except BstError as e:
-                        last_error = e
-                        continue
-                    success = True
-                    break
-                if not success:
-                    raise last_error
+    # Args:
+    #   previous_sources (list): List of Sources listed prior to this source
+    #
+    def _fetch(self, previous_sources):
+
+        if self.BST_REQUIRES_PREVIOUS_SOURCES_FETCH:
+            self.__ensure_previous_sources(previous_sources)
+            with self.tempdir() as staging_directory:
+                for src in previous_sources:
+                    src._stage(staging_directory)
+                self.__do_fetch(previous_sources_dir=self.__ensure_directory(staging_directory))
         else:
-            alias = self._get_alias()
-            if self.__first_pass:
-                mirrors = project.first_pass_config.mirrors
-            else:
-                mirrors = project.config.mirrors
-            if not mirrors or not alias:
-                self.fetch()
-                return
-
-            context = self._get_context()
-            source_kind = type(self)
-            for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
-                new_source = source_kind(context, project, self.__meta,
-                                         alias_override=(alias, uri))
-                new_source._preflight()
-                try:
-                    new_source.fetch()
-                # FIXME: Need to consider temporary vs. permanent failures,
-                #        and how this works with retries.
-                except BstError as e:
-                    last_error = e
-                    continue
-                return
-            raise last_error
+            self.__do_fetch()
 
     # Wrapper for stage() api which gives the source
     # plugin a fully constructed path considering the
@@ -773,8 +815,19 @@ class Source(Plugin):
 
     # Wrapper for track()
     #
-    def _track(self):
-        new_ref = self.__do_track()
+    # Args:
+    #   previous_sources (list): List of Sources listed prior to this source
+    #
+    def _track(self, previous_sources):
+        if self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK:
+            self.__ensure_previous_sources(previous_sources)
+            with self.tempdir() as staging_directory:
+                for src in previous_sources:
+                    src._stage(staging_directory)
+                new_ref = self.__do_track(previous_sources_dir=self.__ensure_directory(staging_directory))
+        else:
+            new_ref = self.__do_track()
+
         current_ref = self.get_ref()
 
         if new_ref is None:
@@ -786,6 +839,17 @@ class Source(Plugin):
 
         return new_ref
 
+    # _requires_previous_sources()
+    #
+    # If a plugin requires access to previous sources at track or fetch time,
+    # then it cannot be the first source of an elemenet.
+    #
+    # Returns:
+    #   (bool): Whether this source requires access to previous sources
+    #
+    def _requires_previous_sources(self):
+        return self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK or self.BST_REQUIRES_PREVIOUS_SOURCES_FETCH
+
     # Returns the alias if it's defined in the project
     def _get_alias(self):
         alias = self.__expected_alias
@@ -801,8 +865,54 @@ class Source(Plugin):
     #                   Local Private Methods                   #
     #############################################################
 
+    # Tries to call fetch for every mirror, stopping once it succeeds
+    def __do_fetch(self, **kwargs):
+        project = self._get_project()
+        source_fetchers = self.get_source_fetchers()
+        if source_fetchers:
+            for fetcher in source_fetchers:
+                alias = fetcher._get_alias()
+                success = False
+                for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
+                    try:
+                        fetcher.fetch(uri)
+                    # FIXME: Need to consider temporary vs. permanent failures,
+                    #        and how this works with retries.
+                    except BstError as e:
+                        last_error = e
+                        continue
+                    success = True
+                    break
+                if not success:
+                    raise last_error
+        else:
+            alias = self._get_alias()
+            if self.__first_pass:
+                mirrors = project.first_pass_config.mirrors
+            else:
+                mirrors = project.config.mirrors
+            if not mirrors or not alias:
+                self.fetch(**kwargs)
+                return
+
+            context = self._get_context()
+            source_kind = type(self)
+            for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
+                new_source = source_kind(context, project, self.__meta,
+                                         alias_override=(alias, uri))
+                new_source._preflight()
+                try:
+                    new_source.fetch(**kwargs)
+                # FIXME: Need to consider temporary vs. permanent failures,
+                #        and how this works with retries.
+                except BstError as e:
+                    last_error = e
+                    continue
+                return
+            raise last_error
+
     # Tries to call track for every mirror, stopping once it succeeds
-    def __do_track(self):
+    def __do_track(self, **kwargs):
         project = self._get_project()
         # If there are no mirrors, or no aliases to replace, there's nothing to do here.
         alias = self._get_alias()
@@ -811,7 +921,7 @@ class Source(Plugin):
         else:
             mirrors = project.config.mirrors
         if not mirrors or not alias:
-            return self.track()
+            return self.track(**kwargs)
 
         context = self._get_context()
         source_kind = type(self)
@@ -823,7 +933,7 @@ class Source(Plugin):
                                      alias_override=(alias, uri))
             new_source._preflight()
             try:
-                ref = new_source.track()
+                ref = new_source.track(**kwargs)
             # FIXME: Need to consider temporary vs. permanent failures,
             #        and how this works with retries.
             except BstError as e:
@@ -867,3 +977,14 @@ class Source(Plugin):
         _yaml.node_final_assertions(config)
 
         return config
+
+    # Ensures that previous sources have been tracked and fetched.
+    #
+    def __ensure_previous_sources(self, previous_sources):
+        for index, src in enumerate(previous_sources):
+            # BuildStream should track sources in the order they appear so
+            # previous sources should never be in an inconsistent state
+            assert src.get_consistency() != Consistency.INCONSISTENT
+
+            if src.get_consistency() == Consistency.RESOLVED:
+                src._fetch(previous_sources[0:index])

buildstream/storage/_filebaseddirectory.py

@@ -29,25 +29,12 @@ See also: :ref:`sandboxing`.
 
 import os
 import time
-from .._exceptions import BstError, ErrorDomain
-from .directory import Directory
+from .directory import Directory, VirtualDirectoryError
 from ..utils import link_files, copy_files, list_relative_paths, _get_link_mtime, _magic_timestamp
 from ..utils import _set_deterministic_user, _set_deterministic_mtime
 
-
-class VirtualDirectoryError(BstError):
-    """Raised by Directory functions when system calls fail.
-    This will be handled internally by the BuildStream core,
-    if you need to handle this error, then it should be reraised,
-    or either of the :class:`.ElementError` or :class:`.SourceError`
-    exceptions should be raised from this error.
-    """
-    def __init__(self, message, reason=None):
-        super().__init__(message, domain=ErrorDomain.VIRTUAL_FS, reason=reason)
-
-
 # FileBasedDirectory intentionally doesn't call its superclass constuctor,
-# which is mean to be unimplemented.
+# which is meant to be unimplemented.
 # pylint: disable=super-init-not-called
 
 
@@ -108,7 +95,8 @@ class FileBasedDirectory(Directory):
             if create:
                 new_path = os.path.join(self.external_directory, subdirectory_spec[0])
                 os.makedirs(new_path, exist_ok=True)
-                return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create)
+                self.index[subdirectory_spec[0]] = FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create)
+                return self.index[subdirectory_spec[0]]
             else:
                 error = "No entry called '{}' found in the directory rooted at {}"
                 raise VirtualDirectoryError(error.format(subdirectory_spec[0], self.external_directory))
@@ -134,8 +122,12 @@ class FileBasedDirectory(Directory):
 
             for f in import_result.files_written:
                 os.utime(os.path.join(self.external_directory, f), times=(cur_time, cur_time))
+        self._mark_changed()
         return import_result
 
+    def _mark_changed(self):
+        self._directory_read = False
+
     def set_deterministic_mtime(self):
         _set_deterministic_mtime(self.external_directory)
 
@@ -214,3 +206,8 @@ class FileBasedDirectory(Directory):
         # which exposes the sandbox directory; we will have to assume for the time being
         # that people will not abuse __str__.
         return self.external_directory
+
+    def _get_underlying_directory(self) -> str:
+        """ Returns the underlying (real) file system directory this
+        object refers to. """
+        return self.external_directory

buildstream/storage/directory.py

@@ -31,6 +31,19 @@ See also: :ref:`sandboxing`.
 
 """
 
+from .._exceptions import BstError, ErrorDomain
+
+
+class VirtualDirectoryError(BstError):
+    """Raised by Directory functions when system calls fail.
+    This will be handled internally by the BuildStream core,
+    if you need to handle this error, then it should be reraised,
+    or either of the :class:`.ElementError` or :class:`.SourceError`
+    exceptions should be raised from this error.
+    """
+    def __init__(self, message, reason=None):
+        super().__init__(message, domain=ErrorDomain.VIRTUAL_FS, reason=reason)
+
 
 class Directory():
     def __init__(self, external_directory=None):
@@ -153,3 +166,13 @@ class Directory():
 
         """
         raise NotImplementedError()
+
+    def _mark_changed(self):
+        """Internal function to mark this directory as having been changed
+        outside this API. This normally can only happen by calling the
+        Sandbox's `run` method. This does *not* mark everything as modified
+        (i.e. list_modified_paths will not necessarily return the same results
+        as list_relative_paths after calling this.)
+
+        """
+        raise NotImplementedError()

doc/source/core_plugins.rst

@@ -58,6 +58,7 @@ Sources
    sources/ostree
    sources/patch
    sources/deb
+   sources/pip
 
 
 External plugins

doc/source/format_project.rst

@@ -126,22 +126,29 @@ following to your ``project.conf``:
    The ``ref-storage`` configuration is available since :ref:`format version 8 <project_format_version>`
 
 
-Fail on overlaps
-~~~~~~~~~~~~~~~~
-When multiple elements are staged, there's a possibility that different
-elements will try and stage different versions of the same file.
+.. _configurable_warnings:
 
-When ``fail-on-overlap`` is true, if an overlap is detected
-that hasn't been allowed by the element's
-:ref:`overlap whitelist<public_overlap_whitelist>`,
-then an error will be raised and the build will fail.
+Configurable Warnings
+~~~~~~~~~~~~~~~~~~~~~
+Warnings can be configured as fatal using the ``fatal-warnings`` configuration item.
+When a warning is configured as fatal, where a warning would usually be thrown instead an error will be thrown
+causing the build to fail.
 
-otherwise, a warning will be raised indicating which files had overlaps,
-and the order that the elements were overlapped.
+Individual warnings can be configured as fatal by setting ``fatal-warnings`` to a list of warnings.
 
 .. code:: yaml
 
-  fail-on-overlap: true
+  fatal-warnings:
+  - overlaps
+  - ref-not-in-track
+  - <plugin>:<warning>
+
+BuildStream provides a collection of :class:`Core Warnings <buildstream.plugin.CoreWarnings>` which may be raised
+by a variety of plugins. Other configurable warnings are plugin specific and should be noted within their individual documentation.
+
+.. note::
+
+  The ``fatal-warnings`` configuration is available since :ref:`format version 14 <project_format_version>`
 
 
 .. _project_source_aliases:

tests/artifactcache/expiry.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Maat <tristan.maat@codethink.co.uk>
+#
+
 import os
 
 import pytest
@@ -5,7 +24,7 @@ import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 
-from tests.testutils import cli, create_element_size
+from tests.testutils import cli, create_element_size, wait_for_cache_granularity
 
 
 DATA_DIR = os.path.join(
@@ -108,6 +127,8 @@ def test_expiry_order(cli, datafiles, tmpdir):
     res = cli.run(project=project, args=['build', 'target2.bst'])
     res.assert_success()
 
+    wait_for_cache_granularity()
+
     # Now extract dep.bst
     res = cli.run(project=project, args=['checkout', 'dep.bst', checkout])
     res.assert_success()

tests/cachekey/cachekey.py

@@ -37,7 +37,8 @@
 #
 from tests.testutils.runcli import cli
 from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX
-
+from buildstream.plugin import CoreWarnings
+from buildstream import _yaml
 import os
 from collections import OrderedDict
 import pytest
@@ -128,7 +129,6 @@ def assert_cache_keys(project_dir, output):
                              "Use tests/cachekey/update.py to automatically " +
                              "update this test case")
 
-
 ##############################################
 #             Test Entry Point               #
 ##############################################
@@ -167,3 +167,47 @@ def test_cache_key(datafiles, cli):
     ])
     result.assert_success()
     assert_cache_keys(project, result.output)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("first_warnings, second_warnings, identical_keys", [
+    [[], [], True],
+    [[], [CoreWarnings.REF_NOT_IN_TRACK], False],
+    [[CoreWarnings.REF_NOT_IN_TRACK], [], False],
+    [[CoreWarnings.REF_NOT_IN_TRACK], [CoreWarnings.REF_NOT_IN_TRACK], True],
+    [[CoreWarnings.REF_NOT_IN_TRACK, CoreWarnings.OVERLAPS],
+        [CoreWarnings.OVERLAPS, CoreWarnings.REF_NOT_IN_TRACK], True],
+])
+def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings, identical_keys):
+
+    # Builds project, Runs bst show, gathers cache keys
+    def run_get_cache_key(project_name, warnings):
+        config = {
+            'name': project_name,
+            'element-path': 'elements',
+            'fatal-warnings': warnings
+        }
+
+        project_dir = tmpdir.mkdir(project_name)
+        project_config_file = str(project_dir.join('project.conf'))
+        _yaml.dump(_yaml.node_sanitize(config), filename=project_config_file)
+
+        elem_dir = project_dir.mkdir('elements')
+        element_file = str(elem_dir.join('stack.bst'))
+        _yaml.dump({'kind': 'stack'}, filename=element_file)
+
+        result = cli.run(project=str(project_dir), args=[
+            'show',
+            '--format', '%{name}::%{full-key}',
+            'stack.bst'
+        ])
+        return result.output
+
+    # Returns true if all keys are identical
+    def compare_cache_keys(first_keys, second_keys):
+        return not any((x != y for x, y in zip(first_keys, second_keys)))
+
+    first_keys = run_get_cache_key("first", first_warnings)
+    second_keys = run_get_cache_key("second", second_warnings)
+
+    assert compare_cache_keys(first_keys, second_keys) == identical_keys

tests/cachekey/project/elements/build1.expected

-594be3eb2a211f706557b156ec4b0ffb3ca256af35bcd0116b97fdb8c942d1c5
\ No newline at end of file
+90fa9c1b5334aac3bfd8956d92af0a91e2f2bbcbcac73861957165e577bc8768
\ No newline at end of file

tests/cachekey/project/elements/build2.expected

-5038d37bf1714180d160271e688ec1715d69666ca266ed5b1d880abdee36b310
\ No newline at end of file
+a62c29fe4e05820412e391430f61aa88a1a82e138de8cac726a9dc4fcd7ed8b9
\ No newline at end of file

tests/cachekey/project/elements/compose1.expected

-e050172c2d445849a2716720814501f0e294b81d28be8cd911ee294291ec60d0
\ No newline at end of file
+4e4c719242aa45fed398cc2fb8936195a1fcae9326d808de7fee340ae48862ea
\ No newline at end of file

tests/cachekey/project/elements/compose2.expected

-ad985f243163aab22576fea63a0b89f7560e361326cb041872c568c5feaabf5c
\ No newline at end of file
+2fc1dd398f6c6f6e1d7ca48d88557e133d2130278882e14cd1105b15a600cd7a
\ No newline at end of file

tests/cachekey/project/elements/compose3.expected

-ec0fec2c821eb34286e9799a6e8d1f5587f161d1d653bd1dbe385602340d86ae
\ No newline at end of file
+4d7c9e2e1e8cfcc4b300a11693767f41f22c7829db9063dec10856328d03ccc3
\ No newline at end of file

tests/cachekey/project/elements/compose4.expected

-7cfe8f6161c00d8cf10114e7458f0b97eb003a41504ae301b24f45d48d42155b
\ No newline at end of file
+cad8f3b622f4a906f9fc3f5187a7703e2b17dfc550dd5a07479ca3ebffbd5c86
\ No newline at end of file

tests/cachekey/project/elements/compose5.expected

-7f016c3165f2de9161312b74f29513dff7dfdcba5ff8c6897beb5b123eaafd3d
\ No newline at end of file
+4fd21699827aa16da8d7a1525020f9fd45422f0431749510947ff472d76c1802
\ No newline at end of file

tests/cachekey/project/elements/import1.expected

-41ce5a640fdfd7b6ce8a2c3fa1dde7983bc4df0e4c3ca926670118bae3c051fe
\ No newline at end of file
+aa443ea4607d7dd5a0c99646a1b827e3165862772fc1b26e20195aadd2ab8885
\ No newline at end of file

tests/cachekey/project/elements/import2.expected

-d657db503460486a6de80d87f15c6b1fa84b0c4dacabed374acdc70172d4761d
\ No newline at end of file
+18ea6bbb968ca6945c8c2941650f447b8df58673be7270c967c8152730eff036
\ No newline at end of file

tests/cachekey/project/elements/import3.expected

-8de0293a6231dc661cf7229aa5a2f25abdf9a6d38ff70bd6f2562dae51ff05d3
\ No newline at end of file
+34ce4816b0307f0691302460367ab24b4e1f86e61c0e307e68bcd6833946c1f1
\ No newline at end of file

tests/cachekey/project/elements/script1.expected

-d0a6b7d29226b083c404d76d9551a0eee98753058580cd62901f8dfac06ca08d
\ No newline at end of file
+c48922b3d80d36e6d30bed7581aa1473a5e463321b3a19606b603c055d2b4be4
\ No newline at end of file

tests/cachekey/project/sources/bzr1.expected

-51415ebc7d72315c5c7704759025d6a9237e786bfe9c2bda8c51e15840c3470a
\ No newline at end of file
+ee271c8f469cd33330229d8dcc44e26f3480a9f47b55db46f42d1a396a94609e
\ No newline at end of file