buildstream/_project.py

@@ -31,6 +31,7 @@ from ._options import OptionPool
 from ._artifactcache import ArtifactCache
 from ._elementfactory import ElementFactory
 from ._sourcefactory import SourceFactory
+from .plugin import CoreWarnings
 from ._projectrefs import ProjectRefs, ProjectRefStorage
 from ._versions import BST_FORMAT_VERSION
 from ._loader import Loader
@@ -105,7 +106,7 @@ class Project():
         self.first_pass_config = ProjectConfig()
 
         self.junction = junction                 # The junction Element object, if this is a subproject
-        self.fail_on_overlap = False             # Whether overlaps are treated as errors
+
         self.ref_storage = None                  # ProjectRefStorage setting
         self.base_environment = {}               # The base set of environment variables
         self.base_env_nocache = None             # The base nocache mask (list) for the environment
@@ -120,6 +121,8 @@ class Project():
         self._cli_options = cli_options
         self._cache_key = None
 
+        self._fatal_warnings = []             # A list of warnings which should trigger an error
+
         self._shell_command = []      # The default interactive shell command
         self._shell_environment = {}  # Statically set environment vars
         self._shell_host_files = []   # A list of HostMount objects
@@ -456,7 +459,7 @@ class Project():
             'split-rules', 'elements', 'plugins',
             'aliases', 'name',
             'artifacts', 'options',
-            'fail-on-overlap', 'shell',
+            'fail-on-overlap', 'shell', 'fatal-warnings',
             'ref-storage', 'sandbox', 'mirrors'
         ])
 
@@ -478,8 +481,25 @@ class Project():
         # Load project split rules
         self._splits = _yaml.node_get(config, Mapping, 'split-rules')
 
-        # Fail on overlap
-        self.fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap')
+        # Fatal warnings
+        self._fatal_warnings = _yaml.node_get(config, list, 'fatal-warnings', default_value=[])
+
+        # Support backwards compatibility for fail-on-overlap
+        fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap', default_value=None)
+
+        if (CoreWarnings.OVERLAPS not in self._fatal_warnings) and fail_on_overlap:
+            self._fatal_warnings.append(CoreWarnings.OVERLAPS)
+
+        # Deprecation check
+        if fail_on_overlap is not None:
+            self._context.message(
+                Message(
+                    None,
+                    MessageType.WARN,
+                    "Use of fail-on-overlap within project.conf " +
+                    "is deprecated. Consider using fatal-warnings instead."
+                )
+            )
 
         # Load project.refs if it exists, this may be ignored.
         if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
@@ -712,3 +732,17 @@ class Project():
                 # paths are passed in relative to the project, but must be absolute
                 origin_dict['path'] = os.path.join(self.directory, path)
             destination.append(origin_dict)
+
+    # _warning_is_fatal():
+    #
+    # Returns true if the warning in question should be considered fatal based on
+    # the project configuration.
+    #
+    # Args:
+    #   warning_str (str): The warning configuration string to check against
+    #
+    # Returns:
+    #    (bool): True if the warning should be considered fatal and cause an error.
+    #
+    def _warning_is_fatal(self, warning_str):
+        return warning_str in self._fatal_warnings

buildstream/_versions.py

@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 14
+BST_FORMAT_VERSION = 15
 
 
 # The base BuildStream artifact version
@@ -33,4 +33,4 @@ BST_FORMAT_VERSION = 14
 # or if buildstream was changed in a way which can cause
 # the same cache key to produce something that is no longer
 # the same.
-BST_CORE_ARTIFACT_VERSION = 4
+BST_CORE_ARTIFACT_VERSION = 5

buildstream/data/projectconfig.yaml

@@ -13,10 +13,6 @@ element-path: .
 # Store source references in element files
 ref-storage: inline
 
-# Overlaps are just warnings
-fail-on-overlap: False
-
-
 # Variable Configuration
 #
 variables:

buildstream/element.py

@@ -94,6 +94,7 @@ from . import _cachekey
 from . import _signals
 from . import _site
 from ._platform import Platform
+from .plugin import CoreWarnings
 from .sandbox._config import SandboxConfig
 
 from .storage.directory import Directory
@@ -746,32 +747,23 @@ class Element(Plugin):
                 ignored[dep.name] = result.ignored
 
         if overlaps:
-            overlap_error = overlap_warning = False
-            error_detail = warning_detail = "Staged files overwrite existing files in staging area:\n"
+            overlap_warning = False
+            warning_detail = "Staged files overwrite existing files in staging area:\n"
             for f, elements in overlaps.items():
-                overlap_error_elements = []
                 overlap_warning_elements = []
                 # The bottom item overlaps nothing
                 overlapping_elements = elements[1:]
                 for elm in overlapping_elements:
                     element = self.search(scope, elm)
-                    element_project = element._get_project()
                     if not element.__file_is_whitelisted(f):
-                        if element_project.fail_on_overlap:
-                            overlap_error_elements.append(elm)
-                            overlap_error = True
-                        else:
                         overlap_warning_elements.append(elm)
                         overlap_warning = True
 
                 warning_detail += _overlap_error_detail(f, overlap_warning_elements, elements)
-                error_detail += _overlap_error_detail(f, overlap_error_elements, elements)
 
             if overlap_warning:
-                self.warn("Non-whitelisted overlaps detected", detail=warning_detail)
-            if overlap_error:
-                raise ElementError("Non-whitelisted overlaps detected and fail-on-overlaps is set",
-                                   detail=error_detail, reason="overlap-error")
+                self.warn("Non-whitelisted overlaps detected", detail=warning_detail,
+                          warning_token=CoreWarnings.OVERLAPS)
 
         if ignored:
             detail = "Not staging files which would replace non-empty directories:\n"
@@ -2054,9 +2046,7 @@ class Element(Plugin):
                 'cache': type(self.__artifacts).__name__
             }
 
-            # fail-on-overlap setting cannot affect elements without dependencies
-            if project.fail_on_overlap and dependencies:
-                self.__cache_key_dict['fail-on-overlap'] = True
+            self.__cache_key_dict['fatal-warnings'] = sorted(project._fatal_warnings)
 
         cache_key_dict = self.__cache_key_dict.copy()
         cache_key_dict['dependencies'] = dependencies

buildstream/plugin.py

@@ -47,6 +47,23 @@ it is mandatory to implement the following abstract methods:
   Once all configuration has been loaded and preflight checks have passed,
   this method is used to inform the core of a plugin's unique configuration.
 
+Configurable Warnings
+---------------------
+Warnings raised through calling :func:`Plugin.warn() <buildstream.plugin.Plugin.warn>` can provide an optional
+parameter ``warning_token``, this will raise a :class:`PluginError` if the warning is configured as fatal within
+the project configuration.
+
+Configurable warnings will be prefixed with :func:`Plugin.get_kind() <buildstream.plugin.Plugin.get_kind>`
+within buildstream and must be prefixed as such in project configurations. For more detail on project configuration
+see :ref:`Configurable Warnings <configurable_warnings>`.
+
+It is important to document these warnings in your plugin documentation to allow users to make full use of them
+while configuring their projects.
+
+Example
+~~~~~~~
+If the :class:`git <buildstream.plugins.sources.git.GitSource>` plugin uses the warning ``"inconsistent-submodule"``
+then it could be referenced in project configuration as ``"git:inconsistent-submodule"``.
 
 Plugin Structure
 ----------------
@@ -166,7 +183,6 @@ class Plugin():
         # Infer the kind identifier
         modulename = type(self).__module__
         self.__kind = modulename.split('.')[-1]
-
         self.debug("Created: {}".format(self))
 
     def __del__(self):
@@ -473,14 +489,28 @@ class Plugin():
         """
         self.__message(MessageType.INFO, brief, detail=detail)
 
-    def warn(self, brief, *, detail=None):
-        """Print a warning message
+    def warn(self, brief, *, detail=None, warning_token=None):
+        """Print a warning message, checks warning_token against project configuration
 
         Args:
            brief (str): The brief message
            detail (str): An optional detailed message, can be multiline output
+           warning_token (str): An optional configurable warning assosciated with this warning,
+                                this will cause PluginError to be raised if this warning is configured as fatal.
+                                (*Since 1.4*)
+
+        Raises:
+           (:class:`.PluginError`): When warning_token is considered fatal by the project configuration
         """
-        self.__message(MessageType.WARN, brief, detail=detail)
+        if warning_token:
+            warning_token = _prefix_warning(self, warning_token)
+            brief = "[{}]: {}".format(warning_token, brief)
+            project = self._get_project()
+
+            if project._warning_is_fatal(warning_token):
+                raise PluginError(message="{}\n{}".format(brief, detail), reason=warning_token)
+
+        self.__message(MessageType.WARN, brief=brief, detail=detail)
 
     def log(self, brief, *, detail=None):
         """Log a message into the plugin's log file
@@ -709,6 +739,32 @@ class Plugin():
             return self.name
 
 
+class CoreWarnings():
+    """CoreWarnings()
+
+    Some common warnings which are raised by core functionalities within BuildStream are found in this class.
+    """
+
+    OVERLAPS = "overlaps"
+    """
+    This warning will be produced when buildstream detects an overlap on an element
+        which is not whitelisted. See :ref:`Overlap Whitelist <public_overlap_whitelist>`
+    """
+
+    REF_NOT_IN_TRACK = "ref-not-in-track"
+    """
+    This warning will be produced when a source is configured with a reference
+    which is found to be invalid based on the configured track
+    """
+
+
+__CORE_WARNINGS = [
+    value
+    for name, value in CoreWarnings.__dict__.items()
+    if not name.startswith("__")
+]
+
+
 # Hold on to a lookup table by counter of all instantiated plugins.
 # We use this to send the id back from child processes so we can lookup
 # corresponding element/source in the master process.
@@ -739,6 +795,23 @@ def _plugin_lookup(unique_id):
     return __PLUGINS_TABLE[unique_id]
 
 
+# _prefix_warning():
+#
+# Prefix a warning with the plugin kind. CoreWarnings are not prefixed.
+#
+# Args:
+#   plugin (Plugin): The plugin which raised the warning
+#   warning (str): The warning to prefix
+#
+# Returns:
+#    (str): A prefixed warning
+#
+def _prefix_warning(plugin, warning):
+    if any((warning is core_warning for core_warning in __CORE_WARNINGS)):
+        return warning
+    return "{}:{}".format(plugin.get_kind(), warning)
+
+
 # No need for unregister, WeakValueDictionary() will remove entries
 # in itself when the referenced plugins are garbage collected.
 def _plugin_register(plugin):

buildstream/plugins/sources/git.py

@@ -68,6 +68,12 @@ git - stage files from a git repository
        url: upstream:baz.git
        checkout: False
 
+**Configurable Warnings:**
+
+This plugin provides the following configurable warnings:
+
+- 'git:inconsistent-submodule' - A submodule was found to be missing from the underlying git repository.
+
 """
 
 import os
@@ -84,6 +90,9 @@ from buildstream import utils
 
 GIT_MODULES = '.gitmodules'
 
+# Warnings
+INCONSISTENT_SUBMODULE = "inconsistent-submodules"
+
 
 # Because of handling of submodules, we maintain a GitMirror
 # for the primary git source and also for each submodule it
@@ -283,7 +292,7 @@ class GitMirror(SourceFetcher):
                      "underlying git repository with `git submodule add`."
 
             self.source.warn("{}: Ignoring inconsistent submodule '{}'"
-                             .format(self.source, submodule), detail=detail)
+                             .format(self.source, submodule), detail=detail, warning_token=INCONSISTENT_SUBMODULE)
 
             return None
 

doc/source/format_project.rst

@@ -126,22 +126,29 @@ following to your ``project.conf``:
    The ``ref-storage`` configuration is available since :ref:`format version 8 <project_format_version>`
 
 
-Fail on overlaps
-~~~~~~~~~~~~~~~~
-When multiple elements are staged, there's a possibility that different
-elements will try and stage different versions of the same file.
+.. _configurable_warnings:
 
-When ``fail-on-overlap`` is true, if an overlap is detected
-that hasn't been allowed by the element's
-:ref:`overlap whitelist<public_overlap_whitelist>`,
-then an error will be raised and the build will fail.
+Configurable Warnings
+~~~~~~~~~~~~~~~~~~~~~
+Warnings can be configured as fatal using the ``fatal-warnings`` configuration item.
+When a warning is configured as fatal, where a warning would usually be thrown instead an error will be thrown
+causing the build to fail.
 
-otherwise, a warning will be raised indicating which files had overlaps,
-and the order that the elements were overlapped.
+Individual warnings can be configured as fatal by setting ``fatal-warnings`` to a list of warnings.
 
 .. code:: yaml
 
-  fail-on-overlap: true
+  fatal-warnings:
+  - overlaps
+  - ref-not-in-track
+  - <plugin>:<warning>
+
+BuildStream provides a collection of :class:`Core Warnings <buildstream.plugin.CoreWarnings>` which may be raised
+by a variety of plugins. Other configurable warnings are plugin specific and should be noted within their individual documentation.
+
+.. note::
+
+  The ``fatal-warnings`` configuration is available since :ref:`format version 14 <project_format_version>`
 
 
 .. _project_source_aliases:

tests/cachekey/cachekey.py

@@ -37,7 +37,8 @@
 #
 from tests.testutils.runcli import cli
 from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX
-
+from buildstream.plugin import CoreWarnings
+from buildstream import _yaml
 import os
 from collections import OrderedDict
 import pytest
@@ -128,7 +129,6 @@ def assert_cache_keys(project_dir, output):
                              "Use tests/cachekey/update.py to automatically " +
                              "update this test case")
 
-
 ##############################################
 #             Test Entry Point               #
 ##############################################
@@ -167,3 +167,47 @@ def test_cache_key(datafiles, cli):
     ])
     result.assert_success()
     assert_cache_keys(project, result.output)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("first_warnings, second_warnings, identical_keys", [
+    [[], [], True],
+    [[], [CoreWarnings.REF_NOT_IN_TRACK], False],
+    [[CoreWarnings.REF_NOT_IN_TRACK], [], False],
+    [[CoreWarnings.REF_NOT_IN_TRACK], [CoreWarnings.REF_NOT_IN_TRACK], True],
+    [[CoreWarnings.REF_NOT_IN_TRACK, CoreWarnings.OVERLAPS],
+        [CoreWarnings.OVERLAPS, CoreWarnings.REF_NOT_IN_TRACK], True],
+])
+def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings, identical_keys):
+
+    # Builds project, Runs bst show, gathers cache keys
+    def run_get_cache_key(project_name, warnings):
+        config = {
+            'name': project_name,
+            'element-path': 'elements',
+            'fatal-warnings': warnings
+        }
+
+        project_dir = tmpdir.mkdir(project_name)
+        project_config_file = str(project_dir.join('project.conf'))
+        _yaml.dump(_yaml.node_sanitize(config), filename=project_config_file)
+
+        elem_dir = project_dir.mkdir('elements')
+        element_file = str(elem_dir.join('stack.bst'))
+        _yaml.dump({'kind': 'stack'}, filename=element_file)
+
+        result = cli.run(project=str(project_dir), args=[
+            'show',
+            '--format', '%{name}::%{full-key}',
+            'stack.bst'
+        ])
+        return result.output
+
+    # Returns true if all keys are identical
+    def compare_cache_keys(first_keys, second_keys):
+        return not any((x != y for x, y in zip(first_keys, second_keys)))
+
+    first_keys = run_get_cache_key("first", first_warnings)
+    second_keys = run_get_cache_key("second", second_warnings)
+
+    assert compare_cache_keys(first_keys, second_keys) == identical_keys

tests/cachekey/project/elements/build1.expected

-594be3eb2a211f706557b156ec4b0ffb3ca256af35bcd0116b97fdb8c942d1c5
\ No newline at end of file
+90fa9c1b5334aac3bfd8956d92af0a91e2f2bbcbcac73861957165e577bc8768
\ No newline at end of file

tests/cachekey/project/elements/build2.expected

-5038d37bf1714180d160271e688ec1715d69666ca266ed5b1d880abdee36b310
\ No newline at end of file
+a62c29fe4e05820412e391430f61aa88a1a82e138de8cac726a9dc4fcd7ed8b9
\ No newline at end of file

tests/cachekey/project/elements/compose1.expected

-e050172c2d445849a2716720814501f0e294b81d28be8cd911ee294291ec60d0
\ No newline at end of file
+4e4c719242aa45fed398cc2fb8936195a1fcae9326d808de7fee340ae48862ea
\ No newline at end of file

tests/cachekey/project/elements/compose2.expected

-ad985f243163aab22576fea63a0b89f7560e361326cb041872c568c5feaabf5c
\ No newline at end of file
+2fc1dd398f6c6f6e1d7ca48d88557e133d2130278882e14cd1105b15a600cd7a
\ No newline at end of file

tests/cachekey/project/elements/compose3.expected

-ec0fec2c821eb34286e9799a6e8d1f5587f161d1d653bd1dbe385602340d86ae
\ No newline at end of file
+4d7c9e2e1e8cfcc4b300a11693767f41f22c7829db9063dec10856328d03ccc3
\ No newline at end of file

tests/cachekey/project/elements/compose4.expected

-7cfe8f6161c00d8cf10114e7458f0b97eb003a41504ae301b24f45d48d42155b
\ No newline at end of file
+cad8f3b622f4a906f9fc3f5187a7703e2b17dfc550dd5a07479ca3ebffbd5c86
\ No newline at end of file

tests/cachekey/project/elements/compose5.expected

-7f016c3165f2de9161312b74f29513dff7dfdcba5ff8c6897beb5b123eaafd3d
\ No newline at end of file
+4fd21699827aa16da8d7a1525020f9fd45422f0431749510947ff472d76c1802
\ No newline at end of file

tests/cachekey/project/elements/import1.expected

-41ce5a640fdfd7b6ce8a2c3fa1dde7983bc4df0e4c3ca926670118bae3c051fe
\ No newline at end of file
+aa443ea4607d7dd5a0c99646a1b827e3165862772fc1b26e20195aadd2ab8885
\ No newline at end of file

tests/cachekey/project/elements/import2.expected

-d657db503460486a6de80d87f15c6b1fa84b0c4dacabed374acdc70172d4761d
\ No newline at end of file
+18ea6bbb968ca6945c8c2941650f447b8df58673be7270c967c8152730eff036
\ No newline at end of file

tests/cachekey/project/elements/import3.expected

-8de0293a6231dc661cf7229aa5a2f25abdf9a6d38ff70bd6f2562dae51ff05d3
\ No newline at end of file
+34ce4816b0307f0691302460367ab24b4e1f86e61c0e307e68bcd6833946c1f1
\ No newline at end of file

tests/cachekey/project/elements/script1.expected

-d0a6b7d29226b083c404d76d9551a0eee98753058580cd62901f8dfac06ca08d
\ No newline at end of file
+c48922b3d80d36e6d30bed7581aa1473a5e463321b3a19606b603c055d2b4be4
\ No newline at end of file

tests/cachekey/project/sources/bzr1.expected

-51415ebc7d72315c5c7704759025d6a9237e786bfe9c2bda8c51e15840c3470a
\ No newline at end of file
+ee271c8f469cd33330229d8dcc44e26f3480a9f47b55db46f42d1a396a94609e
\ No newline at end of file