Commit df67f9c0 authored by Qinusty's avatar Qinusty
Browse files

_project.py: Add fatal-warnings configuration item 

This allows for users to configure fatal-warnings to be either a list of
warnings or simply True to trigger all configurable warnings as errors.

This commit deprecates the use of fail-on-overlap within project.conf,
this will now use the fatal-warnings configuration item.
parent 8656a65d

buildstream/_frontend/app.py

+7 −0
Original line number Diff line number Diff line
@@ -230,6 +230,13 @@ class App(): 
        # Propagate pipeline feedback to the user

        self.context.set_message_handler(self._message_handler)



        # Deprecation check now that message handler is initialized

        if self.project._fail_on_overlap is not None:

            self._message(MessageType.WARN,

                          "Use of fail-on-overlap within project.conf" +

                          "is deprecated. Please use fatal-warnings instead.")

        self._message(MessageType.DEBUG, str(self.project._fatal_warnings))



        # Now that we have a logger and message handler,

        # we can override the global exception hook.

        sys.excepthook = self._global_exception_handler

buildstream/_project.py

+76 −3
Original line number Diff line number Diff line
@@ -39,6 +39,9 @@ from ._versions import BST_FORMAT_VERSION 
# Project Configuration file

_PROJECT_CONF_FILE = 'project.conf'



# This should be updated as warnings are added for `fatal-warnings: True`

_ALL_WARNINGS = ['overlaps']





# HostMount()

#
@@ -85,7 +88,7 @@ class Project(): 


        self.options = None                      # OptionPool

        self.junction = junction                 # The junction Element object, if this is a subproject

        self.fail_on_overlap = False             # Whether overlaps are treated as errors



        self.ref_storage = None                  # ProjectRefStorage setting

        self.base_variables = {}                 # The base set of variables

        self.base_environment = {}               # The base set of environment variables
@@ -108,6 +111,9 @@ class Project(): 
        self._source_format_versions = {}

        self._element_format_versions = {}



        self._fail_on_overlap = False    # Whether to fail on overlaps or not # Deprecated use _warning_is_fatal()

        self._fatal_warnings = []        # A list of warnings which should trigger an error



        self._shell_command = []      # The default interactive shell command

        self._shell_environment = {}  # Statically set environment vars

        self._shell_host_files = []   # A list of HostMount objects
@@ -232,6 +238,18 @@ class Project(): 
        mirror_list.append(self._aliases[alias])

        return mirror_list



    # fail_on_overlap

    #

    # Property added to continue support of Project.fail_on_overlap after

    # introduction of fatal-warnings configuration item.

    #

    # Returns:

    #    (bool): True if the configuration specifies that overlaps should

    #            cause errors instead of warnings.

    @property

    def fail_on_overlap(self):

        return self._fail_on_overlap or self._warning_is_fatal("overlaps")



    # _load():

    #

    # Loads the project configuration file in the project directory.
@@ -278,7 +296,7 @@ class Project(): 
            'split-rules', 'elements', 'plugins',

            'aliases', 'name',

            'artifacts', 'options',

            'fail-on-overlap', 'shell',

            'fail-on-overlap', 'shell', 'fatal-warnings',

            'ref-storage', 'sandbox', 'mirrors',

        ])
@@ -404,7 +422,22 @@ class Project(): 
        self._splits = _yaml.node_get(config, Mapping, 'split-rules')



        # Fail on overlap

        self.fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap')

        self._fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap', default_value=None)



        # Fatal warnings

        p = _yaml.node_get_provenance(config, 'fatal-warnings')

        try:  # Check for bool type

            fatal_warnings = _yaml.node_get(config, bool, 'fatal-warnings', default_value=False)

        except (ValueError, LoadError) as e:

            try:  # Check for list type

                fatal_warnings = _yaml.node_get(config, list, 'fatal-warnings', default_value=[])

            except (ValueError, LoadError):

                raise LoadError(LoadErrorReason.INVALID_DATA,

                                "{}: Invalid value specified for 'fatal-warnings', ".format(p) +

                                "must be list or bool.")



        # Validate and set fatal warnings

        self._set_fatal_warnings(fatal_warnings, p)



        # Use separate file for storing source references

        self.ref_storage = _yaml.node_get(config, str, 'ref-storage')
@@ -532,3 +565,43 @@ class Project(): 
            directory = parent_dir



        return directory



    # _warning_is_fatal():

    #

    # Returns true if the warning in question should be considered fatal based on

    # the project configuration.

    #

    # Args:

    #   warning_str (str): The warning configuration string to check against

    #

    # Returns:

    #    (bool): True if the warning should be considered fatal and cause an error.

    #

    def _warning_is_fatal(self, warning_str):

        return warning_str in self._fatal_warnings



    # _set_fatal_warnings()

    #

    # Validates and sets the self._fatal_warnings

    #

    # Args:

    #   warnings (list|bool): The warnings to set self._fatal_warnings to.

    #                         If True, _ALL_WARNINGS is used. If False, [] is used.

    #   provenance (str): The provenance assosciated with the warnings parameter.

    #

    def _set_fatal_warnings(self, warnings, provenance="[Unknown Provenance]"):

        if isinstance(warnings, bool):

            self._fatal_warnings = _ALL_WARNINGS if warnings else []

            return

        elif warnings is None:

            return



        # Check for unknown warnings.

        unknown_warnings = filter(lambda x: x not in _ALL_WARNINGS, warnings)

        if list(unknown_warnings):  # Restrict 'fatal-warnings' to known warnings

            raise LoadError(LoadErrorReason.INVALID_DATA,

                            ("{}: Invalid warning provided to fatal-warnings ({})\n" +

                             "Valid options are: ({})")

                            .format(provenance, ", ".join(unknown_warnings), ", ".join(_ALL_WARNINGS)))

        else:

            self._fatal_warnings = warnings

buildstream/_yaml.py

+2 −2
Original line number Diff line number Diff line
@@ -368,9 +368,9 @@ def node_get(node, expected_type, key, indices=None, default_value=_get_sentinel 
        try:

            if (expected_type == bool and isinstance(value, str)):

                # Dont coerce booleans to string, this makes "False" strings evaluate to True

                if value == 'true' or value == 'True':

                if value in ('true', 'True'):

                    value = True

                elif value == 'false' or value == 'False':

                elif value in ('false', 'False'):

                    value = False

                else:

                    raise ValueError()

buildstream/data/projectconfig.yaml

+9 −1
Original line number Diff line number Diff line
@@ -14,7 +14,15 @@ element-path: . 
ref-storage: inline



# Overlaps are just warnings

fail-on-overlap: False

# This has been DEPRECATED in favour of fatal-warnings

#fail-on-overlap: False



# Allows a collection of warnings to be configured to be raised as errors.

# Setting this value to true will enable all possible fatal-warnings

# fatal-warnings: True



# fatal-warnings:

# - overlaps





# Variable Configuration

tests/frontend/overlaps.py

+14 −8
Original line number Diff line number Diff line
@@ -16,26 +16,31 @@ project_template = { 
}





def gen_project(project_dir, fail_on_overlap):

def gen_project(project_dir, fail_on_overlap, use_fatal_warnings=True):

    template = dict(project_template)

    if use_fatal_warnings:

        template["fatal-warnings"] = ["overlaps"] if fail_on_overlap else []

    else:

        template["fail-on-overlap"] = fail_on_overlap

    projectfile = os.path.join(project_dir, "project.conf")

    _yaml.dump(template, projectfile)





@pytest.mark.datafiles(DATA_DIR)

def test_overlaps(cli, datafiles):

@pytest.mark.parametrize("use_fatal_warnings", [True, False])

def test_overlaps(cli, datafiles, use_fatal_warnings):

    project_dir = str(datafiles)

    gen_project(project_dir, False)

    gen_project(project_dir, False, use_fatal_warnings)

    result = cli.run(project=project_dir, silent=True, args=[

        'build', 'collect.bst'])

    result.assert_success()





@pytest.mark.datafiles(DATA_DIR)

def test_overlaps_error(cli, datafiles):

@pytest.mark.parametrize("use_fatal_warnings", [True, False])

def test_overlaps_error(cli, datafiles, use_fatal_warnings):

    project_dir = str(datafiles)

    gen_project(project_dir, True)

    gen_project(project_dir, True, use_fatal_warnings)

    result = cli.run(project=project_dir, silent=True, args=[

        'build', 'collect.bst'])

    result.assert_main_error(ErrorDomain.STREAM, None)
@@ -74,11 +79,12 @@ def test_overlaps_whitelist_on_overlapper(cli, datafiles): 




@pytest.mark.datafiles(DATA_DIR)

def test_overlaps_script(cli, datafiles):

@pytest.mark.parametrize("use_fatal_warnings", [True, False])

def test_overlaps_script(cli, datafiles, use_fatal_warnings):

    # Test overlaps with script element to test

    # Element.stage_dependency_artifacts() with Scope.RUN

    project_dir = str(datafiles)

    gen_project(project_dir, False)

    gen_project(project_dir, False, use_fatal_warnings)

    result = cli.run(project=project_dir, silent=True, args=[

        'build', 'script.bst'])

    result.assert_success()