Commit c1cef0cc authored by Tom Coldrick's avatar Tom Coldrick
Browse files

bst-fmt: Allow greater control over order 

Allow authors of plugins to utilise the Plugin.keyorder attribute to
define an order for the sub-headers in config. Modify `bst fmt` to
modify the yaml into that order. Add canonical keyorder for every plugin
already in core.

In terms of implementation, this adds a custom yaml dumper subclassed
from ruamel.yaml's RoundTripDumper, which will dump a dict in the order
defined in the keyorder attribute. The Plugin class is given a keyorder
attribute, which defines the order in which keys should be dumped. The
element and source classes add the common fields used in them, so as to
minimise work for plugin authors. Plugin authors should add their custom
keys at configure time.

This causes some stripping of comments, which seems to be due to a bug
in the ruamel.yaml RoundTripDumper, as that also strips the same
comments. It also will remove blank spaces, again probably due to
limitations in ruamel.
parent 574e5a78

buildstream/_yaml.py

+27 −2
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ from pathlib import Path 


from ruamel import yaml

from ruamel.yaml.representer import SafeRepresenter, RoundTripRepresenter

from ruamel.yaml.nodes import MappingNode, ScalarNode

from ruamel.yaml.constructor import RoundTripConstructor

from ._exceptions import LoadError, LoadErrorReason
@@ -50,6 +51,29 @@ class ProvenanceFile(): 
        self.shortname = shortname

        self.project = project



# A custom yaml dumper to reorder keys in dicts to a canonical order, defined

# in each plugin

class BstFormatter(yaml.RoundTripDumper):



    keyorder = []



    @classmethod

    def _iter_in_global_order(cls, mapping):

        for key in cls.keyorder:

            if key in mapping.keys():

                yield key, mapping[key]

        for key in sorted(mapping.keys()):

            if key not in self.keyorder:

                yield key, mapping[key]



    @classmethod

    def _represent_dict(cls, dumper, mapping):

        return cls.represent_mapping('tag:yaml.org,2002:map', cls._iter_in_global_order(mapping))



    def __init__(self, *args, **kwargs):

        yaml.RoundTripDumper.__init__(self, *args, **kwargs)

        self.no_newline = False

        self.add_representer(dict, self._represent_dict)



# Provenance tracks the origin of a given node in the parsed dictionary.

#
@@ -244,15 +268,16 @@ def load_data(data, file=None, copy_tree=False): 
# Args:

#    node (dict): A node previously loaded with _yaml.load() above

#    filename (str): The YAML file to load

#    dumper (yaml.Dumper): The yaml dumper to be used

#

def dump(node, filename=None):

def dump(node, filename=None, dumper=yaml.RoundTripDumper):

    with ExitStack() as stack:

        if filename:

            from . import utils

            f = stack.enter_context(utils.save_file_atomic(filename, 'w'))

        else:

            f = sys.stdout

        yaml.round_trip_dump(node, f)

        yaml.round_trip_dump(node, f, Dumper=dumper)





# node_decorated_copy()

buildstream/buildelement.py

+1 −0
Original line number Diff line number Diff line
@@ -172,6 +172,7 @@ class BuildElement(Element): 
        for command_name in _legacy_command_steps:

            if command_name in _command_steps:

                self.__commands[command_name] = self.__get_commands(node, command_name)

                self.keyorder.append(command_name)

            else:

                self.__commands[command_name] = []

buildstream/element.py

+16 −1
Original line number Diff line number Diff line
@@ -269,6 +269,11 @@ class Element(Plugin): 
                # This will taint the artifact, disable pushing.

                self.__sandbox_config_supported = False



        self.keyorder = ['kind', 'description', 'depends', 'variables',

                         'environment', 'config', 'public', 'sandbox', 'sources'] \

                         + self.keyorder





    def __lt__(self, other):

        return self.name < other.name
@@ -1338,7 +1343,17 @@ class Element(Plugin): 
    #

    def _format(self):

        provenance = self._get_provenance()

        _yaml.dump(provenance.toplevel, provenance.filename.name)



        _yaml.BstFormatter.keyorder = self.keyorder



        # We need to add the key orders for each source into the

        # global keyorder, as sources are dumped with the element.

        for s in self.sources():

            for key in s.keyorder:

                if key not in _yaml.BstFormatter.keyorder:

                    _yaml.BstFormatter.keyorder += s.keyorder



        _yaml.dump(provenance.toplevel, provenance.filename.name, dumper=_yaml.BstFormatter)



    # _prepare_sandbox():

    #

buildstream/plugin.py

+2 −0
Original line number Diff line number Diff line
@@ -188,6 +188,8 @@ class Plugin(): 
        self.__kind = modulename.split('.')[-1]

        self.debug("Created: {}".format(self))



        self.keyorder = []



    def __del__(self):

        # Dont send anything through the Message() pipeline at destruction time,

        # any subsequent lookup of plugin by unique id would raise KeyError.

buildstream/plugins/elements/compose.py

+2 −0
Original line number Diff line number Diff line
@@ -70,6 +70,8 @@ class ComposeElement(Element): 
        self.exclude = self.node_get_member(node, list, 'exclude')

        self.include_orphans = self.node_get_member(node, bool, 'include-orphans')



        self.keyorder += ['integrate', 'include', 'exclude', 'include-orphans']



    def preflight(self):

        pass