_workspaces.py 13.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
#!/usr/bin/env python3
#
#  Copyright (C) 2018 Codethink Limited
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2 of the License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
#  Authors:
#        Tristan Maat <tristan.maat@codethink.co.uk>

import os
from . import utils
from . import _yaml

from ._exceptions import LoadError, LoadErrorReason


28
BST_WORKSPACE_FORMAT_VERSION = 3
29

30 31
# Hold on to a list of members which get serialized
_WORKSPACE_MEMBERS = [
32
    'prepared',
33 34 35 36 37
    'path',
    'last_successful',
    'running_files'
]

38 39 40 41 42 43 44 45 46 47 48

# Workspace()
#
# An object to contain various helper functions and data required for
# workspaces.
#
# last_successful, path and running_files are intended to be public
# properties, but may be best accessed using this classes' helper
# methods.
#
# Args:
49
#    toplevel_project (Project): Top project. Will be used for resolving relative workspace paths.
50
#    path (str): The path that should host this workspace
51
#    last_successful (str): The key of the last successful build of this workspace
52 53 54
#    running_files (dict): A dict mapping dependency elements to files
#                          changed between failed builds. Should be
#                          made obsolete with failed build artifacts.
55 56
#
class Workspace():
57
    def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
58
        self.prepared = prepared
59
        self.last_successful = last_successful
60
        self.path = path
61
        self.running_files = running_files if running_files is not None else {}
62

63
        self._toplevel_project = toplevel_project
64
        self._key = None
65

66
    # to_dict()
67
    #
68
    # Convert this object to a dict for serialization purposes
69 70 71 72
    #
    # Returns:
    #     (dict) A dict representation of the workspace
    #
73
    def to_dict(self):
74
        return {key: val for key, val in self.__dict__.items()
75 76 77 78 79 80 81 82 83
                if key in _WORKSPACE_MEMBERS and val is not None}

    # from_dict():
    #
    # Loads a new workspace from a simple dictionary, the dictionary
    # is expected to be generated from Workspace.to_dict(), or manually
    # when loading from a YAML file.
    #
    # Args:
84
    #    toplevel_project (Project): Top project. Will be used for resolving relative workspace paths.
85 86 87 88 89 90
    #    dictionary: A simple dictionary object
    #
    # Returns:
    #    (Workspace): A newly instantiated Workspace
    #
    @classmethod
91
    def from_dict(cls, toplevel_project, dictionary):
92 93

        # Just pass the dictionary as kwargs
94
        return cls(toplevel_project, **dictionary)
95

96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
    # differs()
    #
    # Checks if two workspaces are different in any way.
    #
    # Args:
    #    other (Workspace): Another workspace instance
    #
    # Returns:
    #    True if the workspace differs from 'other', otherwise False
    #
    def differs(self, other):

        for member in _WORKSPACE_MEMBERS:
            member_a = getattr(self, member)
            member_b = getattr(other, member)

            if member_a != member_b:
                return True

        return False

117 118 119 120 121 122
    # invalidate_key()
    #
    # Invalidate the workspace key, forcing a recalculation next time
    # it is accessed.
    #
    def invalidate_key(self):
123
        self._key = None
124 125 126 127 128 129 130 131 132

    # stage()
    #
    # Stage the workspace to the given directory.
    #
    # Args:
    #    directory (str) - The directory into which to stage this workspace
    #
    def stage(self, directory):
133
        fullpath = self.get_absolute_path()
134 135 136 137 138 139
        if os.path.isdir(fullpath):
            utils.copy_files(fullpath, directory)
        else:
            destfile = os.path.join(directory, os.path.basename(self.path))
            utils.safe_copy(fullpath, destfile)

140 141 142 143 144 145
    # add_running_files()
    #
    # Append a list of files to the running_files for the given
    # dependency. Duplicate files will be ignored.
    #
    # Args:
146
    #     dep_name (str) - The dependency name whose files to append to
147 148
    #     files (str) - A list of files to append
    #
149 150
    def add_running_files(self, dep_name, files):
        if dep_name in self.running_files:
151
            # ruamel.py cannot serialize sets in python3.4
152 153
            to_add = set(files) - set(self.running_files[dep_name])
            self.running_files[dep_name].extend(to_add)
154
        else:
155
            self.running_files[dep_name] = list(files)
156 157 158 159 160 161 162 163

    # clear_running_files()
    #
    # Clear all running files associated with this workspace.
    #
    def clear_running_files(self):
        self.running_files = {}

164 165 166 167 168 169 170 171 172 173 174 175 176
    # get_key()
    #
    # Get a unique key for this workspace.
    #
    # Args:
    #    recalculate (bool) - Whether to recalculate the key
    #
    # Returns:
    #    (str) A unique key for this workspace
    #
    def get_key(self, recalculate=False):
        def unique_key(filename):
            try:
177 178
                stat = os.lstat(filename)
            except OSError as e:
179
                raise LoadError(LoadErrorReason.MISSING_FILE,
180 181 182 183
                                "Failed to stat file in workspace: {}".format(e))

            # Use the mtime of any file with sub second precision
            return stat.st_mtime_ns
184

185 186
        if recalculate or self._key is None:
            fullpath = self.get_absolute_path()
187 188 189 190 191 192 193 194

            # Get a list of tuples of the the project relative paths and fullpaths
            if os.path.isdir(fullpath):
                filelist = utils.list_relative_paths(fullpath)
                filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
            else:
                filelist = [(self.path, fullpath)]

195
            self._key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist]
196

197
        return self._key
198 199 200 201 202 203

    # get_absolute_path():
    #
    # Returns: The absolute path of the element's workspace.
    #
    def get_absolute_path(self):
204
        return os.path.join(self._toplevel_project.directory, self.path)
205 206 207 208 209 210 211


# Workspaces()
#
# A class to manage Workspaces for multiple elements.
#
# Args:
212
#    toplevel_project (Project): Top project used to resolve paths.
213 214
#
class Workspaces():
215 216 217
    def __init__(self, toplevel_project):
        self._toplevel_project = toplevel_project
        self._bst_directory = os.path.join(toplevel_project.directory, ".bst")
218
        self._workspaces = self._load_config()
219

220
    # list()
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
    #
    # Generator function to enumerate workspaces.
    #
    # Yields:
    #    A tuple in the following format: (str, Workspace), where the
    #    first element is the name of the workspaced element.
    def list(self):
        for element, _ in _yaml.node_items(self._workspaces):
            yield (element, self._workspaces[element])

    # create_workspace()
    #
    # Create a workspace in the given path for the given element.
    #
    # Args:
236
    #    element_name (str) - The element name to create a workspace for
237 238
    #    path (str) - The path in which the workspace should be kept
    #
239
    def create_workspace(self, element_name, path):
240
        self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
241

242
        return self._workspaces[element_name]
243

244
    # get_workspace()
245 246 247 248 249
    #
    # Get the path of the workspace source associated with the given
    # element's source at the given index
    #
    # Args:
250
    #    element_name (str) - The element name whose workspace to return
251 252 253 254
    #
    # Returns:
    #    (None|Workspace)
    #
255 256
    def get_workspace(self, element_name):
        if element_name not in self._workspaces:
257
            return None
258
        return self._workspaces[element_name]
259

260 261 262 263 264 265 266 267 268 269 270 271 272 273
    # update_workspace()
    #
    # Update the datamodel with a new Workspace instance
    #
    # Args:
    #    element_name (str): The name of the element to update a workspace for
    #    workspace_dict (Workspace): A serialized workspace dictionary
    #
    # Returns:
    #    (bool): Whether the workspace has changed as a result
    #
    def update_workspace(self, element_name, workspace_dict):
        assert element_name in self._workspaces

274
        workspace = Workspace.from_dict(self._toplevel_project, workspace_dict)
275 276 277 278 279 280
        if self._workspaces[element_name].differs(workspace):
            self._workspaces[element_name] = workspace
            return True

        return False

281 282 283 284 285 286 287
    # delete_workspace()
    #
    # Remove the workspace from the workspace element. Note that this
    # does *not* remove the workspace from the stored yaml
    # configuration, call save_config() afterwards.
    #
    # Args:
288
    #    element_name (str) - The element name whose workspace to delete
289
    #
290 291
    def delete_workspace(self, element_name):
        del self._workspaces[element_name]
292 293 294 295 296 297 298 299

    # save_config()
    #
    # Dump the current workspace element to the project configuration
    # file. This makes any changes performed with delete_workspace or
    # create_workspace permanent
    #
    def save_config(self):
300 301
        assert utils._is_main_process()

302 303 304
        config = {
            'format-version': BST_WORKSPACE_FORMAT_VERSION,
            'workspaces': {
305
                element: workspace.to_dict()
306 307 308
                for element, workspace in _yaml.node_items(self._workspaces)
            }
        }
309
        os.makedirs(self._bst_directory, exist_ok=True)
310
        _yaml.dump(_yaml.node_sanitize(config),
311
                   self._get_filename())
312 313 314

    # _load_config()
    #
315
    # Loads and parses the workspace configuration
316 317
    #
    # Returns:
318
    #    (dict) The extracted workspaces
319
    #
320
    # Raises: LoadError if there was a problem with the workspace config
321
    #
322
    def _load_config(self):
323
        workspace_file = self._get_filename()
324
        try:
325 326 327 328 329 330 331
            node = _yaml.load(workspace_file)
        except LoadError as e:
            if e.reason == LoadErrorReason.MISSING_FILE:
                # Return an empty dict if there was no workspace file
                return {}

            raise
332

333
        return self._parse_workspace_config(node)
334

335
    # _parse_workspace_config_format()
336 337 338 339 340 341 342 343 344 345 346 347 348
    #
    # If workspace config is in old-style format, i.e. it is using
    # source-specific workspaces, try to convert it to element-specific
    # workspaces.
    #
    # Args:
    #    workspaces (dict): current workspace config, usually output of _load_workspace_config()
    #
    # Returns:
    #    (dict) The extracted workspaces
    #
    # Raises: LoadError if there was a problem with the workspace config
    #
349
    def _parse_workspace_config(self, workspaces):
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364
        version = _yaml.node_get(workspaces, int, "format-version", default_value=0)

        if version == 0:
            # Pre-versioning format can be of two forms
            for element, config in _yaml.node_items(workspaces):
                if isinstance(config, str):
                    pass

                elif isinstance(config, dict):
                    sources = list(_yaml.node_items(config))
                    if len(sources) > 1:
                        detail = "There are multiple workspaces open for '{}'.\n" + \
                                 "This is not supported anymore.\n" + \
                                 "Please remove this element from '{}'."
                        raise LoadError(LoadErrorReason.INVALID_DATA,
365
                                        detail.format(element, self._get_filename()))
366 367 368 369 370 371 372 373

                    workspaces[element] = sources[0][1]

                else:
                    raise LoadError(LoadErrorReason.INVALID_DATA,
                                    "Workspace config is in unexpected format.")

            res = {
374
                element: Workspace(self._toplevel_project, path=config)
375 376 377
                for element, config in _yaml.node_items(workspaces)
            }

378
        elif version >= 1 and version <= BST_WORKSPACE_FORMAT_VERSION:
379
            workspaces = _yaml.node_get(workspaces, dict, "workspaces", default_value={})
380
            res = {element: self._load_workspace(node)
381 382 383 384 385 386 387 388 389
                   for element, node in _yaml.node_items(workspaces)}

        else:
            raise LoadError(LoadErrorReason.INVALID_DATA,
                            "Workspace configuration format version {} not supported."
                            "Your version of buildstream may be too old. Max supported version: {}"
                            .format(version, BST_WORKSPACE_FORMAT_VERSION))

        return res
390 391 392 393 394 395 396 397 398 399 400

    # _load_workspace():
    #
    # Loads a new workspace from a YAML node
    #
    # Args:
    #    node: A YAML Node
    #
    # Returns:
    #    (Workspace): A newly instantiated Workspace
    #
401
    def _load_workspace(self, node):
402
        dictionary = {
403
            'prepared': _yaml.node_get(node, bool, 'prepared', default_value=False),
404 405 406 407
            'path': _yaml.node_get(node, str, 'path'),
            'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None),
            'running_files': _yaml.node_get(node, dict, 'running_files', default_value=None),
        }
408 409 410 411 412 413 414 415 416 417
        return Workspace.from_dict(self._toplevel_project, dictionary)

    # _get_filename():
    #
    # Get the workspaces.yml file path.
    #
    # Returns:
    #    (str): The path to workspaces.yml file.
    def _get_filename(self):
        return os.path.join(self._bst_directory, "workspaces.yml")