source.py 21.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
#!/usr/bin/env python3
#
#  Copyright (C) 2016 Codethink Limited
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2 of the License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
#  Authors:
#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
20
"""
21
Source - Base source class
22
==========================
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68


.. _core_source_abstract_methods:

Abstract Methods
----------------
For loading and configuration purposes, Sources must implement the
:ref:`Plugin base class abstract methods <core_plugin_abstract_methods>`.

Sources expose the following abstract methods. Unless explicitly mentioned,
these methods are mandatory to implement.

* :func:`Source.get_consistency() <buildstream.source.Source.get_consistency>`

  Report the sources consistency state.

* :func:`Source.load_ref() <buildstream.source.Source.load_ref>`

  Load the ref from a specific YAML node

* :func:`Source.get_ref() <buildstream.source.Source.get_ref>`

  Fetch the source ref

* :func:`Source.set_ref() <buildstream.source.Source.set_ref>`

  Set a new ref explicitly

* :func:`Source.track() <buildstream.source.Source.track>`

  Automatically derive a new ref from a symbolic tracking branch

* :func:`Source.fetch() <buildstream.source.Source.fetch>`

  Fetch the actual payload for the currently set ref

* :func:`Source.stage() <buildstream.source.Source.stage>`

  Stage the sources for a given ref at a specified location

* :func:`Source.init_workspace() <buildstream.source.Source.init_workspace>`

  Stage sources in a local directory for use as a workspace.

  **Optional**: If left unimplemented, this will default to calling
  :func:`Source.stage() <buildstream.source.Source.stage>`
69
"""
70

71
import os
72
from collections import Mapping
73
from contextlib import contextmanager
74

75
from . import Plugin
Gökçen Nurlu's avatar
Gökçen Nurlu committed
76
from . import _yaml, utils
77
from ._exceptions import BstError, ImplError, ErrorDomain
78
from ._projectrefs import ProjectRefStorage
79

80

81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
class Consistency():
    INCONSISTENT = 0
    """Inconsistent

    Inconsistent sources have no explicit reference set. They cannot
    produce a cache key, be fetched or staged. They can only be tracked.
    """

    RESOLVED = 1
    """Resolved

    Resolved sources have a reference and can produce a cache key and
    be fetched, however they cannot be staged.
    """

    CACHED = 2
    """Cached

    Cached sources have a reference which is present in the local
    source cache. Only cached sources can be staged.
    """


104
class SourceError(BstError):
105 106
    """This exception should be raised by :class:`.Source` implementations
    to report errors to the user.
107

108
    Args:
109 110
       message (str): The breif error description to report to the user
       detail (str): A possibly multiline, more detailed error message
111
       reason (str): An optional machine readable reason string, used for test cases
112
    """
113 114
    def __init__(self, message, *, detail=None, reason=None):
        super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason)
115 116


117
class Source(Plugin):
118 119 120
    """Source()

    Base Source class.
121 122 123 124

    All Sources derive from this class, this interface defines how
    the core will be interacting with Sources.
    """
125 126 127
    __defaults = {}          # The defaults from the project
    __defaults_set = False   # Flag, in case there are not defaults at all

128
    def __init__(self, context, project, meta):
129
        provenance = _yaml.node_get_provenance(meta.config)
130 131
        super().__init__("{}-{}".format(meta.element_name, meta.element_index),
                         context, project, provenance, "source")
132

133 134
        self.__element_name = meta.element_name         # The name of the element owning this source
        self.__element_index = meta.element_index       # The index of the source in the owning element's source list
135
        self.__element_kind = meta.element_kind         # The kind of the element owning this source
136
        self.__directory = meta.directory               # Staging relative directory
137
        self.__consistency = Consistency.INCONSISTENT   # Cached consistency state
138

139 140 141 142 143
        # Collect the composited element configuration and
        # ask the element to configure itself.
        self.__init_defaults()
        self.__config = self.__extract_config(meta)
        self.configure(self.__config)
144

145 146 147 148 149 150 151
    COMMON_CONFIG_KEYS = ['kind', 'directory']
    """Common source config keys

    Source config keys that must not be accessed in configure(), and
    should be checked for using node_validate().
    """

152 153 154
    #############################################################
    #                      Abstract Methods                     #
    #############################################################
155
    def get_consistency(self):
156 157 158
        """Report whether the source has a resolved reference

        Returns:
159
           (:class:`.Consistency`): The source consistency
160
        """
161
        raise ImplError("Source plugin '{}' does not implement get_consistency()".format(self.get_kind()))
162

163 164 165 166 167 168
    def load_ref(self, node):
        """Loads the *ref* for this Source from the specified *node*.

        Args:
           node (dict): The YAML node to load the ref from

169 170 171 172 173 174 175 176
        .. note::

           The *ref* for the Source is expected to be read at
           :func:`Plugin.configure() <buildstream.plugin.Plugin.configure>` time,
           this will only be used for loading refs from alternative locations
           than in the `element.bst` file where the given Source object has
           been declared.

177 178 179 180
        *Since: 1.2*
        """
        raise ImplError("Source plugin '{}' does not implement load_ref()".format(self.get_kind()))

181 182 183 184
    def get_ref(self):
        """Fetch the internal ref, however it is represented

        Returns:
185 186 187
           (simple object): The internal source reference, or ``None``

        .. note::
188 189 190 191 192

           The reference is the user provided (or track resolved) value
           the plugin uses to represent a specific input, like a commit
           in a VCS or a tarball's checksum. Usually the reference is a string,
           but the plugin may choose to represent it with a tuple or such.
193 194 195

           Implementations *must* return a ``None`` value in the case that
           the ref was not loaded. E.g. a ``(None, None)`` tuple is not acceptable.
196
        """
197
        raise ImplError("Source plugin '{}' does not implement get_ref()".format(self.get_kind()))
198 199 200

    def set_ref(self, ref, node):
        """Applies the internal ref, however it is represented
201 202

        Args:
203
           ref (simple object): The internal source reference to set, or ``None``
204 205 206
           node (dict): The same dictionary which was previously passed
                        to :func:`~buildstream.source.Source.configure`

207 208
        See :func:`~buildstream.source.Source.get_ref` for a discussion on
        the *ref* parameter.
209 210 211 212 213

        .. note::

           Implementors must support the special ``None`` value here to
           allow clearing any existing ref.
214
        """
215
        raise ImplError("Source plugin '{}' does not implement set_ref()".format(self.get_kind()))
216

217 218
    def track(self):
        """Resolve a new ref from the plugin's track option
219

220 221 222 223 224 225
        Returns:
           (simple object): A new internal source reference, or None

        If the backend in question supports resolving references from
        a symbolic tracking branch or tag, then this should be implemented
        to perform this task on behalf of ``build-stream track`` commands.
226

227 228 229 230
        This usually requires fetching new content from a remote origin
        to see if a new ref has appeared for your branch or tag. If the
        backend store allows one to query for a new ref from a symbolic
        tracking data without downloading then that is desirable.
231

232 233
        See :func:`~buildstream.source.Source.get_ref` for a discussion on
        the *ref* parameter.
234
        """
235 236
        # Allow a non implementation
        return None
237 238 239 240 241 242

    def fetch(self):
        """Fetch remote sources and mirror them locally, ensuring at least
        that the specific reference is cached locally.

        Raises:
243
           :class:`.SourceError`
244

245
        Implementors should raise :class:`.SourceError` if the there is some
246 247
        network error or if the source reference could not be matched.
        """
248
        raise ImplError("Source plugin '{}' does not implement fetch()".format(self.get_kind()))
249 250 251 252 253 254 255

    def stage(self, directory):
        """Stage the sources to a directory

        Args:
           directory (str): Path to stage the source

256 257 258
        Raises:
           :class:`.SourceError`

259 260
        Implementors should assume that *directory* already exists
        and stage already cached sources to the passed directory.
261 262 263

        Implementors should raise :class:`.SourceError` when encountering
        some system error.
264
        """
265
        raise ImplError("Source plugin '{}' does not implement stage()".format(self.get_kind()))
266

267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
    def init_workspace(self, directory):
        """Initialises a new workspace

        Args:
           directory (str): Path of the workspace to init

        Raises:
           :class:`.SourceError`

        Default implementation is to call
        :func:`~buildstream.source.Source.stage`.

        Implementors overriding this method should assume that *directory*
        already exists.

        Implementors should raise :class:`.SourceError` when encountering
        some system error.
        """
        self.stage(directory)

287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
    #############################################################
    #                       Public Methods                      #
    #############################################################
    def get_mirror_directory(self):
        """Fetches the directory where this source should store things

        Returns:
           (str): The directory belonging to this source
        """

        # Create the directory if it doesnt exist
        context = self._get_context()
        directory = os.path.join(context.sourcedir, self.get_kind())
        os.makedirs(directory, exist_ok=True)
        return directory

    def translate_url(self, url):
        """Translates the given url which may be specified with an alias
        into a fully qualified url.

        Args:
           url (str): A url, which may be using an alias

        Returns:
           str: The fully qualified url, with aliases resolved
        """
        project = self._get_project()
        return project.translate_url(url)

    def get_project_directory(self):
        """Fetch the project base directory

        This is useful for sources which need to load resources
        stored somewhere inside the project.

        Returns:
           str: The project base directory
        """
        project = self._get_project()
        return project.directory

    @contextmanager
    def tempdir(self):
        """Context manager for working in a temporary directory

        Yields:
           (str): A path to a temporary directory

        This should be used by source plugins directly instead of the tempfile
        module. This one will automatically cleanup in case of termination by
        catching the signal before os._exit(). It will also use the 'mirror
        directory' as expected for a source.
        """
        mirrordir = self.get_mirror_directory()
        with utils._tempdir(dir=mirrordir) as tempdir:
            yield tempdir

344 345 346 347
    #############################################################
    #            Private Methods used in BuildStream            #
    #############################################################

348 349 350 351 352 353 354 355 356
    # Wrapper around preflight() method
    #
    def _preflight(self):
        try:
            self.preflight()
        except BstError as e:
            # Prepend provenance to the error
            raise SourceError("{}: {}".format(self, e), reason=e.reason) from e

357
    # Update cached consistency for a source
358
    #
359 360 361 362 363
    # This must be called whenever the state of a source may have changed.
    #
    def _update_state(self):

        if self.__consistency < Consistency.CACHED:
364 365 366

            # Source consistency interrogations are silent.
            context = self._get_context()
367
            with context.silence():
368
                self.__consistency = self.get_consistency()
369

370 371 372
    # Return cached consistency
    #
    def _get_consistency(self):
373
        return self.__consistency
374

375 376
    # Wrapper function around plugin provided fetch method
    #
377 378
    def _fetch(self):
        self.fetch()
379

380 381 382 383 384
    # Wrapper for stage() api which gives the source
    # plugin a fully constructed path considering the
    # 'directory' option
    #
    def _stage(self, directory):
385
        staging_directory = self.__ensure_directory(directory)
386

387
        self.stage(staging_directory)
388

389 390
    # Wrapper for init_workspace()
    def _init_workspace(self, directory):
391
        directory = self.__ensure_directory(directory)
392 393 394

        self.init_workspace(directory)

395 396
    # _get_unique_key():
    #
397 398
    # Wrapper for get_unique_key() api
    #
399
    # Args:
400
    #    include_source (bool): Whether to include the delegated source key
401
    #
402
    def _get_unique_key(self, include_source):
403 404 405
        key = {}

        key['directory'] = self.__directory
406
        if include_source:
407 408 409
            key['unique'] = self.get_unique_key()

        return key
410

411
    # Wrapper for set_ref(), also returns whether it changed.
412
    #
413 414 415 416 417 418 419 420 421
    def _set_ref(self, ref, node):
        current_ref = self.get_ref()
        changed = False

        # This comparison should work even for tuples and lists,
        # but we're mostly concerned about simple strings anyway.
        if current_ref != ref:
            changed = True

422 423 424 425 426
        # Set the ref regardless of whether it changed, the
        # TrackQueue() will want to update a specific node with
        # the ref, regardless of whether the original has changed.
        self.set_ref(ref, node)

427 428
        return changed

429 430 431 432 433 434 435 436 437 438 439 440 441 442
    # _project_refs():
    #
    # Gets the appropriate ProjectRefs object for this source,
    # which depends on whether the owning element is a junction
    #
    # Args:
    #    project (Project): The project to check
    #
    def _project_refs(self, project):
        element_kind = self.__element_kind
        if element_kind == 'junction':
            return project.junction_refs
        return project.refs

443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463
    # _load_ref():
    #
    # Loads the ref for the said source.
    #
    # Raises:
    #    (SourceError): If the source does not implement load_ref()
    #
    # Returns:
    #    (ref): A redundant ref specified inline for a project.refs using project
    #
    # This is partly a wrapper around `Source.load_ref()`, it will decide
    # where to load the ref from depending on which project the source belongs
    # to and whether that project uses a project.refs file.
    #
    # Note the return value is used to construct a summarized warning in the
    # case that the toplevel project uses project.refs and also lists refs
    # which will be ignored.
    #
    def _load_ref(self):
        context = self._get_context()
        project = self._get_project()
464
        toplevel = context.get_toplevel_project()
465 466 467 468 469 470 471 472 473 474 475 476 477 478
        redundant_ref = None

        element_name = self.__element_name
        element_idx = self.__element_index

        def do_load_ref(node):
            try:
                self.load_ref(ref_node)
            except ImplError as e:
                raise SourceError("{}: Storing refs in project.refs is not supported by '{}' sources"
                                  .format(self, self.get_kind()),
                                  reason="unsupported-load-ref") from e

        # If the main project overrides the ref, use the override
479
        if project is not toplevel and toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
480 481
            refs = self._project_refs(toplevel)
            ref_node = refs.lookup_ref(project.name, element_name, element_idx)
482 483 484 485 486 487 488 489
            if ref_node is not None:
                do_load_ref(ref_node)

        # If the project itself uses project.refs, clear the ref which
        # was already loaded via Source.configure(), as this would
        # violate the rule of refs being either in project.refs or in
        # the elements themselves.
        #
490
        elif project.ref_storage == ProjectRefStorage.PROJECT_REFS:
491 492 493 494 495 496 497

            # First warn if there is a ref already loaded, and reset it
            redundant_ref = self.get_ref()
            if redundant_ref is not None:
                self.set_ref(None, {})

            # Try to load the ref
498 499
            refs = self._project_refs(project)
            ref_node = refs.lookup_ref(project.name, element_name, element_idx)
500 501 502 503 504 505 506 507 508 509 510 511 512 513
            if ref_node is not None:
                do_load_ref(ref_node)

        return redundant_ref

    # _save_ref()
    #
    # Persists the ref for this source. This will decide where to save the
    # ref, or refuse to persist it, depending on active ref-storage project
    # settings.
    #
    # Args:
    #    new_ref (smth): The new reference to save
    #
514 515 516
    # Returns:
    #    (bool): Whether the ref has changed
    #
517 518 519 520 521 522 523
    # Raises:
    #    (SourceError): In the case we encounter errors saving a file to disk
    #
    def _save_ref(self, new_ref):

        context = self._get_context()
        project = self._get_project()
524
        toplevel = context.get_toplevel_project()
525
        toplevel_refs = self._project_refs(toplevel)
526
        provenance = self._get_provenance()
527 528 529 530 531 532 533 534

        element_name = self.__element_name
        element_idx = self.__element_index

        #
        # Step 1 - Obtain the node
        #
        if project is toplevel:
535
            if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
536
                node = toplevel_refs.lookup_ref(project.name, element_name, element_idx, write=True)
537
            else:
538
                node = provenance.node
539
        else:
540
            if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
541
                node = toplevel_refs.lookup_ref(project.name, element_name, element_idx, write=True)
542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
            else:
                node = {}

        #
        # Step 2 - Set the ref in memory, and determine changed state
        #
        changed = self._set_ref(new_ref, node)

        def do_save_refs(refs):
            try:
                refs.save()
            except OSError as e:
                raise SourceError("{}: Error saving source reference to 'project.refs': {}"
                                  .format(self, e),
                                  reason="save-ref-error") from e

        #
        # Step 3 - Apply the change in project data
        #
        if project is toplevel:
562
            if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
563
                do_save_refs(toplevel_refs)
564 565 566
            else:
                # Save the ref in the originating file
                #
567
                fullname = os.path.join(toplevel.element_path, provenance.filename)
568
                try:
569
                    _yaml.dump(provenance.toplevel, fullname)
570 571
                except OSError as e:
                    raise SourceError("{}: Error saving source reference to '{}': {}"
572
                                      .format(self, provenance.filename, e),
573 574
                                      reason="save-ref-error") from e
        else:
575
            if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
576
                do_save_refs(toplevel_refs)
577 578 579
            else:
                self.warn("{}: Not persisting new reference in junctioned project".format(self))

580 581
        return changed

582 583 584 585 586
    # Wrapper for track()
    #
    def _track(self):
        new_ref = self.track()
        current_ref = self.get_ref()
587

588 589 590 591
        if new_ref is None:
            # No tracking, keep current ref
            new_ref = current_ref

592 593
        if current_ref != new_ref:
            self.info("Found new revision: {}".format(new_ref))
594

595
        return new_ref
596 597 598 599

    #############################################################
    #                   Local Private Methods                   #
    #############################################################
600 601 602 603 604 605 606 607 608 609 610 611 612 613 614

    # Ensures a fully constructed path and returns it
    def __ensure_directory(self, directory):

        if self.__directory is not None:
            directory = os.path.join(directory, self.__directory.lstrip(os.sep))

        try:
            os.makedirs(directory, exist_ok=True)
        except OSError as e:
            raise SourceError("Failed to create staging directory: {}"
                              .format(e),
                              reason="ensure-stage-dir-fail") from e
        return directory

615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
    def __init_defaults(self):
        if not self.__defaults_set:
            project = self._get_project()
            sources = project.source_overrides
            type(self).__defaults = sources.get(self.get_kind(), {})
            type(self).__defaults_set = True

    # This will resolve the final configuration to be handed
    # off to source.configure()
    #
    def __extract_config(self, meta):
        config = _yaml.node_get(self.__defaults, Mapping, 'config', default_value={})
        config = _yaml.node_chain_copy(config)

        _yaml.composite(config, meta.config)
        _yaml.node_final_assertions(config)

        return config