Commit b49d16f3 authored by Jonathan Maw's avatar Jonathan Maw
Browse files

git.py: Use the configured list of submodules when fetching with aliases 

When the source's URL uses an alias (i.e. Source.translate_url()
actually changes the URL), we use the list of submodules defined in the
config as the list of submodules used.
This is because the source would otherwise have to fetch the main repo
to know which submodules it actually has.
This is inconvenient when using mirroring, because when the upstream
repo is unreachable it'll fail when it doesn't have to.

GitMirrors can now be instantiated without having their path and ref
defined, provided the 'parent' mirror is defined. It will retrieve the
path and ref as-needed instead.

If the source's URL does not use an alias, it will behave as normally.
parent d3db49ec

buildstream/plugins/sources/git.py

+108 −29
Original line number Diff line number Diff line
@@ -91,16 +91,18 @@ GIT_MODULES = '.gitmodules' 
#

class GitMirror(SourceFetcher):



    def __init__(self, source, path, url, ref):

    def __init__(self, source, path, url, ref, *, parent=None):



        super().__init__()

        self.source = source

        self.path = path

        self.parent = parent

        self.url = url

        self.ref = ref

        self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))

        self.mark_download_url(url)



        self._path = path

        self._ref = ref



    # Ensures that the mirror exists

    def ensure(self, alias_override=None):
@@ -167,7 +169,8 @@ class GitMirror(SourceFetcher): 
        self.assert_ref()



    def has_ref(self):

        if not self.ref:

        ref = self.get_ref()

        if not ref:

            return False



        # If the mirror doesnt exist, we also dont have the ref
@@ -175,13 +178,13 @@ class GitMirror(SourceFetcher): 
            return False



        # Check if the ref is really there

        rc = self.source.call([self.source.host_git, 'cat-file', '-t', self.ref], cwd=self.mirror)

        rc = self.source.call([self.source.host_git, 'cat-file', '-t', ref], cwd=self.mirror)

        return rc == 0



    def assert_ref(self):

        if not self.has_ref():

            raise SourceError("{}: expected ref '{}' was not found in git repository: '{}'"

                              .format(self.source, self.ref, self.url))

                              .format(self.source, self.get_ref(), self.url))



    def latest_commit(self, tracking):

        _, output = self.source.check_output(
@@ -191,7 +194,8 @@ class GitMirror(SourceFetcher): 
        return output.rstrip('\n')



    def stage(self, directory):

        fullpath = os.path.join(directory, self.path)

        fullpath = os.path.join(directory, self.get_path())

        ref = self.get_ref()



        # Using --shared here avoids copying the objects into the checkout, in any

        # case we're just checking out a specific commit and then removing the .git/
@@ -200,16 +204,17 @@ class GitMirror(SourceFetcher): 
                         fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),

                         fail_temporarily=True)



        self.source.call([self.source.host_git, 'checkout', '--force', self.ref],

                         fail="Failed to checkout git ref {}".format(self.ref),

        self.source.call([self.source.host_git, 'checkout', '--force', ref],

                         fail="Failed to checkout git ref {}".format(ref),

                         cwd=fullpath)



        # Remove .git dir

        shutil.rmtree(os.path.join(fullpath, ".git"))



    def init_workspace(self, directory):

        fullpath = os.path.join(directory, self.path)

        fullpath = os.path.join(directory, self.get_path())

        url = self.source.translate_url(self.url)

        ref = self.get_ref()



        self.source.call([self.source.host_git, 'clone', '--no-checkout', self.mirror, fullpath],

                         fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath),
@@ -219,13 +224,13 @@ class GitMirror(SourceFetcher): 
                         fail='Failed to add remote origin "{}"'.format(url),

                         cwd=fullpath)



        self.source.call([self.source.host_git, 'checkout', '--force', self.ref],

                         fail="Failed to checkout git ref {}".format(self.ref),

        self.source.call([self.source.host_git, 'checkout', '--force', ref],

                         fail="Failed to checkout git ref {}".format(ref),

                         cwd=fullpath)



    # List the submodules (path/url tuples) present at the given ref of this repo

    def submodule_list(self):

        modules = "{}:{}".format(self.ref, GIT_MODULES)

    def _read_gitmodules(self):

        ref = self.get_ref()

        modules = "{}:{}".format(ref, GIT_MODULES)

        exit_code, output = self.source.check_output(

            [self.source.host_git, 'show', modules], cwd=self.mirror)
@@ -236,7 +241,7 @@ class GitMirror(SourceFetcher): 
        elif exit_code != 0:

            raise SourceError(

                "{plugin}: Failed to show gitmodules at ref {ref}".format(

                    plugin=self, ref=self.ref))

                    plugin=self, ref=ref))



        content = '\n'.join([l.strip() for l in output.splitlines()])
@@ -247,6 +252,11 @@ class GitMirror(SourceFetcher): 
        for section in parser.sections():

            # validate section name against the 'submodule "foo"' pattern

            if re.match(r'submodule "(.*)"', section):

                yield (parser, section)



    # List the submodules (path/url tuples) present at the given ref of this repo

    def submodule_list(self):

        for parser, section in self._read_gitmodules():

            path = parser.get(section, 'path')

            url = parser.get(section, 'url')
@@ -256,7 +266,7 @@ class GitMirror(SourceFetcher): 
    # at the given ref of this mirror.

    def submodule_ref(self, submodule, ref=None):

        if not ref:

            ref = self.ref

            ref = self.get_ref()



        # list objects in the parent repo tree to find the commit

        # object that corresponds to the submodule
@@ -287,6 +297,28 @@ class GitMirror(SourceFetcher): 


            return None



    def get_submodule_path(self, url):

        for parser, section in self._read_gitmodules():

            parsed_url = parser.get(section, 'url')

            if parsed_url == url:

                return parser.get(section, 'path')



        raise SourceError("{}: No submodule found with url '{}'".format(self.source, url))



    def get_path(self):

        if self._path is None:

            self._path = self.parent.get_submodule_path()



        return self._path



    def get_ref(self):

        # The top-level GitMirror may have ref as None, submodules don't.

        if self._ref is None and self.parent:

            path = self.get_path()

            self._ref = self.parent.submodule_ref(path)



        return self._ref





class GitSource(Source):

    # pylint: disable=attribute-defined-outside-init
@@ -303,6 +335,8 @@ class GitSource(Source): 
        self.checkout_submodules = self.node_get_member(node, bool, 'checkout-submodules', True)

        self.submodules = []



        self.using_source_fetchers = (self.original_url != self.translate_url(self.original_url))



        # Parse a dict of submodule overrides, stored in the submodule_overrides

        # and submodule_checkout_overrides dictionaries.

        self.submodule_overrides = {}
@@ -311,6 +345,11 @@ class GitSource(Source): 
        for path, _ in self.node_items(modules):

            submodule = self.node_get_member(modules, Mapping, path)

            url = self.node_get_member(submodule, str, 'url', None)



            if self.using_source_fetchers:

                submodule_mirror = GitMirror(self, None, url, None, parent=self.mirror)

                self.submodules.append(submodule_mirror)



            self.submodule_overrides[path] = url

            if 'checkout' in submodule:

                checkout = self.node_get_member(submodule, bool, 'checkout')
@@ -326,7 +365,7 @@ class GitSource(Source): 
        # Here we want to encode the local name of the repository and

        # the ref, if the user changes the alias to fetch the same sources

        # from another location, it should not effect the cache key.

        key = [self.original_url, self.mirror.ref]

        key = [self.original_url, self.mirror.get_ref()]



        # Only modify the cache key with checkout_submodules if it's something

        # other than the default behaviour.
@@ -346,18 +385,18 @@ class GitSource(Source): 
    def get_consistency(self):

        if self.have_all_refs():

            return Consistency.CACHED

        elif self.mirror.ref is not None:

        elif self.mirror.get_ref() is not None:

            return Consistency.RESOLVED

        return Consistency.INCONSISTENT



    def load_ref(self, node):

        self.mirror.ref = self.node_get_member(node, str, 'ref', None)

        self.mirror._ref = self.node_get_member(node, str, 'ref', None)



    def get_ref(self):

        return self.mirror.ref

        return self.mirror.get_ref()



    def set_ref(self, ref, node):

        node['ref'] = self.mirror.ref = ref

        node['ref'] = self.mirror._ref = ref



    def track(self):
@@ -376,6 +415,24 @@ class GitSource(Source): 


        return ret



    def fetch(self):



        with self.timed_activity("Fetching {}".format(self.mirror.url), silent_nested=True):



            # Here we are only interested in ensuring that our mirror contains

            # the self.mirror.ref commit.

            self.mirror.ensure()

            if not self.mirror.has_ref():

                self.mirror.fetch()



            self.mirror.assert_ref()



            # Here after performing any fetches, we need to also ensure that

            # we've cached the desired refs in our mirrors of submodules.

            #

            self.refresh_submodules()

            self.fetch_submodules()



    def init_workspace(self, directory):

        # XXX: may wish to refactor this as some code dupe with stage()

        self.refresh_submodules()
@@ -399,8 +456,9 @@ class GitSource(Source): 
        with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True):

            self.mirror.stage(directory)

            for mirror in self.submodules:

                if mirror.path in self.submodule_checkout_overrides:

                    checkout = self.submodule_checkout_overrides[mirror.path]

                mirror_path = mirror.get_path()

                if mirror_path in self.submodule_checkout_overrides:

                    checkout = self.submodule_checkout_overrides[mirror_path]

                else:

                    checkout = self.checkout_submodules
@@ -408,7 +466,10 @@ class GitSource(Source): 
                    mirror.stage(directory)



    def get_source_fetchers(self):

        self.refresh_submodules()

        # If the url does not contain an alias, then it does not need SourceFetchers

        if self.mirror.url == self.translate_url(self.mirror.url):

            return []

        else:

            return [self.mirror] + self.submodules



    ###########################################################
@@ -432,6 +493,11 @@ class GitSource(Source): 
    # Assumes that we have our mirror and we have the ref which we point to

    #

    def refresh_submodules(self):



        # When using source fetchers, the submodule list is defined by the 'submodules' config field

        if self.using_source_fetchers:

            return



        self.mirror.ensure()

        submodules = []
@@ -454,6 +520,19 @@ class GitSource(Source): 


        self.submodules = submodules



    # Ensures that we have mirrored git repositories for all

    # the submodules existing at the given commit of the main git source.

    #

    # Also ensure that these mirrors have the required commits

    # referred to at the given commit of the main git source.

    #

    def fetch_submodules(self):

        for mirror in self.submodules:

            mirror.ensure()

            if not mirror.has_ref():

                mirror.fetch()

                mirror.assert_ref()





# Plugin entry point

def setup():