git.py: Improve mirror support

When implementing fetching from mirrors, I encountered some problems
with the git source:

1. The mirror URL was using translate_url()'s output, so if a different
   alias was used, then fetching from the mirror would go to a different
   directory, and be inaccessible.
2. After fixing that, fetching was unable to pull from a URL other than
   the one used at repository creation, meaning it wouldn't actually
   pull from the mirror.

buildstream/plugins/sources/git.py

@@ -78,7 +78,7 @@ from io import StringIO
 
 from configparser import RawConfigParser
 
-from buildstream import Source, SourceError, Consistency
+from buildstream import Source, SourceError, Consistency, SourceFetcher
 from buildstream import utils
 
 GIT_MODULES = '.gitmodules'
@@ -88,18 +88,20 @@ GIT_MODULES = '.gitmodules'
 # for the primary git source and also for each submodule it
 # might have at a given time
 #
-class GitMirror():
+class GitMirror(SourceFetcher):
 
     def __init__(self, source, path, url, ref):
 
+        super().__init__()
         self.source = source
         self.path = path
-        self.url = source.translate_url(url)
+        self.url = url
         self.ref = ref
-        self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(self.url))
+        self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
+        self.mark_download_url(url)
 
     # Ensures that the mirror exists
-    def ensure(self):
+    def ensure(self, alias_override=None):
 
         # Unfortunately, git does not know how to only clone just a specific ref,
         # so we have to download all of those gigs even if we only need a couple
@@ -112,22 +114,47 @@ class GitMirror():
             # system configured tmpdir is not on the same partition.
             #
             with self.source.tempdir() as tmpdir:
-                self.source.call([self.source.host_git, 'clone', '--mirror', '-n', self.url, tmpdir],
-                                 fail="Failed to clone git repository {}".format(self.url),
+                url = self.source.translate_url(self.url, alias_override=alias_override)
+                self.source.call([self.source.host_git, 'clone', '--mirror', '-n', url, tmpdir],
+                                 fail="Failed to clone git repository {}".format(url),
                                  fail_temporarily=True)
 
                 try:
                     shutil.move(tmpdir, self.mirror)
                 except (shutil.Error, OSError) as e:
                     raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}'"
-                                      .format(self.source, self.url, tmpdir, self.mirror)) from e
+                                      .format(self.source, url, tmpdir, self.mirror)) from e
+
+    def _fetch(self, alias_override=None):
+        url = self.source.translate_url(self.url, alias_override=alias_override)
+
+        if alias_override:
+            remote_name = utils.url_directory_name(alias_override)
+            _, remotes = self.source.check_output(
+                [self.source.host_git, 'remote'],
+                fail="Failed to retrieve list of remotes in {}".format(self.mirror),
+                cwd=self.mirror
+            )
+            if remote_name not in remotes:
+                self.source.call(
+                    [self.source.host_git, 'remote', 'add', remote_name, url],
+                    fail="Failed to add remote {} with url {}".format(remote_name, url),
+                    cwd=self.mirror
+                )
+        else:
+            remote_name = "origin"
 
-    def fetch(self):
-        self.source.call([self.source.host_git, 'fetch', 'origin', '--prune'],
-                         fail="Failed to fetch from remote git repository: {}".format(self.url),
+        self.source.call([self.source.host_git, 'fetch', remote_name, '--prune'],
+                         fail="Failed to fetch from remote git repository: {}".format(url),
                          fail_temporarily=True,
                          cwd=self.mirror)
 
+    def fetch(self, alias_override=None):
+        self.ensure(alias_override)
+        if not self.has_ref():
+            self._fetch(alias_override)
+        self.assert_ref()
+
     def has_ref(self):
         if not self.ref:
             return False
@@ -171,13 +198,14 @@ class GitMirror():
 
     def init_workspace(self, directory):
         fullpath = os.path.join(directory, self.path)
+        url = self.source.translate_url(self.url)
 
         self.source.call([self.source.host_git, 'clone', '--no-checkout', self.mirror, fullpath],
                          fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath),
                          fail_temporarily=True)
 
-        self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', self.url],
-                         fail='Failed to add remote origin "{}"'.format(self.url),
+        self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', url],
+                         fail='Failed to add remote origin "{}"'.format(url),
                          cwd=fullpath)
 
         self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
@@ -277,6 +305,8 @@ class GitSource(Source):
                 checkout = self.node_get_member(submodule, bool, 'checkout')
                 self.submodule_checkout_overrides[path] = checkout
 
+        self.mark_download_url(self.original_url)
+
     def preflight(self):
         # Check if git is installed, get the binary at the same time
         self.host_git = utils.get_host_tool('git')
@@ -328,31 +358,13 @@ class GitSource(Source):
                                  .format(self.tracking, self.mirror.url),
                                  silent_nested=True):
             self.mirror.ensure()
-            self.mirror.fetch()
+            self.mirror._fetch()
 
             # Update self.mirror.ref and node.ref from the self.tracking branch
             ret = self.mirror.latest_commit(self.tracking)
 
         return ret
 
-    def fetch(self):
-
-        with self.timed_activity("Fetching {}".format(self.mirror.url), silent_nested=True):
-
-            # Here we are only interested in ensuring that our mirror contains
-            # the self.mirror.ref commit.
-            self.mirror.ensure()
-            if not self.mirror.has_ref():
-                self.mirror.fetch()
-
-            self.mirror.assert_ref()
-
-            # Here after performing any fetches, we need to also ensure that
-            # we've cached the desired refs in our mirrors of submodules.
-            #
-            self.refresh_submodules()
-            self.fetch_submodules()
-
     def init_workspace(self, directory):
         # XXX: may wish to refactor this as some code dupe with stage()
         self.refresh_submodules()
@@ -384,6 +396,10 @@ class GitSource(Source):
                 if checkout:
                     mirror.stage(directory)
 
+    def get_source_fetchers(self):
+        self.refresh_submodules()
+        return [self.mirror] + self.submodules
+
     ###########################################################
     #                     Local Functions                     #
     ###########################################################
@@ -405,6 +421,7 @@ class GitSource(Source):
     # Assumes that we have our mirror and we have the ref which we point to
     #
     def refresh_submodules(self):
+        self.mirror.ensure()
         submodules = []
 
         # XXX Here we should issue a warning if either:
@@ -426,19 +443,6 @@ class GitSource(Source):
 
         self.submodules = submodules
 
-    # Ensures that we have mirrored git repositories for all
-    # the submodules existing at the given commit of the main git source.
-    #
-    # Also ensure that these mirrors have the required commits
-    # referred to at the given commit of the main git source.
-    #
-    def fetch_submodules(self):
-        for mirror in self.submodules:
-            mirror.ensure()
-            if not mirror.has_ref():
-                mirror.fetch()
-                mirror.assert_ref()
-
 
 # Plugin entry point
 def setup():