buildstream/_artifactcache/cascache.py

@@ -348,19 +348,29 @@ class CASCache(ArtifactCache):
         return pushed
 
     def push_directory(self, project, directory):
+        """ Push the given virtual directory to all remotes.
+
+        Args:
+            project (Project): The current project
+            directory (Directory): A virtual directory object to push.
+
+        Raises: ArtifactError if no push remotes are configured.
+        """
 
         push_remotes = [r for r in self._remotes[project] if r.spec.push]
 
+        if not push_remotes:
+            raise ArtifactError("CASCache: push_directory was called, but no remote artifact " +
+                                "servers are configured as push remotes.")
+
         if directory.ref is None:
-            return None
+            return
 
         for remote in push_remotes:
             remote.init()
 
             self._send_directory(remote, directory.ref)
 
-        return directory.ref
-
     def push_message(self, project, message):
 
         push_remotes = [r for r in self._remotes[project] if r.spec.push]

buildstream/element.py

@@ -2137,14 +2137,11 @@ class Element(Plugin):
         project = self._get_project()
         platform = Platform.get_platform()
 
-        if self.__remote_execution_url and self.BST_VIRTUAL_DIRECTORY:
-            if not self.__artifacts.has_push_remotes(element=self):
-                # Give an early warning if remote execution will not work
-                raise ElementError("Artifact {} is configured to use remote execution but has no push remotes. "
-                                   .format(self.name) +
-                                   "The remote artifact server(s) may not be correctly configured or contactable.")
+        if (directory is not None and
+            self.__remote_execution_url and
+            self.BST_VIRTUAL_DIRECTORY):
 
-            self.info("Using a remote sandbox for artifact {}".format(self.name))
+            self.info("Using a remote sandbox for artifact {} with directory '{}'".format(self.name, directory))
 
             sandbox = SandboxRemote(context, project,
                                     directory,

buildstream/sandbox/_sandboxremote.py

@@ -173,8 +173,8 @@ class SandboxRemote(Sandbox):
         platform = Platform.get_platform()
         cascache = platform.artifactcache
         # Now, push that key (without necessarily needing a ref) to the remote.
-        vdir_digest = cascache.push_directory(self._get_project(), upload_vdir)
-        if not vdir_digest or not cascache.verify_digest_pushed(self._get_project(), vdir_digest):
+        cascache.push_directory(self._get_project(), upload_vdir)
+        if not cascache.verify_digest_pushed(self._get_project(), upload_vdir.ref):
             raise SandboxError("Failed to verify that source has been pushed to the remote artifact cache.")
 
         # Set up environment and working directory

buildstream/source.py

@@ -930,6 +930,38 @@ class Source(Plugin):
     #                   Local Private Methods                   #
     #############################################################
 
+    # __clone_for_uri()
+    #
+    # Clone the source with an alternative URI setup for the alias
+    # which this source uses.
+    #
+    # This is used for iteration over source mirrors.
+    #
+    # Args:
+    #    uri (str): The alternative URI for this source's alias
+    #
+    # Returns:
+    #    (Source): A new clone of this Source, with the specified URI
+    #              as the value of the alias this Source has marked as
+    #              primary with either mark_download_url() or
+    #              translate_url().
+    #
+    def __clone_for_uri(self, uri):
+        project = self._get_project()
+        context = self._get_context()
+        alias = self._get_alias()
+        source_kind = type(self)
+
+        clone = source_kind(context, project, self.__meta, alias_override=(alias, uri))
+
+        # Do the necessary post instantiation routines here
+        #
+        clone._preflight()
+        clone._load_ref()
+        clone._update_state()
+
+        return clone
+
     # Tries to call fetch for every mirror, stopping once it succeeds
     def __do_fetch(self, **kwargs):
         project = self._get_project()
@@ -968,12 +1000,8 @@ class Source(Plugin):
                 self.fetch(**kwargs)
                 return
 
-            context = self._get_context()
-            source_kind = type(self)
             for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
-                new_source = source_kind(context, project, self.__meta,
-                                         alias_override=(alias, uri))
-                new_source._preflight()
+                new_source = self.__clone_for_uri(uri)
                 try:
                     new_source.fetch(**kwargs)
                 # FIXME: Need to consider temporary vs. permanent failures,
@@ -1006,9 +1034,7 @@ class Source(Plugin):
         # NOTE: We are assuming here that tracking only requires substituting the
         #       first alias used
         for uri in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass)):
-            new_source = source_kind(context, project, self.__meta,
-                                     alias_override=(alias, uri))
-            new_source._preflight()
+            new_source = self.__clone_for_uri(uri)
             try:
                 ref = new_source.track(**kwargs)
             # FIXME: Need to consider temporary vs. permanent failures,

buildstream/storage/__init__.py

@@ -19,4 +19,5 @@
 #        Jim MacArthur <jim.macarthur@codethink.co.uk>
 
 from ._filebaseddirectory import FileBasedDirectory
+from ._casbaseddirectory import CasBasedDirectory
 from .directory import Directory

tests/artifactcache/push.py

@@ -228,9 +228,9 @@ def _test_push_directory(user_config_file, project_dir, artifact_dir, artifact_d
         directory = CasBasedDirectory(context, ref=artifact_digest)
 
         # Push the CasBasedDirectory object
-        directory_digest = cas.push_directory(project, directory)
+        cas.push_directory(project, directory)
 
-        queue.put(directory_digest.hash)
+        queue.put(directory.ref.hash)
     else:
         queue.put("No remote configured")
 

tests/storage/virtual_directory_import.py

+import os
+import pytest
+from tests.testutils import cli
+
+from buildstream.storage import CasBasedDirectory
+
+
+class FakeContext():
+    def __init__(self):
+        self.config_cache_quota = "65536"
+
+    def get_projects(self):
+        return []
+
+# This is a set of example file system contents. The test attempts to import
+# each on top of each other to test importing works consistently.
+# Each tuple is defined as (<filename>, <type>, <content>). Type can be
+# 'F' (file), 'S' (symlink) or 'D' (directory) with content being the contents
+# for a file or the destination for a symlink.
+root_filesets = [
+    [('a/b/c/textfile1', 'F', 'This is textfile 1\n')],
+    [('a/b/c/textfile1', 'F', 'This is the replacement textfile 1\n')],
+    [('a/b/d', 'D', '')],
+    [('a/b/c', 'S', '/a/b/d')],
+    [('a/b/d', 'D', ''), ('a/b/c', 'S', '/a/b/d')],
+]
+
+empty_hash_ref = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+
+def generate_import_roots(directory):
+    for fileset in range(1, len(root_filesets) + 1):
+        rootname = "root{}".format(fileset)
+        rootdir = os.path.join(directory, "content", rootname)
+
+        for (path, typesymbol, content) in root_filesets[fileset - 1]:
+            if typesymbol == 'F':
+                (dirnames, filename) = os.path.split(path)
+                os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
+                with open(os.path.join(rootdir, dirnames, filename), "wt") as f:
+                    f.write(content)
+            elif typesymbol == 'D':
+                os.makedirs(os.path.join(rootdir, path), exist_ok=True)
+            elif typesymbol == 'S':
+                (dirnames, filename) = os.path.split(path)
+                os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
+                os.symlink(content, os.path.join(rootdir, path))
+
+
+def file_contents(path):
+    with open(path, "r") as f:
+        result = f.read()
+    return result
+
+
+def file_contents_are(path, contents):
+    return file_contents(path) == contents
+
+
+def create_new_vdir(root_number, fake_context, tmpdir):
+    d = CasBasedDirectory(fake_context)
+    d.import_files(os.path.join(tmpdir, "content", "root{}".format(root_number)))
+    assert d.ref.hash != empty_hash_ref
+    return d
+
+
+def combinations(integer_range):
+    for x in integer_range:
+        for y in integer_range:
+            yield (x, y)
+
+
+def resolve_symlinks(path, root):
+    """ A function to resolve symlinks inside 'path' components apart from the last one.
+        For example, resolve_symlinks('/a/b/c/d', '/a/b')
+        will return '/a/b/f/d' if /a/b/c is a symlink to /a/b/f. The final component of
+        'path' is not resolved, because we typically want to inspect the symlink found
+        at that path, not its target.
+
+    """
+    components = path.split(os.path.sep)
+    location = root
+    for i in range(0, len(components) - 1):
+        location = os.path.join(location, components[i])
+        if os.path.islink(location):
+            # Resolve the link, add on all the remaining components
+            target = os.path.join(os.readlink(location))
+            tail = os.path.sep.join(components[i + 1:])
+
+            if target.startswith(os.path.sep):
+                # Absolute link - relative to root
+                location = os.path.join(root, target, tail)
+            else:
+                # Relative link - relative to symlink location
+                location = os.path.join(location, target)
+            return resolve_symlinks(location, root)
+    # If we got here, no symlinks were found. Add on the final component and return.
+    location = os.path.join(location, components[-1])
+    return location
+
+
+def directory_not_empty(path):
+    return os.listdir(path)
+
+
+@pytest.mark.parametrize("original,overlay", combinations([1, 2, 3, 4, 5]))
+def test_cas_import(cli, tmpdir, original, overlay):
+    tmpdir = str(tmpdir)  # Avoids LocalPath issues on some systems
+    fake_context = FakeContext()
+    fake_context.artifactdir = tmpdir
+    # Create some fake content
+    generate_import_roots(tmpdir)
+
+    d = create_new_vdir(original, fake_context, tmpdir)
+    d2 = create_new_vdir(overlay, fake_context, tmpdir)
+    d.import_files(d2)
+    d.export_files(os.path.join(tmpdir, "output"))
+
+    for item in root_filesets[overlay - 1]:
+        (path, typename, content) = item
+        realpath = resolve_symlinks(path, os.path.join(tmpdir, "output"))
+        if typename == 'F':
+            if os.path.isdir(realpath) and directory_not_empty(realpath):
+                # The file should not have overwritten the directory in this case.
+                pass
+            else:
+                assert os.path.isfile(realpath), "{} did not exist in the combined virtual directory".format(path)
+                assert file_contents_are(realpath, content)
+        elif typename == 'S':
+            if os.path.isdir(realpath) and directory_not_empty(realpath):
+                # The symlink should not have overwritten the directory in this case.
+                pass
+            else:
+                assert os.path.islink(realpath)
+                assert os.readlink(realpath) == content
+        elif typename == 'D':
+            # Note that isdir accepts symlinks to dirs, so a symlink to a dir is acceptable.
+            assert os.path.isdir(realpath)