.gitlab-ci.yml

@@ -49,10 +49,6 @@ tests-debian-9:
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:9-master-46405991
   <<: *tests
 
-tests-fedora-27:
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:27-master-46405991
-  <<: *tests
-
 tests-fedora-28:
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:28-master-46405991
   <<: *tests
@@ -95,7 +91,7 @@ overnight-fedora-28-aarch64:
 tests-unix:
   # Use fedora here, to a) run a test on fedora and b) ensure that we
   # can get rid of ostree - this is not possible with debian-8
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:27-master-46405991
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:28-master-46405991
   <<: *tests
   variables:
     BST_FORCE_BACKEND: "unix"
@@ -291,7 +287,6 @@ coverage:
     - cp -a .coverage-reports/ ./coverage-report
   dependencies:
   - tests-debian-9
-  - tests-fedora-27
   - tests-fedora-28
   - tests-fedora-missing-deps
   - tests-ubuntu-18.04

buildstream/data/projectconfig.yaml

@@ -90,10 +90,16 @@ split-rules:
   # built element to run, this includes stripped executables
   # and shared libraries by default.
   runtime:
+  - |
+    %{bindir}
   - |
     %{bindir}/*
+  - |
+    %{sbindir}
   - |
     %{sbindir}/*
+  - |
+    %{libexecdir}
   - |
     %{libexecdir}/*
   - |

buildstream/plugins/elements/import.py

@@ -42,6 +42,10 @@ class ImportElement(Element):
     BST_VIRTUAL_DIRECTORY = True
 
     def configure(self, node):
+        self.node_validate(node, [
+            'source', 'target'
+        ])
+
         self.source = self.node_subst_member(node, 'source')
         self.target = self.node_subst_member(node, 'target')
 

buildstream/plugins/sources/local.py

@@ -97,7 +97,7 @@ class LocalSource(Source):
         with self.timed_activity("Staging local files at {}".format(self.path)):
 
             if os.path.isdir(self.fullpath):
-                files = list(utils.list_relative_paths(self.fullpath, list_dirs=True))
+                files = list(utils.list_relative_paths(self.fullpath))
                 utils.copy_files(self.fullpath, directory, files=files)
             else:
                 destfile = os.path.join(directory, os.path.basename(self.path))
@@ -133,11 +133,11 @@ def unique_key(filename):
 
     # Return some hard coded things for files which
     # have no content to calculate a key for
-    if os.path.isdir(filename):
-        return "0"
-    elif os.path.islink(filename):
+    if os.path.islink(filename):
         # For a symbolic link, use the link target as its unique identifier
         return os.readlink(filename)
+    elif os.path.isdir(filename):
+        return "0"
 
     return utils.sha256sum(filename)
 

buildstream/sandbox/sandbox.py

@@ -525,11 +525,11 @@ class Sandbox():
     #         (bool): Whether a command exists inside the sandbox.
     def _has_command(self, command, env=None):
         if os.path.isabs(command):
-            return os.path.exists(os.path.join(
+            return os.path.lexists(os.path.join(
                 self._root, command.lstrip(os.sep)))
 
         for path in env.get('PATH').split(':'):
-            if os.path.exists(os.path.join(
+            if os.path.lexists(os.path.join(
                     self._root, path.lstrip(os.sep), command)):
                 return True
 

buildstream/storage/_casbaseddirectory.py

@@ -795,29 +795,16 @@ class CasBasedDirectory(Directory):
         Return value: List(str) - list of all paths
         """
 
-        symlink_list = filter(lambda i: isinstance(i[1].pb_object, remote_execution_pb2.SymlinkNode),
-                              self.index.items())
-        file_list = list(filter(lambda i: isinstance(i[1].pb_object, remote_execution_pb2.FileNode),
+        file_list = list(filter(lambda i: not isinstance(i[1].buildstream_object, CasBasedDirectory),
                                 self.index.items()))
         directory_list = filter(lambda i: isinstance(i[1].buildstream_object, CasBasedDirectory),
                                 self.index.items())
 
-        # We need to mimic the behaviour of os.walk, in which symlinks
-        # to directories count as directories and symlinks to file or
-        # broken symlinks count as files. os.walk doesn't follow
-        # symlinks, so we don't recurse.
-        for (k, v) in sorted(symlink_list):
-            target = self._resolve(k, absolute_symlinks_resolve=True)
-            if isinstance(target, CasBasedDirectory):
-                yield os.path.join(relpath, k)
-            else:
-                file_list.append((k, v))
-
-        if file_list == [] and relpath != "":
+        if relpath != "":
             yield relpath
-        else:
-            for (k, v) in sorted(file_list):
-                yield os.path.join(relpath, k)
+
+        for (k, v) in sorted(file_list):
+            yield os.path.join(relpath, k)
 
         for (k, v) in sorted(directory_list):
             yield from v.buildstream_object.list_relative_paths(relpath=os.path.join(relpath, k))

buildstream/utils.py

@@ -111,27 +111,37 @@ class FileListResult():
         return ret
 
 
-def list_relative_paths(directory, *, list_dirs=True):
+def list_relative_paths(directory):
     """A generator for walking directory relative paths
 
     This generator is useful for checking the full manifest of
     a directory.
 
-    Note that directories will be yielded only if they are
-    empty.
-
     Symbolic links will not be followed, but will be included
     in the manifest.
 
     Args:
        directory (str): The directory to list files in
-       list_dirs (bool): Whether to list directories
 
     Yields:
        Relative filenames in `directory`
     """
     for (dirpath, dirnames, filenames) in os.walk(directory):
 
+        # os.walk does not decend into symlink directories, which
+        # makes sense because otherwise we might have redundant
+        # directories, or end up descending into directories outside
+        # of the walk() directory.
+        #
+        # But symlinks to directories are still identified as
+        # subdirectories in the walked `dirpath`, so we extract
+        # these symlinks from `dirnames` and add them to `filenames`.
+        #
+        for d in dirnames:
+            fullpath = os.path.join(dirpath, d)
+            if os.path.islink(fullpath):
+                filenames.append(d)
+
         # Modifying the dirnames directly ensures that the os.walk() generator
         # allows us to specify the order in which they will be iterated.
         dirnames.sort()
@@ -143,26 +153,9 @@ def list_relative_paths(directory, *, list_dirs=True):
         # `directory`, prefer to have no prefix in that case.
         basepath = relpath if relpath != '.' and dirpath != directory else ''
 
-        # os.walk does not decend into symlink directories, which
-        # makes sense because otherwise we might have redundant
-        # directories, or end up descending into directories outside
-        # of the walk() directory.
-        #
-        # But symlinks to directories are still identified as
-        # subdirectories in the walked `dirpath`, so we extract
-        # these symlinks from `dirnames`
-        #
-        if list_dirs:
-            for d in dirnames:
-                fullpath = os.path.join(dirpath, d)
-                if os.path.islink(fullpath):
-                    yield os.path.join(basepath, d)
-
-        # We've decended into an empty directory, in this case we
-        # want to include the directory itself, but not in any other
-        # case.
-        if list_dirs and not filenames:
-            yield relpath
+        # First yield the walked directory itself, except for the root
+        if basepath != '':
+            yield basepath
 
         # List the filenames in the walked directory
         for f in filenames:

tests/cachekey/project/elements/build1.expected

-dadb8f86874f714b4f6d4c9025332934efb7e85c38f6a68b1267746ae8f43f24
+298bb7caab56c60bc12f59df15db2b1107a604fdb0ce3fa5729aa23d6d37f6e4
\ No newline at end of file

tests/cachekey/project/elements/build2.expected

-f81cefce283dd3581ba2fc865ff9c2763119274b114b12edb4e87196cfff8b2a
+c745f07458a2488180ae7f5b0601a38853dcdfd2b6771c18ecbb622f1af10cf1
\ No newline at end of file

tests/cachekey/project/elements/compose1.expected

-36cb2ac57281343959266e87913eb690f4c68980d7267160ff5f071db778719d
\ No newline at end of file
+d759c8b7262c4782c12b425bfca0bc8f4c30656d91c98e309c10f11d4179b101
\ No newline at end of file

tests/cachekey/project/elements/compose2.expected

-30ffe86750a497052f8fc868c47a08644cd579f8e86f59be9b04fa5530b6a9b6
\ No newline at end of file
+01e22d37b82c9b35c35d3d8485e05fe6de526d3af00137bbff51a309b1fa3401
\ No newline at end of file

tests/cachekey/project/elements/compose3.expected

-1bac1d17cc2aed85ab14daf264f955b2204823799bc5bac5ff77c79b32d0c08b
\ No newline at end of file
+22b846d1928155b5a21497e8048af2945ee03a826338f91d4f00d74597cc3ef7
\ No newline at end of file

tests/cachekey/project/elements/compose4.expected

-99690e3a915f4c5b6f76a5bcee8bffe74d077a4398ac29623be36bca26703290
\ No newline at end of file
+0e00f203ce9bc4163be0ed304fcf2abb6ee39aad4ee9a12d80719708a4327337
\ No newline at end of file

tests/cachekey/project/elements/compose5.expected

-a3e25dabe35a2909920f94c9e457c977b00dfa49c161b82c47ad55e761349c6a
\ No newline at end of file
+3d8dd9fef508c27961edeb21dd469d04e7ea097c86a98e7c3bf725975548748f
\ No newline at end of file

tests/cachekey/project/elements/import1.expected

-791aaae474dce95f98c849d748088697334a9b4bfcb6225c59804efe03e803c9
\ No newline at end of file
+02f863aedaf175f4b4bb9979a538bd1fb6eabd0ef9d77a633b141c072d957df5
\ No newline at end of file

tests/cachekey/project/elements/import2.expected

-3aaf0565ffbeb2faa4e48230d07ef839b9bc2ff012780ca9b5f6b9c968f539b2
\ No newline at end of file
+80ee70ad27ee4640400533585d1b2313b1abdb0034f4159dad42f545310d9c24
\ No newline at end of file

tests/cachekey/project/elements/import3.expected

-7cadfb9b592af06fa765389278d919cb8e29515ad057710563da9cd1c36e2ce9
\ No newline at end of file
+b30d79b763ba3048945784d06482ee378d2642dff712ecffae737aa6f9391b9f
\ No newline at end of file

tests/cachekey/project/elements/script1.expected

-14db0665eed7c3d4cf3685c2e386a9e494465927a5ad962c98b92856be3d88b8
\ No newline at end of file
+fb69c9237156bdeb0f948078f735189e9d8a4c4dca3c67b6f1ed5fa314f88707
\ No newline at end of file

tests/cachekey/project/sources/bzr1.expected

-faadf63e4025e4a068532d01296b65db38dbf2c95625695832f7c601b2134375
\ No newline at end of file
+5a8144078d50ff5507d7260c2aa7c62fc6272554767392cd827277e224385cb9
\ No newline at end of file

tests/cachekey/project/sources/git1.expected

-3dc6b7c47ddb3ced44c42d22f4c192b9245112197172d2f9ce9498e4614bbca4
\ No newline at end of file
+21968271111bb43e57417d24481742c2fe52fddd798af4aacca0f604c54b6b94
\ No newline at end of file