buildstream/_versions.py

@@ -33,4 +33,5 @@ BST_FORMAT_VERSION = 12
 # or if buildstream was changed in a way which can cause
 # the same cache key to produce something that is no longer
 # the same.
-BST_CORE_ARTIFACT_VERSION = 2
+
+BST_CORE_ARTIFACT_VERSION = ('bst-1.2', 3)

buildstream/plugins/sources/local.py

@@ -37,6 +37,7 @@ local - stage local files and directories
 """
 
 import os
+import stat
 from buildstream import Source, Consistency
 from buildstream import utils
 
@@ -94,12 +95,35 @@ class LocalSource(Source):
         # Dont use hardlinks to stage sources, they are not write protected
         # in the sandbox.
         with self.timed_activity("Staging local files at {}".format(self.path)):
+
             if os.path.isdir(self.fullpath):
-                utils.copy_files(self.fullpath, directory)
+                files = list(utils.list_relative_paths(self.fullpath, list_dirs=True))
+                utils.copy_files(self.fullpath, directory, files=files)
             else:
                 destfile = os.path.join(directory, os.path.basename(self.path))
+                files = [os.path.basename(self.path)]
                 utils.safe_copy(self.fullpath, destfile)
 
+            for f in files:
+                # Non empty directories are not listed by list_relative_paths
+                dirs = f.split(os.sep)
+                for i in range(1, len(dirs)):
+                    d = os.path.join(directory, *(dirs[:i]))
+                    assert os.path.isdir(d) and not os.path.islink(d)
+                    os.chmod(d, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+
+                path = os.path.join(directory, f)
+                if os.path.islink(path):
+                    pass
+                elif os.path.isdir(path):
+                    os.chmod(path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+                else:
+                    st = os.stat(path)
+                    if st.st_mode & stat.S_IXUSR:
+                        os.chmod(path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+                    else:
+                        os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
+
 
 # Create a unique key for a file
 def unique_key(filename):

buildstream/plugins/sources/remote.py

@@ -49,6 +49,7 @@ remote - stage files from remote urls
 
 """
 import os
+import stat
 from buildstream import SourceError, utils
 from ._downloadablefilesource import DownloadableFileSource
 
@@ -75,6 +76,7 @@ class RemoteSource(DownloadableFileSource):
         dest = os.path.join(directory, self.filename)
         with self.timed_activity("Staging remote file to {}".format(dest)):
             utils.safe_copy(self._get_mirror_file(), dest)
+            os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
 
 
 def setup():

buildstream/plugins/sources/zip.py

@@ -49,10 +49,17 @@ zip - stage files from zip archives
    # To extract the root of the archive directly, this can be set
    # to an empty string.
    base-dir: '*'
+
+.. attention::
+
+   File permissions are not preserved. All extracted directories have
+   permissions 0755 and all extracted files have permissions 0644.
+
 """
 
 import os
 import zipfile
+import stat
 
 from buildstream import SourceError
 from buildstream import utils
@@ -74,6 +81,9 @@ class ZipSource(DownloadableFileSource):
         return super().get_unique_key() + [self.base_dir]
 
     def stage(self, directory):
+        exec_rights = (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) & ~(stat.S_IWGRP | stat.S_IWOTH)
+        noexec_rights = exec_rights & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
         try:
             with zipfile.ZipFile(self._get_mirror_file()) as archive:
                 base_dir = None
@@ -81,9 +91,27 @@ class ZipSource(DownloadableFileSource):
                     base_dir = self._find_base_dir(archive, self.base_dir)
 
                 if base_dir:
-                    archive.extractall(path=directory, members=self._extract_members(archive, base_dir))
+                    members = self._extract_members(archive, base_dir)
                 else:
-                    archive.extractall(path=directory)
+                    members = archive.namelist()
+
+                for member in members:
+                    written = archive.extract(member, path=directory)
+
+                    # zipfile.extract might create missing directories
+                    rel = os.path.relpath(written, start=directory)
+                    assert not os.path.isabs(rel)
+                    rel = os.path.dirname(rel)
+                    while rel:
+                        os.chmod(os.path.join(directory, rel), exec_rights)
+                        rel = os.path.dirname(rel)
+
+                    if os.path.islink(written):
+                        pass
+                    elif os.path.isdir(written):
+                        os.chmod(written, exec_rights)
+                    else:
+                        os.chmod(written, noexec_rights)
 
         except (zipfile.BadZipFile, zipfile.LargeZipFile, OSError) as e:
             raise SourceError("{}: Error staging source: {}".format(self, e)) from e

buildstream/utils.py

@@ -1010,6 +1010,15 @@ def _call(*popenargs, terminate=False, **kwargs):
 
     process = None
 
+    old_preexec_fn = kwargs.get('preexec_fn')
+    if 'preexec_fn' in kwargs:
+        del kwargs['preexec_fn']
+
+    def preexec_fn():
+        os.umask(stat.S_IWGRP | stat.S_IWOTH)
+        if old_preexec_fn is not None:
+            old_preexec_fn()
+
     # Handle termination, suspend and resume
     def kill_proc():
         if process:
@@ -1054,7 +1063,7 @@ def _call(*popenargs, terminate=False, **kwargs):
             os.killpg(group_id, signal.SIGCONT)
 
     with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc):
-        process = subprocess.Popen(*popenargs, **kwargs)
+        process = subprocess.Popen(*popenargs, preexec_fn=preexec_fn, **kwargs)
         output, _ = process.communicate()
         exit_code = process.poll()
 

tests/cachekey/project/elements/build1.expected

-3db51572837956b28ffbc4aabdce659b4a1d91dcbb8b75954210346959ed5fa9
\ No newline at end of file
+e7de3dd12a1e5307e07859ddf2192443a0ccb1ff48e0adcc6c18f9edc2bd0d7d
\ No newline at end of file

tests/cachekey/project/elements/build2.expected

-bcde6fc389b7d8bb7788989b68f68653ab8ed658117012c0611f218f4a585d38
\ No newline at end of file
+d74957e0f20a7664e9ceed6cc2ba6c140bd8d8d0712d02066feb442638e8e6ed
\ No newline at end of file

tests/cachekey/project/elements/compose1.expected

-6736bbcc055e1801a19288d3a64b622e0b9223164f8ad2ce842b18a4eaa0cfb9
\ No newline at end of file
+f8b69ac5ce84a8e8db30f9ae58d7560054d41da311176f74047694ec1203d7e8
\ No newline at end of file

tests/cachekey/project/elements/compose2.expected

-9294428a0b5c0d44fdb3ab0f883ee87f9e62d51f96c7de1e5e81ed5e3934d403
\ No newline at end of file
+4f542af0ebf3136b0affe42cb5574b7cf1034db6fb60d272ab2304e1c99b0d6f
\ No newline at end of file

tests/cachekey/project/elements/compose3.expected

-4f1569b9a6317280e6299f9f7f706a6adcc89603030cde51d529dd6dfe2851be
\ No newline at end of file
+93863a5513f3b59a107a3ba23a6e47b38738e7c99ac462d2379308dab9910d8f
\ No newline at end of file

tests/cachekey/project/elements/compose4.expected

-4c83744bec21c8c38bce2d48396b8df1eb4df7b2f155424016bd012743efd808
\ No newline at end of file
+e66827c4f0beffbb9eff52522539b10945ad108bd5ad722107e5dfbce7d064ef
\ No newline at end of file

tests/cachekey/project/elements/compose5.expected

-97385aa2192ef0295dd2601e78491d8bdf6b74e98938d0f8011747c2caf3a5c6
\ No newline at end of file
+74a1fcb3c1c7829962398cc104a30d52aeb38af9fe8631a6b77112a1fe99b653
\ No newline at end of file

tests/cachekey/project/elements/import1.expected

-99c8f61d415de3a6c96e48299fda5554bf4bbaf56bb4b5acd85861ab37ede0c3
\ No newline at end of file
+619692c94c9b499eaa23358b45fd75b4526d9f5b6d3a6061faad9b6726510db3
\ No newline at end of file

tests/cachekey/project/elements/import2.expected

-5f5884c5e4bb7066eede3a135e49753ec06b757a30983513a7a4e0cdd2a8f402
\ No newline at end of file
+8e1ee1f99738be5162e97194d9aa72aef0a9d3458218747f9721102f2d7104d7
\ No newline at end of file

tests/cachekey/project/elements/import3.expected

-e11f93ec629bc3556e15bd374e67a0b5e34350e1e9b1d1f98f8de984a27bbead
\ No newline at end of file
+5d9cfb59d10bb98ca17c52cf8862b84a39202b1d83074a8b8dd3da83a0303c19
\ No newline at end of file

tests/cachekey/project/elements/script1.expected

-d8388b756de5c8441375ba32cedd9560a65a8f9a85e41038837d342c8fb10004
\ No newline at end of file
+7df04616b29ee538ec5e290dcfd7fdfce9cacdbaf224597856272aec3939d5c8
\ No newline at end of file

tests/cachekey/project/sources/bzr1.expected

-519ee88fcca7fea091245713ec68baa048e3d876ea22559d4b2035d3d2ab2494
\ No newline at end of file
+a2682d5e230ea207054fef05eecc1bb8ebc856ae12984ca5ab187d648551e917
\ No newline at end of file

tests/cachekey/project/sources/git1.expected

-a5424aa7cc25f0ada9ac1245b33d55d078559ae6c50b10bea3db9acb964b058c
\ No newline at end of file
+572276304251917e3ce611b19a6c95cb984d989274405344d307e463c53c6b41
\ No newline at end of file

tests/cachekey/project/sources/git2.expected

-93bf7344c118664f0d7f2b8e5a6731b2a95de6df83ba7fa2a2ab28227b0b3e8b
\ No newline at end of file
+e1c86b2e7a5e87f01a7ea10beacff0c9d2771b39e295729e76db42f2133ad478
\ No newline at end of file

tests/cachekey/project/sources/local1.expected

-99c8f61d415de3a6c96e48299fda5554bf4bbaf56bb4b5acd85861ab37ede0c3
\ No newline at end of file
+619692c94c9b499eaa23358b45fd75b4526d9f5b6d3a6061faad9b6726510db3
\ No newline at end of file