Add option to source-checkout to generate build scripts

This commit is part of the work towards #672

buildstream/_frontend/cli.py

@@ -681,11 +681,12 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
 @click.option('--tar', 'tar', default=False, is_flag=True,
               help='Create a tarball from the element\'s sources instead of a '
                    'file tree.')
-@click.argument('element',
-                type=click.Path(readable=False))
+@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
+@click.argument('element', type=click.Path(readable=False))
 @click.argument('location', type=click.Path())
 @click.pass_obj
-def source_checkout(app, element, location, force, deps, fetch_, except_, tar):
+def source_checkout(app, element, location, force, deps, fetch_, except_,
+                    tar, build_scripts):
     """Checkout sources of an element to the specified location
     """
     with app.initialized():
@@ -695,7 +696,8 @@ def source_checkout(app, element, location, force, deps, fetch_, except_, tar):
                                    deps=deps,
                                    fetch=fetch_,
                                    except_targets=except_,
-                                   tar=tar)
+                                   tar=tar,
+                                   include_build_scripts=build_scripts)
 
 
 ##################################################################

buildstream/_stream.py

@@ -440,7 +440,8 @@ class Stream():
                         deps='none',
                         fetch=False,
                         except_targets=(),
-                        tar=False):
+                        tar=False,
+                        include_build_scripts=False):
 
         self._check_location_writable(location, force=force, tar=tar)
 
@@ -456,10 +457,8 @@ class Stream():
 
         # Stage all sources determined by scope
         try:
-            if tar:
-                self._create_source_tarball(location, elements)
-            else:
-                self._write_element_sources(location, elements)
+            self._source_checkout(elements, location, force, deps, fetch,
+                                  except_targets, tar, include_build_scripts)
         except BstError as e:
             raise StreamError("Error while writing sources"
                               ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
@@ -748,7 +747,7 @@ class Stream():
             ]
 
             self._write_element_sources(os.path.join(tempdir, "source"), elements)
-            self._write_build_script(tempdir, elements)
+            self._write_master_build_script(tempdir, elements)
             self._collect_sources(tempdir, tar_location,
                                   target.normal_name, compression)
 
@@ -1135,6 +1134,7 @@ class Stream():
     # Helper function for source_checkout()
     def _source_checkout(self, elements,
                          location=None,
+                         force=False,
                          deps='none',
                          fetch=False,
                          except_targets=(),
@@ -1205,17 +1205,29 @@ class Stream():
                 os.makedirs(element_source_dir)
                 element._stage_sources_at(element_source_dir)
 
-    # Create a tarball containing the sources of each element in elements
-    def _create_source_tarball(self, directory, elements):
-        with tarfile.open(name=directory, mode='w') as tf:
-            with TemporaryDirectory() as tmpdir:
-                self._write_element_sources(tmpdir, elements)
-                for item in os.listdir(tmpdir):
-                    file_to_add = os.path.join(tmpdir, item)
+    # Create a tarball from the content of directory
+    def _create_tarball(self, directory, tar_name):
+        try:
+            with tarfile.open(name=tar_name, mode='w') as tf:
+                for item in os.listdir(str(directory)):
+                    file_to_add = os.path.join(directory, item)
                     tf.add(file_to_add, arcname=item)
+        except OSError as e:
+            # If we have a partially constructed tar file, clean up after ourselves
+            try:
+                os.remove(tar_name)
+            except OSError:
+                pass
+            raise StreamError("Failed to create tar archive: {}".format(e)) from e
+
+    # Write all the build_scripts for elements in the directory location
+    def _write_build_scripts(self, location, elements):
+        for element in elements:
+            self._write_element_script(location, element)
+        self._write_master_build_script(location, elements)
 
     # Write a master build script to the sandbox
-    def _write_build_script(self, directory, elements):
+    def _write_master_build_script(self, directory, elements):
 
         module_string = ""
         for element in elements:

tests/frontend/source_checkout.py

@@ -154,3 +154,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
         assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
     else:
         result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
+    project_path = os.path.join(datafiles.dirname, datafiles.basename)
+    element_name = 'source-bundle/source-bundle-hello.bst'
+    normal_name = 'source-bundle-source-bundle-hello'
+    checkout = os.path.join(str(tmpdir), 'source-checkout')
+
+    args = ['source-checkout', '--include-build-scripts', element_name, checkout]
+    result = cli.run(project=project_path, args=args)
+    result.assert_success()
+
+    # There sould be a script for each element (just one in this case) and a top level build script
+    expected_scripts = ['build.sh', 'build-' + normal_name]
+    for script in expected_scripts:
+        assert script in os.listdir(checkout)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
+    project_path = os.path.join(datafiles.dirname, datafiles.basename)
+    element_name = 'source-bundle/source-bundle-hello.bst'
+    normal_name = 'source-bundle-source-bundle-hello'
+    tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
+
+    args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
+    result = cli.run(project=project_path, args=args)
+    result.assert_success()
+
+    expected_scripts = ['build.sh', 'build-' + normal_name]
+
+    with tarfile.open(tar_file, 'r') as tf:
+        for script in expected_scripts:
+            assert script in tf.getnames()