buildstream/_frontend/cli.py

@@ -3,6 +3,7 @@ import sys
 
 import click
 from .. import _yaml
+from .. import _manifest
 from .._exceptions import BstError, LoadError, AppError
 from .._versions import BST_FORMAT_VERSION
 from .complete import main_bashcomplete, complete_path, CompleteUnhandled
@@ -289,6 +290,15 @@ def init(app, project_name, format_version, element_path, force):
 ##################################################################
 #                          Build Command                         #
 ##################################################################
+def _validate_manifest_path(ctx, param, value):
+    if not value:
+        return
+    if value.lower().endswith(".yaml") or value.lower().endswith(".yml"):
+        return os.path.abspath(value)
+    else:
+        raise click.BadParameter("Manifest files are outputted as YAML\n\t" +
+                                 "Please provide a path with a valid file extension (yml, yaml)")
+
 @cli.command(short_help="Build elements in a pipeline")
 @click.option('--all', 'all_', default=False, is_flag=True,
               help="Build elements that would not be needed for the current build plan")
@@ -305,10 +315,16 @@ def init(app, project_name, format_version, element_path, force):
               help="Allow tracking to cross junction boundaries")
 @click.option('--track-save', default=False, is_flag=True,
               help="Deprecated: This is ignored")
+@click.option('--build-manifest', default=False, is_flag=True,
+              help="Produces a build manifest containing elements and sources.")
+@click.option('--manifest-path', default=None, type=click.Path(readable=False),
+              help="Provides a path for a build manifest to be written to.",
+              callback=_validate_manifest_path)
 @click.argument('elements', nargs=-1,
                 type=click.Path(readable=False))
 @click.pass_obj
-def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):
+def build(app, elements, all_, track_, track_save, build_manifest,
+          manifest_path, track_all, track_except, track_cross_junctions):
     """Build elements in a pipeline"""
 
     if (track_except or track_cross_junctions) and not (track_ or track_all):
@@ -329,6 +345,13 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
                          track_cross_junctions=track_cross_junctions,
                          build_all=all_)
 
+        if build_manifest and not manifest_path:
+            manifest_path = os.path.join(app.project.directory,
+                                         "build_manifest.yaml")
+        if manifest_path:
+            _manifest.generate(app.context, app.stream.total_elements,
+                               app._session_start, manifest_path)
+
 
 ##################################################################
 #                          Fetch Command                         #

buildstream/_manifest.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Josh Smith <josh.smith@codethink.co.uk>
+
+from ruamel import yaml
+from . import __version__ as bst_version
+from . import _yaml
+from ._message import MessageType, Message
+
+# generate():
+#
+# Generates a manfiest file from the collection of elements.
+#
+# Args:
+#    context (Context): Application context
+#    elements (list of Element): Collection of elements to build the manifest from
+#    build_start_datetime (datetime): The start of the build assosciated with this manifest
+#    manifest_path (str): Absolute path to write the manifest file to.
+#
+def generate(context, elements, build_start_datetime, manifest_path):
+    with context.timed_activity("Building Manifest"):
+        manifest = _build(elements, build_start_datetime)
+
+    _yaml.dump(manifest, manifest_path)
+    context.message(Message(None, MessageType.STATUS,
+                            "Manifest saved to {}".format(manifest_path)))
+
+# _build():
+#
+# Builds a manifest (dictionary) using the provided elements to be stored.
+#
+# Args:
+#    elements (list of Element): Collection of elements to build the manifest from
+#    build_start_datetime (datetime): The start of the build assosciated with this manifest
+#
+# Returns:
+#    (CommentedMap): A dictionary containing the entire
+#                    manifest produced from the provided elements.
+#
+def _build(elements, build_start_datetime):
+    manifest = yaml.comments.CommentedMap()
+
+    # Add BuildStream Version
+    manifest['BuildStream_Version'] = "{}".format(bst_version)
+    # Add Build Date
+    manifest['Build_Date'] = build_start_datetime.isoformat()
+    manifest['Elements'] = yaml.comments.CommentedMap()
+
+    # Sort elements
+    elements = sorted(elements, key=lambda e: e.normal_name)
+
+    # Add Elements
+    for elem in elements:
+        manifest['Elements'][elem.normal_name] = _build_element(elem)
+
+    return manifest
+
+# _build_element():
+#
+# Builds a manifest segment for an individual element.
+#
+# Args:
+#    element (Element): Element to extract information from
+#
+# Returns:
+#    (CommentedMap): A dictionary containing the information
+#                    extracted from the provided element
+#
+def _build_element(element):
+    element_dict = yaml.comments.CommentedMap()
+    sources = yaml.comments.CommentedMap()
+    # Add Cache Key
+    cache_key = element._get_cache_key()
+    if cache_key:
+        element_dict["Cache_Key"] = cache_key
+
+    # Add sources
+    for source in element.sources():
+        src = _build_source(source)
+        if src:
+            source_desc = "{}({})".format(source._get_element_index(), type(source).__name__)
+            sources[source_desc] = src
+    if sources:
+        element_dict['Sources'] = sources
+
+
+    return element_dict
+
+# _build_source():
+#
+# Builds a manifest segment for an individual source.
+#
+# Args:
+#    source (Source): Source to extract information from
+#
+# Returns:
+#    (CommentedMap): A dictionary containing the information
+#                    extracted from the provided source
+#
+def _build_source(source):
+    src = yaml.comments.CommentedMap()
+    if hasattr(source, "url") and source.url:
+        src["url"] = source.url
+    if hasattr(source, "ref") and source.ref:
+        src["ref"] = source.ref
+    if hasattr(source, "path") and source.path:
+        src["path"] = source.path
+
+    return src if src else None

buildstream/source.py

@@ -786,6 +786,9 @@ class Source(Plugin):
         else:
             return None
 
+    def _get_element_index(self):
+        return self.__element_index
+
     #############################################################
     #                   Local Private Methods                   #
     #############################################################

tests/manifest/manifest.py

+import pytest
+import os
+from ruamel import yaml
+
+from tests.testutils import cli
+
+# Project directory
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "project",
+)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("specify_path, build_manifest", [
+    (True, True), (True, False), (False, True)
+])
+def test_manifest_created(tmpdir, cli, datafiles, specify_path, build_manifest):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    manifest_path = os.path.join(str(tmpdir), "build_manifest.yaml")
+
+    args = ['build', "base.bst"]
+
+    if specify_path:
+        args += ["--manifest-path", manifest_path]
+    if build_manifest:
+        args.append("--build-manifest")
+
+    result = cli.run(project=project, args=args)
+    result.assert_success()
+
+    with open(manifest_path) as f:
+        manifest = yaml.load(f, Loader=yaml.loader.RoundTripLoader)
+
+    assert len(manifest["Elements"]["base"]["Sources"]) == 1
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("extension, valid", [
+    (".yaml", True),
+    (".yml", True),
+    (".bst", False),
+    (".ynl", False),
+    (".xml", False),
+    (".mnf", False),
+    (".txt", False),
+    (".abc", False),
+    (".json", False)
+])
+def test_manifest_extensions(tmpdir, cli, datafiles, extension, valid):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    manifest_path = os.path.join(str(tmpdir), "build_manifest{}" + extension)
+
+    result = cli.run(project=project, args=['build', "base.bst", "--manifest-path", manifest_path])
+
+    if valid:
+        result.assert_success()
+    else:
+        assert result.exit_code == 2

tests/manifest/project/elements/base.bst

+kind: import
+description: Custom foo element
+
+sources:
+  - kind: local
+    path: files/hello
\ No newline at end of file

tests/manifest/project/project.conf

+# Project config for frontend build test
+name: test
+
+element-path: elements