NEWS

@@ -2,6 +2,10 @@
 buildstream 1.3.1
 =================
 
+  o All elements must now be suffixed with `.bst`
+    Attempting to use an element that does not have the `.bst` extension,
+    will result in a warning.
+
   o BREAKING CHANGE: The 'manual' element lost its default 'MAKEFLAGS' and 'V'
     environment variables. There is already a 'make' element with the same
     variables. Note that this is a breaking change, it will require users to

buildstream/_frontend/cli.py

@@ -109,7 +109,11 @@ def complete_target(args, incomplete):
     if element_directory:
         base_directory = os.path.join(base_directory, element_directory)
 
-    return complete_path("File", incomplete, base_directory=base_directory)
+    complete_list = []
+    for p in complete_path("File", incomplete, base_directory=base_directory):
+        if p.endswith(".bst ") or p.endswith("/"):
+            complete_list.append(p)
+    return complete_list
 
 
 def override_completions(cmd, cmd_param, args, incomplete):
@@ -469,6 +473,10 @@ def push(app, elements, deps, remote):
     The default destination is the highest priority configured cache. You can
     override this by passing a different cache URL with the `--remote` flag.
 
+    If bst has been configured to include build trees on artifact pulls,
+    an attempt will be made to pull any required build trees to avoid the
+    skipping of partial artifacts being pushed.
+
     Specify `--deps` to control which artifacts to push:
 
     \b

buildstream/_loader/loader.py

@@ -36,6 +36,8 @@ from .types import Symbol, Dependency
 from .loadelement import LoadElement
 from . import MetaElement
 from . import MetaSource
+from ..plugin import CoreWarnings
+from .._message import Message, MessageType
 
 
 # Loader():
@@ -97,6 +99,7 @@ class Loader():
     # Returns: The toplevel LoadElement
     def load(self, targets, rewritable=False, ticker=None, fetch_subprojects=False):
 
+        invalid_elements = []
         for filename in targets:
             if os.path.isabs(filename):
                 # XXX Should this just be an assertion ?
@@ -106,6 +109,14 @@ class Loader():
                                 "path to the base project directory: {}"
                                 .format(filename, self._basedir))
 
+            if not filename.endswith(".bst") and not filename.endswith("/"):
+                invalid_elements.append(filename)
+
+        if invalid_elements:
+            self._warn("Target elements '{}' do not have expected file extension `.bst` "
+                       "Improperly named elements will not be discoverable by commands"
+                       .format(invalid_elements),
+                       warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX)
         # First pass, recursively load files and populate our table of LoadElements
         #
         deps = []
@@ -269,7 +280,12 @@ class Loader():
         self._elements[filename] = element
 
         # Load all dependency files for the new LoadElement
+        invalid_elements = []
         for dep in element.deps:
+            if not dep.name.endswith(".bst"):
+                invalid_elements.append(dep.name)
+                continue
+
             if dep.junction:
                 self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
                 loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
@@ -284,6 +300,11 @@ class Loader():
                                 "{}: Cannot depend on junction"
                                 .format(dep.provenance))
 
+        if invalid_elements:
+            self._warn("The following dependencies do not have expected file extension `.bst`: {} "
+                       "Improperly named elements will not be discoverable by commands"
+                       .format(invalid_elements),
+                       warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX)
         return element
 
     # _check_circular_deps():
@@ -639,3 +660,22 @@ class Loader():
             loader = self._get_loader(junction_path[-2], rewritable=rewritable, ticker=ticker,
                                       fetch_subprojects=fetch_subprojects)
             return junction_path[-2], junction_path[-1], loader
+
+    # Print a warning message, checks warning_token against project configuration
+    #
+    # Args:
+    #     brief (str): The brief message
+    #     warning_token (str): An optional configurable warning assosciated with this warning,
+    #                          this will cause PluginError to be raised if this warning is configured as fatal.
+    #                          (*Since 1.4*)
+    #
+    # Raises:
+    #     (:class:`.LoadError`): When warning_token is considered fatal by the project configuration
+    #
+    def _warn(self, brief, *, warning_token=None):
+        if warning_token:
+            if self.project._warning_is_fatal(warning_token):
+                raise LoadError(warning_token, brief)
+
+        message = Message(None, MessageType.WARN, brief)
+        self._context.message(message)

buildstream/_project.py

@@ -446,6 +446,9 @@ class Project():
         self.config.options = OptionPool(self.element_path)
         self.first_pass_config.options = OptionPool(self.element_path)
 
+        # Fatal warnings
+        self._fatal_warnings = _yaml.node_get(pre_config_node, list, 'fatal-warnings', default_value=[])
+
         self.loader = Loader(self._context, self,
                              parent=parent_loader,
                              tempdir=tempdir)
@@ -506,9 +509,6 @@ class Project():
         # Load project split rules
         self._splits = _yaml.node_get(config, Mapping, 'split-rules')
 
-        # Fatal warnings
-        self._fatal_warnings = _yaml.node_get(config, list, 'fatal-warnings', default_value=[])
-
         # Support backwards compatibility for fail-on-overlap
         fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap', default_value=None)
 

buildstream/_stream.py

@@ -327,6 +327,10 @@ class Stream():
     # If `remote` specified as None, then regular configuration will be used
     # to determine where to push artifacts to.
     #
+    # If any of the given targets are missing their expected buildtree artifact,
+    # a pull queue will be created if user context and available remotes allow for
+    # attempting to fetch them.
+    #
     def push(self, targets, *,
              selection=PipelineSelection.NONE,
              remote=None):
@@ -345,8 +349,17 @@ class Stream():
             raise StreamError("No artifact caches available for pushing artifacts")
 
         self._pipeline.assert_consistent(elements)
-        self._add_queue(PushQueue(self._scheduler))
-        self._enqueue_plan(elements)
+
+        # Check if we require a pull queue, with given artifact state and context
+        require_buildtrees = self._buildtree_pull_required(elements)
+        if require_buildtrees:
+            self._message(MessageType.INFO, "Attempting to fetch missing artifact buildtrees")
+            self._add_queue(PullQueue(self._scheduler))
+            self._enqueue_plan(require_buildtrees)
+
+        push_queue = PushQueue(self._scheduler)
+        self._add_queue(push_queue)
+        self._enqueue_plan(elements, queue=push_queue)
         self._run()
 
     # checkout()
@@ -1237,3 +1250,28 @@ class Stream():
             parts.append(element.normal_name)
 
         return os.path.join(directory, *reversed(parts))
+
+    # _buildtree_pull_required()
+    #
+    # Check if current task, given config, requires element buildtree artifact
+    #
+    # Args:
+    #    elements (list): elements to check if buildtrees are required
+    #
+    # Returns:
+    #    (list): elements requiring buildtrees
+    #
+    def _buildtree_pull_required(self, elements):
+        required_list = []
+
+        # If context is set to not pull buildtrees, or no fetch remotes, return empty list
+        if not (self._context.pull_buildtrees or self._artifacts.has_fetch_remotes()):
+            return required_list
+
+        for element in elements:
+            # Check if element is partially cached without its buildtree, as the element
+            # artifact may not be cached at all
+            if element._cached() and not element._cached_buildtree():
+                required_list.append(element)
+
+        return required_list

buildstream/element.py

@@ -1422,7 +1422,7 @@ class Element(Plugin):
                                                  .format(workspace.get_absolute_path())):
                             workspace.stage(temp_staging_directory)
                 # Check if we have a cached buildtree to use
-                elif self.__cached_buildtree():
+                elif self._cached_buildtree():
                     artifact_base, _ = self.__extract()
                     import_dir = os.path.join(artifact_base, 'buildtree')
                 else:
@@ -1808,7 +1808,7 @@ class Element(Plugin):
 
         # Do not push elements that aren't cached, or that are cached with a dangling buildtree
         # artifact unless element type is expected to have an an empty buildtree directory
-        if not self.__cached_buildtree():
+        if not self._cached_buildtree():
             return True
 
         # Do not push tainted artifact
@@ -1998,6 +1998,29 @@ class Element(Plugin):
     def _get_source_element(self):
         return self
 
+    # _cached_buildtree()
+    #
+    # Check if element artifact contains expected buildtree. An
+    # element's buildtree artifact will not be present if the rest
+    # of the partial artifact is not cached.
+    #
+    # Returns:
+    #     (bool): True if artifact cached with buildtree, False if
+    #             element not cached or missing expected buildtree.
+    #
+    def _cached_buildtree(self):
+        context = self._get_context()
+
+        if not self._cached():
+            return False
+
+        key_strength = _KeyStrength.STRONG if context.get_strict() else _KeyStrength.WEAK
+        if not self.__artifacts.contains_subdir_artifact(self, self._get_cache_key(strength=key_strength),
+                                                         'buildtree'):
+            return False
+
+        return True
+
     #############################################################
     #                   Private Local Methods                   #
     #############################################################
@@ -2764,27 +2787,6 @@ class Element(Plugin):
 
         return True
 
-    # __cached_buildtree():
-    #
-    # Check if cached element artifact contains expected buildtree
-    #
-    # Returns:
-    #     (bool): True if artifact cached with buildtree, False if
-    #             element not cached or missing expected buildtree
-    #
-    def __cached_buildtree(self):
-        context = self._get_context()
-
-        if not self._cached():
-            return False
-        elif context.get_strict():
-            if not self.__artifacts.contains_subdir_artifact(self, self.__strict_cache_key, 'buildtree'):
-                return False
-        elif not self.__artifacts.contains_subdir_artifact(self, self.__weak_cache_key, 'buildtree'):
-            return False
-
-        return True
-
     # __pull_directories():
     #
     # Which directories to include or exclude given the current

buildstream/plugin.py

@@ -784,6 +784,12 @@ class CoreWarnings():
     which is found to be invalid based on the configured track
     """
 
+    BAD_ELEMENT_SUFFIX = "bad-element-suffix"
+    """
+    This warning will be produced when an element whose name does not end in .bst
+    is referenced either on the command line or by another element
+    """
+
 
 __CORE_WARNINGS = [
     value

man/bst-artifact-server.1

+.TH "BST-ARTIFACT-SERVER" "1" "29-Nov-2018" "" "bst-artifact-server Manual"
+.SH NAME
+bst-artifact-server \- CAS Artifact Server
+.SH SYNOPSIS
+.B bst-artifact-server
+[OPTIONS] REPO
+.SH OPTIONS
+.TP
+\fB\-p,\fP \-\-port INTEGER
+Port number  [required]
+.TP
+\fB\-\-server\-key\fP TEXT
+Private server key for TLS (PEM-encoded)
+.TP
+\fB\-\-server\-cert\fP TEXT
+Public server certificate for TLS (PEM-encoded)
+.TP
+\fB\-\-client\-certs\fP TEXT
+Public client certificates for TLS (PEM-encoded)
+.TP
+\fB\-\-enable\-push\fP
+Allow clients to upload blobs and update artifact cache
\ No newline at end of file

man/bst-build.1

-.TH "BST BUILD" "1" "26-Apr-2018" "" "bst build Manual"
+.TH "BST BUILD" "1" "29-Nov-2018" "" "bst build Manual"
 .SH NAME
 bst\-build \- Build elements in a pipeline
 .SH SYNOPSIS

man/bst-checkout.1

-.TH "BST CHECKOUT" "1" "26-Apr-2018" "" "bst checkout Manual"
+.TH "BST CHECKOUT" "1" "29-Nov-2018" "" "bst checkout Manual"
 .SH NAME
 bst\-checkout \- Checkout a built artifact
 .SH SYNOPSIS
@@ -12,7 +12,7 @@ Checkout a built artifact to the specified location
 \fB\-f,\fP \-\-force
 Allow files to be overwritten
 .TP
-\fB\-f,\fP \-\-deps
+\fB\-d,\fP \-\-deps [run|none]
 The dependencies to checkout (default: run)
 .TP
 \fB\-\-integrate\fP / \-\-no\-integrate
@@ -22,5 +22,4 @@ Whether to run integration commands
 Checkout hardlinks instead of copies (handle with care)
 .TP
 \fB\-\-tar\fP
-Create a tarball from the artifact contents instead of a file tree. If
-LOCATION is '-', the tarball will be dumped to the standard output.
+Create a tarball from the artifact contents instead of a file tree. If LOCATION is '-', the tarball will be dumped to the standard output.
\ No newline at end of file

man/bst-fetch.1

-.TH "BST FETCH" "1" "26-Apr-2018" "" "bst fetch Manual"
+.TH "BST FETCH" "1" "29-Nov-2018" "" "bst fetch Manual"
 .SH NAME
 bst\-fetch \- Fetch sources in a pipeline
 .SH SYNOPSIS
@@ -6,14 +6,14 @@ bst\-fetch \- Fetch sources in a pipeline
 [OPTIONS] [ELEMENTS]...
 .SH DESCRIPTION
 Fetch sources required to build the pipeline
-
+.PP
 By default this will only try to fetch sources which are
 required for the build plan of the specified target element,
 omitting sources for any elements which are already built
 and available in the artifact cache.
-
+.PP
 Specify `--deps` to control which sources to fetch:
-
+.PP
 
     none:  No dependencies, just the element itself
     plan:  Only dependencies required for the build plan

man/bst-help.1

+.TH "BST HELP" "1" "29-Nov-2018" "" "bst help Manual"
+.SH NAME
+bst\-help \- Print usage information
+.SH SYNOPSIS
+.B bst help
+[OPTIONS] COMMAND
+.SH DESCRIPTION
+Print usage information about a given command
+    
\ No newline at end of file

man/bst-init.1

-.TH "BST INIT" "1" "26-Apr-2018" "" "bst init Manual"
+.TH "BST INIT" "1" "29-Nov-2018" "" "bst init Manual"
 .SH NAME
 bst\-init \- Initialize a new BuildStream project
 .SH SYNOPSIS
@@ -6,10 +6,10 @@ bst\-init \- Initialize a new BuildStream project
 [OPTIONS]
 .SH DESCRIPTION
 Initialize a new BuildStream project
-
+.PP
 Creates a new BuildStream project.conf in the project
 directory.
-
+.PP
 Unless `--project-name` is specified, this will be an
 interactive session.
 .SH OPTIONS
@@ -18,7 +18,7 @@ interactive session.
 The project name to use
 .TP
 \fB\-\-format\-version\fP INTEGER
-The required format version (default: 8)
+The required format version (default: 18)
 .TP
 \fB\-\-element\-path\fP PATH
 The subdirectory to store elements in (default: elements)

man/bst-pull.1

-.TH "BST PULL" "1" "26-Apr-2018" "" "bst pull Manual"
+.TH "BST PULL" "1" "29-Nov-2018" "" "bst pull Manual"
 .SH NAME
 bst\-pull \- Pull a built artifact
 .SH SYNOPSIS
@@ -6,13 +6,13 @@ bst\-pull \- Pull a built artifact
 [OPTIONS] [ELEMENTS]...
 .SH DESCRIPTION
 Pull a built artifact from the configured remote artifact cache.
-
+.PP
 By default the artifact will be pulled one of the configured caches
 if possible, following the usual priority order. If the `--remote` flag
 is given, only the specified cache will be queried.
-
+.PP
 Specify `--deps` to control which artifacts to pull:
-
+.PP
 
     none:  No dependencies, just the element itself
     all:   All dependencies

man/bst-push.1

-.TH "BST PUSH" "1" "26-Apr-2018" "" "bst push Manual"
+.TH "BST PUSH" "1" "29-Nov-2018" "" "bst push Manual"
 .SH NAME
 bst\-push \- Push a built artifact
 .SH SYNOPSIS
@@ -6,12 +6,12 @@ bst\-push \- Push a built artifact
 [OPTIONS] [ELEMENTS]...
 .SH DESCRIPTION
 Push a built artifact to a remote artifact cache.
-
+.PP
 The default destination is the highest priority configured cache. You can
 override this by passing a different cache URL with the `--remote` flag.
-
+.PP
 Specify `--deps` to control which artifacts to push:
-
+.PP
 
     none:  No dependencies, just the element itself
     all:   All dependencies

man/bst-shell.1

-.TH "BST SHELL" "1" "26-Apr-2018" "" "bst shell Manual"
+.TH "BST SHELL" "1" "29-Nov-2018" "" "bst shell Manual"
 .SH NAME
 bst\-shell \- Shell into an element's sandbox environment
 .SH SYNOPSIS
@@ -6,18 +6,18 @@ bst\-shell \- Shell into an element's sandbox environment
 [OPTIONS] ELEMENT [COMMAND]...
 .SH DESCRIPTION
 Run a command in the target element's sandbox environment
-
+.PP
 This will stage a temporary sysroot for running the target
 element, assuming it has already been built and all required
 artifacts are in the local cache.
-
+.PP
 Use the --build option to create a temporary sysroot for
 building the element instead.
-
+.PP
 Use the --sysroot option with an existing failed build
 directory or with a checkout of the given target, in order
 to use a specific sysroot.
-
+.PP
 If no COMMAND is specified, the default is to attempt
 to run an interactive shell.
 .SH OPTIONS

man/bst-show.1

-.TH "BST SHOW" "1" "26-Apr-2018" "" "bst show Manual"
+.TH "BST SHOW" "1" "29-Nov-2018" "" "bst show Manual"
 .SH NAME
 bst\-show \- Show elements in the pipeline
 .SH SYNOPSIS
@@ -6,25 +6,25 @@ bst\-show \- Show elements in the pipeline
 [OPTIONS] [ELEMENTS]...
 .SH DESCRIPTION
 Show elements in the pipeline
-
+.PP
 By default this will show all of the dependencies of the
 specified target element.
-
+.PP
 Specify `--deps` to control which elements to show:
-
+.PP
 
     none:  No dependencies, just the element itself
     plan:  Dependencies required for a build plan
     run:   Runtime dependencies, including the element itself
     build: Build time dependencies, excluding the element itself
     all:   All dependencies
-
+.PP
 
 FORMAT
 ~~~~~~
 The --format option controls what should be printed for each element,
 the following symbols can be used in the format string:
-
+.PP
 
     %{name}           The element name
     %{key}            The abbreviated cache key (if all sources are consistent)
@@ -36,17 +36,17 @@ the following symbols can be used in the format string:
     %{public}         Public domain data
     %{workspaced}     If the element is workspaced
     %{workspace-dirs} A list of workspace directories
-
+.PP
 The value of the %{symbol} without the leading '%' character is understood
 as a pythonic formatting string, so python formatting features apply,
 examle:
-
+.PP
 
     bst show target.bst --format \
         'Name: %{name: ^20} Key: %{key: ^8} State: %{state}'
-
+.PP
 If you want to use a newline in a format string in bash, use the '$' modifier:
-
+.PP
 
     bst show target.bst --format \
         $'---------- %{name} ----------\n%{vars}'
@@ -62,7 +62,4 @@ The dependencies to show (default: all)
 Staging or alphabetic ordering of dependencies
 .TP
 \fB\-f,\fP \-\-format FORMAT
-Format string for each element
-.TP
-\fB\-\-downloadable\fP
-Refresh downloadable state
\ No newline at end of file
+Format string for each element
\ No newline at end of file

man/bst-source-bundle.1

-.TH "BST SOURCE-BUNDLE" "1" "26-Apr-2018" "" "bst source-bundle Manual"
+.TH "BST SOURCE-BUNDLE" "1" "29-Nov-2018" "" "bst source-bundle Manual"
 .SH NAME
 bst\-source-bundle \- Produce a build bundle to be manually executed
 .SH SYNOPSIS
 .B bst source-bundle
-[OPTIONS] TARGET
+[OPTIONS] ELEMENT
 .SH DESCRIPTION
 Produce a source bundle to be manually executed
     
@@ -16,10 +16,10 @@ Elements to except from the tarball
 Compress the tar file using the given algorithm.
 .TP
 \fB\-\-track\fP
-Track new source references before building
+Track new source references before bundling
 .TP
 \fB\-f,\fP \-\-force
-Overwrite files existing in checkout directory
+Overwrite an existing tarball
 .TP
 \fB\-\-directory\fP TEXT
 The directory to write the tarball to
\ No newline at end of file

man/bst-source-checkout.1

+.TH "BST SOURCE-CHECKOUT" "1" "29-Nov-2018" "" "bst source-checkout Manual"
+.SH NAME
+bst\-source-checkout \- Checkout sources for an element
+.SH SYNOPSIS
+.B bst source-checkout
+[OPTIONS] ELEMENT LOCATION
+.SH DESCRIPTION
+Checkout sources of an element to the specified location
+    
+.SH OPTIONS
+.TP
+\fB\-\-except\fP PATH
+Except certain dependencies
+.TP
+\fB\-d,\fP \-\-deps [build|none|run|all]
+The dependencies whose sources to checkout (default: none)
+.TP
+\fB\-\-fetch\fP
+Fetch elements if they are not fetched
\ No newline at end of file

man/bst-track.1

-.TH "BST TRACK" "1" "26-Apr-2018" "" "bst track Manual"
+.TH "BST TRACK" "1" "29-Nov-2018" "" "bst track Manual"
 .SH NAME
 bst\-track \- Track new source references
 .SH SYNOPSIS
@@ -7,12 +7,12 @@ bst\-track \- Track new source references
 .SH DESCRIPTION
 Consults the specified tracking branches for new versions available
 to build and updates the project with any newly available references.
-
+.PP
 By default this will track just the specified element, but you can also
 update a whole tree of dependencies in one go.
-
+.PP
 Specify `--deps` to control which sources to track:
-
+.PP
 
     none:  No dependencies, just the specified elements
     all:   All dependencies of all specified elements