buildstream/_artifactcache/cascache.py

@@ -25,6 +25,7 @@ import signal
 import stat
 import tempfile
 import uuid
+import errno
 from urllib.parse import urlparse
 
 import grpc
@@ -82,7 +83,8 @@ class CASCache(ArtifactCache):
 
         tree = self.resolve_ref(ref, update_mtime=True)
 
-        dest = os.path.join(self.extractdir, element._get_project().name, element.normal_name, tree.hash)
+        dest = os.path.join(self.extractdir, element._get_project().name,
+                            element.normal_name, tree.hash)
         if os.path.isdir(dest):
             # artifact has already been extracted
             return dest
@@ -100,7 +102,7 @@ class CASCache(ArtifactCache):
                 #
                 # If rename fails with these errors, another process beat
                 # us to it so just ignore.
-                if e.errno not in [os.errno.ENOTEMPTY, os.errno.EEXIST]:
+                if e.errno not in [errno.ENOTEMPTY, errno.EEXIST]:
                     raise ArtifactError("Failed to extract artifact for ref '{}': {}"
                                         .format(ref, e)) from e
 

buildstream/_frontend/app.py

@@ -269,6 +269,9 @@ class App():
                 else:
                     self._message(MessageType.FAIL, session_name, elapsed=elapsed)
 
+                    # Notify session failure
+                    self._notify("{} failed".format(session_name), "{}".format(e))
+
                 if self._started:
                     self._print_summary()
 
@@ -286,6 +289,9 @@ class App():
                 if self._started:
                     self._print_summary()
 
+                # Notify session success
+                self._notify("{} succeeded".format(session_name), "")
+
     # init_project()
     #
     # Initialize a new BuildStream project, either with the explicitly passed options,
@@ -419,6 +425,12 @@ class App():
     #                      Local Functions                     #
     ############################################################
 
+    # Local function for calling the notify() virtual method
+    #
+    def _notify(self, title, text):
+        if self.interactive:
+            self.notify(title, text)
+
     # Local message propagator
     #
     def _message(self, message_type, message, **kwargs):
@@ -571,8 +583,8 @@ class App():
             while choice not in ['continue', 'quit', 'terminate', 'retry']:
                 click.echo(summary, err=True)
 
-                self.notify("BuildStream failure", "{} on element {}"
-                            .format(failure.action_name, element.name))
+                self._notify("BuildStream failure", "{} on element {}"
+                             .format(failure.action_name, element.name))
 
                 try:
                     choice = click.prompt("Choice:", default='continue', err=True,

buildstream/_pipeline.py

@@ -359,23 +359,14 @@ class Pipeline():
 
         if inconsistent:
             detail = "Exact versions are missing for the following elements:\n\n"
-
-            missingTrack = 0
             for element in inconsistent:
-                detail += "  " + element._get_full_name()
+                detail += "  Element: {} is inconsistent\n".format(element._get_full_name())
                 for source in element.sources():
-                    if not source._get_consistency() and not source.get_ref():
-                        if hasattr(source, 'tracking') and source.tracking is None:
-                            detail += ": Source {} is missing ref and track. ".format(source._get_full_name()) + \
-                                      "Please specify a ref or branch/tag to track."
-                            missingTrack = 1
-
-                detail += "\n"
+                    if source._get_consistency() == Consistency.INCONSISTENT:
+                        detail += "    Source {} is missing ref\n".format(source)
+                detail += '\n'
+            detail += "Try tracking these elements first with `bst track`\n"
 
-            if missingTrack:
-                detail += "\nThen track these elements with `bst track`\n"
-            else:
-                detail += "\nTry tracking these elements first with `bst track`\n"
             raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline")
 
     #############################################################

buildstream/plugins/sources/git.py

@@ -74,6 +74,9 @@ This plugin provides the following configurable warnings:
 
 - 'git:inconsistent-submodule' - A submodule was found to be missing from the underlying git repository.
 
+This plugin also utilises the following configurable core plugin warnings:
+
+- 'ref-not-in-track' - The provided ref was not found in the provided track in the element's git repository.
 """
 
 import os
@@ -87,6 +90,7 @@ from configparser import RawConfigParser
 
 from buildstream import Source, SourceError, Consistency, SourceFetcher
 from buildstream import utils
+from buildstream.plugin import CoreWarnings
 
 GIT_MODULES = '.gitmodules'
 
@@ -199,7 +203,7 @@ class GitMirror(SourceFetcher):
             cwd=self.mirror)
         return output.rstrip('\n')
 
-    def stage(self, directory):
+    def stage(self, directory, track=None):
         fullpath = os.path.join(directory, self.path)
 
         # Using --shared here avoids copying the objects into the checkout, in any
@@ -213,10 +217,14 @@ class GitMirror(SourceFetcher):
                          fail="Failed to checkout git ref {}".format(self.ref),
                          cwd=fullpath)
 
+        # Check that the user specified ref exists in the track if provided & not already tracked
+        if track:
+            self.assert_ref_in_track(fullpath, track)
+
         # Remove .git dir
         shutil.rmtree(os.path.join(fullpath, ".git"))
 
-    def init_workspace(self, directory):
+    def init_workspace(self, directory, track=None):
         fullpath = os.path.join(directory, self.path)
         url = self.source.translate_url(self.url)
 
@@ -232,6 +240,10 @@ class GitMirror(SourceFetcher):
                          fail="Failed to checkout git ref {}".format(self.ref),
                          cwd=fullpath)
 
+        # Check that the user specified ref exists in the track if provided & not already tracked
+        if track:
+            self.assert_ref_in_track(fullpath, track)
+
     # List the submodules (path/url tuples) present at the given ref of this repo
     def submodule_list(self):
         modules = "{}:{}".format(self.ref, GIT_MODULES)
@@ -296,6 +308,28 @@ class GitMirror(SourceFetcher):
 
             return None
 
+    # Assert that ref exists in track, if track has been specified.
+    def assert_ref_in_track(self, fullpath, track):
+        _, branch = self.source.check_output([self.source.host_git, 'branch', '--list', track,
+                                              '--contains', self.ref],
+                                             cwd=fullpath,)
+        if branch:
+            return True
+        else:
+            _, tag = self.source.check_output([self.source.host_git, 'tag', '--list', track,
+                                               '--contains', self.ref],
+                                              cwd=fullpath,)
+            if tag:
+                return True
+
+        detail = "The ref provided for the element does not exist locally in the provided track branch / tag " + \
+                 "'{}'.\nYou may wish to track the element to update the ref from '{}' ".format(track, track) + \
+                 "with `bst track`,\nor examine the upstream at '{}' for the specific ref.".format(self.url)
+
+        self.source.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n"
+                         .format(self.source, self.ref, track, self.url),
+                         detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK)
+
 
 class GitSource(Source):
     # pylint: disable=attribute-defined-outside-init
@@ -309,6 +343,13 @@ class GitSource(Source):
         self.original_url = self.node_get_member(node, str, 'url')
         self.mirror = GitMirror(self, '', self.original_url, ref)
         self.tracking = self.node_get_member(node, str, 'track', None)
+
+        # At this point we now know if the source has a ref and/or a track.
+        # If it is missing both then we will be unable to track or build.
+        if self.mirror.ref is None and self.tracking is None:
+            raise SourceError("{}: Git sources require a ref and/or track".format(self),
+                              reason="missing-track-and-ref")
+
         self.checkout_submodules = self.node_get_member(node, bool, 'checkout-submodules', True)
         self.submodules = []
 
@@ -326,6 +367,7 @@ class GitSource(Source):
                 self.submodule_checkout_overrides[path] = checkout
 
         self.mark_download_url(self.original_url)
+        self.tracked = False
 
     def preflight(self):
         # Check if git is installed, get the binary at the same time
@@ -389,6 +431,8 @@ class GitSource(Source):
             # Update self.mirror.ref and node.ref from the self.tracking branch
             ret = self.mirror.latest_commit(self.tracking)
 
+        # Set tracked attribute, parameter for if self.mirror.assert_ref_in_track is needed
+        self.tracked = True
         return ret
 
     def init_workspace(self, directory):
@@ -396,7 +440,7 @@ class GitSource(Source):
         self.refresh_submodules()
 
         with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
-            self.mirror.init_workspace(directory)
+            self.mirror.init_workspace(directory, track=(self.tracking if not self.tracked else None))
             for mirror in self.submodules:
                 mirror.init_workspace(directory)
 
@@ -412,7 +456,7 @@ class GitSource(Source):
         # Stage the main repo in the specified directory
         #
         with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True):
-            self.mirror.stage(directory)
+            self.mirror.stage(directory, track=(self.tracking if not self.tracked else None))
             for mirror in self.submodules:
                 if mirror.path in self.submodule_checkout_overrides:
                     checkout = self.submodule_checkout_overrides[mirror.path]

buildstream/plugins/sources/ostree.py

@@ -71,7 +71,13 @@ class OSTreeSource(Source):
         self.ref = self.node_get_member(node, str, 'ref', None)
         self.tracking = self.node_get_member(node, str, 'track', None)
         self.mirror = os.path.join(self.get_mirror_directory(),
-                                   utils.url_directory_name(self.url))
+                                   utils.url_directory_name(self.original_url))
+
+        # At this point we now know if the source has a ref and/or a track.
+        # If it is missing both then we will be unable to track or build.
+        if self.ref is None and self.tracking is None:
+            raise SourceError("{}: OSTree sources require a ref and/or track".format(self),
+                              reason="missing-track-and-ref")
 
         # (optional) Not all repos are signed. But if they are, get the gpg key
         self.gpg_key_path = None
@@ -104,10 +110,11 @@ class OSTreeSource(Source):
             return None
 
         self.ensure()
+        remote_name = self.ensure_remote(self.url)
         with self.timed_activity("Fetching tracking ref '{}' from origin: {}"
                                  .format(self.tracking, self.url)):
             try:
-                _ostree.fetch(self.repo, ref=self.tracking, progress=self.progress)
+                _ostree.fetch(self.repo, remote=remote_name, ref=self.tracking, progress=self.progress)
             except OSTreeError as e:
                 raise SourceError("{}: Failed to fetch tracking ref '{}' from origin {}\n\n{}"
                                   .format(self, self.tracking, self.url, e)) from e
@@ -116,11 +123,12 @@ class OSTreeSource(Source):
 
     def fetch(self):
         self.ensure()
+        remote_name = self.ensure_remote(self.url)
         if not _ostree.exists(self.repo, self.ref):
             with self.timed_activity("Fetching remote ref: {} from origin: {}"
                                      .format(self.ref, self.url)):
                 try:
-                    _ostree.fetch(self.repo, ref=self.ref, progress=self.progress)
+                    _ostree.fetch(self.repo, remote=remote_name, ref=self.ref, progress=self.progress)
                 except OSTreeError as e:
                     raise SourceError("{}: Failed to fetch ref '{}' from origin: {}\n\n{}"
                                       .format(self, self.ref, self.url, e)) from e
@@ -171,14 +179,22 @@ class OSTreeSource(Source):
             self.status("Creating local mirror for {}".format(self.url))
 
             self.repo = _ostree.ensure(self.mirror, True)
-            gpg_key = None
-            if self.gpg_key_path:
-                gpg_key = 'file://' + self.gpg_key_path
 
-            try:
-                _ostree.configure_remote(self.repo, "origin", self.url, key_url=gpg_key)
-            except OSTreeError as e:
-                raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e
+    def ensure_remote(self, url):
+        if self.original_url == self.url:
+            remote_name = 'origin'
+        else:
+            remote_name = utils.url_directory_name(url)
+
+        gpg_key = None
+        if self.gpg_key_path:
+            gpg_key = 'file://' + self.gpg_key_path
+
+        try:
+            _ostree.configure_remote(self.repo, remote_name, url, key_url=gpg_key)
+        except OSTreeError as e:
+            raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e
+        return remote_name
 
     def progress(self, percent, message):
         self.status(message)

doc/examples/autotools/project.conf

@@ -10,4 +10,4 @@ element-path: elements
 # Define some aliases for the tarballs we download
 aliases:
   alpine: https://gnome7.codethink.co.uk/tarballs/
-  gnu: https://ftpmirror.gnu.org/gnu/automake/
+  gnu: http://ftpmirror.gnu.org/gnu/automake/

tests/artifactcache/expiry.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Maat <tristan.maat@codethink.co.uk>
+#
+
 import os
 
 import pytest
@@ -5,7 +24,7 @@ import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 
-from tests.testutils import cli, create_element_size
+from tests.testutils import cli, create_element_size, wait_for_cache_granularity
 
 
 DATA_DIR = os.path.join(
@@ -108,6 +127,8 @@ def test_expiry_order(cli, datafiles, tmpdir):
     res = cli.run(project=project, args=['build', 'target2.bst'])
     res.assert_success()
 
+    wait_for_cache_granularity()
+
     # Now extract dep.bst
     res = cli.run(project=project, args=['checkout', 'dep.bst', checkout])
     res.assert_success()

tests/frontend/mirror.py

@@ -466,10 +466,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
 def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
-    if kind == 'ostree':
-        # FIXME: Mirroring fallback fails with ostree
-        pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
-
     bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
     upstream_repodir = os.path.join(str(tmpdir), 'upstream')
     mirror_repodir = os.path.join(str(tmpdir), 'mirror')

tests/frontend/push.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#           Sam Thursfield <sam.thursfield@codethink.co.uk>
+#           Jürg Billeter <juerg.billeter@codethink.co.uk>
+#
+
 import os
 import pytest
 
 from buildstream._exceptions import ErrorDomain
 from tests.testutils import cli, create_artifact_share, create_element_size
-from tests.testutils import generate_junction
+from tests.testutils import generate_junction, wait_for_cache_granularity
 from . import configure_project
 
 
@@ -327,6 +349,8 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
         # Ensure element1 is cached locally
         assert cli.get_element_state(project, 'element1.bst') == 'cached'
 
+        wait_for_cache_granularity()
+
         # Create and build the element3 (of 5 MB)
         create_element_size('element3.bst', project, element_path, [], int(5e6))
         result = cli.run(project=project, args=['build', 'element3.bst'])

tests/frontend/workspace.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#           Tristan Maat <tristan.maat@codethink.co.uk>
+#           Chandan Singh <csingh43@bloomberg.net>
+#           Phillip Smyth <phillip.smyth@codethink.co.uk>
+#           Jonathan Maw <jonathan.maw@codethink.co.uk>
+#           Richard Maw <richard.maw@codethink.co.uk>
+#
+
 import os
 import pytest
 import shutil
 import subprocess
 from ruamel.yaml.comments import CommentedSet
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
@@ -507,6 +532,8 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
     assert cli.get_element_state(project, element_name) == 'cached'
     assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
 
+    wait_for_cache_granularity()
+
     # Modify the workspace in various different ways, ensuring we
     # properly detect the changes.
     #

tests/sources/git.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#           Jonathan Maw <jonathan.maw@codethink.co.uk>
+#           William Salmon <will.salmon@codethink.co.uk>
+#
+
 import os
 import pytest
 
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
+from buildstream.plugin import CoreWarnings
 
 from tests.testutils import cli, create_repo
 from tests.testutils.site import HAVE_GIT
@@ -383,21 +406,78 @@ def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
     _yaml.dump(element, os.path.join(project, 'target.bst'))
 
     # Track will encounter an inconsistent submodule without any ref
-    result = cli.run(project=project, args=['track', 'target.bst'])
-    result.assert_main_error(ErrorDomain.STREAM, None)
-    result.assert_task_error(ErrorDomain.SOURCE, 'track-attempt-no-track')
-
-    # Assert that we are just fine without it, and emit a warning to the user.
-    assert "FAILURE git source at" in result.stderr
-    assert "Without a tracking branch ref can not be updated. Please " + \
-        "provide a ref or a track." in result.stderr
-
-    # Track will encounter an inconsistent submodule without any ref
-    result = cli.run(project=project, args=['build', 'target.bst'])
-    result.assert_main_error(ErrorDomain.PIPELINE, 'inconsistent-pipeline')
+    result = cli.run(project=project, args=['show', 'target.bst'])
+    result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref")
     result.assert_task_error(None, None)
 
     # Assert that we are just fine without it, and emit a warning to the user.
     assert "Exact versions are missing for the following elements" in result.stderr
     assert "is missing ref and track." in result.stderr
     assert "Then track these elements with `bst track`" in result.stderr
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+def test_ref_not_in_track_warn(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    # Create the repo from 'repofiles', create a branch without latest commit
+    repo = create_repo('git', str(tmpdir))
+    ref = repo.create(os.path.join(project, 'repofiles'))
+
+    gitsource = repo.source_config(ref=ref)
+
+    # Overwrite the track value to the added branch
+    gitsource['track'] = 'foo'
+
+    # Write out our test target
+    element = {
+        'kind': 'import',
+        'sources': [
+            gitsource
+        ]
+    }
+    _yaml.dump(element, os.path.join(project, 'target.bst'))
+
+    # Assert the warning is raised as ref is not in branch foo.
+    # Assert warning not error to the user, when not set as fatal.
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    assert "The ref provided for the element does not exist locally" in result.stderr
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+def test_ref_not_in_track_warn_error(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    # Add fatal-warnings ref-not-in-track to project.conf
+    project_template = {
+        "name": "foo",
+        "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK]
+    }
+
+    _yaml.dump(project_template, os.path.join(project, 'project.conf'))
+
+    # Create the repo from 'repofiles', create a branch without latest commit
+    repo = create_repo('git', str(tmpdir))
+    ref = repo.create(os.path.join(project, 'repofiles'))
+
+    gitsource = repo.source_config(ref=ref)
+
+    # Overwrite the track value to the added branch
+    gitsource['track'] = 'foo'
+
+    # Write out our test target
+    element = {
+        'kind': 'import',
+        'sources': [
+            gitsource
+        ]
+    }
+    _yaml.dump(element, os.path.join(project, 'target.bst'))
+
+    # Assert that build raises a warning here that is captured
+    # as plugin error, due to the fatal warning being set
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    result.assert_main_error(ErrorDomain.STREAM, None)
+    result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.REF_NOT_IN_TRACK)

tests/sources/ostree.py

+#
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: William Salmon <will.salmon@codethink.co.uk>
+#
+
+import os
+import pytest
+
+from buildstream._exceptions import ErrorDomain
+from buildstream import _yaml
+
+from tests.testutils import cli, create_repo
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'ostree',
+)
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    # Create the repo from 'repofiles' subdir
+    repo = create_repo('ostree', str(tmpdir))
+    ref = repo.create(os.path.join(project, 'repofiles'))
+
+    # Write out our test target
+    ostreesource = repo.source_config(ref=None)
+    ostreesource.pop('track')
+    element = {
+        'kind': 'import',
+        'sources': [
+            ostreesource
+        ]
+    }
+
+    _yaml.dump(element, os.path.join(project, 'target.bst'))
+
+    # Track will encounter an inconsistent submodule without any ref
+    result = cli.run(project=project, args=['show', 'target.bst'])
+    result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref")
+    result.assert_task_error(None, None)

tests/sources/ostree/template/project.conf

+# Basic project
+name: foo

tests/testutils/__init__.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#           Tristan Maat <tristan.maat@codethink.co.uk>
+#           Sam Thursfield <sam.thursfield@codethink.co.uk>
+#           James Ennis <james.ennis@codethink.co.uk>
+#           Valentin David <valentin.david@codethink.co.uk>
+#           William Salmon <will.salmon@codethink.co.uk>
+#
+
 from .runcli import cli, cli_integration
 from .repo import create_repo, ALL_REPO_KINDS
 from .artifactshare import create_artifact_share
 from .element_generators import create_element_size
 from .junction import generate_junction
+from .runner_integration import wait_for_cache_granularity

tests/testutils/runner_integration.py

+#
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#         Will Salmon <will.salmon@codethink.co.uk>
+
+import time
+
+
+def wait_for_cache_granularity():
+    # This isn't called very often so has minimal impact on test runtime.
+    # If this changes it may be worth while adding a more sophisticated approach.
+    """
+    Mitigate the coarse granularity of the gitlab runners mtime
+
+    This function waits for the mtime to increment so that the cache can sort by mtime and
+    get the most recent results.
+    """
+    time.sleep(1.1)