buildstream/_artifactcache/artifactcache.py

@@ -80,6 +80,8 @@ class ArtifactCache():
         self.context = context
         self.required_artifacts = set()
         self.extractdir = os.path.join(context.artifactdir, 'extract')
+        self.tmpdir = os.path.join(context.artifactdir, 'tmp')
+
         self.max_size = context.cache_quota
         self.estimated_size = None
 
@@ -89,7 +91,8 @@ class ArtifactCache():
         self._local = False
         self.cache_size = None
 
-        os.makedirs(context.artifactdir, exist_ok=True)
+        os.makedirs(self.extractdir, exist_ok=True)
+        os.makedirs(self.tmpdir, exist_ok=True)
 
     ################################################
     #  Methods implemented on the abstract class   #

buildstream/_artifactcache/cascache.py

@@ -56,7 +56,8 @@ class CASCache(ArtifactCache):
         super().__init__(context)
 
         self.casdir = os.path.join(context.artifactdir, 'cas')
-        os.makedirs(os.path.join(self.casdir, 'tmp'), exist_ok=True)
+        os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
+        os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
 
         self._enable_push = enable_push
 
@@ -85,8 +86,6 @@ class CASCache(ArtifactCache):
             # artifact has already been extracted
             return dest
 
-        os.makedirs(self.extractdir, exist_ok=True)
-
         with tempfile.TemporaryDirectory(prefix='tmp', dir=self.extractdir) as tmpdir:
             checkoutdir = os.path.join(tmpdir, ref)
             self._checkout(checkoutdir, tree)
@@ -394,7 +393,7 @@ class CASCache(ArtifactCache):
         try:
             h = hashlib.sha256()
             # Always write out new file to avoid corruption if input file is modified
-            with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
+            with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
                 # Set mode bits to 0644
                 os.chmod(out.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
 
@@ -764,7 +763,7 @@ class CASCache(ArtifactCache):
             # already in local cache
             return
 
-        with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
+        with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
             self._fetch_blob(remote, tree, out)
 
             directory = remote_execution_pb2.Directory()
@@ -778,7 +777,7 @@ class CASCache(ArtifactCache):
                     # already in local cache
                     continue
 
-                with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as f:
+                with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
                     self._fetch_blob(remote, filenode.digest, f)
 
                     digest = self.add_object(path=f.name)

buildstream/_artifactcache/casserver.py

@@ -161,7 +161,7 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
         offset = 0
         finished = False
         resource_name = None
-        with tempfile.NamedTemporaryFile(dir=os.path.join(self.cas.casdir, 'tmp')) as out:
+        with tempfile.NamedTemporaryFile(dir=self.cas.tmpdir) as out:
             for request in request_iterator:
                 assert not finished
                 assert request.write_offset == offset

buildstream/_fuse/fuse.py

@@ -757,7 +757,11 @@ class FUSE(object):
         if self.raw_fi:
             return self.operations('create', path, mode, fi)
         else:
-            fi.fh = self.operations('create', path, mode)
+            # This line is different from upstream to fix issues
+            # reading file opened with O_CREAT|O_RDWR.
+            # See issue #143.
+            fi.fh = self.operations('create', path, mode, fi.flags)
+            # END OF MODIFICATION
             return 0
 
     def ftruncate(self, path, length, fip):

buildstream/_fuse/hardlinks.py

@@ -185,12 +185,12 @@ class SafeHardlinkOps(Operations):
 
         return os.open(full_path, flags)
 
-    def create(self, path, mode, fi=None):
+    def create(self, path, mode, flags):
         full_path = self._full_path(path)
 
         # If it already exists, ensure it's a copy first
         self._ensure_copy(full_path)
-        return os.open(full_path, os.O_WRONLY | os.O_CREAT, mode)
+        return os.open(full_path, flags, mode)
 
     def read(self, path, length, offset, fh):
         os.lseek(fh, offset, os.SEEK_SET)

buildstream/_pipeline.py

@@ -358,10 +358,15 @@ class Pipeline():
                     inconsistent.append(element)
 
         if inconsistent:
-            detail = "Exact versions are missing for the following elements\n" + \
-                     "Try tracking these elements first with `bst track`\n\n"
+            detail = "Exact versions are missing for the following elements:\n\n"
+
             for element in inconsistent:
-                detail += "  " + element._get_full_name() + "\n"
+                detail += "  " + element._get_full_name()
+                for source in element.sources():
+                    if not source._get_consistency() and not source.get_ref():
+                        detail += ": Source {} is missing ref\n".format(source)
+            detail += "\nTry tracking these elements first with `bst track`\n"
+
             raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline")
 
     #############################################################

buildstream/plugins/sources/git.py

@@ -80,6 +80,7 @@ from configparser import RawConfigParser
 
 from buildstream import Source, SourceError, Consistency, SourceFetcher
 from buildstream import utils
+from buildstream._exceptions import LoadError, LoadErrorReason
 
 GIT_MODULES = '.gitmodules'
 
@@ -296,6 +297,12 @@ class GitSource(Source):
         self.original_url = self.node_get_member(node, str, 'url')
         self.mirror = GitMirror(self, '', self.original_url, ref)
         self.tracking = self.node_get_member(node, str, 'track', None)
+
+        # At this point we now know if the source has a ref and/or a track.
+        # If it is missing both then we will be unable to track or build.
+        if self.mirror.ref is None and self.tracking is None:
+            raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Git sources require a ref and/or track".format(self))
+
         self.checkout_submodules = self.node_get_member(node, bool, 'checkout-submodules', True)
         self.submodules = []
 
@@ -359,6 +366,12 @@ class GitSource(Source):
 
         # If self.tracking is not specified it's not an error, just silently return
         if not self.tracking:
+            # Is there a better way to check if a ref is given.
+            if self.mirror.ref is None:
+                detail = 'Without a tracking branch ref can not be updated. Please ' + \
+                         'provide a ref or a track.'
+                raise SourceError("{}: No track or ref".format(self),
+                                  detail=detail, reason="track-attempt-no-track")
             return None
 
         with self.timed_activity("Tracking {} from {}"

tests/sources/git.py

 import os
 import pytest
 
-from buildstream._exceptions import ErrorDomain
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
 from buildstream import _yaml
 
 from tests.testutils import cli, create_repo
@@ -359,3 +359,41 @@ def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles):
 
     # Assert that we are just fine without it, and emit a warning to the user.
     assert "Ignoring inconsistent submodule" in result.stderr
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    # Create the repo from 'repofiles' subdir
+    repo = create_repo('git', str(tmpdir))
+    ref = repo.create(os.path.join(project, 'repofiles'))
+
+    # Write out our test target
+    gitsource = repo.source_config(ref=None)
+    gitsource.pop('track')
+    element = {
+        'kind': 'import',
+        'sources': [
+            gitsource
+        ]
+    }
+
+    _yaml.dump(element, os.path.join(project, 'target.bst'))
+
+    # Track will encounter an inconsistent submodule without any ref
+    result = cli.run(project=project, args=['track', 'target.bst'])
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
+    result.assert_task_error(None, None)
+
+    # Assert that we are just fine without it, and emit a warning to the user.
+    assert "Git sources require a ref and/or track" in result.stderr
+
+    # Track will encounter an inconsistent submodule without any ref
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
+    result.assert_task_error(None, None)
+
+    # Assert that we are just fine without it, and emit a warning to the user.
+    assert "Git sources require a ref and/or track" in result.stderr