Make cache clients not fail when a blob is not available.

We plan to make cache incomplete. That is some blobs are missing.  For
most of cases we will delete references when requested if they are
incomplete. But there will be corner cases where objects are removed
after the reference is requested.

buildstream/_artifactcache/cascache.py

@@ -43,6 +43,13 @@ from .._exceptions import CASError
 _MAX_PAYLOAD_BYTES = 1024 * 1024
 
 
+class BlobNotFound(CASError):
+
+    def __init__(self, blob, msg):
+        self.blob = blob
+        super().__init__(msg)
+
+
 # A CASCache manages a CAS repository as specified in the Remote Execution API.
 #
 # Args:
@@ -219,6 +226,8 @@ class CASCache():
                 raise CASError("Failed to pull ref {}: {}".format(ref, e)) from e
             else:
                 return False
+        except BlobNotFound as e:
+            return False
 
     # pull_tree():
     #
@@ -1113,6 +1122,9 @@ class _CASBatchRead():
         batch_response = self._remote.cas.BatchReadBlobs(self._request)
 
         for response in batch_response.responses:
+            if response.status.code == code_pb2.NOT_FOUND:
+                raise BlobNotFound(response.digest.hash, "Failed to download blob {}: {}".format(
+                    response.digest.hash, response.status.code))
             if response.status.code != code_pb2.OK:
                 raise CASError("Failed to download blob {}: {}".format(
                     response.digest.hash, response.status.code))