Commit d3db3ddf authored by Valentin David's avatar Valentin David
Browse files

Re-check disk space for each write and handle ENOSPC in artifact server. 

There were some race conditions in the way we store artifacts in the
server. Now we verify we still have space on disk for every write. We
also handle ENOSPC to reallocate space or if we cannot properly fail
the connection.

This should help for #609.
parent 6e19a26a

buildstream/_artifactcache/casserver.py

+18 −7
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ import signal 
import sys

import tempfile

import uuid

import errno



import click

import grpc
@@ -190,19 +191,29 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer): 
                    if client_digest is None:

                        context.set_code(grpc.StatusCode.NOT_FOUND)

                        return response

                elif request.resource_name:

                    # If it is set on subsequent calls, it **must** match the value of the first request.

                    if request.resource_name != resource_name:

                        context.set_code(grpc.StatusCode.FAILED_PRECONDITION)

                        return response



                while True:

                    try:

                        _clean_up_cache(self.cas, client_digest.size_bytes)

                        _clean_up_cache(self.cas, client_digest.size_bytes - offset)

                    except ArtifactTooLargeException as e:

                        context.set_code(grpc.StatusCode.RESOURCE_EXHAUSTED)

                        context.set_details(str(e))

                        return response

                elif request.resource_name:

                    # If it is set on subsequent calls, it **must** match the value of the first request.

                    if request.resource_name != resource_name:

                        context.set_code(grpc.StatusCode.FAILED_PRECONDITION)

                        return response

                    try:

                        out.write(request.data)

                        break

                    except OSError as e:

                        # Multiple upload can happen in the same time

                        if e.errno == errno.ENOSPC:

                            continue

                        else:

                            raise



                offset += len(request.data)

                if request.finish_write:

                    if client_digest.size_bytes != offset: