buildgrid/client/cas.py

@@ -283,7 +283,7 @@ class Downloader:
             try:
                 batch_response = self.__cas_stub.BatchReadBlobs(batch_request)
                 for response in batch_response.responses:
-                    assert response.digest.hash in digests
+                    assert response.digest in digests
 
                     read_blobs.append(response.data)
 

buildgrid/server/cas/storage/disk.py

@@ -21,7 +21,6 @@ A CAS storage provider that stores files as blobs on disk.
 """
 
 import os
-import pathlib
 import tempfile
 
 from .storage_abc import StorageABC
@@ -30,28 +29,41 @@ from .storage_abc import StorageABC
 class DiskStorage(StorageABC):
 
     def __init__(self, path):
-        self._path = pathlib.Path(path)
-        os.makedirs(str(self._path / "temp"), exist_ok=True)
+        if not os.path.isabs(path):
+            self.__root_path = os.path.abspath(path)
+        else:
+            self.__root_path = path
+        self.__cas_path = os.path.join(self.__root_path, 'cas')
+
+        self.objects_path = os.path.join(self.__cas_path, 'objects')
+        self.temp_path = os.path.join(self.__root_path, 'tmp')
+
+        os.makedirs(self.objects_path, exist_ok=True)
+        os.makedirs(self.temp_path, exist_ok=True)
 
     def has_blob(self, digest):
-        return (self._path / (digest.hash + "_" + str(digest.size_bytes))).exists()
+        return os.path.exists(self._get_object_path(digest))
 
     def get_blob(self, digest):
         try:
-            return (self._path / (digest.hash + "_" + str(digest.size_bytes))).open('rb')
+            return open(self._get_object_path(digest), 'rb')
         except FileNotFoundError:
             return None
 
-    def begin_write(self, _digest):
-        return tempfile.NamedTemporaryFile("wb", dir=str(self._path / "temp"))
+    def begin_write(self, digest):
+        return tempfile.NamedTemporaryFile("wb", dir=self.temp_path)
 
     def commit_write(self, digest, write_session):
-        # Atomically move the temporary file into place.
-        path = self._path / (digest.hash + "_" + str(digest.size_bytes))
-        os.replace(write_session.name, str(path))
+        object_path = self._get_object_path(digest)
+
         try:
-            write_session.close()
-        except FileNotFoundError:
-            # We moved the temporary file to a new location, so when Python
-            # tries to delete its old location, it'll fail.
+            os.makedirs(os.path.dirname(object_path), exist_ok=True)
+            os.link(write_session.name, object_path)
+        except FileExistsError:
+            # Object is already there!
             pass
+
+        write_session.close()
+
+    def _get_object_path(self, digest):
+        return os.path.join(self.objects_path, digest.hash[:2], digest.hash[2:])

docs/source/data/cas-example-server.conf

+server:
+  - !channel
+    port: 50051
+    insecure_mode: true
+
+instances:
+  - name: main
+
+    storages:
+      - !disk-storage &main-storage
+	path: !expand-path $HOME/cas
+
+    services:
+      - !cas
+	storage: *main-storage
+      - !bytestream
+	storage: *main-storage
+      - !reference-cache
+	storage: *main-storage
+	max_cached_refs: 512

docs/source/using.rst

-
 .. _using:
 
 Using
@@ -12,3 +11,4 @@ This section covers how to run an use the BuildGrid build service.
    using_internal.rst
    using_bazel.rst
    using_buildstream.rst
+   using_cas_server.rst

docs/source/using_bazel.rst

@@ -98,7 +98,7 @@ has ``gcc`` installed, run:
 
 The ``--remote`` option is used to specify the server location (running on the
 same machine here, and listening to port 50051). The ``--parent`` option is used
-to specify the server instance you except the bot to be attached to. Refer to
+to specify the server instance you expect the bot to be attached to. Refer to
 the :ref:`CLI reference section <invoking-bgd-bot-temp-directory>` for command
 line interface details.
 
@@ -120,7 +120,7 @@ You can finally have Bazel to build the example workspace by running:
 
    bazel build //main:hello-world
 
-You can verify that the example has been successfully build by running the
+You can verify that the example has been successfully built by running the
 generated executable. Simply invoke:
 
 .. code-block:: sh

docs/source/using_buildstream.rst

@@ -149,7 +149,7 @@ that the ``buildbox`` tool is functional on your machine (refer to
 
 The ``--remote`` option is used to specify the server location (running on the
 same machine here, and listening to port 50051). The ``--parent`` option is
-used to specify the server instance you except the bot to be attached to (empty
+used to specify the server instance you expect the bot to be attached to (empty
 here). ``--fuse-dir`` and ``--local-cas`` are specific to the ``buildbox`` bot
 and respectively specify the sandbox mount point and local CAS cache locations.
 Refer to the :ref:`CLI reference section <invoking-bgd-bot-buildbox>` for
@@ -178,7 +178,7 @@ You can finally have BuildStream to build the example project by running:
 
    bst build hello.bst
 
-You can verify that the example has been successfully build by running the
+You can verify that the example has been successfully built by running the
 generated executable. Simply invoke:
 
 .. code-block:: sh

docs/source/using_cas_server.rst

+.. _cas-server:
+
+CAS server
+==========
+
+It is possible to configure BuildGrid with just a Content Addressable Storage service.
+
+.. note::
+
+   This service can be equivalent to `BuildStream's Artifact Server`_ if the `Reference Storage Service`_ is included.
+
+.. _cas-configuration:
+
+Configuration
+-------------
+
+Here is an example project configuration. It also implements an optional API called the `Reference Storage Service`_, which if used, allows the user to store a ``Digest`` behind a user defined ``key``.
+
+.. literalinclude:: ./data/cas-example-server.conf
+   :language: yaml
+
+.. hint::
+
+   Use ``- name: ""`` if using with BuildStream, as instance names are not supported for that tool yet.
+
+This defines a single ``main`` instance of the ``CAS``, ``Bytestream`` and ``Reference Storage`` service on port ``55051``. It is backed onto disk storage and will populate the folder ``$HOME/cas``. To start the server, simply type into your terminal:
+
+.. code-block:: sh
+
+   bgd server start example.conf
+
+The server should now be available to use.
+
+.. _BuildStream's Artifact Server: https://buildstream.gitlab.io/buildstream/install_artifacts.html
+.. _Reference Storage Service: https://gitlab.com/BuildGrid/buildgrid/blob/master/buildgrid/_protos/buildstream/v2/buildstream.proto

setup.py

@@ -97,7 +97,8 @@ tests_require = [
 ]
 
 docs_require = [
-    'sphinx',
+    # rtd-theme broken in Sphinx >= 1.8, this breaks search functionality.
+    'sphinx == 1.7.8',
     'sphinx-click',
     'sphinx-rtd-theme',
     'sphinxcontrib-apidoc',