Rearranged files into more sensible folders.

Updated imports to relfect this also.

buildgrid/_app/commands/cmd_server.py

@@ -30,7 +30,7 @@ from buildgrid.server.cas.storage.disk import DiskStorage
 from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
 from buildgrid.server.cas.storage.s3 import S3Storage
 from buildgrid.server.cas.storage.with_cache import WithCacheStorage
-from buildgrid.server.execution.action_cache import ActionCache
+from buildgrid.server.actioncache.storage import ActionCache
 
 from ..cli import pass_context
 

buildgrid/server/execution/action_cache.py → buildgrid/server/actioncache/storage.py

@@ -21,7 +21,7 @@ Implements an in-memory action Cache
 """
 
 
-from ..cas.reference_cache import ReferenceCache
+from ..referencestorage.storage import ReferenceCache
 
 
 class ActionCache(ReferenceCache):

buildgrid/server/buildgrid_server.py

@@ -30,12 +30,14 @@ from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2_grp
 from buildgrid._protos.google.longrunning import operations_pb2_grpc
 
 from .buildgrid_instance import BuildGridInstance
-from .cas.bytestream_service import ByteStreamService
-from .cas.content_addressable_storage_service import ContentAddressableStorageService
-from .execution.action_cache_service import ActionCacheService
-from .execution.execution_service import ExecutionService
-from .execution.operations_service import OperationsService
-from .worker.bots_service import BotsService
+from .cas.service import ByteStreamService, ContentAddressableStorageService
+from .actioncache.service import ActionCacheService
+from .execution.service import ExecutionService
+from .operations.service import OperationsService
+from .execution.instance import ExecutionInstance
+from .scheduler import Scheduler
+from .bots.service import BotsService
+from .bots.instance import BotsInterface
 
 
 class BuildGridServer:

buildgrid/server/cas/content_addressable_storage_service.py

-# Copyright (C) 2018 Bloomberg LP
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#  <http://www.apache.org/licenses/LICENSE-2.0>
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-"""
-ContentAddressableStorageService
-==================
-
-Implements the Content Addressable Storage API, which provides methods
-to check for missing CAS blobs and update them in bulk.
-"""
-
-from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
-from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc as re_pb2_grpc
-
-
-class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServicer):
-
-    def __init__(self, storage):
-        self._storage = storage
-
-    def FindMissingBlobs(self, request, context):
-        # Only one instance for now.
-        storage = self._storage
-        return re_pb2.FindMissingBlobsResponse(
-            missing_blob_digests=storage.missing_blobs(request.blob_digests))
-
-    def BatchUpdateBlobs(self, request, context):
-        # Only one instance for now.
-        storage = self._storage
-        requests = []
-        for request_proto in request.requests:
-            requests.append((request_proto.digest, request_proto.data))
-        response = re_pb2.BatchUpdateBlobsResponse()
-        for (digest, _), status in zip(requests, storage.bulk_update_blobs(requests)):
-            response_proto = response.responses.add()
-            response_proto.digest.CopyFrom(digest)
-            response_proto.status.CopyFrom(status)
-        return response

buildgrid/server/cas/bytestream_service.py → buildgrid/server/cas/service.py

@@ -14,21 +14,47 @@
 
 
 """
-ByteStreamService
+CAS services
 ==================
 
-Implements the ByteStream API, which clients can use to read and write
-CAS blobs.
+Implements the Content Addressable Storage API and ByteStream API.
 """
 
+
 import grpc
 
 from buildgrid._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
 from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc as re_pb2_grpc
 
 from ...settings import HASH
 
 
+class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServicer):
+
+    def __init__(self, storage):
+        self._storage = storage
+
+    def FindMissingBlobs(self, request, context):
+        # Only one instance for now.
+        storage = self._storage
+        return re_pb2.FindMissingBlobsResponse(
+            missing_blob_digests=storage.missing_blobs(request.blob_digests))
+
+    def BatchUpdateBlobs(self, request, context):
+        # Only one instance for now.
+        storage = self._storage
+        requests = []
+        for request_proto in request.requests:
+            requests.append((request_proto.digest, request_proto.data))
+        response = re_pb2.BatchUpdateBlobsResponse()
+        for (digest, _), status in zip(requests, storage.bulk_update_blobs(requests)):
+            response_proto = response.responses.add()
+            response_proto.digest.CopyFrom(digest)
+            response_proto.status.CopyFrom(status)
+        return response
+
+
 class ByteStreamService(bytestream_pb2_grpc.ByteStreamServicer):
 
     BLOCK_SIZE = 1 * 1024 * 1024  # 1 MB block size

tests/action_cache.py

@@ -17,8 +17,8 @@
 
 import pytest
 
+from buildgrid.server.actioncache.storage import ActionCache
 from buildgrid.server.cas.storage import lru_memory_cache
-from buildgrid.server.execution import action_cache
 from buildgrid.server._exceptions import NotFoundError
 from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
 
@@ -28,8 +28,8 @@ def cas():
     return lru_memory_cache.LRUMemoryCache(1024 * 1024)
 
 
-def test_null_action_cache(cas):
-    cache = action_cache.ActionCache(cas, 0)
+def test_null_cas_action_cache(cas):
+    cache = ActionCache(cas, 0)
 
     action_digest1 = remote_execution_pb2.Digest(hash='alpha', size_bytes=4)
     dummy_result = remote_execution_pb2.ActionResult()
@@ -39,8 +39,8 @@ def test_null_action_cache(cas):
         cache.get_action_result(action_digest1)
 
 
-def test_action_cache_expiry(cas):
-    cache = action_cache.ActionCache(cas, 2)
+def test_expiry(cas):
+    cache = ActionCache(cas, 2)
 
     action_digest1 = remote_execution_pb2.Digest(hash='alpha', size_bytes=4)
     action_digest2 = remote_execution_pb2.Digest(hash='bravo', size_bytes=4)
@@ -62,8 +62,8 @@ def test_action_cache_expiry(cas):
     assert cache.get_action_result(action_digest3) is not None
 
 
-def test_action_cache_checks_cas(cas):
-    cache = action_cache.ActionCache(cas, 50)
+def test_checks_cas(cas):
+    cache = ActionCache(cas, 50)
 
     action_digest1 = remote_execution_pb2.Digest(hash='alpha', size_bytes=4)
     action_digest2 = remote_execution_pb2.Digest(hash='bravo', size_bytes=4)

tests/cas/test_services.py

@@ -24,8 +24,8 @@ import pytest
 from buildgrid._protos.google.bytestream import bytestream_pb2
 from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
 from buildgrid.server.cas.storage.storage_abc import StorageABC
-from buildgrid.server.cas.bytestream_service import ByteStreamService
-from buildgrid.server.cas.content_addressable_storage_service import ContentAddressableStorageService
+from buildgrid.server.cas.service import ByteStreamService
+from buildgrid.server.cas.service import ContentAddressableStorageService
 from buildgrid.settings import HASH
 
 

tests/integration/action_cache_service.py

@@ -26,7 +26,8 @@ import pytest
 
 from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
 from buildgrid.server.cas.storage import lru_memory_cache
-from buildgrid.server.execution import action_cache, action_cache_service
+from buildgrid.server.actioncache.storage import ActionCache
+from buildgrid.server.actioncache.service import ActionCacheService
 
 
 # Can mock this
@@ -42,11 +43,11 @@ def cas():
 
 @pytest.fixture
 def cache(cas):
-    yield action_cache.ActionCache(cas, 50)
+    yield ActionCache(cas, 50)
 
 
 def test_simple_action_result(cache, context):
-    service = action_cache_service.ActionCacheService(cache)
+    service = ActionCacheService(cache)
     action_digest = remote_execution_pb2.Digest(hash='sample', size_bytes=4)
 
     # Check that before adding the ActionResult, attempting to fetch it fails
@@ -67,8 +68,8 @@ def test_simple_action_result(cache, context):
 
 
 def test_disabled_update_action_result(cache, context):
-    disabled_push = action_cache.ActionCache(cas, 50, False)
-    service = action_cache_service.ActionCacheService(disabled_push)
+    disabled_push = ActionCache(cas, 50, False)
+    service = ActionCacheService(disabled_push)
 
     request = remote_execution_pb2.UpdateActionResultRequest()
     service.UpdateActionResult(request, context)