scheduler.py: Add max time for jobs to block

This stops the issue of grpc calls timeouts defaulting to the max int64 value.
MAX_JOB_BLOCK_TIME is in the settings file.

buildgrid/server/scheduler.py

@@ -25,6 +25,7 @@ from queue import Queue, Empty
 
 from buildgrid._enums import LeaseState, OperationStage
 from buildgrid._exceptions import NotFoundError
+from buildgrid.settings import MAX_JOB_BLOCK_TIME
 
 
 class Scheduler:
@@ -134,11 +135,13 @@ class Scheduler:
             worker_capabilities (dict): a set of key-value pairs decribing the
                 worker properties, configuration and state at the time of the
                 request.
-            timeout (int): time to block waiting on job queue
+            timeout (int): time to block waiting on job queue, caps if longer
+                than MAX_JOB_BLOCK_TIME
         """
         if not timeout and self.queue.empty():
             return []
 
+        timeout = timeout if timeout < MAX_JOB_BLOCK_TIME else MAX_JOB_BLOCK_TIME
         try:
             job = (self.queue.get(block=True, timeout=timeout)
                    if timeout

buildgrid/settings.py

@@ -33,3 +33,7 @@ MAX_REQUEST_COUNT = 500
 
 # time in seconds to pad timeouts
 NETWORK_TIMEOUT = 5
+
+# Hard limit for waiting on jobs, avoids grpc timeouts not being set defaulting
+# the interval to the max int64 value
+MAX_JOB_BLOCK_TIME = 300