From a7aed65a09a96ee242a0a0cc60d8dadd2dfc8664 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BCrg=20Billeter?= <j@bitron.ch>
Date: Mon, 11 Feb 2019 09:40:34 +0100
Subject: [PATCH 1/2] utils.py: Increase buffer size in sha256sum()
Increasing buffer size from 4 kB to 64 kB speeds up read() bandwidth by
factor 4, according to a very simple benchmark.
---
buildstream/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 76f95637e2..b739ca658c 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -235,7 +235,7 @@ def sha256sum(filename):
try:
h = hashlib.sha256()
with open(filename, "rb") as f:
- for chunk in iter(lambda: f.read(4096), b""):
+ for chunk in iter(lambda: f.read(65536), b""):
h.update(chunk)
except OSError as e:
--
GitLab
From 8b9e1d24a6210bc0945f8c39f1dd8be42b9ca53b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BCrg=20Billeter?= <j@bitron.ch>
Date: Mon, 11 Feb 2019 09:56:15 +0100
Subject: [PATCH 2/2] _cas/cascache.py: Increase buffer size in add_object()
Increasing buffer size from 4 kB to 64 kB speeds up read() bandwidth by
factor 4, according to a very simple benchmark.
---
buildstream/_cas/cascache.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 5a6251815a..9d7a121f43 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -35,6 +35,8 @@ from .._exceptions import CASCacheError
from .casremote import BlobNotFound, _CASBatchRead, _CASBatchUpdate
+_BUFFER_SIZE = 65536
+
# A CASCache manages a CAS repository as specified in the Remote Execution API.
#
@@ -371,7 +373,7 @@ class CASCache():
with contextlib.ExitStack() as stack:
if path is not None and link_directly:
tmp = stack.enter_context(open(path, 'rb'))
- for chunk in iter(lambda: tmp.read(4096), b""):
+ for chunk in iter(lambda: tmp.read(_BUFFER_SIZE), b""):
h.update(chunk)
else:
tmp = stack.enter_context(utils._tempnamedfile(dir=self.tmpdir))
@@ -380,7 +382,7 @@ class CASCache():
if path:
with open(path, 'rb') as f:
- for chunk in iter(lambda: f.read(4096), b""):
+ for chunk in iter(lambda: f.read(_BUFFER_SIZE), b""):
h.update(chunk)
tmp.write(chunk)
else:
--
GitLab