...
  View open merge request
Commits (13)
......@@ -58,7 +58,8 @@ tests:
# Use pre-created source cache to avoid measuring the network fetch time.
mkdir -p ~/.config
echo "sourcedir: /mnt/baserock-source" > ~/.config/buildstream.conf
cp /root/usr_config_bst.yml ~/.config/buildstream.conf
echo "sourcedir: /mnt/baserock-source" >> ~/.config/buildstream.conf
# Disable the artifact cache to ensure a full build.
sed -e '/artifacts:/,/^$/ d' -i project.conf
......@@ -69,7 +70,7 @@ tests:
bst_measure \
-o /root/measurements.json \
-l /root/log.txt \
bst --log-file='/root/log.txt' --config='/root/usr_config_bst.yml' build gnu-toolchain/stage1-binutils.bst
bst --log-file='/root/log.txt' build gnu-toolchain/stage1-binutils.bst
# Or, for a more complete test: bst build systems/minimal-system-image-x86_64.bst
repeats: 3
......
......@@ -206,7 +206,7 @@ def verify_token_data(token_data, last_result, results_path, repo_path):
if token_data['build']['bm_sha'] not in [
commit.hexsha for commit in bench_mark_repo.iter_commits(
token_data['build']['bm_branch'])]:
logging.error('SHA of previous benchmarking Head not in repo branch')
logging.error('SHA of previous benchmarking Head not in repo branch: %', token_data['build']['bm_sha'])
return False
except git.exc.GitError as err: # pylint: disable=no-member
logging.error("Unable to verify benchmark sha: %s", err)
......
#!/usr/bin/env python3
#
# Copyright (C) 2019 Codethink Limited
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
# Authors:
# Tristan Maat <tristan.maat@codethink.co.uk>
# Lachlan Mackenzie <lachlan.mackenzie@codethink.co.uk>
"""Parse a set of measurements."""
from typing import Dict, Tuple
from .result import Result
def parse(tests: Dict, json_ref: str) -> Dict[Tuple[str, int, str, str, str], Result]:
"""Parse results from a bst_measure results file.
A bst_measure results file is structured broadly as follows:
Common versioning elements (timestamps, host info, benchmarking sha)
Buildstream versions being tested (name, docker image, docker versioning, buildstream info
unique reference taking into account source repository)
tests ->
name (e.g. "Startup time")
results ->
version (Matches the unique reference in the above version section)
Measurements ->
total time (stopwatch time for bst command)
max-rss-kb (maximum memory usage)
bs-sha (the sha of the builstream ref being tested)
bs-ref (the buildstream reference)
logs (the logs for the buildstream command - optional)
repeats (the number of times the test should be carried out - this
should equate to the number of measurements)
version (Another version)
Measurements -> ......
name (Another test)
results ->
version (again matching the unique version section)
Measurements ->
etc .....
repeats ....
version (Another unique version)
EOF
We want to end up with a dictionary that looks like this:
{(test.name, test.timestamp, version.tag, version.sha): Result}
e.g. {("Build Foo", 1559757649.402028,
"master.https...gitlab.com.buildstream.buildstream",
bf8c33a1d5389e5f2e2869052a7a5c5e0d829e17):
Result: (reported_times: [ 289.081, 308.042, 262.25 ],
reported_memory_usages: [ 85580, 85596, 85512 ],
reported_logs: [ "<logging 1>", "<logging 2>", "<logging 3>" })}
"""
final = {}
common_versioning_fields = ["start_timestamp",
"end_timestamp",
"host_info",
"benchmarking_sha",
"versions"]
common_versioning = {k: v for k, v in tests.items() if k in common_versioning_fields}
for test in tests["tests"]:
for result in parse_versions(test, common_versioning, json_ref):
final[result[0:4]] = result[5]
return final
def parse_versions(test: Dict, common_versioning: Dict, json_ref: str) \
-> Tuple[str, int, str, str, str, Result]:
"""Parse benchmark test results and incoporate common versioning information.
This takes versioning for a benchmark file run and test version (start time, buildstream
reference and buildstream sha) together with a generated results class and returns a single
instance that represents a result together with corresponding versioning information.
"""
common_version = {}
for _common_version in common_versioning["versions"]:
ref = _common_version["unique_ref"]
common_version[ref] = _common_version
for result_set in test["results"]:
unique_ref = result_set["version"]
yield test["name"], common_versioning["start_timestamp"], \
common_version[unique_ref]["buildstream_ref"], \
common_version[unique_ref]["buildstream_commit"], \
json_ref, Result.from_json(result_set)
#!/usr/bin/env python3
#
# Copyright (C) 2019 Codethink Limited
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
# Authors:
# Tristan Maat <tristan.maat@codethink.co.uk>
# Lachlan Mackenzie <lachlan.mackenzie@codethink.co.uk>
"""Result definitions and helper functions."""
import statistics
from typing import Dict
class Result():
"""A test result.
This reports averaged results and their standard deviations over a
number of test runs.
"""
def __init__(self,
reported_times: [float],
reported_memory_usages: [int],
reported_logs: [str]):
"""Create the result object.
Args:
reported_times: A list of reported times
reported_memory_usages: A list of reported memory usages
"""
assert len(reported_times) == len(reported_memory_usages),\
"Inconsistent number of repetitions."
# Rather than computing the times here, we keep our values
# around, and compute the averages on access. This makes
# working with this class a bit more flexible, and makes it
# easier to spot what the values mean.
self._real_times = reported_times
self._memory_usages = reported_memory_usages
self._logs = reported_logs
def __repr__(self):
return "Times: {}, Memory Usages: {} \n Logs: \n {}" \
.format(str(self._real_times), str(self._memory_usages), str(self._logs))
@property
def real_time(self):
"""Get the average real time (in seconds).
This time *includes* the time we spent running in the
background because the OS swapped us out for something else.
"""
return statistics.mean(self._real_times)
@property
def real_time_stdev(self):
"""Get the standard deviation of our time measurements."""
# We use pstdev here because we're interested in the stdev of
# the runs we performed (so we know how reliable our results
# are), not the potential spread in execution times in
# general.
return statistics.pstdev(self._real_times)
@property
def memory_usage(self):
"""Get the average memory usage (in kb).
This is the maximum resident size (i.e., at the point at which
we used the most memory, how much memory we were using).
"""
return statistics.mean(self._memory_usages)
@property
def memory_usage_stdev(self):
"""Get the standard deviation of our time measurements."""
# See real_time_stdev
return statistics.pstdev(self._memory_usages)
@property
def logs(self):
"""Get the logs of our measurements."""
return self._logs
# TODO: Be more specific about what the dict can contain.
@classmethod
def from_json(cls, json: Dict) -> 'Result':
"""Create a result object from json data."""
reported_times = [repetition["total-time"] for repetition in json["measurements"]]
reported_memory_usages = [repetition["max-rss-kb"] for repetition in json["measurements"]]
reported_logs = [repetition["logs"] for repetition in json["measurements"]]
return cls(reported_times, reported_memory_usages, reported_logs)
# Tree(logs) can also be done as an @property and in this
# class (maybe nicer), though I'd recommend caching the result
# once it's been computed