...
 
Commits (65)
......@@ -148,7 +148,8 @@ disable=print-statement,
too-many-instance-attributes,
too-few-public-methods,
missing-docstring,
fixme
fixme,
consider-iterating-dictionary
# Enable the message, report, category or checker with the given id(s). You can
......
This diff is collapsed.
#!/usr/bin/env python3
#
# Copyright (C) 2019 Codethink Limited
# Copyright (C) 2019 Bloomberg Finance LP
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
# Authors:
# Lachlan Mackenzie <lachlan.mackenzie@codethink.co.uk>
"""
This command line executable acts as a method to analyse build log files for
performance data. The paths for a number of benchmarking log files are passed
in and these are then chewed down for statistical information. The collective
results set are then compared and a summary generated for overall performance
based upon the collective set of information generated.
Options are available to:
Write out extracted build logs from the benchmarkings logs
Save intermediate statistics that are used to generate the final summary.
Output further debug.
Inputs:
input_path - path(s) to benchmarking log files
debug - select to output debug
write_logs - path for file to write extracted logs to.
save_stats - path for file to contain intermediate statistical results.
"""
import json
import logging
from collections import namedtuple
import argparse
import statistics
from contrib.logfile_analyser import process_data, accumulate_trees, stat_nodes
from contrib.logfile_analyser import create_stat_tree, print_time_tree
def main():
parser = argparse.ArgumentParser(description="Chew down log entries in results files and "
"create tree of results.")
parser.add_argument("results_paths",
help="Path for generated results to process",
nargs="*",
default=["results.json"])
parser.add_argument("-d", "--debug",
help="Display debug data",
action='store_true')
parser.add_argument("-w", "--write_logs",
help="Write out extracted logs to given file",
type=str)
parser.add_argument("-i", "--display_tree",
help="Display tree data",
action='store_true')
parser.add_argument("-s", "--save_stats",
help="Write out statistical results to given file",
type=str)
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO)
if args.write_logs:
log_file = args.write_logs
else:
log_file = None
if args.save_stats:
stats_file = args.save_stats
else:
stats_file = None
if args.display_tree:
display_tree = args.display_tree
else:
display_tree = None
processed_results = [process_file(f, write_logs=log_file,
save_stats=stats_file,
display_tree=display_tree)
for f in args.results_paths]
process_results(processed_results)
UniqueKeyT = namedtuple('UniqueKeyT', ['name', 'version', 'end_timestamp'])
UniqueKeyVT = namedtuple('UniqueKeyVT', ['name', 'version'])
TAndVStat = namedtuple('TAndVStat', ['mean_times_all', \
'mean_times_outer', \
'mean_total_all', \
'mean_total_outer'])
def process_results(processed_results):
test_and_version_results = {}
for result_set in processed_results:
unique_results = {}
for result in result_set:
unique_key = UniqueKeyT(result.name, result.version,
result.end_timestamp)
if unique_key in unique_results:
unique_results[unique_key].append(result)
else:
unique_results[unique_key] = [result]
for unique in unique_results.keys():
mean_times_outer = []
mean_times_all = []
mean_total_outer = []
mean_total_all = []
for entry in unique_results[unique]:
logging.debug("Entry: %s", entry)
mean_times_outer.append(entry.average_time_at_leaf_elements)
mean_times_all.append(entry.average_time_at_all_elements)
mean_total_outer.append(entry.total_leaf_time_a)
mean_total_outer.append(entry.total_leaf_time_b)
mean_total_all.append(entry.total_time_set_b)
mean_total_all.append(entry.total_time_set_a)
unique_to_test_and_version = UniqueKeyVT(unique.name,
unique.version)
stat = TAndVStat(statistics.mean(mean_times_all),
statistics.mean(mean_times_outer),
statistics.mean(mean_total_all),
statistics.mean(mean_total_outer))
if unique_to_test_and_version in test_and_version_results:
test_and_version_results[unique_to_test_and_version].append(stat)
else:
test_and_version_results[unique_to_test_and_version] = [stat]
for key in test_and_version_results.keys():
mean_times_outer = []
mean_times_all = []
mean_total_outer = []
mean_total_all = []
for entry in test_and_version_results[key]:
mean_times_outer.append(entry.mean_times_outer)
mean_times_all.append(entry.mean_times_all)
mean_total_outer.append(entry.mean_total_outer)
mean_total_all.append(entry.mean_total_all)
if (
len(mean_times_outer) != 1 and len(mean_times_all) != 1 and
len(mean_total_outer) != 1 and len(mean_total_all) != 1
):
print("For test: {}, version: {}, mean outer: {}, std dev outer: {}, mean all: {}, \
std dev all: {}, total time outer mean: {}, std dev total outer: {}, \
total time all mean is {}, std dev total all: {}"
.format(key.name, key.version, statistics.mean(mean_times_outer), \
statistics.stdev(mean_times_outer), statistics.mean(mean_times_all), \
statistics.stdev(mean_times_all), statistics.mean(mean_total_outer), \
statistics.stdev(mean_total_outer), statistics.mean(mean_total_all), \
statistics.stdev(mean_total_all)))
else:
print("For test: {}, version: {}, mean outer: {}, mean all: {}, \
total time outer mean: {}, total time all mean is {}" \
.format(key.name, key.version, statistics.mean(mean_times_outer), \
statistics.mean(mean_times_all), statistics.mean(mean_total_outer), \
statistics.mean(mean_total_all),))
def process_file(results_file, write_logs=None, save_stats=None, display_tree=None):
logs = []
stats_strings = []
stats_rtns = []
with open(results_file, "r") as f:
data = json.load(f)
try:
for test in data["tests"]:
for result in test['results']:
result_set = []
if "measurements" not in result:
logging.error("Ignoring missing results from: %s, test %s",
str(results_file), str(test['name']))
continue
for measurement in result['measurements']:
if measurement["logs"] != "":
result_set.append(process_data(measurement["logs"],
display_tree=display_tree))
if write_logs:
logs.append(measurement["logs"])
else:
break
acc_results = accumulate_trees(result_set)
stats = stat_nodes(acc_results)
for stat_result in stats:
print("For {}, mean {}, std dev {}"
.format(stat_result, stats[stat_result][0], stats[stat_result][1]))
stats_tree = create_stat_tree(stats)
if stats_tree:
print_time_tree(stats_tree)
stats_rtns.append(stats_tree)
except ValueError:
logging.error("Unable to resolve tests in: %s", results_file)
if write_logs:
with open(write_logs, "a") as f:
for index, item in enumerate(logs):
f.write("*** Log Entry {} ***\n\n".format(index))
f.write(item)
f.write('\n')
if save_stats:
with open(save_stats, "a") as f:
for item in stats_strings:
f.write(item)
f.write('\n')
return stats_rtns
if __name__ == "__main__":
main()
......@@ -203,10 +203,11 @@ def verify_token_data(token_data, last_result, results_path, repo_path):
if not bench_mark_repo.git.rev_parse('--verify', token_data['build']['bm_branch']):
logging.error('Branch not in repo: %s', token_data['build']['bs_branch'])
return False
if token_data['build']['bm_sha'] not in [
commit.hexsha for commit in bench_mark_repo.iter_commits(
token_data['build']['bm_branch'])]:
logging.error('SHA of previous benchmarking Head not in repo branch')
if token_data['build']['bm_sha'] not in [commit.hexsha for commit
in bench_mark_repo.iter_commits(token_data['build']
['bm_branch'])]:
logging.error('SHA of previous benchmarking Head not in repo branch: %s',
token_data['build']['bm_sha'])
return False
except git.exc.GitError as err: # pylint: disable=no-member
logging.error("Unable to verify benchmark sha: %s", err)
......