bst_benchmark_local.py 9.29 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48
#!/usr/bin/env python3

#
#  Copyright (C) 2019 Codethink Limited
#  Copyright (C) 2019 Bloomberg Finance LP
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2 of the License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
#  Authors:
#        Lachlan Mackenzie <lachlan.mackenzie@codethink.co.uk>

import tempfile
import argparse
import os
import logging
from shutil import copyfile
import sys

import git

import bst_benchmarks.main
import generate_benchmark_config
import digest_results

# This command line executable acts as a method to generate a locally run
# benchmark CI run. A number of elements are passed in to check that
# configuration settings are tenable - path to repository, buildstream branch,
# SHAs to be tested (defaults to HEAD of branch if not specified), specific tests
# to be carried out (defaults to default.benchmark if not set), output file
# (defaults to stdout if not set), whether master should be tested as a comparable
# (default is yes)
#
# output_file - the path where the bespoke benchmark file is to be placed
# repo_path - path to the local buildstream repo.
# bs_branch - the branch of buildstream being considered.
# shas - shas to be tested.

49

50 51 52 53 54
def main():
   # Define all the defaults explicitly
   repo_path = 'https://gitlab.com/BuildStream/buildstream.git'
   bs_branch = 'master'
   shas_to_be_tested = []
55
   docker_tag = "30-latest"
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
   docker_image = 'registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora'
   tests_to_run = []
   debug = False
   keep_results = False
   keep_path = "results.json"

   def make_help(message):
      return message + " (Default: %(default)s)"

   # Resolve commandline arguments
   parser = argparse.ArgumentParser(description="Automatically set up test environments for a set "
                                    "of buildstream benchmarks and run them locally using docker.")
   parser.add_argument("tests_to_run",
                       help=make_help("The benchmark tests to run"),
                       nargs="*",
                       default=["bst_benchmarks/default.benchmark"])
   parser.add_argument("-o", "--output_file",
                       help=make_help("The file to write benchmark results to"),
                       type=argparse.FileType("w"),
                       default="-")
   parser.add_argument("-r", "--repo_path",
                       help=make_help("The repository from which to clone branches to compare "
                                      "against the defaults set in the test. Note that this can"
                                      " be a local path using file://"),
                       default=repo_path,
                       type=str)
   parser.add_argument("-b", "--bs_branch",
                       help=make_help("The branch to clone from the set repository"),
                       default=bs_branch,
                       type=str)
   parser.add_argument("-s", "--shas_to_be_tested",
                       help=make_help("SHAs to clone from the set repository"),
                       action='append')
   parser.add_argument("-d", "--docker_tag",
                       help=make_help("The tag to use for the buildstream image"),
                       default=docker_tag,
                       type=str)
   parser.add_argument("-p", "--docker_image",
                       help=make_help("The docker image to use"),
                       default=docker_image,
                       type=str)
   parser.add_argument("-g", "--debug",
                       help=make_help("Show debug messages"),
                       default=debug,
                       action='store_true')
   parser.add_argument("-k", "--keep_results",
                       help=make_help("Retain raw benchmarking results"),
                       default=keep_results,
                       action='store_true')
   parser.add_argument('--results-file',
                       help=make_help("The file to store benchmarking results in; Implies"
                                      " --keep-results"),
                       default=keep_path,
                       type=str)
   args = parser.parse_args()

   if bool(args.output_file):
      output_file = args.output_file

   if bool(args.repo_path):
      repo_path = args.repo_path

   if bool(args.bs_branch):
      bs_branch = args.bs_branch

   if bool(args.shas_to_be_tested):
      shas_to_be_tested = args.shas_to_be_tested

   if bool(args.docker_tag):
      docker_tag = args.docker_tag

   if bool(args.tests_to_run):
      tests_to_run = args.tests_to_run

   if bool(args.debug):
      debug = True
      logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
   else:
      logging.basicConfig(stream=sys.stderr, level=logging.INFO)

   if bool(args.keep_results):
      keep_results = args.keep_results

   if bool(args.results_file):
      keep_path = os.path.abspath(args.results_file)
      keep_results = True

   commits = list()

   # Create a temporary directory for all work
   with tempfile.TemporaryDirectory(prefix='temp_staging_location') as temp_staging_area:
      # Get a reference to the requested repository cloning if remote
      try:
         if os.path.exists(repo_path):
            logging.info("Repo path resolves locally: %s", repo_path)
            repo = git.Repo.init(repo_path, bare=False)
         else:
            logging.info("Repo path resolves remotely: %s", repo_path)
            repo = git.Repo.clone_from(repo_path, temp_staging_area)
155
      except git.exc.GitError as err:  # pylint: disable=no-member
156 157 158 159 160 161 162 163 164 165 166 167
         logging.error("Unable to access git repository: %s", err)
         sys.exit(1)

      # Iterate the commits in the requested branch and add to list to be
      # processed if they are as per the command line selection. Keep a
      # list of all SHAs that have been found
      shas_found = []
      try:
         for commit in repo.iter_commits(bs_branch):
            if commit.hexsha in shas_to_be_tested:
               commits.append(commit)
               shas_found.append(commit.hexsha)
168
      except git.exc.GitCommandError as err:  # pylint: disable=no-member
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
         logging.error("Could not find commits in repository '%s' for branch '%s':\n%s",
                       repo_path, bs_branch, err)
         sys.exit(1)

      # Check list of found SHAs against original list and flag any missing
      shas_not_found = [sha for sha in shas_to_be_tested if sha not in shas_found]
      if shas_not_found:
         logging.error("SHA(s) could not be found: %s", shas_not_found)
         sys.exit(1)

      # Create a temporary file reference for the benchmark versioning configuration
      # file that will be processed together with the selected benchmark test(s).
      output_tmp_file = os.path.join(temp_staging_area, 'output_file.benchmark')

      # Generate the bechmark versioning configuration file for selected parameters
      try:
         generate_benchmark_config.generate_benchmark_configuration(
            output_file=output_tmp_file,
            list_of_shas=commits,
            docker_version=docker_tag,
            bs_branch=bs_branch,
            bs_path=repo_path,
            docker_path=docker_image)
      # pylint: disable=broad-except
      except Exception as err:
         logging.error("Creating benchmarking configuration failed:\n%s", err)
         sys.exit(1)

      test_set = []
      # Add tests to run
      for test in tests_to_run:
         if test in test_set:
201
            logging.error("Duplicate benchmarking test will be ignored: %s", test)
202 203 204 205 206 207 208 209 210 211
         else:
            test_set.append(test)

      # Generate the commandline parameters for the benchmarking versioning together
      # with the selected benchmarking test(s).
      test_set.append(output_tmp_file)

      results_tmp_file = os.path.join(temp_staging_area, 'tmp_result')

      try:
212
         bst_benchmarks.main.run(config_files=test_set, _debug=True,
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
                                 keep_images=False, reuse_images=False,
                                 output_file=results_tmp_file)
      # pylint: disable=broad-except
      except Exception as err:
         logging.error("Benchmarking failed:\n%s", err)
         sys.exit(1)

      # Copy results to keep path if set
      if keep_results:
         os.makedirs(os.path.dirname(keep_path), exist_ok=True)
         copyfile(results_tmp_file, keep_path)

      # Create temporary file for results digest
      tmp_output = os.path.join(temp_staging_area, 'tmp_result')

      # Create temporary file for results digest
      tmp_error = os.path.join(temp_staging_area, 'tmp_error')

231 232
      # TODO: _error_file is not actually in use
      digest_results.parse(files=[results_tmp_file], output_file=tmp_output, _error_file=tmp_error)
233 234 235 236 237 238 239 240 241

      # Write output to requested outfile
      with open(tmp_output, "r") as fin:
         output_file.write(fin.read())
      output_file.close()


if __name__ == "__main__":
   main()