From 8070282c62d2b667e8378efbe2c4151a00a3d9c4 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 9 Jun 2019 18:27:40 +0200 Subject: [PATCH 01/20] Test coverage for each individual test By adding --individual-test-coverage the coverage of each test is printed together with the total line coverage to an extra file --- suite_validation/__init__.py | 14 ++++++ suite_validation/execution.py | 28 ++++++++++- suite_validation/execution_utils.py | 1 + suite_validation/test_coverage.py | 74 +++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 suite_validation/test_coverage.py diff --git a/suite_validation/__init__.py b/suite_validation/__init__.py index a6bf802..019e365 100644 --- a/suite_validation/__init__.py +++ b/suite_validation/__init__.py @@ -25,6 +25,7 @@ import shutil import zipfile from suite_validation import execution from suite_validation import execution_utils as eu +from suite_validation import test_coverage as test_cov __VERSION__ = "v1.1-dev" @@ -110,6 +111,14 @@ def get_parser(): required=False, ) + parser.add_argument( + "--individual-test-coverage", + dest="individual_test_cov", + action="store_true", + default=False, + help="print coverage of each test to file", + ) + parser.add_argument( "--verbose", dest="verbose", @@ -238,6 +247,7 @@ def main(): exec_results = eu.SuiteExecutionResult() harness_file = os.path.join(args.output_dir, "harness.c") executable = os.path.join(args.output_dir, "a.out") + compute_individuals = args.individual_test_cov try: executor = execution.SuiteExecutor( args.stop_after_success, @@ -246,6 +256,7 @@ def main(): overwrite_files=args.overwrite, harness_file_target=harness_file, compile_target=executable, + compute_individuals=compute_individuals ) executor.run(args.file, args.test_suite, args.machine_model, exec_results) @@ -282,6 +293,9 @@ def main(): [str(c) + "\n" for c in exec_results.coverage_sequence] ) + if exec_results.coverage_tests: + test_cov.write_individual_test_coverages_to_output(args.output_dir, args.file, exec_results) + print() print("---Results---") print("Tests run:", len(exec_results.results)) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index 937d007..2403509 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -26,6 +26,7 @@ import zipfile from lxml import etree from suite_validation import execution_utils as eu +from suite_validation import test_coverage as test_cov HARNESS_FILE_NAME = "harness.c" ARCHITECTURE_TAG = "architecture" @@ -307,7 +308,7 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): data_file = os.path.basename(data_file) # data file is in cwd if os.path.exists(data_file): - cmd = ["gcov", "-nbc", data_file] + cmd = ["gcov", "-bc", data_file] res = eu.execute(cmd, quiet=True) full_cov = res.stdout.splitlines() @@ -364,6 +365,7 @@ class SuiteExecutor: compute_sequence=False, overwrite_files=True, isolate_tests=True, + compute_individuals=True, ): self._stop_after_success = stop_after_found_error self._timelimit = timelimit_per_run @@ -373,6 +375,7 @@ class SuiteExecutor: self._compute_sequence = compute_sequence self._overwrite_files = overwrite_files self._isolate_tests = isolate_tests + self._compute_individual_test_coverages = compute_individuals def run(self, program_file, test_suite, machine_model, result_target=None): """Execute the given tests on the given program. @@ -433,6 +436,10 @@ class SuiteExecutor: # this method call raises an ExecutionError if the given test suite is invalid test_vectors = self._get_described_vectors(test_suite) + if self._compute_individual_test_coverages: + self._execute_individual_tests(program_file, test_vectors, executor, result_target) + test_vectors = self._get_described_vectors(test_suite) + self._execute_tests(program_file, test_vectors, executor, result_target) return result_target @@ -502,6 +509,25 @@ class SuiteExecutor: program_file ) + @staticmethod + def _execute_individual_tests(program_file, test_vectors, executor, result_target): + + for tv in test_vectors: + executor.run(program_file, tv) + + lines_executed, branches_executed, branches_taken = executor.get_coverage( + program_file + ) + + gcov_file = os.path.basename(program_file) + ".gcov" + hit_counter_dic = test_cov.get_hit_counter_dic_from_gcov_file(gcov_file) + test_coverage = test_cov.TestCoverage(program_file, tv, hit_counter_dic, lines_executed, + branches_executed, branches_taken) + result_target.coverage_tests.append(test_coverage) + data_file = executor.harness_file[:-1] + "gcda" + data_file = os.path.basename(data_file) + cmd = ["rm", data_file] + eu.execute(cmd, quiet=True) def _parse_xml_if_testcase(xml_lines): curr_content = [] diff --git a/suite_validation/execution_utils.py b/suite_validation/execution_utils.py index 80164f5..d89e4b5 100644 --- a/suite_validation/execution_utils.py +++ b/suite_validation/execution_utils.py @@ -142,6 +142,7 @@ class SuiteExecutionResult: self.branches_taken = "0%" self.successful_test = None self.coverage_sequence = list() + self.coverage_tests = list() class ExecutionResult: diff --git a/suite_validation/test_coverage.py b/suite_validation/test_coverage.py new file mode 100644 index 0000000..64c7f2c --- /dev/null +++ b/suite_validation/test_coverage.py @@ -0,0 +1,74 @@ +import os + +UNREACHED_LINE = "#####" +NO_COUNTABLE_PROGRAM_LINE = "-" + + +class TestCoverage: + def __init__(self, file_name, test_vector, hit_counter_dic, lines_executed, branches_executed, branches_taken): + self.filename = file_name + self.test_vector = test_vector + self.hit_counter_dic = hit_counter_dic + self.lines_executed = lines_executed + self.branches_executed = branches_executed + self.branches_taken = branches_taken + + +def get_hit_counter_dic_from_gcov_file(filename): + hit_counter_dic = {} + with open(filename) as file: + for line in file: + line = line.strip() + line = line.replace(" ", "") + chunks = line.split(":") + if len(chunks) == 1: + continue + hit_counter = chunks[0] + countable_program_line = True + if hit_counter == NO_COUNTABLE_PROGRAM_LINE: + hit_counter = 0 + countable_program_line = False + if hit_counter == UNREACHED_LINE: + hit_counter = 0 + program_line = chunks[1] + if program_line.isdigit() and int(program_line) > 0: + if countable_program_line: + hit_counter_dic[int(program_line)] = int(hit_counter) + return hit_counter_dic + + +def write_individual_test_coverages_to_output(outputdir, program, exec_results): + hit_counter_file = os.path.join(outputdir, "individual-test-coverage") + with open(hit_counter_file, "w") as outp: + outp.write("Program: " + program + "\n\n") + for test_coverage in exec_results.coverage_tests: + outp.write(str(test_coverage.test_vector) + "\n") + outp.write("Hit counter:" + "\n") + for program_line, hit_counter in test_coverage.hit_counter_dic.items(): + outp.write(str(program_line) + ": " + str(hit_counter) + "\n") + outp.write("Lines covered:" + test_coverage.lines_executed + "\n") + outp.write("Branch conditions executed:" + test_coverage.branches_executed + "\n") + outp.write("Branches covered:" + test_coverage.branches_taken + "\n\n") + + test_suite_hit_counter_dic = {} + + for test_coverage in exec_results.coverage_tests: + for program_line in test_coverage.hit_counter_dic.keys(): + if not program_line in test_suite_hit_counter_dic: + test_suite_hit_counter_dic[program_line] = 0 + test_suite_hit_counter_dic[program_line] += test_coverage.hit_counter_dic[program_line] + + outp.write("Total Coverage" + "\n") + outp.write("Hit counter:" + "\n") + lines = 0 + covered_lines = 0 + for program_line, hit_counter in test_suite_hit_counter_dic.items(): + outp.write(str(program_line) + ": " + str(hit_counter) + "\n") + lines += 1 + if hit_counter > 0: + covered_lines += 1 + covered = 0 + if lines != 0: + covered = 100 * float(covered_lines) / float(lines) + outp.write("Lines covered:" + str(covered) + "%" + " (of " + str(lines) + ")") + outp.close() -- GitLab From 23f7409bc55c97c3ffd54973bb335898b03a45e2 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 10 Jun 2019 13:47:00 +0200 Subject: [PATCH 02/20] Code refactoring to fix pipeline failure Check if gcov file available. Use seperate method for printing execution results. --- suite_validation/__init__.py | 31 ++++++++++------- suite_validation/execution.py | 15 ++++++-- suite_validation/test_coverage.py | 58 ++++++++++++++++++++----------- 3 files changed, 67 insertions(+), 37 deletions(-) diff --git a/suite_validation/__init__.py b/suite_validation/__init__.py index 019e365..9917c76 100644 --- a/suite_validation/__init__.py +++ b/suite_validation/__init__.py @@ -231,6 +231,19 @@ def parse_coverage_goal_file(goal_file: str) -> str: ) return eu.COVERAGE_GOALS[goal] +def _print_suite_execution_results(exec_results): + print("---Results---") + print("Tests run:", len(exec_results.results)) + print("Lines covered:", exec_results.lines_executed) + print("Branch conditions executed:", exec_results.branches_executed) + print("Branches covered:", exec_results.branches_taken) + + if any(r == execution.COVERS for r in exec_results.results): + verdict = "TRUE" + else: + verdict = "UNKNOWN" + print("Result:", verdict) + def main(): args = parse() @@ -256,7 +269,7 @@ def main(): overwrite_files=args.overwrite, harness_file_target=harness_file, compile_target=executable, - compute_individuals=compute_individuals + compute_individuals=compute_individuals, ) executor.run(args.file, args.test_suite, args.machine_model, exec_results) @@ -294,17 +307,9 @@ def main(): ) if exec_results.coverage_tests: - test_cov.write_individual_test_coverages_to_output(args.output_dir, args.file, exec_results) + test_cov.write_individual_test_coverages_to_output( + args.output_dir, args.file, exec_results + ) print() - print("---Results---") - print("Tests run:", len(exec_results.results)) - print("Lines covered:", exec_results.lines_executed) - print("Branch conditions executed:", exec_results.branches_executed) - print("Branches covered:", exec_results.branches_taken) - - if any(r == execution.COVERS for r in exec_results.results): - verdict = "TRUE" - else: - verdict = "UNKNOWN" - print("Result:", verdict) + _print_suite_execution_results(exec_results) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index 2403509..80b2454 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -437,7 +437,9 @@ class SuiteExecutor: test_vectors = self._get_described_vectors(test_suite) if self._compute_individual_test_coverages: - self._execute_individual_tests(program_file, test_vectors, executor, result_target) + self._execute_individual_tests( + program_file, test_vectors, executor, result_target + ) test_vectors = self._get_described_vectors(test_suite) self._execute_tests(program_file, test_vectors, executor, result_target) @@ -521,14 +523,21 @@ class SuiteExecutor: gcov_file = os.path.basename(program_file) + ".gcov" hit_counter_dic = test_cov.get_hit_counter_dic_from_gcov_file(gcov_file) - test_coverage = test_cov.TestCoverage(program_file, tv, hit_counter_dic, lines_executed, - branches_executed, branches_taken) + test_coverage = test_cov.TestCoverage( + program_file, + tv, + hit_counter_dic, + lines_executed, + branches_executed, + branches_taken, + ) result_target.coverage_tests.append(test_coverage) data_file = executor.harness_file[:-1] + "gcda" data_file = os.path.basename(data_file) cmd = ["rm", data_file] eu.execute(cmd, quiet=True) + def _parse_xml_if_testcase(xml_lines): curr_content = [] for line_number, line in enumerate(xml_lines): diff --git a/suite_validation/test_coverage.py b/suite_validation/test_coverage.py index 64c7f2c..fdb3477 100644 --- a/suite_validation/test_coverage.py +++ b/suite_validation/test_coverage.py @@ -1,11 +1,20 @@ import os +import logging UNREACHED_LINE = "#####" NO_COUNTABLE_PROGRAM_LINE = "-" class TestCoverage: - def __init__(self, file_name, test_vector, hit_counter_dic, lines_executed, branches_executed, branches_taken): + def __init__( + self, + file_name, + test_vector, + hit_counter_dic, + lines_executed, + branches_executed, + branches_taken, + ): self.filename = file_name self.test_vector = test_vector self.hit_counter_dic = hit_counter_dic @@ -16,24 +25,27 @@ class TestCoverage: def get_hit_counter_dic_from_gcov_file(filename): hit_counter_dic = {} - with open(filename) as file: - for line in file: - line = line.strip() - line = line.replace(" ", "") - chunks = line.split(":") - if len(chunks) == 1: - continue - hit_counter = chunks[0] - countable_program_line = True - if hit_counter == NO_COUNTABLE_PROGRAM_LINE: - hit_counter = 0 - countable_program_line = False - if hit_counter == UNREACHED_LINE: - hit_counter = 0 - program_line = chunks[1] - if program_line.isdigit() and int(program_line) > 0: - if countable_program_line: - hit_counter_dic[int(program_line)] = int(hit_counter) + if os.path.exists(filename): + with open(filename) as file: + for line in file: + line = line.strip() + line = line.replace(" ", "") + chunks = line.split(":") + if len(chunks) == 1: + continue + hit_counter = chunks[0] + countable_program_line = True + if hit_counter == NO_COUNTABLE_PROGRAM_LINE: + hit_counter = 0 + countable_program_line = False + if hit_counter == UNREACHED_LINE: + hit_counter = 0 + program_line = chunks[1] + if program_line.isdigit() and int(program_line) > 0: + if countable_program_line: + hit_counter_dic[int(program_line)] = int(hit_counter) + else: + logging.debug("File '%s' does not exist. Returning empty hit counter", filename) return hit_counter_dic @@ -47,7 +59,9 @@ def write_individual_test_coverages_to_output(outputdir, program, exec_results): for program_line, hit_counter in test_coverage.hit_counter_dic.items(): outp.write(str(program_line) + ": " + str(hit_counter) + "\n") outp.write("Lines covered:" + test_coverage.lines_executed + "\n") - outp.write("Branch conditions executed:" + test_coverage.branches_executed + "\n") + outp.write( + "Branch conditions executed:" + test_coverage.branches_executed + "\n" + ) outp.write("Branches covered:" + test_coverage.branches_taken + "\n\n") test_suite_hit_counter_dic = {} @@ -56,7 +70,9 @@ def write_individual_test_coverages_to_output(outputdir, program, exec_results): for program_line in test_coverage.hit_counter_dic.keys(): if not program_line in test_suite_hit_counter_dic: test_suite_hit_counter_dic[program_line] = 0 - test_suite_hit_counter_dic[program_line] += test_coverage.hit_counter_dic[program_line] + test_suite_hit_counter_dic[ + program_line + ] += test_coverage.hit_counter_dic[program_line] outp.write("Total Coverage" + "\n") outp.write("Hit counter:" + "\n") -- GitLab From f474b1822e7ed6ffab3b494018ce169d285ea05f Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 12 Jun 2019 21:15:54 +0200 Subject: [PATCH 03/20] Individual test coverage by using lcov gcda file is called using lcov. Resulting coverage info file is analyzed for test coverage and then moved to subfolder to combine it with the summarized previous coverage info files. gcda file is after each test deleted to ensure that each test delivers an independent coverage. At the end summarized info file delivers coverage info for whole test suite. --- suite_validation/__init__.py | 2 +- suite_validation/execution.py | 161 +++++++++++++++--- suite_validation/test_coverage.py | 270 ++++++++++++++++++++++-------- 3 files changed, 343 insertions(+), 90 deletions(-) diff --git a/suite_validation/__init__.py b/suite_validation/__init__.py index 9917c76..9615947 100644 --- a/suite_validation/__init__.py +++ b/suite_validation/__init__.py @@ -307,7 +307,7 @@ def main(): ) if exec_results.coverage_tests: - test_cov.write_individual_test_coverages_to_output( + test_cov.write_test_coverages_to_output( args.output_dir, args.file, exec_results ) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index 80b2454..db4f590 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -36,6 +36,13 @@ UNKNOWN = "unknown" ERROR = "error" ABORTED = "abort" +HARNESS_GCDA_FILE = "harness.gcda" + +LCOV_SUBFOLDER_TRACE_FILE = "tmp_tracefiles" +LCOV_SUMMARY_TRACE_FILE = "tracefile_summary.info" +LCOV_CURRENT_TRACE_FILE = "current_test.info" +LCOV_WITH_BRANCH_COVERAGE = "--rc lcov_branch_coverage=1" + class ExecutionError(Exception): def __init__(self, msg): @@ -297,6 +304,67 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): logging.info("Aborted test run is not considered for coverage") return result + def get_coverage_for_individual_test(self, program_name): + if self.harness_file: + assert self.harness_file.endswith(".c") + data_file = self.harness_file[:-1] + "gcda" + data_file = os.path.basename(data_file) # data file is in cwd + if os.path.exists(data_file): + cmd = ( + "lcov -c --rc lcov_branch_coverage=1 -d . -o " + + LCOV_CURRENT_TRACE_FILE + ) + os.system(cmd) + if os.path.exists(LCOV_CURRENT_TRACE_FILE): + test_coverage = test_cov.get_test_coverage_from_lcov_file( + program_name, LCOV_CURRENT_TRACE_FILE + ) + self.move_tracefile_into_subfolder_and_combine_with_previous( + LCOV_CURRENT_TRACE_FILE + ) + return test_coverage + logging.warning( + "Trace file '%s' not created. Returning empty test coverage.", + LCOV_CURRENT_TRACE_FILE, + ) + + else: + logging.warning( + "Coverage requested without any execution. Returning empty test coverage." + ) + return test_cov.TestCoverage(program_name) + + @staticmethod + def move_tracefile_into_subfolder_and_combine_with_previous(trace_file): + trace_file_summary = ( + LCOV_SUBFOLDER_TRACE_FILE + os.sep + LCOV_SUMMARY_TRACE_FILE + ) + + if not os.path.isdir(LCOV_SUBFOLDER_TRACE_FILE): + cmd = ["mkdir", LCOV_SUBFOLDER_TRACE_FILE] + eu.execute(cmd, quiet=True) + + if os.path.exists(trace_file_summary): + cmd = ( + "lcov --rc lcov_branch_coverage=1" + + " -a " + + trace_file + + " -a " + + trace_file_summary + + " -o " + + trace_file_summary + ) + os.system(cmd) + summarize_coverage_info = "--summary" + cmd = ["lcov", summarize_coverage_info, trace_file_summary] + eu.execute(cmd, quiet=True) + else: + cmd = ["mv", trace_file, trace_file_summary] + eu.execute(cmd, quiet=True) + + _remove_current_tracefile() + _remove_harness_gcda_file() + def get_coverage(self, program_file): lines_executed = None branches_executed = None @@ -440,10 +508,8 @@ class SuiteExecutor: self._execute_individual_tests( program_file, test_vectors, executor, result_target ) - test_vectors = self._get_described_vectors(test_suite) - - self._execute_tests(program_file, test_vectors, executor, result_target) - + else: + self._execute_tests(program_file, test_vectors, executor, result_target) return result_target @staticmethod @@ -511,31 +577,80 @@ class SuiteExecutor: program_file ) - @staticmethod - def _execute_individual_tests(program_file, test_vectors, executor, result_target): + def _execute_individual_tests( + self, program_file, test_vectors, executor, result_target + ): + + # old trace file in working directory might still exist + _remove_current_tracefile() + # old tmp folder might still exist + _remove_tracefile_tmp_folder() + # old gcda file might exist + _remove_harness_gcda_file() + program_name = os.path.basename(program_file) + + # get coverage of each individual test for tv in test_vectors: - executor.run(program_file, tv) + result = executor.run(program_file, tv) + coverage_test = executor.get_coverage_for_individual_test(program_name) + coverage_test.set_result(result) + coverage_test.set_test_vector(tv) + result_target.coverage_tests.append(coverage_test) + result_target.results.append(result) + + if result == COVERS: + result_target.successful_test = tv + if self._stop_after_success: + logging.info("Stopping. Error found for test %s", tv) + break + + # get test coverage of summarized trace file and add it to the result + trace_file_summary = ( + LCOV_SUBFOLDER_TRACE_FILE + os.sep + LCOV_SUMMARY_TRACE_FILE + ) + if os.path.exists(trace_file_summary): - lines_executed, branches_executed, branches_taken = executor.get_coverage( - program_file + test_coverage_summary = test_cov.get_test_coverage_from_lcov_file( + program_name, trace_file_summary ) - gcov_file = os.path.basename(program_file) + ".gcov" - hit_counter_dic = test_cov.get_hit_counter_dic_from_gcov_file(gcov_file) - test_coverage = test_cov.TestCoverage( - program_file, - tv, - hit_counter_dic, - lines_executed, - branches_executed, - branches_taken, + lines_executed = test_coverage_summary.compute_line_coverage() + branches_executed = ( + test_coverage_summary.compute_branch_conditions_executed() ) - result_target.coverage_tests.append(test_coverage) - data_file = executor.harness_file[:-1] + "gcda" - data_file = os.path.basename(data_file) - cmd = ["rm", data_file] - eu.execute(cmd, quiet=True) + branches_taken = test_coverage_summary.compute_branch_coverage() + + result_target.lines_executed = str(lines_executed) + "%" + result_target.branches_executed = str(branches_executed) + "%" + result_target.branches_taken = str(branches_taken) + "%" + + _remove_tracefile_tmp_folder() + + else: + logging.warning( + "Summary trace file '%s' does not exist in '%s'", + LCOV_SUMMARY_TRACE_FILE, + LCOV_SUBFOLDER_TRACE_FILE, + ) + + +def _remove_tracefile_tmp_folder(): + if os.path.isdir(LCOV_SUBFOLDER_TRACE_FILE): + cmd = ["rm", "-r", LCOV_SUBFOLDER_TRACE_FILE] + eu.execute(cmd, quiet=True) + + +def _remove_current_tracefile(): + if os.path.exists(LCOV_CURRENT_TRACE_FILE): + cmd = ["rm", LCOV_CURRENT_TRACE_FILE] + eu.execute(cmd, quiet=True) + + +def _remove_harness_gcda_file(): + if os.path.exists(HARNESS_GCDA_FILE): + cmd = ["rm", HARNESS_GCDA_FILE] + eu.execute(cmd, quiet=True) def _parse_xml_if_testcase(xml_lines): diff --git a/suite_validation/test_coverage.py b/suite_validation/test_coverage.py index fdb3477..d088c0c 100644 --- a/suite_validation/test_coverage.py +++ b/suite_validation/test_coverage.py @@ -1,90 +1,228 @@ +from enum import Enum import os import logging -UNREACHED_LINE = "#####" -NO_COUNTABLE_PROGRAM_LINE = "-" +ZERO = 0.00 +FILE_NAME_TEST_COVERAGES = "individual-test-coverages" + + +class LcovPrefix(Enum): + FILEPATH = "SF:" + BRANCH_LINE_CONDITION_HIT_COUNTER = "BRDA:" + BRANCHES_FOUND = "BRF:" + BRANCHES_HIT = "BRH:" + LINE_HIT_COUNTER = "DA:" + LINES_NONZERO_HIT_COUNTER = "LH:" + LINES_FOUND = "LF:" + END_OF_RECORD = "end_of_record" + + +class LcovSector(Enum): + BEFORE_TEST_RECORD = 0 + IN_TEST_RECORD = 1 class TestCoverage: def __init__( self, file_name, - test_vector, - hit_counter_dic, - lines_executed, - branches_executed, - branches_taken, + lines_hit_counter_dic=None, + lines_hit=0, + lines_found=0, + branch_condition_hit_counter_dic=None, + branches_hit=0, + branches_found=0, ): self.filename = file_name + self.lines_hit_counter_dic = lines_hit_counter_dic + self.lines_hit = lines_hit + self.lines_found = lines_found + self.branch_condition_hit_counter_dic = branch_condition_hit_counter_dic + self.branches_hit = branches_hit + self.branches_found = branches_found + self.test_vector = "" + self.result = "" + if self.lines_hit_counter_dic is None: + self.lines_hit_counter_dic = {} + if self.branch_condition_hit_counter_dic is None: + self.branch_condition_hit_counter_dic = {} + + def set_test_vector(self, test_vector): self.test_vector = test_vector - self.hit_counter_dic = hit_counter_dic - self.lines_executed = lines_executed - self.branches_executed = branches_executed - self.branches_taken = branches_taken + + def set_result(self, result): + self.result = result + + def compute_line_coverage(self): + if self.lines_found <= 0: + return ZERO + return round(100 * float(self.lines_hit) / float(self.lines_found), 2) + + def compute_branch_conditions_executed(self): + possible_branch_conditions_executions = ( + len(self.branch_condition_hit_counter_dic.keys()) * 2 + ) + lines_with_branch_condition_executed = 0 + for line_with_branch_condition in self.branch_condition_hit_counter_dic.keys(): + conditions_executed = self.branch_condition_hit_counter_dic[ + line_with_branch_condition + ] + if conditions_executed[0]: + lines_with_branch_condition_executed += 1 + if conditions_executed[1]: + lines_with_branch_condition_executed += 1 + if possible_branch_conditions_executions <= 0: + return ZERO + return round( + 100 + * float(lines_with_branch_condition_executed) + / float(possible_branch_conditions_executions), + 2, + ) + + def compute_branch_coverage(self): + if self.branches_found <= 0: + return ZERO + return round(100 * float(self.branches_hit) / float(self.branches_found), 2) + + +def remove_prefix(line, prefix): + return line[len(prefix) :] + + +def _examine_branch_line_condition(branch_line, branch_condition_hit_counter_dic): + chunks = branch_line.split(",") + # chunks should be [program_line, block-number, branch-number, taken] + assert len(chunks) == 4 + program_line = chunks[0] + branch_number = chunks[2] + taken = chunks[3] + if program_line.isdigit() and branch_number.isdigit(): + program_line = int(program_line) + branch_number = int(branch_number) + else: + logging.error( + "Trace file corrupted. Program line or branch number not a number" + ) + return + if program_line not in branch_condition_hit_counter_dic.keys(): + branch_condition_hit_counter_dic[program_line] = [False, False] + if taken.isdigit() and int(taken) >= 1: + # a branch is fully executed when at least the condition is one time satisfied and one time not + # branch number even when condition not satisfied + # branch number odd when condition satisfied + if branch_number % 2 == 0: + branch_condition_hit_counter_dic[program_line][0] = True + else: + branch_condition_hit_counter_dic[program_line][1] = True -def get_hit_counter_dic_from_gcov_file(filename): - hit_counter_dic = {} - if os.path.exists(filename): - with open(filename) as file: +def get_test_coverage_from_lcov_file(program_name, trace_file): + lines_hit_counter_dic = {} + lines_hit = 0 + lines_found = 0 + branch_condition_hit_counter_dic = {} + branches_hit = 0 + branches_found = 0 + if os.path.exists(trace_file): + lcov_sector = LcovSector.BEFORE_TEST_RECORD.value + with open(trace_file) as file: for line in file: line = line.strip() - line = line.replace(" ", "") - chunks = line.split(":") - if len(chunks) == 1: - continue - hit_counter = chunks[0] - countable_program_line = True - if hit_counter == NO_COUNTABLE_PROGRAM_LINE: - hit_counter = 0 - countable_program_line = False - if hit_counter == UNREACHED_LINE: - hit_counter = 0 - program_line = chunks[1] - if program_line.isdigit() and int(program_line) > 0: - if countable_program_line: - hit_counter_dic[int(program_line)] = int(hit_counter) + if lcov_sector == LcovSector.BEFORE_TEST_RECORD.value: + if line.startswith(LcovPrefix.FILEPATH.value): + absolute_file_path = remove_prefix( + line, LcovPrefix.FILEPATH.value + ) + if os.path.basename(absolute_file_path) == program_name: + lcov_sector = LcovSector.IN_TEST_RECORD.value + + elif lcov_sector == LcovSector.IN_TEST_RECORD.value: + + if line.startswith( + LcovPrefix.BRANCH_LINE_CONDITION_HIT_COUNTER.value + ): + branch_line_information = remove_prefix( + line, LcovPrefix.BRANCH_LINE_CONDITION_HIT_COUNTER.value + ) + _examine_branch_line_condition( + branch_line_information, branch_condition_hit_counter_dic + ) + elif line.startswith(LcovPrefix.BRANCHES_FOUND.value): + branches_found = int( + remove_prefix(line, LcovPrefix.BRANCHES_FOUND.value) + ) + elif line.startswith(LcovPrefix.BRANCHES_HIT.value): + branches_hit = int( + remove_prefix(line, LcovPrefix.BRANCHES_HIT.value) + ) + elif line.startswith(LcovPrefix.LINE_HIT_COUNTER.value): + line_with_counter = remove_prefix( + line, LcovPrefix.LINE_HIT_COUNTER.value + ) + chunks = line_with_counter.split(",") + # chunks should be [program-line, hit-counter] + assert len(chunks) == 2 + lines_hit_counter_dic[chunks[0]] = chunks[1] + elif line.startswith(LcovPrefix.LINES_FOUND.value): + lines_found = int( + remove_prefix(line, LcovPrefix.LINES_FOUND.value) + ) + elif line.startswith(LcovPrefix.LINES_NONZERO_HIT_COUNTER.value): + lines_hit = int( + remove_prefix( + line, LcovPrefix.LINES_NONZERO_HIT_COUNTER.value + ) + ) + elif line.startswith(LcovPrefix.END_OF_RECORD.value): + break + + else: + break + else: - logging.debug("File '%s' does not exist. Returning empty hit counter", filename) - return hit_counter_dic + logging.debug( + "File '%s' does not exist. Returning empty test coverage", trace_file + ) + + return TestCoverage( + trace_file, + lines_hit_counter_dic, + lines_hit, + lines_found, + branch_condition_hit_counter_dic, + branches_hit, + branches_found, + ) -def write_individual_test_coverages_to_output(outputdir, program, exec_results): - hit_counter_file = os.path.join(outputdir, "individual-test-coverage") - with open(hit_counter_file, "w") as outp: - outp.write("Program: " + program + "\n\n") +def write_test_coverages_to_output(output_dir, program, exec_results): + output_file = os.path.join(output_dir, FILE_NAME_TEST_COVERAGES) + with open(output_file, "w") as outp: + outp.write("Program: " + program + "\n") for test_coverage in exec_results.coverage_tests: - outp.write(str(test_coverage.test_vector) + "\n") - outp.write("Hit counter:" + "\n") - for program_line, hit_counter in test_coverage.hit_counter_dic.items(): - outp.write(str(program_line) + ": " + str(hit_counter) + "\n") - outp.write("Lines covered:" + test_coverage.lines_executed + "\n") + outp.write("\n") + outp.write("Test input: " + str(test_coverage.test_vector) + "\n") + outp.write("Test result: " + test_coverage.result + "\n") outp.write( - "Branch conditions executed:" + test_coverage.branches_executed + "\n" + "Lines covered: " + + as_percent_expression(test_coverage.compute_line_coverage()) + + "\n" ) - outp.write("Branches covered:" + test_coverage.branches_taken + "\n\n") + outp.write( + "Branch conditions executed: " + + as_percent_expression( + test_coverage.compute_branch_conditions_executed() + ) + + "\n" + ) + outp.write( + "Branches covered: " + + as_percent_expression(test_coverage.compute_branch_coverage()) + + "\n" + ) + outp.close() - test_suite_hit_counter_dic = {} - for test_coverage in exec_results.coverage_tests: - for program_line in test_coverage.hit_counter_dic.keys(): - if not program_line in test_suite_hit_counter_dic: - test_suite_hit_counter_dic[program_line] = 0 - test_suite_hit_counter_dic[ - program_line - ] += test_coverage.hit_counter_dic[program_line] - - outp.write("Total Coverage" + "\n") - outp.write("Hit counter:" + "\n") - lines = 0 - covered_lines = 0 - for program_line, hit_counter in test_suite_hit_counter_dic.items(): - outp.write(str(program_line) + ": " + str(hit_counter) + "\n") - lines += 1 - if hit_counter > 0: - covered_lines += 1 - covered = 0 - if lines != 0: - covered = 100 * float(covered_lines) / float(lines) - outp.write("Lines covered:" + str(covered) + "%" + " (of " + str(lines) + ")") - outp.close() +def as_percent_expression(value): + return str(value) + "%" -- GitLab From 60d0315f0c105dfdeecde55ce04a8f3ab61b6891 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 13 Jun 2019 11:37:06 +0200 Subject: [PATCH 04/20] Applying lcov commands with Popen --- suite_validation/execution.py | 40 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index db4f590..f5c0d08 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -41,7 +41,7 @@ HARNESS_GCDA_FILE = "harness.gcda" LCOV_SUBFOLDER_TRACE_FILE = "tmp_tracefiles" LCOV_SUMMARY_TRACE_FILE = "tracefile_summary.info" LCOV_CURRENT_TRACE_FILE = "current_test.info" -LCOV_WITH_BRANCH_COVERAGE = "--rc lcov_branch_coverage=1" +LCOV_WITH_BRANCH_COVERAGE = "lcov_branch_coverage=1" class ExecutionError(Exception): @@ -310,11 +310,17 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): data_file = self.harness_file[:-1] + "gcda" data_file = os.path.basename(data_file) # data file is in cwd if os.path.exists(data_file): - cmd = ( - "lcov -c --rc lcov_branch_coverage=1 -d . -o " - + LCOV_CURRENT_TRACE_FILE - ) - os.system(cmd) + cmd = [ + "lcov", + "-c", + "--rc", + LCOV_WITH_BRANCH_COVERAGE, + "-d", + ".", + "-o", + LCOV_CURRENT_TRACE_FILE, + ] + eu.execute(cmd, quiet=True) if os.path.exists(LCOV_CURRENT_TRACE_FILE): test_coverage = test_cov.get_test_coverage_from_lcov_file( program_name, LCOV_CURRENT_TRACE_FILE @@ -345,16 +351,18 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): eu.execute(cmd, quiet=True) if os.path.exists(trace_file_summary): - cmd = ( - "lcov --rc lcov_branch_coverage=1" - + " -a " - + trace_file - + " -a " - + trace_file_summary - + " -o " - + trace_file_summary - ) - os.system(cmd) + cmd = [ + "lcov", + "--rc", + LCOV_WITH_BRANCH_COVERAGE, + "-a", + trace_file, + "-a", + trace_file_summary, + "-o", + trace_file_summary, + ] + eu.execute(cmd, quiet=True) summarize_coverage_info = "--summary" cmd = ["lcov", summarize_coverage_info, trace_file_summary] eu.execute(cmd, quiet=True) -- GitLab From 89d6ac1e6729e96fe2aa16a298015dd99ce648ce Mon Sep 17 00:00:00 2001 From: Thomas Lemberger Date: Thu, 13 Jun 2019 12:36:29 +0200 Subject: [PATCH 05/20] Update dockerfiles with lcov to make CI pass --- test/Dockerfile.python-3.5 | 2 +- test/Dockerfile.python-3.6 | 2 +- test/Dockerfile.python-3.7 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Dockerfile.python-3.5 b/test/Dockerfile.python-3.5 index 49482fe..949ddd8 100644 --- a/test/Dockerfile.python-3.5 +++ b/test/Dockerfile.python-3.5 @@ -8,7 +8,7 @@ FROM python:3.5 -RUN apt update && apt install -y gcc-multilib g++-multilib +RUN apt update && apt install -y gcc-multilib g++-multilib lcov RUN pip install -U pip && pip install \ coverage \ diff --git a/test/Dockerfile.python-3.6 b/test/Dockerfile.python-3.6 index f53f98d..17d230c 100644 --- a/test/Dockerfile.python-3.6 +++ b/test/Dockerfile.python-3.6 @@ -8,7 +8,7 @@ FROM python:3.6 -RUN apt update && apt install -y gcc-multilib g++-multilib +RUN apt update && apt install -y gcc-multilib g++-multilib lcov RUN pip install -U pip && pip install \ coverage \ diff --git a/test/Dockerfile.python-3.7 b/test/Dockerfile.python-3.7 index 1f72fd0..74fd0a8 100644 --- a/test/Dockerfile.python-3.7 +++ b/test/Dockerfile.python-3.7 @@ -8,7 +8,7 @@ FROM python:3.7 -RUN apt update && apt install -y gcc-multilib g++-multilib +RUN apt update && apt install -y gcc-multilib g++-multilib lcov RUN pip install -U pip && pip install \ coverage \ -- GitLab From a8b303e4d8319d9457c0201a2835b1cc821c3813 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 14 Jun 2019 13:13:01 +0200 Subject: [PATCH 06/20] Merged individual tests into _execute_tests --- suite_validation/__init__.py | 6 +- .../{test_coverage.py => coverage.py} | 22 ++- suite_validation/execution.py | 160 ++++++++++-------- 3 files changed, 111 insertions(+), 77 deletions(-) rename suite_validation/{test_coverage.py => coverage.py} (90%) diff --git a/suite_validation/__init__.py b/suite_validation/__init__.py index 9615947..8c43974 100644 --- a/suite_validation/__init__.py +++ b/suite_validation/__init__.py @@ -25,7 +25,7 @@ import shutil import zipfile from suite_validation import execution from suite_validation import execution_utils as eu -from suite_validation import test_coverage as test_cov +from suite_validation import coverage as cov __VERSION__ = "v1.1-dev" @@ -307,9 +307,7 @@ def main(): ) if exec_results.coverage_tests: - test_cov.write_test_coverages_to_output( - args.output_dir, args.file, exec_results - ) + cov.write_test_coverages_to_dir(args.output_dir, args.file, exec_results) print() _print_suite_execution_results(exec_results) diff --git a/suite_validation/test_coverage.py b/suite_validation/coverage.py similarity index 90% rename from suite_validation/test_coverage.py rename to suite_validation/coverage.py index d088c0c..2fd5972 100644 --- a/suite_validation/test_coverage.py +++ b/suite_validation/coverage.py @@ -1,9 +1,29 @@ +# tbf-testsuite-validator is tool for validation and execution of test suites. +# This file is part of tbf-testsuite-validator. +# +# Copyright (C) 2018 Dirk Beyer +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for coverage of individual tests""" + from enum import Enum import os import logging ZERO = 0.00 FILE_NAME_TEST_COVERAGES = "individual-test-coverages" +LINES_COVERED = "Lines covered" class LcovPrefix(Enum): @@ -196,7 +216,7 @@ def get_test_coverage_from_lcov_file(program_name, trace_file): ) -def write_test_coverages_to_output(output_dir, program, exec_results): +def write_test_coverages_to_dir(output_dir, program, exec_results): output_file = os.path.join(output_dir, FILE_NAME_TEST_COVERAGES) with open(output_file, "w") as outp: outp.write("Program: " + program + "\n") diff --git a/suite_validation/execution.py b/suite_validation/execution.py index f5c0d08..069fa7f 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -26,7 +26,7 @@ import zipfile from lxml import etree from suite_validation import execution_utils as eu -from suite_validation import test_coverage as test_cov +from suite_validation import coverage as cov HARNESS_FILE_NAME = "harness.c" ARCHITECTURE_TAG = "architecture" @@ -304,7 +304,8 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): logging.info("Aborted test run is not considered for coverage") return result - def get_coverage_for_individual_test(self, program_name): + def get_coverage_for_individual_test(self, program_file): + program_name = os.path.basename(program_file) if self.harness_file: assert self.harness_file.endswith(".c") data_file = self.harness_file[:-1] + "gcda" @@ -322,7 +323,7 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): ] eu.execute(cmd, quiet=True) if os.path.exists(LCOV_CURRENT_TRACE_FILE): - test_coverage = test_cov.get_test_coverage_from_lcov_file( + test_coverage = cov.get_test_coverage_from_lcov_file( program_name, LCOV_CURRENT_TRACE_FILE ) self.move_tracefile_into_subfolder_and_combine_with_previous( @@ -338,7 +339,7 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): logging.warning( "Coverage requested without any execution. Returning empty test coverage." ) - return test_cov.TestCoverage(program_name) + return cov.TestCoverage(program_name) @staticmethod def move_tracefile_into_subfolder_and_combine_with_previous(trace_file): @@ -370,8 +371,46 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): cmd = ["mv", trace_file, trace_file_summary] eu.execute(cmd, quiet=True) - _remove_current_tracefile() - _remove_harness_gcda_file() + @staticmethod + def get_coverage_of_summarized_trace_file(program_file): + lines_executed = None + branches_executed = None + branches_taken = None + + trace_file_summary = ( + LCOV_SUBFOLDER_TRACE_FILE + os.sep + LCOV_SUMMARY_TRACE_FILE + ) + if os.path.exists(trace_file_summary): + program_name = os.path.basename(program_file) + test_coverage_summary = cov.get_test_coverage_from_lcov_file( + program_name, trace_file_summary + ) + + lines_executed = test_coverage_summary.compute_line_coverage() + branches_executed = ( + test_coverage_summary.compute_branch_conditions_executed() + ) + branches_taken = test_coverage_summary.compute_branch_coverage() + + lines_executed = str(lines_executed) + "%" + branches_executed = str(branches_executed) + "%" + branches_taken = str(branches_taken) + "%" + + else: + logging.warning( + "Summary trace file '%s' does not exist in '%s'. Returning defaults.", + LCOV_SUMMARY_TRACE_FILE, + LCOV_SUBFOLDER_TRACE_FILE, + ) + + if not lines_executed: + lines_executed = "0%" + if not branches_executed: + branches_executed = "0%" + if not branches_taken: + branches_taken = "0%" + + return lines_executed, branches_executed, branches_taken def get_coverage(self, program_file): lines_executed = None @@ -512,12 +551,17 @@ class SuiteExecutor: # this method call raises an ExecutionError if the given test suite is invalid test_vectors = self._get_described_vectors(test_suite) - if self._compute_individual_test_coverages: - self._execute_individual_tests( - program_file, test_vectors, executor, result_target - ) - else: - self._execute_tests(program_file, test_vectors, executor, result_target) + # old trace file in working directory might still exist + _remove_current_tracefile() + # old tmp folder might still exist + _remove_tracefile_tmp_folder() + # old gcda, gcno or gcov files might exist and can affect coverage computation with lcov + _remove_coverage_analysis_files_in_working_directory() + + self._execute_tests(program_file, test_vectors, executor, result_target) + + _remove_tracefile_tmp_folder() + return result_target @staticmethod @@ -562,11 +606,27 @@ class SuiteExecutor: for tv in test_vectors: next_result = executor.run(program_file, tv) - if self._compute_sequence: - result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage( + if self._compute_individual_test_coverages: + coverage_test = executor.get_coverage_for_individual_test( program_file ) + coverage_test.set_result(next_result) + coverage_test.set_test_vector(tv) + result_target.coverage_tests.append(coverage_test) + + _remove_current_tracefile() + _remove_harness_gcda_file() + if self._compute_sequence: + # if we use individual tests with lcov we extract the info from the summarized file + if self._compute_individual_test_coverages: + result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage_of_summarized_trace_file( + program_file + ) + else: + result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage( + program_file + ) result_target.coverage_sequence.append( float(result_target.branches_taken.split("%")[0]) ) @@ -581,66 +641,22 @@ class SuiteExecutor: finally: if not self._compute_sequence: - result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage( - program_file - ) - - def _execute_individual_tests( - self, program_file, test_vectors, executor, result_target - ): - - # old trace file in working directory might still exist - _remove_current_tracefile() - # old tmp folder might still exist - _remove_tracefile_tmp_folder() - # old gcda file might exist - _remove_harness_gcda_file() - - program_name = os.path.basename(program_file) - - # get coverage of each individual test - for tv in test_vectors: - result = executor.run(program_file, tv) - coverage_test = executor.get_coverage_for_individual_test(program_name) - coverage_test.set_result(result) - coverage_test.set_test_vector(tv) - result_target.coverage_tests.append(coverage_test) - result_target.results.append(result) - - if result == COVERS: - result_target.successful_test = tv - if self._stop_after_success: - logging.info("Stopping. Error found for test %s", tv) - break - - # get test coverage of summarized trace file and add it to the result - trace_file_summary = ( - LCOV_SUBFOLDER_TRACE_FILE + os.sep + LCOV_SUMMARY_TRACE_FILE - ) - if os.path.exists(trace_file_summary): - - test_coverage_summary = test_cov.get_test_coverage_from_lcov_file( - program_name, trace_file_summary - ) - - lines_executed = test_coverage_summary.compute_line_coverage() - branches_executed = ( - test_coverage_summary.compute_branch_conditions_executed() - ) - branches_taken = test_coverage_summary.compute_branch_coverage() - - result_target.lines_executed = str(lines_executed) + "%" - result_target.branches_executed = str(branches_executed) + "%" - result_target.branches_taken = str(branches_taken) + "%" + if self._compute_individual_test_coverages: + result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage_of_summarized_trace_file( + program_file + ) + else: + result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage( + program_file + ) - _remove_tracefile_tmp_folder() - else: - logging.warning( - "Summary trace file '%s' does not exist in '%s'", - LCOV_SUMMARY_TRACE_FILE, - LCOV_SUBFOLDER_TRACE_FILE, - ) +def _remove_coverage_analysis_files_in_working_directory(): + files = [f for f in os.listdir(".") if re.match(r".*\.[gcov|gcno|gcda]", f)] + if files: + for file in files: + cmd = ["rm", "-r", file] + eu.execute(cmd, quiet=True) def _remove_tracefile_tmp_folder(): -- GitLab From eef5f391c33c430ff0140fbfccd39f1a1bba3e1c Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 14 Jun 2019 13:37:48 +0200 Subject: [PATCH 07/20] Write individual test coverages into testsuite folder --- suite_validation/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/suite_validation/__init__.py b/suite_validation/__init__.py index 8c43974..2f06501 100644 --- a/suite_validation/__init__.py +++ b/suite_validation/__init__.py @@ -307,7 +307,9 @@ def main(): ) if exec_results.coverage_tests: - cov.write_test_coverages_to_dir(args.output_dir, args.file, exec_results) + if not os.path.exists(testsuite_folder): + os.mkdir(testsuite_folder) + cov.write_test_coverages_to_dir(testsuite_folder, args.file, exec_results) print() _print_suite_execution_results(exec_results) -- GitLab From f13480d0cc7f7e70f2a50e2eb8180d98ca3ee341 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 14 Jun 2019 15:35:03 +0200 Subject: [PATCH 08/20] Removed evil regex expression that can destroy git folder Now regex expression is replaced with file names which is safe. Also add option to lcov not to look in subfolders to get data files. Generally, lcov takes all gcda files in the specified folder and subfolders (if not explicitly negated). Old gcov files also affect the results in tracefiles produced by lcov. Hence, deleting of old gcda files or gcov files is necessary in the current directory. --- suite_validation/execution.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index 069fa7f..f4bb590 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -38,10 +38,15 @@ ABORTED = "abort" HARNESS_GCDA_FILE = "harness.gcda" +GCOV_FILE_END = "gcov" +GCDA_FILE_END = "gcda" +GCNO_FILE_END = "gcno" + LCOV_SUBFOLDER_TRACE_FILE = "tmp_tracefiles" LCOV_SUMMARY_TRACE_FILE = "tracefile_summary.info" LCOV_CURRENT_TRACE_FILE = "current_test.info" LCOV_WITH_BRANCH_COVERAGE = "lcov_branch_coverage=1" +LCOV_NO_RECURSION = "--no-recursion" class ExecutionError(Exception): @@ -318,6 +323,7 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): LCOV_WITH_BRANCH_COVERAGE, "-d", ".", + LCOV_NO_RECURSION, "-o", LCOV_CURRENT_TRACE_FILE, ] @@ -556,7 +562,7 @@ class SuiteExecutor: # old tmp folder might still exist _remove_tracefile_tmp_folder() # old gcda, gcno or gcov files might exist and can affect coverage computation with lcov - _remove_coverage_analysis_files_in_working_directory() + _remove_coverages_files_in_working_directory(program_file) self._execute_tests(program_file, test_vectors, executor, result_target) @@ -650,15 +656,6 @@ class SuiteExecutor: program_file ) - -def _remove_coverage_analysis_files_in_working_directory(): - files = [f for f in os.listdir(".") if re.match(r".*\.[gcov|gcno|gcda]", f)] - if files: - for file in files: - cmd = ["rm", "-r", file] - eu.execute(cmd, quiet=True) - - def _remove_tracefile_tmp_folder(): if os.path.isdir(LCOV_SUBFOLDER_TRACE_FILE): cmd = ["rm", "-r", LCOV_SUBFOLDER_TRACE_FILE] @@ -676,6 +673,20 @@ def _remove_harness_gcda_file(): cmd = ["rm", HARNESS_GCDA_FILE] eu.execute(cmd, quiet=True) +def _remove_coverages_files_in_working_directory(program_file): + harness = HARNESS_FILE_NAME[:-1] + program_name = os.path.basename(program_file) + program = program_name[:-1] + c_gcov_file_end = "c." + GCOV_FILE_END + file_endings = [c_gcov_file_end, GCDA_FILE_END, GCNO_FILE_END] + file_names = [harness, program] + for name in file_names: + for ending in file_endings: + file_name = name + ending + if os.path.exists(file_name): + cmd = ["rm", file_name] + eu.execute(cmd, quiet=True) + def _parse_xml_if_testcase(xml_lines): curr_content = [] -- GitLab From d47e38e228f8fbbbb0f2367a27f0b8a74dcf2401 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 14 Jun 2019 15:51:43 +0200 Subject: [PATCH 09/20] Small format change --- suite_validation/execution.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index f4bb590..28ee550 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -656,6 +656,7 @@ class SuiteExecutor: program_file ) + def _remove_tracefile_tmp_folder(): if os.path.isdir(LCOV_SUBFOLDER_TRACE_FILE): cmd = ["rm", "-r", LCOV_SUBFOLDER_TRACE_FILE] @@ -673,6 +674,7 @@ def _remove_harness_gcda_file(): cmd = ["rm", HARNESS_GCDA_FILE] eu.execute(cmd, quiet=True) + def _remove_coverages_files_in_working_directory(program_file): harness = HARNESS_FILE_NAME[:-1] program_name = os.path.basename(program_file) -- GitLab From 2cf113c9b7fbd45cd20bb11c5b413ad168d690b6 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 14 Jun 2019 16:38:07 +0200 Subject: [PATCH 10/20] Changed rm commands to os.remove --- suite_validation/execution.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index 28ee550..2d1e221 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -22,6 +22,7 @@ import xml.etree.ElementTree as ET import re import os import zipfile +import shutil from lxml import etree @@ -659,20 +660,17 @@ class SuiteExecutor: def _remove_tracefile_tmp_folder(): if os.path.isdir(LCOV_SUBFOLDER_TRACE_FILE): - cmd = ["rm", "-r", LCOV_SUBFOLDER_TRACE_FILE] - eu.execute(cmd, quiet=True) + shutil.rmtree(LCOV_SUBFOLDER_TRACE_FILE, ignore_errors=True) def _remove_current_tracefile(): if os.path.exists(LCOV_CURRENT_TRACE_FILE): - cmd = ["rm", LCOV_CURRENT_TRACE_FILE] - eu.execute(cmd, quiet=True) + os.remove(LCOV_CURRENT_TRACE_FILE) def _remove_harness_gcda_file(): if os.path.exists(HARNESS_GCDA_FILE): - cmd = ["rm", HARNESS_GCDA_FILE] - eu.execute(cmd, quiet=True) + os.remove(HARNESS_GCDA_FILE) def _remove_coverages_files_in_working_directory(program_file): @@ -686,8 +684,7 @@ def _remove_coverages_files_in_working_directory(program_file): for ending in file_endings: file_name = name + ending if os.path.exists(file_name): - cmd = ["rm", file_name] - eu.execute(cmd, quiet=True) + os.remove(file_name) def _parse_xml_if_testcase(xml_lines): -- GitLab From 4c6281481ad13b2dbacb2931b553fe2be66bd7bd Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 14 Jun 2019 17:53:35 +0200 Subject: [PATCH 11/20] Use shutil for mv command. Refactored coverage computation. --- suite_validation/coverage.py | 49 +++++++++++++++-------------------- suite_validation/execution.py | 13 +++------- 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/suite_validation/coverage.py b/suite_validation/coverage.py index 2fd5972..0bd862e 100644 --- a/suite_validation/coverage.py +++ b/suite_validation/coverage.py @@ -21,7 +21,6 @@ from enum import Enum import os import logging -ZERO = 0.00 FILE_NAME_TEST_COVERAGES = "individual-test-coverages" LINES_COVERED = "Lines covered" @@ -75,8 +74,8 @@ class TestCoverage: def compute_line_coverage(self): if self.lines_found <= 0: - return ZERO - return round(100 * float(self.lines_hit) / float(self.lines_found), 2) + return 1.0 + return round(float(self.lines_hit) / float(self.lines_found), 2) def compute_branch_conditions_executed(self): possible_branch_conditions_executions = ( @@ -92,18 +91,27 @@ class TestCoverage: if conditions_executed[1]: lines_with_branch_condition_executed += 1 if possible_branch_conditions_executions <= 0: - return ZERO + return 1.0 return round( - 100 - * float(lines_with_branch_condition_executed) + float(lines_with_branch_condition_executed) / float(possible_branch_conditions_executions), 2, ) def compute_branch_coverage(self): if self.branches_found <= 0: - return ZERO - return round(100 * float(self.branches_hit) / float(self.branches_found), 2) + return 1.0 + return round(float(self.branches_hit) / float(self.branches_found), 2) + + def get_coverage_ratios_as_percent_expressions(self): + line_coverage = self.compute_line_coverage() + branch_condition_coverage = self.compute_branch_conditions_executed() + branch_coverage = self.compute_branch_coverage() + return ( + str(line_coverage * 100) + "%", + str(branch_condition_coverage * 100) + "%", + str(branch_coverage * 100) + "%", + ) def remove_prefix(line, prefix): @@ -224,25 +232,10 @@ def write_test_coverages_to_dir(output_dir, program, exec_results): outp.write("\n") outp.write("Test input: " + str(test_coverage.test_vector) + "\n") outp.write("Test result: " + test_coverage.result + "\n") - outp.write( - "Lines covered: " - + as_percent_expression(test_coverage.compute_line_coverage()) - + "\n" - ) - outp.write( - "Branch conditions executed: " - + as_percent_expression( - test_coverage.compute_branch_conditions_executed() - ) - + "\n" - ) - outp.write( - "Branches covered: " - + as_percent_expression(test_coverage.compute_branch_coverage()) - + "\n" + lines_executed, branches_executed, branches_taken = ( + test_coverage.get_coverage_ratios_as_percent_expressions() ) + outp.write("Lines covered: " + lines_executed + "\n") + outp.write("Branch conditions executed: " + branches_executed + "\n") + outp.write("Branches covered: " + branches_taken + "\n") outp.close() - - -def as_percent_expression(value): - return str(value) + "%" diff --git a/suite_validation/execution.py b/suite_validation/execution.py index 2d1e221..cbe11cd 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -375,8 +375,7 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): cmd = ["lcov", summarize_coverage_info, trace_file_summary] eu.execute(cmd, quiet=True) else: - cmd = ["mv", trace_file, trace_file_summary] - eu.execute(cmd, quiet=True) + shutil.move(trace_file, trace_file_summary) @staticmethod def get_coverage_of_summarized_trace_file(program_file): @@ -393,15 +392,9 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): program_name, trace_file_summary ) - lines_executed = test_coverage_summary.compute_line_coverage() - branches_executed = ( - test_coverage_summary.compute_branch_conditions_executed() + lines_executed, branches_executed, branches_taken = ( + test_coverage_summary.get_coverage_ratios_as_percent_expressions() ) - branches_taken = test_coverage_summary.compute_branch_coverage() - - lines_executed = str(lines_executed) + "%" - branches_executed = str(branches_executed) + "%" - branches_taken = str(branches_taken) + "%" else: logging.warning( -- GitLab From 305516cc6e3dab49e50a8144aa5da13318deb922 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 15 Jun 2019 15:38:54 +0200 Subject: [PATCH 12/20] Simplified remove method of coverage files in current directory Also round to 4 numbers when computing the coverage ratio. --- suite_validation/coverage.py | 6 +++--- suite_validation/execution.py | 21 ++++++++------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/suite_validation/coverage.py b/suite_validation/coverage.py index 0bd862e..d1b4753 100644 --- a/suite_validation/coverage.py +++ b/suite_validation/coverage.py @@ -75,7 +75,7 @@ class TestCoverage: def compute_line_coverage(self): if self.lines_found <= 0: return 1.0 - return round(float(self.lines_hit) / float(self.lines_found), 2) + return round(float(self.lines_hit) / float(self.lines_found), 4) def compute_branch_conditions_executed(self): possible_branch_conditions_executions = ( @@ -95,13 +95,13 @@ class TestCoverage: return round( float(lines_with_branch_condition_executed) / float(possible_branch_conditions_executions), - 2, + 4, ) def compute_branch_coverage(self): if self.branches_found <= 0: return 1.0 - return round(float(self.branches_hit) / float(self.branches_found), 2) + return round(float(self.branches_hit) / float(self.branches_found), 4) def get_coverage_ratios_as_percent_expressions(self): line_coverage = self.compute_line_coverage() diff --git a/suite_validation/execution.py b/suite_validation/execution.py index cbe11cd..010f4be 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -556,7 +556,7 @@ class SuiteExecutor: # old tmp folder might still exist _remove_tracefile_tmp_folder() # old gcda, gcno or gcov files might exist and can affect coverage computation with lcov - _remove_coverages_files_in_working_directory(program_file) + _remove_coverages_files_in_working_directory() self._execute_tests(program_file, test_vectors, executor, result_target) @@ -666,18 +666,13 @@ def _remove_harness_gcda_file(): os.remove(HARNESS_GCDA_FILE) -def _remove_coverages_files_in_working_directory(program_file): - harness = HARNESS_FILE_NAME[:-1] - program_name = os.path.basename(program_file) - program = program_name[:-1] - c_gcov_file_end = "c." + GCOV_FILE_END - file_endings = [c_gcov_file_end, GCDA_FILE_END, GCNO_FILE_END] - file_names = [harness, program] - for name in file_names: - for ending in file_endings: - file_name = name + ending - if os.path.exists(file_name): - os.remove(file_name) +def _remove_coverages_files_in_working_directory(): + extensions = (GCOV_FILE_END, GCDA_FILE_END, GCNO_FILE_END) + files = [ + f for f in os.listdir(os.curdir) if os.path.isfile(f) and f.endswith(extensions) + ] + for file in files: + os.remove(file) def _parse_xml_if_testcase(xml_lines): -- GitLab From 7ae4ebf3accb47ed988c301f2538af8a1bc95f87 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 17 Jun 2019 15:48:58 +0200 Subject: [PATCH 13/20] Produce coverage using clang and only lcov. Tests work locally. Code might need some refactoring. Commit includes also changes in harness.c file which have already been committed on the master. --- suite_validation/execution.py | 125 +++++++++++----------------------- 1 file changed, 39 insertions(+), 86 deletions(-) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index 010f4be..a1c31e9 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -43,11 +43,20 @@ GCOV_FILE_END = "gcov" GCDA_FILE_END = "gcda" GCNO_FILE_END = "gcno" -LCOV_SUBFOLDER_TRACE_FILE = "tmp_tracefiles" +LCOV_SUBFOLDER_TRACE_FILE = "tracefiles" LCOV_SUMMARY_TRACE_FILE = "tracefile_summary.info" LCOV_CURRENT_TRACE_FILE = "current_test.info" LCOV_WITH_BRANCH_COVERAGE = "lcov_branch_coverage=1" LCOV_NO_RECURSION = "--no-recursion" +LCOV_USED_GCOV_TOOL = "/bin/llvm-gcov" + +LCOV_COMMAND_PREFIX = [ + "lcov", + "--gcov-tool", + LCOV_USED_GCOV_TOOL, + "--rc", + LCOV_WITH_BRANCH_COVERAGE, +] class ExecutionError(Exception): @@ -170,10 +179,10 @@ class ExecutionRunner: self._overwrite = overwrite_files def _get_compile_cmd( - self, program_file, harness_file, output_file, c_version="gnu11" + self, program_file, harness_file, output_file, c_version="gnu90" ): mm_arg = "-m64" if self.machine_model == eu.MACHINE_MODEL_64 else "-m32" - cmd = ["gcc"] + cmd = ["clang"] cmd += [ "-std={}".format(c_version), mm_arg, @@ -310,18 +319,15 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): logging.info("Aborted test run is not considered for coverage") return result - def get_coverage_for_individual_test(self, program_file): + def get_current_test_coverage(self, program_file): program_name = os.path.basename(program_file) if self.harness_file: assert self.harness_file.endswith(".c") data_file = self.harness_file[:-1] + "gcda" data_file = os.path.basename(data_file) # data file is in cwd if os.path.exists(data_file): - cmd = [ - "lcov", + cmd = LCOV_COMMAND_PREFIX + [ "-c", - "--rc", - LCOV_WITH_BRANCH_COVERAGE, "-d", ".", LCOV_NO_RECURSION, @@ -333,15 +339,11 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): test_coverage = cov.get_test_coverage_from_lcov_file( program_name, LCOV_CURRENT_TRACE_FILE ) - self.move_tracefile_into_subfolder_and_combine_with_previous( - LCOV_CURRENT_TRACE_FILE - ) return test_coverage logging.warning( "Trace file '%s' not created. Returning empty test coverage.", LCOV_CURRENT_TRACE_FILE, ) - else: logging.warning( "Coverage requested without any execution. Returning empty test coverage." @@ -359,10 +361,7 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): eu.execute(cmd, quiet=True) if os.path.exists(trace_file_summary): - cmd = [ - "lcov", - "--rc", - LCOV_WITH_BRANCH_COVERAGE, + cmd = LCOV_COMMAND_PREFIX + [ "-a", trace_file, "-a", @@ -371,73 +370,28 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): trace_file_summary, ] eu.execute(cmd, quiet=True) - summarize_coverage_info = "--summary" - cmd = ["lcov", summarize_coverage_info, trace_file_summary] + cmd = ["lcov", "--summary", trace_file_summary] eu.execute(cmd, quiet=True) else: shutil.move(trace_file, trace_file_summary) - @staticmethod - def get_coverage_of_summarized_trace_file(program_file): - lines_executed = None - branches_executed = None - branches_taken = None + def get_coverage(self, program_file): trace_file_summary = ( LCOV_SUBFOLDER_TRACE_FILE + os.sep + LCOV_SUMMARY_TRACE_FILE ) + if os.path.exists(trace_file_summary): program_name = os.path.basename(program_file) test_coverage_summary = cov.get_test_coverage_from_lcov_file( program_name, trace_file_summary ) - - lines_executed, branches_executed, branches_taken = ( - test_coverage_summary.get_coverage_ratios_as_percent_expressions() - ) - else: - logging.warning( - "Summary trace file '%s' does not exist in '%s'. Returning defaults.", - LCOV_SUMMARY_TRACE_FILE, - LCOV_SUBFOLDER_TRACE_FILE, - ) - - if not lines_executed: - lines_executed = "0%" - if not branches_executed: - branches_executed = "0%" - if not branches_taken: - branches_taken = "0%" - - return lines_executed, branches_executed, branches_taken - - def get_coverage(self, program_file): - lines_executed = None - branches_executed = None - branches_taken = None - - if self.harness_file: - assert self.harness_file.endswith(".c") - data_file = self.harness_file[:-1] + "gcda" - data_file = os.path.basename(data_file) # data file is in cwd + test_coverage_summary = self.get_current_test_coverage(program_file) - if os.path.exists(data_file): - cmd = ["gcov", "-bc", data_file] - res = eu.execute(cmd, quiet=True) - full_cov = res.stdout.splitlines() - - program_name = os.path.basename(program_file) - for number, line in enumerate(full_cov): - if line.startswith("File") and program_name in line: - lines_executed = self._get_gcov_val(full_cov[number + 1]) - branches_executed = self._get_gcov_val(full_cov[number + 2]) - branches_taken = self._get_gcov_val(full_cov[number + 3]) - break - else: - logging.debug( - "Coverage requested without any execution. Returning defaults." - ) + lines_executed, branches_executed, branches_taken = ( + test_coverage_summary.get_coverage_ratios_as_percent_expressions() + ) if not lines_executed: lines_executed = "0%" @@ -554,14 +508,12 @@ class SuiteExecutor: # old trace file in working directory might still exist _remove_current_tracefile() # old tmp folder might still exist - _remove_tracefile_tmp_folder() + _remove_tracefile_folder() # old gcda, gcno or gcov files might exist and can affect coverage computation with lcov _remove_coverages_files_in_working_directory() self._execute_tests(program_file, test_vectors, executor, result_target) - _remove_tracefile_tmp_folder() - return result_target @staticmethod @@ -606,23 +558,29 @@ class SuiteExecutor: for tv in test_vectors: next_result = executor.run(program_file, tv) + coverage_test = None + if self._compute_individual_test_coverages: - coverage_test = executor.get_coverage_for_individual_test( - program_file - ) + coverage_test = executor.get_current_test_coverage(program_file) coverage_test.set_result(next_result) coverage_test.set_test_vector(tv) result_target.coverage_tests.append(coverage_test) + if os.path.exists(LCOV_CURRENT_TRACE_FILE): + executor.move_tracefile_into_subfolder_and_combine_with_previous( + LCOV_CURRENT_TRACE_FILE + ) + _remove_current_tracefile() _remove_harness_gcda_file() if self._compute_sequence: # if we use individual tests with lcov we extract the info from the summarized file - if self._compute_individual_test_coverages: - result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage_of_summarized_trace_file( - program_file + if coverage_test: + result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = ( + coverage_test.get_coverage_ratios_as_percent_expressions() ) + else: result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage( program_file @@ -641,17 +599,12 @@ class SuiteExecutor: finally: if not self._compute_sequence: - if self._compute_individual_test_coverages: - result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage_of_summarized_trace_file( - program_file - ) - else: - result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage( - program_file - ) + result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage( + program_file + ) -def _remove_tracefile_tmp_folder(): +def _remove_tracefile_folder(): if os.path.isdir(LCOV_SUBFOLDER_TRACE_FILE): shutil.rmtree(LCOV_SUBFOLDER_TRACE_FILE, ignore_errors=True) -- GitLab From dc22e84e9570368f6e12ab7242bc10b1567e4882 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 17 Jun 2019 17:25:44 +0200 Subject: [PATCH 14/20] Added clang to docker files --- test/Dockerfile.python-3.5 | 2 +- test/Dockerfile.python-3.6 | 2 +- test/Dockerfile.python-3.7 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Dockerfile.python-3.5 b/test/Dockerfile.python-3.5 index 949ddd8..155d3b7 100644 --- a/test/Dockerfile.python-3.5 +++ b/test/Dockerfile.python-3.5 @@ -8,7 +8,7 @@ FROM python:3.5 -RUN apt update && apt install -y gcc-multilib g++-multilib lcov +RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang RUN pip install -U pip && pip install \ coverage \ diff --git a/test/Dockerfile.python-3.6 b/test/Dockerfile.python-3.6 index 17d230c..d0765eb 100644 --- a/test/Dockerfile.python-3.6 +++ b/test/Dockerfile.python-3.6 @@ -8,7 +8,7 @@ FROM python:3.6 -RUN apt update && apt install -y gcc-multilib g++-multilib lcov +RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang RUN pip install -U pip && pip install \ coverage \ diff --git a/test/Dockerfile.python-3.7 b/test/Dockerfile.python-3.7 index 74fd0a8..6e62e83 100644 --- a/test/Dockerfile.python-3.7 +++ b/test/Dockerfile.python-3.7 @@ -8,7 +8,7 @@ FROM python:3.7 -RUN apt update && apt install -y gcc-multilib g++-multilib lcov +RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang RUN pip install -U pip && pip install \ coverage \ -- GitLab From 95d292ceacdf814d6735a6bf8387708f60ed8c62 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 17 Jun 2019 17:38:31 +0200 Subject: [PATCH 15/20] Added llvm-gcov to bin folder --- bin/llvm-gcov | 2 ++ suite_validation/__init__.py | 1 + suite_validation/execution.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 bin/llvm-gcov diff --git a/bin/llvm-gcov b/bin/llvm-gcov new file mode 100644 index 0000000..b5a9cc8 --- /dev/null +++ b/bin/llvm-gcov @@ -0,0 +1,2 @@ +#!/bin/bash +llvm-cov gcov $* diff --git a/suite_validation/__init__.py b/suite_validation/__init__.py index 2f06501..d8e37cc 100644 --- a/suite_validation/__init__.py +++ b/suite_validation/__init__.py @@ -231,6 +231,7 @@ def parse_coverage_goal_file(goal_file: str) -> str: ) return eu.COVERAGE_GOALS[goal] + def _print_suite_execution_results(exec_results): print("---Results---") print("Tests run:", len(exec_results.results)) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index a1c31e9..fa7a8c5 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -48,7 +48,7 @@ LCOV_SUMMARY_TRACE_FILE = "tracefile_summary.info" LCOV_CURRENT_TRACE_FILE = "current_test.info" LCOV_WITH_BRANCH_COVERAGE = "lcov_branch_coverage=1" LCOV_NO_RECURSION = "--no-recursion" -LCOV_USED_GCOV_TOOL = "/bin/llvm-gcov" +LCOV_USED_GCOV_TOOL = "llvm-gcov" LCOV_COMMAND_PREFIX = [ "lcov", -- GitLab From 6575bd94d8c3e5f8e56399fbbf217b3e689980df Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 17 Jun 2019 18:02:03 +0200 Subject: [PATCH 16/20] llvm-gcov was deleted in the execution by mistake --- bin/llvm-gcov | 0 suite_validation/execution.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 bin/llvm-gcov diff --git a/bin/llvm-gcov b/bin/llvm-gcov old mode 100644 new mode 100755 diff --git a/suite_validation/execution.py b/suite_validation/execution.py index fa7a8c5..7463dc3 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -39,9 +39,9 @@ ABORTED = "abort" HARNESS_GCDA_FILE = "harness.gcda" -GCOV_FILE_END = "gcov" -GCDA_FILE_END = "gcda" -GCNO_FILE_END = "gcno" +GCOV_FILE_END = ".gcov" +GCDA_FILE_END = ".gcda" +GCNO_FILE_END = ".gcno" LCOV_SUBFOLDER_TRACE_FILE = "tracefiles" LCOV_SUMMARY_TRACE_FILE = "tracefile_summary.info" -- GitLab From ab7fb0c40a26fa7c93c19e922c32b67634ce677a Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 17 Jun 2019 18:38:56 +0200 Subject: [PATCH 17/20] Added llvm to dependencies --- test/Dockerfile.python-3.5 | 2 +- test/Dockerfile.python-3.6 | 2 +- test/Dockerfile.python-3.7 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Dockerfile.python-3.5 b/test/Dockerfile.python-3.5 index 155d3b7..b355552 100644 --- a/test/Dockerfile.python-3.5 +++ b/test/Dockerfile.python-3.5 @@ -8,7 +8,7 @@ FROM python:3.5 -RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang +RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang llvm RUN pip install -U pip && pip install \ coverage \ diff --git a/test/Dockerfile.python-3.6 b/test/Dockerfile.python-3.6 index d0765eb..7e1300d 100644 --- a/test/Dockerfile.python-3.6 +++ b/test/Dockerfile.python-3.6 @@ -8,7 +8,7 @@ FROM python:3.6 -RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang +RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang llvm RUN pip install -U pip && pip install \ coverage \ diff --git a/test/Dockerfile.python-3.7 b/test/Dockerfile.python-3.7 index 6e62e83..bae99d5 100644 --- a/test/Dockerfile.python-3.7 +++ b/test/Dockerfile.python-3.7 @@ -8,7 +8,7 @@ FROM python:3.7 -RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang +RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang llvm RUN pip install -U pip && pip install \ coverage \ -- GitLab From d98cbb18a59046b30b3a2ce3087449ca8f5314ed Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 17 Jun 2019 19:18:55 +0200 Subject: [PATCH 18/20] Hopefully fixed path to llvm-gcov --- suite_validation/execution.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index 7463dc3..3401af9 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -48,7 +48,9 @@ LCOV_SUMMARY_TRACE_FILE = "tracefile_summary.info" LCOV_CURRENT_TRACE_FILE = "current_test.info" LCOV_WITH_BRANCH_COVERAGE = "lcov_branch_coverage=1" LCOV_NO_RECURSION = "--no-recursion" -LCOV_USED_GCOV_TOOL = "llvm-gcov" + +MODULE_DIRECTORY = os.path.join(os.path.dirname(__file__), os.path.pardir) +LCOV_USED_GCOV_TOOL = os.path.join(MODULE_DIRECTORY, "bin/llvm-gcov") LCOV_COMMAND_PREFIX = [ "lcov", -- GitLab From fc2d2f624541ce22538b7a365fa246094644480d Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 17 Jun 2019 20:35:48 +0200 Subject: [PATCH 19/20] Reset to gnu11 when compiling the files --- suite_validation/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index 3401af9..37aad82 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -181,7 +181,7 @@ class ExecutionRunner: self._overwrite = overwrite_files def _get_compile_cmd( - self, program_file, harness_file, output_file, c_version="gnu90" + self, program_file, harness_file, output_file, c_version="gnu11" ): mm_arg = "-m64" if self.machine_model == eu.MACHINE_MODEL_64 else "-m32" cmd = ["clang"] -- GitLab From ace26a800cb5530661529ff94ad15cf5488752fc Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 17 Jun 2019 20:50:49 +0200 Subject: [PATCH 20/20] Not delete tracefiles when --no-overwrite specified Also round on 2 digits when computing percent coverage --- suite_validation/coverage.py | 6 +++--- suite_validation/execution.py | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/suite_validation/coverage.py b/suite_validation/coverage.py index d1b4753..667a4f8 100644 --- a/suite_validation/coverage.py +++ b/suite_validation/coverage.py @@ -108,9 +108,9 @@ class TestCoverage: branch_condition_coverage = self.compute_branch_conditions_executed() branch_coverage = self.compute_branch_coverage() return ( - str(line_coverage * 100) + "%", - str(branch_condition_coverage * 100) + "%", - str(branch_coverage * 100) + "%", + str(round(line_coverage * 100, 2)) + "%", + str(round(branch_condition_coverage * 100, 2)) + "%", + str(round(branch_coverage * 100, 2)) + "%", ) diff --git a/suite_validation/execution.py b/suite_validation/execution.py index 37aad82..2e3ce93 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -507,12 +507,13 @@ class SuiteExecutor: # this method call raises an ExecutionError if the given test suite is invalid test_vectors = self._get_described_vectors(test_suite) - # old trace file in working directory might still exist - _remove_current_tracefile() - # old tmp folder might still exist - _remove_tracefile_folder() - # old gcda, gcno or gcov files might exist and can affect coverage computation with lcov - _remove_coverages_files_in_working_directory() + if self._overwrite_files: + # old trace file in working directory might still exist + _remove_current_tracefile() + # old trace file folder might still exist + _remove_tracefile_folder() + # old gcda, gcno or gcov files might exist + _remove_coverages_files_in_working_directory() self._execute_tests(program_file, test_vectors, executor, result_target) -- GitLab