diff --git a/bin/llvm-gcov b/bin/llvm-gcov new file mode 100755 index 0000000000000000000000000000000000000000..b5a9cc8a1be60fb703abf4705e30fc68b5a77679 --- /dev/null +++ b/bin/llvm-gcov @@ -0,0 +1,2 @@ +#!/bin/bash +llvm-cov gcov $* diff --git a/suite_validation/__init__.py b/suite_validation/__init__.py index a6bf8021db03656486ea8610893cebfcb2f50fc2..d8e37cc0ab6e175f825cf426b3215f4e0bff04b4 100644 --- a/suite_validation/__init__.py +++ b/suite_validation/__init__.py @@ -25,6 +25,7 @@ import shutil import zipfile from suite_validation import execution from suite_validation import execution_utils as eu +from suite_validation import coverage as cov __VERSION__ = "v1.1-dev" @@ -110,6 +111,14 @@ def get_parser(): required=False, ) + parser.add_argument( + "--individual-test-coverage", + dest="individual_test_cov", + action="store_true", + default=False, + help="print coverage of each test to file", + ) + parser.add_argument( "--verbose", dest="verbose", @@ -223,6 +232,20 @@ def parse_coverage_goal_file(goal_file: str) -> str: return eu.COVERAGE_GOALS[goal] +def _print_suite_execution_results(exec_results): + print("---Results---") + print("Tests run:", len(exec_results.results)) + print("Lines covered:", exec_results.lines_executed) + print("Branch conditions executed:", exec_results.branches_executed) + print("Branches covered:", exec_results.branches_taken) + + if any(r == execution.COVERS for r in exec_results.results): + verdict = "TRUE" + else: + verdict = "UNKNOWN" + print("Result:", verdict) + + def main(): args = parse() @@ -238,6 +261,7 @@ def main(): exec_results = eu.SuiteExecutionResult() harness_file = os.path.join(args.output_dir, "harness.c") executable = os.path.join(args.output_dir, "a.out") + compute_individuals = args.individual_test_cov try: executor = execution.SuiteExecutor( args.stop_after_success, @@ -246,6 +270,7 @@ def main(): overwrite_files=args.overwrite, harness_file_target=harness_file, compile_target=executable, + compute_individuals=compute_individuals, ) executor.run(args.file, args.test_suite, args.machine_model, exec_results) @@ -282,15 +307,10 @@ def main(): [str(c) + "\n" for c in exec_results.coverage_sequence] ) + if exec_results.coverage_tests: + if not os.path.exists(testsuite_folder): + os.mkdir(testsuite_folder) + cov.write_test_coverages_to_dir(testsuite_folder, args.file, exec_results) + print() - print("---Results---") - print("Tests run:", len(exec_results.results)) - print("Lines covered:", exec_results.lines_executed) - print("Branch conditions executed:", exec_results.branches_executed) - print("Branches covered:", exec_results.branches_taken) - - if any(r == execution.COVERS for r in exec_results.results): - verdict = "TRUE" - else: - verdict = "UNKNOWN" - print("Result:", verdict) + _print_suite_execution_results(exec_results) diff --git a/suite_validation/coverage.py b/suite_validation/coverage.py new file mode 100644 index 0000000000000000000000000000000000000000..667a4f8043c7d005af2634e5b485d1e35387fa5b --- /dev/null +++ b/suite_validation/coverage.py @@ -0,0 +1,241 @@ +# tbf-testsuite-validator is tool for validation and execution of test suites. +# This file is part of tbf-testsuite-validator. +# +# Copyright (C) 2018 Dirk Beyer +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for coverage of individual tests""" + +from enum import Enum +import os +import logging + +FILE_NAME_TEST_COVERAGES = "individual-test-coverages" +LINES_COVERED = "Lines covered" + + +class LcovPrefix(Enum): + FILEPATH = "SF:" + BRANCH_LINE_CONDITION_HIT_COUNTER = "BRDA:" + BRANCHES_FOUND = "BRF:" + BRANCHES_HIT = "BRH:" + LINE_HIT_COUNTER = "DA:" + LINES_NONZERO_HIT_COUNTER = "LH:" + LINES_FOUND = "LF:" + END_OF_RECORD = "end_of_record" + + +class LcovSector(Enum): + BEFORE_TEST_RECORD = 0 + IN_TEST_RECORD = 1 + + +class TestCoverage: + def __init__( + self, + file_name, + lines_hit_counter_dic=None, + lines_hit=0, + lines_found=0, + branch_condition_hit_counter_dic=None, + branches_hit=0, + branches_found=0, + ): + self.filename = file_name + self.lines_hit_counter_dic = lines_hit_counter_dic + self.lines_hit = lines_hit + self.lines_found = lines_found + self.branch_condition_hit_counter_dic = branch_condition_hit_counter_dic + self.branches_hit = branches_hit + self.branches_found = branches_found + self.test_vector = "" + self.result = "" + if self.lines_hit_counter_dic is None: + self.lines_hit_counter_dic = {} + if self.branch_condition_hit_counter_dic is None: + self.branch_condition_hit_counter_dic = {} + + def set_test_vector(self, test_vector): + self.test_vector = test_vector + + def set_result(self, result): + self.result = result + + def compute_line_coverage(self): + if self.lines_found <= 0: + return 1.0 + return round(float(self.lines_hit) / float(self.lines_found), 4) + + def compute_branch_conditions_executed(self): + possible_branch_conditions_executions = ( + len(self.branch_condition_hit_counter_dic.keys()) * 2 + ) + lines_with_branch_condition_executed = 0 + for line_with_branch_condition in self.branch_condition_hit_counter_dic.keys(): + conditions_executed = self.branch_condition_hit_counter_dic[ + line_with_branch_condition + ] + if conditions_executed[0]: + lines_with_branch_condition_executed += 1 + if conditions_executed[1]: + lines_with_branch_condition_executed += 1 + if possible_branch_conditions_executions <= 0: + return 1.0 + return round( + float(lines_with_branch_condition_executed) + / float(possible_branch_conditions_executions), + 4, + ) + + def compute_branch_coverage(self): + if self.branches_found <= 0: + return 1.0 + return round(float(self.branches_hit) / float(self.branches_found), 4) + + def get_coverage_ratios_as_percent_expressions(self): + line_coverage = self.compute_line_coverage() + branch_condition_coverage = self.compute_branch_conditions_executed() + branch_coverage = self.compute_branch_coverage() + return ( + str(round(line_coverage * 100, 2)) + "%", + str(round(branch_condition_coverage * 100, 2)) + "%", + str(round(branch_coverage * 100, 2)) + "%", + ) + + +def remove_prefix(line, prefix): + return line[len(prefix) :] + + +def _examine_branch_line_condition(branch_line, branch_condition_hit_counter_dic): + chunks = branch_line.split(",") + # chunks should be [program_line, block-number, branch-number, taken] + assert len(chunks) == 4 + program_line = chunks[0] + branch_number = chunks[2] + taken = chunks[3] + if program_line.isdigit() and branch_number.isdigit(): + program_line = int(program_line) + branch_number = int(branch_number) + else: + logging.error( + "Trace file corrupted. Program line or branch number not a number" + ) + return + if program_line not in branch_condition_hit_counter_dic.keys(): + branch_condition_hit_counter_dic[program_line] = [False, False] + if taken.isdigit() and int(taken) >= 1: + # a branch is fully executed when at least the condition is one time satisfied and one time not + # branch number even when condition not satisfied + # branch number odd when condition satisfied + if branch_number % 2 == 0: + branch_condition_hit_counter_dic[program_line][0] = True + else: + branch_condition_hit_counter_dic[program_line][1] = True + + +def get_test_coverage_from_lcov_file(program_name, trace_file): + lines_hit_counter_dic = {} + lines_hit = 0 + lines_found = 0 + branch_condition_hit_counter_dic = {} + branches_hit = 0 + branches_found = 0 + if os.path.exists(trace_file): + lcov_sector = LcovSector.BEFORE_TEST_RECORD.value + with open(trace_file) as file: + for line in file: + line = line.strip() + if lcov_sector == LcovSector.BEFORE_TEST_RECORD.value: + if line.startswith(LcovPrefix.FILEPATH.value): + absolute_file_path = remove_prefix( + line, LcovPrefix.FILEPATH.value + ) + if os.path.basename(absolute_file_path) == program_name: + lcov_sector = LcovSector.IN_TEST_RECORD.value + + elif lcov_sector == LcovSector.IN_TEST_RECORD.value: + + if line.startswith( + LcovPrefix.BRANCH_LINE_CONDITION_HIT_COUNTER.value + ): + branch_line_information = remove_prefix( + line, LcovPrefix.BRANCH_LINE_CONDITION_HIT_COUNTER.value + ) + _examine_branch_line_condition( + branch_line_information, branch_condition_hit_counter_dic + ) + elif line.startswith(LcovPrefix.BRANCHES_FOUND.value): + branches_found = int( + remove_prefix(line, LcovPrefix.BRANCHES_FOUND.value) + ) + elif line.startswith(LcovPrefix.BRANCHES_HIT.value): + branches_hit = int( + remove_prefix(line, LcovPrefix.BRANCHES_HIT.value) + ) + elif line.startswith(LcovPrefix.LINE_HIT_COUNTER.value): + line_with_counter = remove_prefix( + line, LcovPrefix.LINE_HIT_COUNTER.value + ) + chunks = line_with_counter.split(",") + # chunks should be [program-line, hit-counter] + assert len(chunks) == 2 + lines_hit_counter_dic[chunks[0]] = chunks[1] + elif line.startswith(LcovPrefix.LINES_FOUND.value): + lines_found = int( + remove_prefix(line, LcovPrefix.LINES_FOUND.value) + ) + elif line.startswith(LcovPrefix.LINES_NONZERO_HIT_COUNTER.value): + lines_hit = int( + remove_prefix( + line, LcovPrefix.LINES_NONZERO_HIT_COUNTER.value + ) + ) + elif line.startswith(LcovPrefix.END_OF_RECORD.value): + break + + else: + break + + else: + logging.debug( + "File '%s' does not exist. Returning empty test coverage", trace_file + ) + + return TestCoverage( + trace_file, + lines_hit_counter_dic, + lines_hit, + lines_found, + branch_condition_hit_counter_dic, + branches_hit, + branches_found, + ) + + +def write_test_coverages_to_dir(output_dir, program, exec_results): + output_file = os.path.join(output_dir, FILE_NAME_TEST_COVERAGES) + with open(output_file, "w") as outp: + outp.write("Program: " + program + "\n") + for test_coverage in exec_results.coverage_tests: + outp.write("\n") + outp.write("Test input: " + str(test_coverage.test_vector) + "\n") + outp.write("Test result: " + test_coverage.result + "\n") + lines_executed, branches_executed, branches_taken = ( + test_coverage.get_coverage_ratios_as_percent_expressions() + ) + outp.write("Lines covered: " + lines_executed + "\n") + outp.write("Branch conditions executed: " + branches_executed + "\n") + outp.write("Branches covered: " + branches_taken + "\n") + outp.close() diff --git a/suite_validation/execution.py b/suite_validation/execution.py index 937d0077e31f90c6826e8ddba765f29541f801d8..2e3ce938c319c1cd21e19eba01e9cb6a1a08a915 100644 --- a/suite_validation/execution.py +++ b/suite_validation/execution.py @@ -22,10 +22,12 @@ import xml.etree.ElementTree as ET import re import os import zipfile +import shutil from lxml import etree from suite_validation import execution_utils as eu +from suite_validation import coverage as cov HARNESS_FILE_NAME = "harness.c" ARCHITECTURE_TAG = "architecture" @@ -35,6 +37,29 @@ UNKNOWN = "unknown" ERROR = "error" ABORTED = "abort" +HARNESS_GCDA_FILE = "harness.gcda" + +GCOV_FILE_END = ".gcov" +GCDA_FILE_END = ".gcda" +GCNO_FILE_END = ".gcno" + +LCOV_SUBFOLDER_TRACE_FILE = "tracefiles" +LCOV_SUMMARY_TRACE_FILE = "tracefile_summary.info" +LCOV_CURRENT_TRACE_FILE = "current_test.info" +LCOV_WITH_BRANCH_COVERAGE = "lcov_branch_coverage=1" +LCOV_NO_RECURSION = "--no-recursion" + +MODULE_DIRECTORY = os.path.join(os.path.dirname(__file__), os.path.pardir) +LCOV_USED_GCOV_TOOL = os.path.join(MODULE_DIRECTORY, "bin/llvm-gcov") + +LCOV_COMMAND_PREFIX = [ + "lcov", + "--gcov-tool", + LCOV_USED_GCOV_TOOL, + "--rc", + LCOV_WITH_BRANCH_COVERAGE, +] + class ExecutionError(Exception): def __init__(self, msg): @@ -159,7 +184,7 @@ class ExecutionRunner: self, program_file, harness_file, output_file, c_version="gnu11" ): mm_arg = "-m64" if self.machine_model == eu.MACHINE_MODEL_64 else "-m32" - cmd = ["gcc"] + cmd = ["clang"] cmd += [ "-std={}".format(c_version), mm_arg, @@ -296,32 +321,79 @@ class CoverageMeasuringExecutionRunner(ExecutionRunner): logging.info("Aborted test run is not considered for coverage") return result - def get_coverage(self, program_file): - lines_executed = None - branches_executed = None - branches_taken = None - + def get_current_test_coverage(self, program_file): + program_name = os.path.basename(program_file) if self.harness_file: assert self.harness_file.endswith(".c") data_file = self.harness_file[:-1] + "gcda" data_file = os.path.basename(data_file) # data file is in cwd - if os.path.exists(data_file): - cmd = ["gcov", "-nbc", data_file] - res = eu.execute(cmd, quiet=True) - full_cov = res.stdout.splitlines() - - program_name = os.path.basename(program_file) - for number, line in enumerate(full_cov): - if line.startswith("File") and program_name in line: - lines_executed = self._get_gcov_val(full_cov[number + 1]) - branches_executed = self._get_gcov_val(full_cov[number + 2]) - branches_taken = self._get_gcov_val(full_cov[number + 3]) - break + cmd = LCOV_COMMAND_PREFIX + [ + "-c", + "-d", + ".", + LCOV_NO_RECURSION, + "-o", + LCOV_CURRENT_TRACE_FILE, + ] + eu.execute(cmd, quiet=True) + if os.path.exists(LCOV_CURRENT_TRACE_FILE): + test_coverage = cov.get_test_coverage_from_lcov_file( + program_name, LCOV_CURRENT_TRACE_FILE + ) + return test_coverage + logging.warning( + "Trace file '%s' not created. Returning empty test coverage.", + LCOV_CURRENT_TRACE_FILE, + ) + else: + logging.warning( + "Coverage requested without any execution. Returning empty test coverage." + ) + return cov.TestCoverage(program_name) + + @staticmethod + def move_tracefile_into_subfolder_and_combine_with_previous(trace_file): + trace_file_summary = ( + LCOV_SUBFOLDER_TRACE_FILE + os.sep + LCOV_SUMMARY_TRACE_FILE + ) + + if not os.path.isdir(LCOV_SUBFOLDER_TRACE_FILE): + cmd = ["mkdir", LCOV_SUBFOLDER_TRACE_FILE] + eu.execute(cmd, quiet=True) + + if os.path.exists(trace_file_summary): + cmd = LCOV_COMMAND_PREFIX + [ + "-a", + trace_file, + "-a", + trace_file_summary, + "-o", + trace_file_summary, + ] + eu.execute(cmd, quiet=True) + cmd = ["lcov", "--summary", trace_file_summary] + eu.execute(cmd, quiet=True) else: - logging.debug( - "Coverage requested without any execution. Returning defaults." + shutil.move(trace_file, trace_file_summary) + + def get_coverage(self, program_file): + + trace_file_summary = ( + LCOV_SUBFOLDER_TRACE_FILE + os.sep + LCOV_SUMMARY_TRACE_FILE + ) + + if os.path.exists(trace_file_summary): + program_name = os.path.basename(program_file) + test_coverage_summary = cov.get_test_coverage_from_lcov_file( + program_name, trace_file_summary ) + else: + test_coverage_summary = self.get_current_test_coverage(program_file) + + lines_executed, branches_executed, branches_taken = ( + test_coverage_summary.get_coverage_ratios_as_percent_expressions() + ) if not lines_executed: lines_executed = "0%" @@ -364,6 +436,7 @@ class SuiteExecutor: compute_sequence=False, overwrite_files=True, isolate_tests=True, + compute_individuals=True, ): self._stop_after_success = stop_after_found_error self._timelimit = timelimit_per_run @@ -373,6 +446,7 @@ class SuiteExecutor: self._compute_sequence = compute_sequence self._overwrite_files = overwrite_files self._isolate_tests = isolate_tests + self._compute_individual_test_coverages = compute_individuals def run(self, program_file, test_suite, machine_model, result_target=None): """Execute the given tests on the given program. @@ -433,6 +507,14 @@ class SuiteExecutor: # this method call raises an ExecutionError if the given test suite is invalid test_vectors = self._get_described_vectors(test_suite) + if self._overwrite_files: + # old trace file in working directory might still exist + _remove_current_tracefile() + # old trace file folder might still exist + _remove_tracefile_folder() + # old gcda, gcno or gcov files might exist + _remove_coverages_files_in_working_directory() + self._execute_tests(program_file, test_vectors, executor, result_target) return result_target @@ -479,11 +561,33 @@ class SuiteExecutor: for tv in test_vectors: next_result = executor.run(program_file, tv) + coverage_test = None + + if self._compute_individual_test_coverages: + coverage_test = executor.get_current_test_coverage(program_file) + coverage_test.set_result(next_result) + coverage_test.set_test_vector(tv) + result_target.coverage_tests.append(coverage_test) + + if os.path.exists(LCOV_CURRENT_TRACE_FILE): + executor.move_tracefile_into_subfolder_and_combine_with_previous( + LCOV_CURRENT_TRACE_FILE + ) + + _remove_current_tracefile() + _remove_harness_gcda_file() + if self._compute_sequence: - result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage( - program_file - ) + # if we use individual tests with lcov we extract the info from the summarized file + if coverage_test: + result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = ( + coverage_test.get_coverage_ratios_as_percent_expressions() + ) + else: + result_target.lines_executed, result_target.branches_executed, result_target.branches_taken = executor.get_coverage( + program_file + ) result_target.coverage_sequence.append( float(result_target.branches_taken.split("%")[0]) ) @@ -503,6 +607,30 @@ class SuiteExecutor: ) +def _remove_tracefile_folder(): + if os.path.isdir(LCOV_SUBFOLDER_TRACE_FILE): + shutil.rmtree(LCOV_SUBFOLDER_TRACE_FILE, ignore_errors=True) + + +def _remove_current_tracefile(): + if os.path.exists(LCOV_CURRENT_TRACE_FILE): + os.remove(LCOV_CURRENT_TRACE_FILE) + + +def _remove_harness_gcda_file(): + if os.path.exists(HARNESS_GCDA_FILE): + os.remove(HARNESS_GCDA_FILE) + + +def _remove_coverages_files_in_working_directory(): + extensions = (GCOV_FILE_END, GCDA_FILE_END, GCNO_FILE_END) + files = [ + f for f in os.listdir(os.curdir) if os.path.isfile(f) and f.endswith(extensions) + ] + for file in files: + os.remove(file) + + def _parse_xml_if_testcase(xml_lines): curr_content = [] for line_number, line in enumerate(xml_lines): diff --git a/suite_validation/execution_utils.py b/suite_validation/execution_utils.py index 80164f506f3465caac55b74bf37c44743cba431f..d89e4b51beed0d4848d29ebe0738a64b621779c3 100644 --- a/suite_validation/execution_utils.py +++ b/suite_validation/execution_utils.py @@ -142,6 +142,7 @@ class SuiteExecutionResult: self.branches_taken = "0%" self.successful_test = None self.coverage_sequence = list() + self.coverage_tests = list() class ExecutionResult: diff --git a/test/Dockerfile.python-3.5 b/test/Dockerfile.python-3.5 index 49482fe1860acac9b89f21ed7537dcf4af7e42d6..b3555526d2bb0ed4cbb9cbd3efe3cabbb1e667e9 100644 --- a/test/Dockerfile.python-3.5 +++ b/test/Dockerfile.python-3.5 @@ -8,7 +8,7 @@ FROM python:3.5 -RUN apt update && apt install -y gcc-multilib g++-multilib +RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang llvm RUN pip install -U pip && pip install \ coverage \ diff --git a/test/Dockerfile.python-3.6 b/test/Dockerfile.python-3.6 index f53f98d17ace0ba0f118c8624c638234b420f815..7e1300dcdacbf1c71c03ddb74ae2f1388d4ffc16 100644 --- a/test/Dockerfile.python-3.6 +++ b/test/Dockerfile.python-3.6 @@ -8,7 +8,7 @@ FROM python:3.6 -RUN apt update && apt install -y gcc-multilib g++-multilib +RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang llvm RUN pip install -U pip && pip install \ coverage \ diff --git a/test/Dockerfile.python-3.7 b/test/Dockerfile.python-3.7 index 1f72fd0891daa77985c3ec56784776d8fb5152ae..bae99d54e21b45e9a5ae7c3f574bd307040eb0a0 100644 --- a/test/Dockerfile.python-3.7 +++ b/test/Dockerfile.python-3.7 @@ -8,7 +8,7 @@ FROM python:3.7 -RUN apt update && apt install -y gcc-multilib g++-multilib +RUN apt update && apt install -y gcc-multilib g++-multilib lcov clang llvm RUN pip install -U pip && pip install \ coverage \