Parallelize tests execution

parent f731c10a
......@@ -17,11 +17,11 @@ artifacts
tmp/gitlab-test
/.gopath/
testsdefinitions.txt
/.testoutput/
/.cover/
codeclimate.json
coverprofile.html
coverprofile.txt
coverprofile.func.txt
# Ignore the generated binary
/gitlab-runner
......@@ -2,6 +2,7 @@ stages:
- prepare
- prebuild
- test
- coverage
- build
- package
- release
......@@ -29,7 +30,7 @@ image: $CI_IMAGE
- docker:dind
variables: &docker_variables
CI_IMAGE: registry.gitlab.com/gitlab-org/gitlab-runner/ci:1.8.7-0
DOCKER_DRIVER: overlay
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375
tags:
- docker
......@@ -135,6 +136,22 @@ review-docs-cleanup:
script:
- ./trigger-build-docs cleanup
tests definitions:
stage: prebuild
# Using Go 1.10, because the `go test -list` is unavailable in <1.9
# and it's much faster in 1.10. And since there is no real test execution
# in this step, the version difference doesn't matter
image: golang:1.10
script:
- apt update
- apt install make
- source ci/touch_make_dependencies
- make parallel_test_prepare
artifacts:
paths:
- testsdefinitions.txt
expire_in: 7d
#
# Test stage
#
......@@ -154,37 +171,46 @@ code_quality:
- ./scripts/codequality analyze -f json --dev | tee gl-code-quality-report.json
artifacts:
paths: [gl-code-quality-report.json]
expire_in: 1 week
expire_in: 7d
.unit_tests: &unit_tests
coverage: /total:\s+\(statements\)\s+\d+.\d+\%/
artifacts:
paths:
- coverprofile.html
- coverprofile.txt
- coverprofile.func.txt
expire_in: 1 week
unit tests (no race):
<<: *docker
<<: *unit_tests
stage: test
retry: 2
script:
- JOB_NAME=( $CI_JOB_NAME )
- export SUITE_INDEX=${JOB_NAME[-2]}
- export SUITE_TOTAL=${JOB_NAME[-1]}
- source ci/touch_make_dependencies
- docker pull alpine
- docker pull docker:dind
- docker pull docker:git
- make test
- make parallel_test_execute
artifacts:
paths:
- .cover/*
- .testoutput/*
when: always
expire_in: 7d
unit tests:
<<: *docker
.unit_tests_with_race: &unit_tests_with_race
<<: *unit_tests
stage: test
allow_failure: true
script:
- source ci/touch_make_dependencies
- CGO_ENABLED=1 TESTFLAGS="-cover -race" make test
variables:
<<: *docker_variables
CGO_ENABLED: "1"
TESTFLAGS: "-cover -race"
unit tests 0 5: *unit_tests
unit tests 1 5: *unit_tests
unit tests 2 5: *unit_tests
unit tests 3 5: *unit_tests
unit tests 4 5: *unit_tests
unit tests with race 0 5: *unit_tests_with_race
unit tests with race 1 5: *unit_tests_with_race
unit tests with race 2 5: *unit_tests_with_race
unit tests with race 3 5: *unit_tests_with_race
unit tests with race 4 5: *unit_tests_with_race
docs check links:
image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine"
......@@ -224,13 +250,6 @@ integration windows:
- integration
- windows
race conditions detector:
<<: *docker
stage: test
script:
- source ci/touch_make_dependencies
- make check_race_conditions
git 1.7.1:
<<: *except_docs
image: centos:6
......@@ -249,6 +268,30 @@ git 1.7.1:
script:
- make test
#
# Coverage stage
#
test coverage report: &test_coverage_report
stage: coverage
image: golang:1.10
coverage: /regular total:\s+\(statements\)\s+\d+.\d+\%/
script:
- source ci/touch_make_dependencies
- make parallel_test_coverage_report
- make parallel_test_coverage_report TESTFLAGS="-cover -race"
artifacts:
paths:
- out/coverage/*
expire_in: 7d
race conditions detector:
stage: coverage
image: golang:1.10
script:
- source ci/touch_make_dependencies
- make check_race_conditions
#
# Build stage
#
......
......@@ -123,7 +123,19 @@ check_race_conditions:
test: $(PKG_BUILD_DIR) docker
# Running tests...
@./scripts/go_test_with_coverage_report
go test $(OUR_PACKAGES) $(TESTFLAGS)
parallel_test_prepare: $(GOPATH_SETUP)
# Preparing test commands
@./scripts/go_test_with_coverage_report prepare
parallel_test_execute: $(GOPATH_SETUP)
# executing tests
@./scripts/go_test_with_coverage_report execute
parallel_test_coverage_report: $(GOPATH_SETUP)
# Preparing coverage report
@./scripts/go_test_with_coverage_report coverage
install:
go install --ldflags="$(GO_LDFLAGS)" $(PKG)
......
......@@ -5,15 +5,18 @@
# fix existing race conditions
max=33
tmpfile=$(mktemp)
CGO_ENABLED=1 TESTFLAGS="-race" make test 2>&1 | tee -a "$tmpfile"
tmpFile=$(mktemp)
cnt=$(grep -c -e "^WARNING: DATA RACE$" "$tmpfile")
find .testoutput/
grep -E "^WARNING: DATA RACE$" .testoutput/*.race.output.txt > ${tmpFile}
cnt=$(cat ${tmpFile} | wc -l)
echo "Found ${cnt} race conditions. Maximum allowed value is ${max}"
rm "$tmpfile" 2>/dev/null || true
rm "$tmpFile" 2>/dev/null || true
if [ "$cnt" -gt "$max" ]; then
if [ "${cnt}" -gt "${max}" ]; then
echo "Race conditions count increased"
exit 1
fi
......@@ -2,29 +2,152 @@
set -eo pipefail
testsDefinitions="testsdefinitions.txt"
TESTFLAGS=${TESTFLAGS:-"-cover"}
PARALLEL_TESTS_LIMIT=${PARALLEL_TESTS_LIMIT:-10}
SUITE_TOTAL=${SUITE_TOTAL:-1}
SUITE_INDEX=${SUITE_INDEX:-0}
output="regular"
coverMode="count"
if [[ ${TESTFLAGS} = *"-cover"* ]]; then
rm -rf ".cover/"
mkdir -p ".cover"
if [[ ${TESTFLAGS} = *"-race"* ]]; then
output="race"
coverMode="atomic"
fi
if [[ ${TESTFLAGS} = *"-race"* ]]; then
coverMode="atomic"
fi
printMessage() {
echo -e "\033[1m${@}\033[0m"
}
echo "Starting go tests with coverprofile in ${coverMode} mode"
joinBy() {
local IFS="${1}"
shift
echo "${*}"
}
prepareTestCommands() {
[[ ! -f ${testsDefinitions} ]] || rm ${testsDefinitions}
for pkg in ${OUR_PACKAGES}; do
profileFile=".cover/$(echo ${pkg} | tr "/" "-").cover"
testIndex=0
runTests=()
tests=$(go test -list "Test.*" ${pkg} | grep "^Test" || echo "")
if [[ -z "${tests}" ]]; then
continue
fi
counter=0
for test in ${tests}; do
counter=$((counter+1))
runTests+=("${test}")
if [[ ${counter} -ge ${PARALLEL_TESTS_LIMIT} ]]; then
if [[ ${#runTests[@]} -gt 0 ]]; then
echo ${pkg} ${testIndex} $(joinBy "|" "${runTests[@]}") | tee -a ${testsDefinitions}
fi
counter=0
runTests=()
testIndex=$((testIndex+1))
fi
done
if [[ ${#runTests[@]} -gt 0 ]]; then
echo ${pkg} ${testIndex} $(joinBy "|" "${runTests[@]}") | tee -a ${testsDefinitions}
fi
done
}
executeTestCommand() {
local pkg=${1}
local index=${2}
local runTests=${3}
local options=""
local pkgSlug=$(echo ${pkg} | tr "/" "-")
if [[ ${TESTFLAGS} = *"-cover"* ]]; then
mkdir -p ".cover"
mkdir -p ".testoutput"
go test -covermode="${coverMode}" -coverprofile="${profileFile}" ${TESTFLAGS} -v ${pkg}
printMessage "\n\n--- Starting part ${index} of go tests of '${pkg}' package with coverprofile in '${coverMode}' mode:\n"
profileFile=".cover/${pkgSlug}.${index}.${coverMode}.cover.txt"
options="-covermode=${coverMode} -coverprofile=${profileFile}"
else
echo "Starting go test"
fi
testOutputFile=".testoutput/${pkgSlug}.${index}.${output}.output.txt"
exitCode=0
go test ${options} ${TESTFLAGS} -v ${pkg} -run "${runTests}" 2>&1 | tee ${testOutputFile} || exitCode=1
return $exitCode
}
executeTestPart() {
rm -rf ".cover/"
rm -rf ".testoutput/"
numberOfDefinitions=$(cat ${testsDefinitions} | wc -l)
executionSize=$((numberOfDefinitions/SUITE_TOTAL+1))
executionOffset=$((SUITE_INDEX*executionSize+1))
printMessage "Number of definitions: ${numberOfDefinitions}"
printMessage "Suite size: ${SUITE_TOTAL}"
printMessage "Suite index: ${SUITE_INDEX}"
printMessage "Execution size: ${executionSize}"
printMessage "Execution offset: ${executionOffset}"
exitCode=0
tail -n +${executionOffset} ${testsDefinitions} | head -n ${executionSize} | while read pkg index tests; do
executeTestCommand ${pkg} ${index} ${tests} || exitCode=1
done
echo "mode: ${coverMode}" > coverprofile.txt
grep -h -v "^mode:" .cover/*.cover >> coverprofile.txt
exit $exitCode
}
computeCoverageReport() {
local reportDirectory="out/coverage"
local sourceFile="${reportDirectory}/coverprofile.${output}.source.txt"
local htmlReportFile="${reportDirectory}/coverprofile.${output}.html"
local textReportFile="${reportDirectory}/coverprofile.${output}.txt"
mkdir -p ${reportDirectory}
echo "mode: ${coverMode}" > ${sourceFile}
grep -h -v -e "^mode:" -e "executors/docker/bindata.go" .cover/*.${coverMode}.cover.txt >> ${sourceFile}
printMessage "Generating HTML coverage report"
go tool cover -o ${htmlReportFile} -html=${sourceFile}
printMessage "Generating TXT coverage report"
go tool cover -o ${textReportFile} -func=${sourceFile}
printMessage "General coverage percentage:"
total=$(grep "total" "${textReportFile}" || echo "")
if [[ -n "${total}" ]]; then
echo "${output} ${total}"
fi
}
echo "Generating coverprofile.html file"
go tool cover -o coverprofile.html -html=coverprofile.txt
go tool cover -o coverprofile.func.txt -func=coverprofile.txt
grep total coverprofile.func.txt || true
fi
\ No newline at end of file
case "$1" in
prepare)
prepareTestCommands
;;
execute)
executeTestPart
;;
coverage)
computeCoverageReport
;;
esac
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment