Commit 0285fb41 authored by Grant Young's avatar Grant Young

Merge branch 'gy-nfs-update' into 'master'

Add in Test Flags for improved test results based on factors

Closes #200

See merge request !125
parents 96d4ea1e 89ac2fb2
......@@ -19,7 +19,7 @@ check:gpt-docker:
script:
- apk add -u --no-cache docker
- docker build -t gitlab/gitlab-performance-tool:test .
- docker run --rm -e ACCESS_TOKEN=$GPT_CHECK_STAGING_ACCESS_TOKEN gitlab/gitlab-performance-tool:test -e staging.json -t api_v4_user.js -l 2000
- docker run --rm -e ACCESS_TOKEN=$GPT_CHECK_STAGING_ACCESS_TOKEN gitlab/gitlab-performance-tool:test -e staging.json -t api_v4_user.js
except:
- schedules
......
......@@ -7,7 +7,7 @@ check:k6:
stage: check
extends: .k6-base
script:
- ACCESS_TOKEN=$GPT_CHECK_STAGING_ACCESS_TOKEN bin/run-k6 -e staging.json -t api_v4_user.js -l 2000
- ACCESS_TOKEN=$GPT_CHECK_STAGING_ACCESS_TOKEN bin/run-k6 -e staging.json -t api_v4_user.js
except:
- schedules
- tags
......@@ -105,10 +105,9 @@ report-k6-results-wiki:
variables:
- $TEST_TYPE == "k6"
report-k6-tests-wiki:
report-k6-test-info-wiki:
extends: .k6-base
script:
- bin/ci-report-test-info-wiki -p "Current Test Details"
except:
variables:
- $CI_TEST_DOCS_WIKI_REPORT != "true"
only:
- tags
......@@ -10,6 +10,7 @@ require 'table_print'
# Get parent folder(`k6`) path from the current file
k6_dir = File.expand_path('../k6', __dir__)
tp.set(:max_width, 800)
@opts = Optimist.options do
banner "Usage: ci-report-test-info-wiki [options]"
......@@ -26,14 +27,9 @@ raise 'Environment Variable CI_PROJECT_ACCESS_TOKEN must be set to proceed. See
puts "Collecting test info..."
tests = RunK6.get_tests(k6_dir: k6_dir, test_paths: ["tests"], quarantined: true, scenarios: true, unsafe: true)
aggregated_tests = []
tests.each do |test|
aggregated_tests << TestInfo.parse_test_docs_for_info(test)
end
test_types = aggregated_tests.map { |test| test[:type] }.uniq
puts "Collecting known issues..."
aggregated_issues = TestInfo.get_known_issues(k6_dir)
tests_info = TestInfo.get_tests_info(tests)
test_types = tests_info.map { |test_info| test_info[:type] }.uniq
tests_with_issues = tests_info.select { |test_info| !test_info['issues'].nil? }
wiki_report_contents = <<~DOC
GitLab Performance Tool provides several different types of tests:
......@@ -45,16 +41,15 @@ wiki_report_contents = <<~DOC
**Note:** Some endpoints have [known issues](#known-issues). These tests have either been run with a custom lower threshold limit applied or are quarantined until the issue is fixed.
DOC
tp.set(:max_width, 800)
test_types.each do |type|
wiki_report_contents << "\n## #{type.match?(/api/) ? type.upcase : type.capitalize}\n"
aggregated_tests_by_type = aggregated_tests.select { |test| test[:type] == type }
wiki_report_contents << TablePrint::Printer.table_print(aggregated_tests_by_type)
tests_info_by_type = tests_info.select { |test| test[:type] == type }
wiki_report_contents << TablePrint::Printer.table_print(tests_info_by_type, [{ link_md: { display_name: "Tests" } }, :endpoint, :description])
end
wiki_report_contents << "\n## Known Issues\n"
wiki_report_contents << TablePrint::Printer.table_print(aggregated_issues)
wiki_report_contents << TablePrint::Printer.table_print(tests_with_issues, [:name, :issues])
puts "Posting test info to Wiki page..."
headers = {
......
......@@ -16,7 +16,7 @@ require 'tmpdir'
k6_dir = Pathname.new(File.expand_path('../k6', __dir__)).relative_path_from(Dir.pwd)
gpt_version = '1.2.1'
gpt_version = '1.2.2'
puts Rainbow("GitLab Performance Tool (GPT) v#{gpt_version} - Performance test runner for GitLab environments based on k6").color(230, 83, 40)
opts = Optimist.options do
......@@ -29,7 +29,6 @@ opts = Optimist.options do
opt :options, "Name of Options Config file in options directory that the test(s) will be run with. Alternative filepath can also be given.", type: :string, default: "20s_2rps.json"
opt :tests, "Names of Test files or directories to run with. When directory given tests will be recursively added from api, web and git subdirs.", type: :strings, default: ["tests"]
opt :scenarios, "Include any tests inside the test directory's scenarios subfolder when true.", type: :flag, default: false
opt :latency, "Specify a network latency time in MS to increase applicable test thresholds by. Only typically used in slow network environments.", type: :integer, default: 0
opt :quarantined, "Include any tests inside the test directory's quarantined subfolder when true.", type: :flag, default: false
opt :excludes, "List of words used to exclude tests by matching against their names.", type: :strings, default: []
opt :unsafe, "Include any tests that perform unsafe requests (POST, PUT, DELETE, PATCH)", type: :flag, default: false
......@@ -55,7 +54,7 @@ raise "Environment config file '#{opts[:environment]}' not found as given or in
options_file = Dir.glob([opts[:options], "#{k6_dir}/#{opts[:options]}", "#{k6_dir}/options/#{opts[:options]}", "#{ENV['GPT_DOCKER_OPTIONS_DIR'] || ''}/#{opts[:options]}"])[0]
raise "Options config file '#{opts[:options]}' not found as given or in default folder. Exiting..." unless File.file?(options_file.to_s)
env_vars = RunK6.setup_env_vars(env_file: env_file, options_file: options_file, latency: opts[:latency])
env_vars = RunK6.setup_env_vars(env_file: env_file, options_file: options_file)
env_version = RunK6.get_env_version(env_vars: env_vars)
start_time = Time.now
......@@ -80,19 +79,19 @@ tests.each do |test_file|
out_file.puts output
end
aggregated_results << RunK6.parse_k6_results(status: status, output: output).merge({ 'known_issue' => TestInfo.parse_test_docs_for_issues(test_file).key?(:issue) })
aggregated_results << RunK6.get_test_results(test_file: test_file, status: status, output: output)
aggregated_success &&= status
rescue Interrupt
warn Rainbow("Caught the interrupt. Stopping.").yellow
exit
rescue ArgumentError => e
warn Rainbow(e.message).yellow
warn Rainbow(e).yellow
next
rescue NoMethodError
warn Rainbow("Test failed and output couldn't be parsed").yellow
warn Rainbow("Test failed and output couldn't be parsed - \n#{e}").yellow
next
rescue StandardError => e
warn Rainbow("Test failed: #{e.message}").red
warn Rainbow("Test failed: #{e}").red
aggregated_success = false
end
aggregated_score = RunK6.get_results_score(results: aggregated_results, env_vars: env_vars)
......@@ -109,11 +108,11 @@ run_time = (end_time - start_time).round(2)
puts Rainbow("All k6 tests have finished after #{run_time}s!").green
# Output known issues
unless ENV['GPT_SKIP_KNOWN_ISSUES'] == 'true'
known_issues = TestInfo.get_known_issues(k6_dir)
puts "\nKnown issues:\n\nNote that the following endpoints below have known issues. These tests have either been run with a custom lower threshold limit applied or are quarantined until the issue is fixed:\n\n"
tests_with_issues = TestInfo.get_tests_info(tests).select { |test_info| !test_info['issues'].nil? }
unless tests_with_issues.empty? || ENV['GPT_SKIP_KNOWN_ISSUES'] == 'true'
puts "\n█ Known issues\n\nNote that the following endpoints below have known issues. These tests have either been run with a custom lower threshold limit applied or are quarantined until the issue is fixed:\n\n"
tp.set(:max_width, 100)
tp(known_issues)
tp(tests_with_issues, :name, :issues)
puts "\nFull list of issues found both past and present can be found here: https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues\n"
end
......@@ -142,7 +141,7 @@ File.write(results_json_file, results_json.to_json)
# STDOUT \ Text
results_summary = RunK6.generate_results_summary(results_json: results_json)
puts "\nResults summary:\n\n#{results_summary}\n"
puts "\n█ Results summary\n\n#{results_summary}\n"
results_table = RunK6.generate_results_table(results_json: results_json)
puts results_table
......@@ -154,6 +153,6 @@ results_txt_file = File.join(results_dir, "#{results_file_prefix}_results.txt")
# Write results to file but also remove any terminal ANSI codes
File.write(results_txt_file, "#{results_summary}\n#{results_table}\n#{results_footer}".gsub(/\e\[([;\d]+)?m/, ''))
puts "\nResults files:\n#{results_output_file}\n#{results_json_file}\n#{results_txt_file}"
puts "\n█ Results files\n\n#{results_output_file}\n#{results_json_file}\n#{results_txt_file}"
abort("\n" + Rainbow("One or more tests have failed...").red) unless aggregated_success || ENV['GPT_IGNORE_RESULT'] == 'true'
......@@ -56,7 +56,11 @@ As an example, the following is one of our Environment Config Files, [`10k.json`
{
"environment": {
"name": "10k",
"url": "http://10k.testbed.gitlab.net"
"url": "http://10k.testbed.gitlab.net",
"config": {
"latency": "0",
"repo_storage": "gitaly"
}
},
"projects": [
{
......@@ -97,8 +101,13 @@ As an example, the following is one of our Environment Config Files, [`10k.json`
}
```
* The environment's Name and URL.
* Note as a convenience these two settings can also be defined as environment variables, `ENVIRONMENT_NAME` and `ENVIRONMENT_URL` respectively, for overriding as required.
Details for each of the settings are as follows. Some are also available to be configured as environment variables as noted:
* The environment's details:
* `name` - The name of the environment. Mainly used in output and results. (Environment variable: `ENVIRONMENT_NAME`)
* `url` - Full URL of the environment, used by all of the tests and in other areas. (Environment variable: `ENVIRONMENT_URL`)
* `config`- Additional details about the environment that are used to adjust test results accordingly.
* `latency` - The network latency (in ms) between where the Tool will be running and the environment. (Environment variable: `ENVIRONMENT_LATENCY`)
* `repo_storage` - What storage type the environment is using for repository data. Can be `gitaly` or `nfs`. (Environment variable: `ENVIRONMENT_REPO_STORAGE`)
* Details for each project that the tests should target along with it's data. You should aim to have each of these details present here and in the target environment otherwise the specific tests that require them will be skipped automatically:
* `name` - Name of the Project.
* `group` - The name of the Group that contains the intended Project.
......@@ -246,7 +255,7 @@ The recommended way to run the Tool is with our Docker image, [gitlab/gitlab-per
The image will start running the tests when it's called. The full options for running the tool can be seen by getting the help output by running `docker run -it gitlab/gitlab-performance-tool --help`:
```
GitLab Performance Tool (GPT) v1.2.1 - Performance test runner for GitLab environments based on k6
GitLab Performance Tool (GPT) v1.2.2 - Performance test runner for GitLab environments based on k6
Documentation: https://gitlab.com/gitlab-org/quality/performance/blob/master/docs/README.md
......@@ -257,7 +266,6 @@ Options:
-o, --options=<s> Name of Options Config file in options directory that the test(s) will be run with. Alternative filepath can also be given. (Default: 20s_2rps.json)
-t, --tests=<s+> Names of Test files or directories to run with. When directory given tests will be recursively added from api, web and git subdirs. (Default: tests)
-s, --scenarios Include any tests inside the test directory's scenarios subfolder when true.
-l, --latency=<i> Specify a network latency time in MS to increase applicable test thresholds by. Only typically used in slow network environments. (Default: 0)
-q, --quarantined Include any tests inside the test directory's quarantined subfolder when true.
-x, --excludes=<s+> List of words used to exclude tests by matching against their names. (Default: )
-u, --unsafe Include any tests that perform unsafe requests (POST, PUT, DELETE, PATCH)
......@@ -320,7 +328,7 @@ After running the results will be in the tool's `k6/results` folder. More detail
Two factors that can significantly affect the Tool's results are the conditions it's run in - Namely Location (physically in relation to the GitLab Environment) and Network.
The GPT is designed to test the GitLab application's server performance directly at max throughputs. As such, it's recommended to be run as close as possible physically to the GitLab environment and in optimum network conditions. Through this, the results given by the Tool can be trusted to be representative of the application's actual performance and not disrupted by location or network issues. If this isn't possible the Tool provides the option to specify a network latency to take into account when it's processing the results, see the `--latency` option description in the `bin/run-k6` script's help output above for more info.
The GPT is designed to test the GitLab application's server performance directly at max throughputs. As such, it's recommended to be run as close as possible physically to the GitLab environment and in optimum network conditions. Through this, the results given by the Tool can be trusted to be representative of the application's actual performance and not disrupted by location or network issues. If this isn't possible an Environment can be configured to specify a latency that the Tool will take into account. See the [Environments](#environments) section for more info.
That said, the GPT could also be utilized to test your location and network conditions as a secondary test. For example, you could run the Tool from a desired location (and network conditions) with a representative throughput (i.e. the amount of users that would be typical for that location). In this example this would be run after doing the primary test as described above then compared to validate that your location and network conditions are also performing as expected.
......
{
"environment": {
"name": "10k",
"url": "http://10k.testbed.gitlab.net"
"url": "http://10k.testbed.gitlab.net",
"config": {
"latency": "0",
"repo_storage": "gitaly"
}
},
"projects": [
{
......
{
"environment": {
"name": "25k",
"url": "http://25k.testbed.gitlab.net"
"url": "http://25k.testbed.gitlab.net",
"config": {
"latency": "0",
"repo_storage": "gitaly"
}
},
"projects": [
{
......
{
"environment": {
"name": "2k",
"url": "http://2k.testbed.gitlab.net"
"url": "http://2k.testbed.gitlab.net",
"config": {
"latency": "0",
"repo_storage": "gitaly"
}
},
"projects": [
{
......
{
"environment": {
"name": "50k",
"url": "http://50k.testbed.gitlab.net"
"url": "http://50k.testbed.gitlab.net",
"config": {
"latency": "0",
"repo_storage": "gitaly"
}
},
"projects": [
{
......
{
"environment": {
"name": "5k",
"url": "http://5k.testbed.gitlab.net"
"url": "http://5k.testbed.gitlab.net",
"config": {
"latency": "0",
"repo_storage": "gitaly"
}
},
"projects": [
{
......
{
"environment": {
"name": "docker",
"url": "http://docker"
"url": "http://docker",
"config": {
"latency": "0",
"repo_storage": "gitaly"
}
},
"projects": [
{
......
{
"environment": {
"name": "localhost",
"url": "http://localhost:3000"
"url": "http://localhost:3000",
"config": {
"latency": "0",
"repo_storage": "gitaly"
}
},
"projects": [
{
......
{
"environment": {
"name": "staging",
"url": "https://staging.gitlab.com"
"url": "https://staging.gitlab.com",
"config": {
"latency": "2000",
"repo_storage": "gitaly"
}
},
"projects": [
{
......
......@@ -30,7 +30,7 @@ export function getRpsThresholds(modifier=1.0, endpoints=1) {
}
export function getTtfbThreshold(ttfbBase=__ENV.TTFB_THRESHOLD) {
return parseInt(ttfbBase) + parseInt(__ENV.TTFB_LATENCY);
return parseInt(ttfbBase) + parseInt(__ENV.ENVIRONMENT_LATENCY);
}
export function adjustRps(modifier=1.0) {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /groups/:id`
@description: [Get all details of a group](https://docs.gitlab.com/ee/api/groups.html#details-of-a-group)
@issue: TBC
*/
import http from "k6/http";
......@@ -10,7 +11,7 @@ import { Rate } from "k6/metrics";
import { logError, getRpsThresholds, getTtfbThreshold, getProjects, selectProject } from "../../lib/gpt_k6_modules.js";
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let ttfbThreshold = getTtfbThreshold(1000)
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /projects?pagination=keyset&order_by=id&sort=asc`
@description: [Get a list of all visible projects across GitLab for the authenticated user using keyset-pagination](https://docs.gitlab.com/ee/api/projects.html#list-all-projects)
@issue: https://gitlab.com/gitlab-org/gitlab/issues/30181
@gitlab_version: 12.7.0
*/
......@@ -10,8 +11,8 @@ import { group } from "k6";
import { Rate } from "k6/metrics";
import { logError, getRpsThresholds, getTtfbThreshold } from "../../lib/gpt_k6_modules.js";
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = getRpsThresholds(0.2)
export let ttfbThreshold = getTtfbThreshold(5000)
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /projects?order_by=id&sort=asc`
@description: [Get a list of all visible projects across GitLab for the authenticated user](https://docs.gitlab.com/ee/api/projects.html#list-all-projects)
@issue: https://gitlab.com/gitlab-org/gitlab/issues/30181
*/
import http from "k6/http";
......@@ -9,8 +10,8 @@ import { group } from "k6";
import { Rate } from "k6/metrics";
import { logError, getRpsThresholds, getTtfbThreshold } from "../../lib/gpt_k6_modules.js";
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = getRpsThresholds(0.2)
export let ttfbThreshold = getTtfbThreshold(6000)
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -3,6 +3,7 @@
@endpoint: `GET /projects/:id/repository/branches`
@description: [Get a list of repository branches from a project, sorted by name alphabetically](https://docs.gitlab.com/ee/api/branches.html#list-repository-branches)
@issue: https://gitlab.com/gitlab-org/gitlab/issues/30536
@flags: repo_storage
*/
import http from "k6/http";
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /projects/:id/repository/branches/:branch`
@description: [Get a single project repository branch](https://docs.gitlab.com/ee/api/branches.html#get-single-repository-branch)
@flags: repo_storage
*/
import http from "k6/http";
......@@ -11,8 +12,8 @@ import { logError, getRpsThresholds, getTtfbThreshold, getProjects, selectProjec
if (!__ENV.ACCESS_TOKEN) fail('ACCESS_TOKEN has not been set. Skipping...')
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getRpsThresholds(0.5) : getRpsThresholds()
export let ttfbThreshold = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getTtfbThreshold(2500) : getTtfbThreshold()
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /projects/:id/repository/commits`
@description: [Get a list of repository commits in a project](https://docs.gitlab.com/ee/api/commits.html#list-repository-commits)
@flags: repo_storage
*/
import http from "k6/http";
......@@ -11,8 +12,8 @@ import { logError, getRpsThresholds, getTtfbThreshold, getProjects, selectProjec
if (!__ENV.ACCESS_TOKEN) fail('ACCESS_TOKEN has not been set. Skipping...')
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getRpsThresholds(0.5) : getRpsThresholds()
export let ttfbThreshold = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getTtfbThreshold(2500) : getTtfbThreshold()
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /projects/:id/repository/commits/:sha`
@description: [Get a specific commit identified by the commit hash](https://docs.gitlab.com/ee/api/commits.html#get-a-single-commit)
@flags: repo_storage
*/
import http from "k6/http";
......@@ -11,8 +12,8 @@ import { logError, getRpsThresholds, getTtfbThreshold, getProjects, selectProjec
if (!__ENV.ACCESS_TOKEN) fail('ACCESS_TOKEN has not been set. Skipping...')
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getRpsThresholds(0.5) : getRpsThresholds()
export let ttfbThreshold = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getTtfbThreshold(2500) : getTtfbThreshold()
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /projects/:id/repository/commits/:sha/diff`
@description: [Get the diff of a commit in a project](https://docs.gitlab.com/ee/api/commits.html#get-the-diff-of-a-commit)
@flags: repo_storage
*/
import http from "k6/http";
......@@ -11,8 +12,8 @@ import { logError, getRpsThresholds, getTtfbThreshold, getProjects, selectProjec
if (!__ENV.ACCESS_TOKEN) fail('ACCESS_TOKEN has not been set. Skipping...')
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getRpsThresholds(0.5) : getRpsThresholds()
export let ttfbThreshold = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getTtfbThreshold(2500) : getTtfbThreshold()
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -3,6 +3,7 @@
@endpoint: `GET /projects/:id/repository/commits/:sha/signature`
@description: [Get GPG signature of a commit](https://docs.gitlab.com/ee/api/commits.html#get-gpg-signature-of-a-commit)
@gitlab_version: 11.9.0
@flags: repo_storage
*/
import http from "k6/http";
......@@ -12,8 +13,8 @@ import { logError, getRpsThresholds, getTtfbThreshold, getProjects, selectProjec
if (!__ENV.ACCESS_TOKEN) fail('ACCESS_TOKEN has not been set. Skipping...')
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getRpsThresholds(0.5) : getRpsThresholds()
export let ttfbThreshold = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getTtfbThreshold(2500) : getTtfbThreshold()
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /projects/:id/repository/compare?from=commit_sha1&to=commit_sha2`
@description: [Compare commits](https://docs.gitlab.com/ee/api/repositories.html#compare-branches-tags-or-commits)
@flags: repo_storage
*/
import http from "k6/http";
......@@ -11,8 +12,8 @@ import { logError, getRpsThresholds, getTtfbThreshold, getProjects, selectProjec
if (!__ENV.ACCESS_TOKEN) fail('ACCESS_TOKEN has not been set. Skipping...')
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold(600)
export let rpsThresholds = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getRpsThresholds(0.5) : getRpsThresholds()
export let ttfbThreshold = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getTtfbThreshold(2500) : getTtfbThreshold()
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /projects/:id/repository/files/:file_path`
@description: [Get information about file in repository](https://docs.gitlab.com/ee/api/repository_files.html#get-file-from-repository)
@flags: repo_storage
*/
import http from "k6/http";
......@@ -11,8 +12,8 @@ import { logError, getRpsThresholds, getTtfbThreshold, getProjects, selectProjec
if (!__ENV.ACCESS_TOKEN) fail('ACCESS_TOKEN has not been set. Skipping...')
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getRpsThresholds(0.5) : getRpsThresholds()
export let ttfbThreshold = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getTtfbThreshold(2500) : getTtfbThreshold()
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /projects/:id/repository/files/:file_path/raw?ref=master`
@description: [Get raw file from repository](https://docs.gitlab.com/ee/api/repository_files.html#get-raw-file-from-repository)
@flags: repo_storage
*/
import http from "k6/http";
......@@ -11,8 +12,8 @@ import { logError, getRpsThresholds, getTtfbThreshold, getProjects, selectProjec
if (!__ENV.ACCESS_TOKEN) fail('ACCESS_TOKEN has not been set. Skipping...')
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getRpsThresholds(0.5) : getRpsThresholds()
export let ttfbThreshold = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getTtfbThreshold(2500) : getTtfbThreshold()
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /projects/:id/repository/tree`
@description: [Get a list of repository files and directories in a project](https://docs.gitlab.com/ee/api/repositories.html#list-repository-tree)
@flags: repo_storage
*/
import http from "k6/http";
......@@ -11,8 +12,8 @@ import { logError, getRpsThresholds, getTtfbThreshold, getProjects, selectProjec
if (!__ENV.ACCESS_TOKEN) fail('ACCESS_TOKEN has not been set. Skipping...')
export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getRpsThresholds(0.5) : getRpsThresholds()
export let ttfbThreshold = __ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? getTtfbThreshold(2500) : getTtfbThreshold()
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /:group`
@description: Web - Group Page. <br>Controllers: `GroupsController#show`, `Groups::ChildrenController#index`</br>
@issue: TBC
*/
import http from "k6/http";
......@@ -10,10 +11,10 @@ import { Rate } from "k6/metrics";
import { logError, getRpsThresholds, getTtfbThreshold, adjustRps, adjustStageVUs, getProjects, selectProject } from "../../lib/gpt_k6_modules.js";
export let endpointCount = 2
export let webProtoRps = adjustRps(__ENV.WEB_ENDPOINT_THROUGHPUT)
export let webProtoRps = adjustRps(__ENV.WEB_ENDPOINT_THROUGHPUT )
export let webProtoStages = adjustStageVUs(__ENV.WEB_ENDPOINT_THROUGHPUT)
export let rpsThresholds = getRpsThresholds(__ENV.WEB_ENDPOINT_THROUGHPUT, endpointCount)
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = getRpsThresholds(__ENV.WEB_ENDPOINT_THROUGHPUT* 0.7, endpointCount)
export let ttfbThreshold = getTtfbThreshold(5000)
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -2,6 +2,7 @@
/*
@endpoint: `GET /:group/:project/blob/master/:file_path`
@description: Web - Project Blob File. <br>Controllers: `Projects::BlobController#show`, `Projects::BlobController#show.json`</br>
@flags: repo_storage
*/
import http from "k6/http";
......@@ -12,8 +13,8 @@ import { logError, getRpsThresholds, getTtfbThreshold, adjustRps, adjustStageVUs
export let endpointCount = 2
export let webProtoRps = adjustRps(__ENV.WEB_ENDPOINT_THROUGHPUT)
export let webProtoStages = adjustStageVUs(__ENV.WEB_ENDPOINT_THROUGHPUT)
export let rpsThresholds = getRpsThresholds(__ENV.WEB_ENDPOINT_THROUGHPUT, endpointCount)
export let ttfbThreshold = getTtfbThreshold()
export let rpsThresholds = getRpsThresholds(__ENV.WEB_ENDPOINT_THROUGHPUT * (__ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? 0.5 : 1.0), endpointCount)
export let ttfbThreshold = getTtfbThreshold(__ENV.ENVIRONMENT_REPO_STORAGE == "nfs" ? 1000 : 0)
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -14,7 +14,7 @@ export let endpointCount = 7
export let webProtoRps = adjustRps(__ENV.WEB_ENDPOINT_THROUGHPUT)
export let webProtoStages = adjustStageVUs(__ENV.WEB_ENDPOINT_THROUGHPUT)
export let rpsThresholds = getRpsThresholds(__ENV.WEB_ENDPOINT_THROUGHPUT * 0.5, endpointCount)
export let ttfbThreshold = getTtfbThreshold(3000)
export let ttfbThreshold = getTtfbThreshold(4000)
export let successRate = new Rate("successful_requests")
export let options = {
thresholds: {
......
......@@ -44,12 +44,14 @@ module RunK6
File.join(File.dirname(k6_archive.path), 'k6')
end
def setup_env_vars(env_file:, options_file:, latency:)
def setup_env_vars(env_file:, options_file:)
env_vars = {}
env_file_vars = JSON.parse(File.read(env_file))
env_vars['ENVIRONMENT_NAME'] = ENV['ENVIRONMENT_NAME'].dup || env_file_vars['environment']['name']
env_vars['ENVIRONMENT_URL'] = (ENV['ENVIRONMENT_URL'].dup || env_file_vars['environment']['url']).chomp('/')
env_vars['ENVIRONMENT_LATENCY'] = ENV['ENVIRONMENT_LATENCY'].dup || env_file_vars['environment']['config']['latency']
env_vars['ENVIRONMENT_REPO_STORAGE'] = ENV['ENVIRONMENT_REPO_STORAGE'].dup || env_file_vars['environment']['config']['repo_storage']
env_vars['ENVIRONMENT_PROJECTS'] = env_file_vars['projects'].to_json
options_file_vars = JSON.parse(File.read(options_file))
......@@ -63,7 +65,6 @@ module RunK6
env_vars['RPS_THRESHOLD_MULTIPLIER'] ||= '0.8'
env_vars['SUCCESS_RATE_THRESHOLD'] ||= '0.95'
env_vars['TTFB_THRESHOLD'] ||= '500'
env_vars['TTFB_LATENCY'] ||= latency.to_s
env_vars['GIT_ENDPOINT_THROUGHPUT'] ||= '0.1'
env_vars['WEB_ENDPOINT_THROUGHPUT'] ||= '0.1'