Commit d62ad83a authored by Grant Young's avatar Grant Young

Add Git Push wait and Unsafe flag

parent 7bbc2341
......@@ -20,7 +20,7 @@
- echo "registry.gitlab.com/gitlab-org/quality/performance-images/gitlab-ce-performance:$MAJOR_VERSION.$MINOR_VERSION.0-ce.0"
- bin/run-gitlab-docker --image registry.gitlab.com/gitlab-org/quality/performance-images/gitlab-ce-performance:$MAJOR_VERSION.$MINOR_VERSION.0-ce.0
- sleep $(( 30 * CI_NODE_INDEX ))
- bin/run-k6 -s -e $ENVIRONMENT_NAME.json -o $OPTIONS_FILE.json $([ -n "$QUARANTINED" ] && echo "-q")
- bin/run-k6 -s -e $ENVIRONMENT_NAME.json -o $OPTIONS_FILE.json -u $([ -n "$QUARANTINED" ] && echo "-q")
only:
refs:
- schedules
......
......@@ -51,7 +51,7 @@ k6:
stage: test
extends: .k6-base
script:
- bin/run-k6 -s -e $ENVIRONMENT_NAME.json -o $OPTIONS_FILE.json $([ -n "$QUARANTINED" ] && echo "-q")
- bin/run-k6 -s -e $ENVIRONMENT_NAME.json -o $OPTIONS_FILE.json -u $([ -n "$QUARANTINED" ] && echo "-q")
only:
refs:
- schedules
......
......@@ -25,7 +25,7 @@ end
raise 'Environment Variable CI_PROJECT_ACCESS_TOKEN must be set to proceed. See command help for more info' unless ENV['CI_PROJECT_ACCESS_TOKEN']
puts "Collecting test info..."
tests = RunK6.get_tests(k6_dir: k6_dir, test_paths: ["tests"], quarantined: true, scenarios: true, read_only: false)
tests = RunK6.get_tests(k6_dir: k6_dir, test_paths: ["tests"], quarantined: true, scenarios: true, unsafe: true)
aggregated_tests = []
tests.each do |test|
aggregated_tests << TestInfo.parse_test_docs_for_info(test)
......
......@@ -16,7 +16,7 @@ require 'tmpdir'
k6_dir = Pathname.new(File.expand_path('../k6', __dir__)).relative_path_from(Dir.pwd)
gpt_version = '1.2.0'
gpt_version = '1.2.1'
puts Rainbow("GitLab Performance Tool (GPT) v#{gpt_version} - Performance test runner for GitLab environments based on k6").color(230, 83, 40)
opts = Optimist.options do
......@@ -32,7 +32,7 @@ opts = Optimist.options do
opt :latency, "Specify a network latency time in MS to increase applicable test thresholds by. Only typically used in slow network environments.", type: :integer, default: 0
opt :quarantined, "Include any tests inside the test directory's quarantined subfolder when true.", type: :flag, default: false
opt :excludes, "List of words used to exclude tests by matching against their names.", type: :strings, default: []
opt :read_only, "Enable read-only mode to run tests with only GET requests", type: :flag, default: false
opt :unsafe, "Include any tests that perform unsafe requests (POST, PUT, DELETE, PATCH)", type: :flag, default: false
banner "\nEnvironment Variable(s):"
banner " ACCESS_TOKEN A valid GitLab Personal Access Token for the specified environment that's required by various tests. The token should come from a User that has admin access for the project(s) to be tested and have API, read_repository, and write_repository permissions. (Default: nil)"
banner "\nExamples:"
......@@ -59,7 +59,7 @@ env_vars = RunK6.setup_env_vars(env_file: env_file, options_file: options_file,
env_version = RunK6.get_env_version(env_vars: env_vars)
start_time = Time.now
tests = RunK6.get_tests(k6_dir: k6_dir, test_paths: opts[:tests], test_excludes: opts[:excludes], quarantined: opts[:quarantined], scenarios: opts[:scenarios], read_only: opts[:read_only], env_version: env_version['version'], env_vars: env_vars)
tests = RunK6.get_tests(k6_dir: k6_dir, test_paths: opts[:tests], test_excludes: opts[:excludes], quarantined: opts[:quarantined], scenarios: opts[:scenarios], unsafe: opts[:unsafe], env_version: env_version['version'], env_vars: env_vars)
RunK6.prepare_tests(tests: tests, env_vars: env_vars)
results_home = ENV['GPT_DOCKER_RESULTS_DIR'] || "#{k6_dir}/results"
......
......@@ -13,6 +13,7 @@ On this page we'll detail how to configure and run [k6](https://k6.io/) tests ag
* [Options](#options)
* [Tests](#tests)
* [Test Types](#test-types)
* [Unsafe Tests](#unsafe-tests)
* [Running the Tests with the Tool](#running-the-tests-with-the-tool)
* [Docker (Recommended)](#docker-recommended)
* [Linux](#linux)
......@@ -228,6 +229,12 @@ In addition to the above we have experimental [Scenario](../k6/tests/scenarios)
Finally, some specific tests also have individual supporting documentation. These can be found here - [Test Documentation](test_docs/README.md).
#### Unsafe Tests
Some tests provided with the Tool perform [unsafe http requests](https://developer.mozilla.org/en-US/docs/Glossary/safe) (POST, PUT, DELETE or PATCH) and will create data on the environment. As such, these tests are not included by default when running the Tool, with the `--unsafe` flag being provided to include them (refer to the Tools help output above).
These tests are typically fine to run against your environment if you're using the recommended `gitlabhq` project or any other project that can be easily removed after testing has completed. It's unrecommended to run these tests against a permanent project or a live GitLab environment.
## Running the Tests with the Tool
When all of the above configuration is in place the tests can be run with the tool. This can be done in one of two ways - With our [Docker image](https://hub.docker.com/r/gitlab/gitlab-performance-tool) (recommended) or natively on a Linux based system.
......@@ -239,7 +246,7 @@ The recommended way to run the Tool is with our Docker image, [gitlab/gitlab-per
The image will start running the tests when it's called. The full options for running the tool can be seen by getting the help output by running `docker run -it gitlab/gitlab-performance-tool --help`:
```
GitLab Performance Tool (GPT) v1.2.0 - Performance test runner for GitLab environments based on k6
GitLab Performance Tool (GPT) v1.2.1 - Performance test runner for GitLab environments based on k6
Documentation: https://gitlab.com/gitlab-org/quality/performance/blob/master/docs/README.md
......@@ -253,7 +260,7 @@ Options:
-l, --latency=<i> Specify a network latency time in MS to increase applicable test thresholds by. Only typically used in slow network environments. (Default: 0)
-q, --quarantined Include any tests inside the test directory's quarantined subfolder when true.
-x, --excludes=<s+> List of words used to exclude tests by matching against their names. (Default: )
-r, --read-only Enable read-only mode to run tests with only GET requests
-u, --unsafe Include any tests that perform unsafe requests (POST, PUT, DELETE, PATCH)
Environment Variable(s):
ACCESS_TOKEN A valid GitLab Personal Access Token for the specified environment that's required by various tests. The token should come from a User that has admin access for
......
/*global __ENV : true */
import http from "k6/http";
import { fail } from "k6";
import { fail, sleep } from "k6";
import { logError } from "./gpt_k6_modules.js";
//------------------- Git Pull-------------------//
......@@ -117,3 +117,18 @@ export function updateProjectPipelinesSetting(project, state) {
let res = http.put(`${__ENV.ENVIRONMENT_URL}/api/v4/projects/${project['group']}%2F${project['name']}`, formdata, params);
/20(0|1)/.test(res.status) ? console.log(`Project Pipelines setting was ${state}`) : (logError(res), fail(`Error with Project Pipelines setting update.`));
}
export function waitForGitSidekiqQueue() {
let params = { headers: { "Accept": "application/json", "PRIVATE-TOKEN": `${__ENV.ACCESS_TOKEN}` } };
let res, queueSize;
console.log('Waiting for all Sidekiq enqueued jobs to finish before proceeding...')
do {
sleep(5);
res = http.get(`${__ENV.ENVIRONMENT_URL}/api/v4/sidekiq/job_stats`, params);
queueSize = JSON.parse(res.body)['jobs']['enqueued'];
queueSize > 0 ? console.log(`Sidekiq enqueue is currently ${queueSize}. Waiting...`) : console.log(`Sidekiq enqueue is ${queueSize}. Proceeding...`)
}
while (queueSize > 0);
}
......@@ -8,7 +8,7 @@ import http from "k6/http";
import { group, fail } from "k6";
import { Rate } from "k6/metrics";
import { logError, getRpsThresholds, getTtfbThreshold, getProjects, selectProject, checkProjectKeys, adjustRps, adjustStageVUs } from "../../lib/gpt_k6_modules.js";
import { getRefsListGitPush, pushRefsData, checkCommitExists, prepareGitPushData, updateProjectPipelinesSetting } from "../../lib/gpt_git_functions.js";
import { getRefsListGitPush, pushRefsData, checkCommitExists, prepareGitPushData, updateProjectPipelinesSetting, waitForGitSidekiqQueue } from "../../lib/gpt_git_functions.js";
if (!__ENV.ACCESS_TOKEN) fail('ACCESS_TOKEN has not been set. Skipping...')
......@@ -25,7 +25,7 @@ export let options = {
},
rps: gitProtoRps,
stages: gitProtoStages,
teardownTimeout: '30s'
teardownTimeout: '600s'
};
export let authEnvUrl = __ENV.ENVIRONMENT_URL.replace(/(^https?:\/\/)(.*)/, `$1test:${__ENV.ACCESS_TOKEN}@$2`)
......@@ -60,7 +60,7 @@ export default function () {
let refsListResponse = getRefsListGitPush(authEnvUrl, project);
/20(0|1)/.test(refsListResponse.status) ? successRate.add(true) : (successRate.add(false), logError(refsListResponse));
});
if (project.data) {
group("Git - Git Push Data", function () {
let pushResponses = pushRefsData(authEnvUrl, project);
......@@ -76,6 +76,7 @@ export default function () {
}
export function teardown() {
waitForGitSidekiqQueue();
projects.forEach(project => {
// Ensure that all branches were restored to the original `branch_current_head_sha`
let params = {
......
......@@ -83,7 +83,7 @@ module RunK6
GitTest.prepare_git_push_data(env_vars: env_vars) unless tests.grep(/git_push/).empty? || env_vars.empty?
end
def get_tests(k6_dir:, test_paths:, test_excludes: [], quarantined:, scenarios:, read_only:, env_version: '-', env_vars: {})
def get_tests(k6_dir:, test_paths:, test_excludes: [], quarantined:, scenarios:, unsafe:, env_version: '-', env_vars: {})
tests = []
test_paths.each do |test_path|
# Add any tests found within given and default folders matching name
......@@ -106,7 +106,7 @@ module RunK6
tests.reject! { |test| test.include? exclude }
end
tests.select! { |test| TestInfo.test_is_read_only?(test) } if read_only
tests.reject! { |test| TestInfo.test_has_unsafe_requests?(test) } unless unsafe
tests.select! { |test| TestInfo.test_supported_by_version?(test, env_version) } unless env_version == '-'
tests
......
......@@ -8,7 +8,7 @@ module TestInfo
# Get
def get_known_issues(k6_dir)
tests = RunK6.get_tests(k6_dir: k6_dir, test_paths: ["tests"], quarantined: true, scenarios: true, read_only: false)
tests = RunK6.get_tests(k6_dir: k6_dir, test_paths: ["tests"], quarantined: true, scenarios: true, unsafe: true)
aggregated_issues = []
tests.each do |test|
......@@ -64,16 +64,16 @@ module TestInfo
# Check
def test_is_read_only?(test_file)
read_only = true
def test_has_unsafe_requests?(test_file)
write_methods = %w[post put del patch]
File.open(test_file, "r") do |test_file_content|
test_file_content.each_line do |line|
line_has_write_method = write_methods.any? { |write_method| line.include?("http.#{write_method}") }
read_only = false if line_has_write_method
return true if line_has_write_method
end
end
read_only
false
end
def test_supported_by_version?(test_file, gitlab_version_string)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment