Commit b7b9eaf1 authored by Ahmad Hassan's avatar Ahmad Hassan Committed by DJ Mountney

Backup testing infrastructure

parent 522afba7
......@@ -43,6 +43,7 @@ stages:
- staging
- canary
- stable
- specs
- qa
- package
- cleanup
......@@ -101,6 +102,10 @@ review:
- create_secret
- deploy
- echo "export QA_ENVIRONMENT_URL=gitlab-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN" >> variables
- echo "export GITLAB_ROOT_DOMAIN=$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN" >> variables
- echo "export GITLAB_URL=gitlab-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN" >> variables
- echo "export REGISTRY_URL=registry-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN" >> variables
- echo "export S3_ENDPOINT=https://minio-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN" >> variables
artifacts:
paths:
- variables
......@@ -118,7 +123,6 @@ review:
except:
- master
stop_review:
stage: cleanup
variables:
......@@ -214,6 +218,8 @@ stable:
- deploy
- delete canary
- echo "export QA_ENVIRONMENT_URL=gitlab.$AUTO_DEVOPS_DOMAIN" >> variables
- echo "export GITLAB_ROOT_DOMAIN=$AUTO_DEVOPS_DOMAIN" >> variables
- echo "export S3_ENDPOINT=https://minio.$AUTO_DEVOPS_DOMAIN" >> variables
artifacts:
paths:
- variables
......@@ -228,6 +234,47 @@ stable:
- master
kubernetes: active
.specs: &specs
image: gunesmes/docker-capybara-chrome
stage: specs
services:
- docker:dind
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375
GITLAB_PASSWORD: $ROOT_PASSWORD
RELEASE_NAME: $CI_ENVIRONMENT_SLUG
S3_CONFIG_PATH: /etc/gitlab/minio
script:
- source variables
- apt-get update && apt-get install -y --no-install-recommends curl ca-certificates
- curl -sSL https://get.docker.com/ | sh
- curl -LsO https://storage.googleapis.com/kubernetes-release/release/v1.9.3/bin/linux/amd64/kubectl
- chmod +x kubectl
- mv kubectl /usr/local/bin/kubectl
- mkdir -p /etc/gitlab/minio
- kubectl get secret ${CI_ENVIRONMENT_SLUG}-minio-secret -o jsonpath='{.data.accesskey}' | base64 --decode > /etc/gitlab/minio/accesskey
- kubectl get secret ${CI_ENVIRONMENT_SLUG}-minio-secret -o jsonpath='{.data.secretkey}' | base64 --decode > /etc/gitlab/minio/secretkey
- bundle install --without non_test
- bundle exec rspec -c -f d spec
review_specs:
<<: *specs
environment:
name: review/$CI_COMMIT_REF_NAME
url: https://gitlab-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN
except:
refs:
- master
production_specs:
<<: *specs
environment:
name: production
url: https://gitlab.$AUTO_DEVOPS_DOMAIN
only:
refs:
- master
qa:
image: registry.gitlab.com/gitlab-org/gitlab-omnibus-builder:ruby_docker-0.0.7
stage: qa
......@@ -353,6 +400,7 @@ changelog_manager:
--set gitlab.sidekiq.resources.requests.cpu=200m \
--set gitlab.sidekiq.maxReplicas=2 \
--set gitlab.gitlab-shell.resources.requests.cpu=100m \
--set gitlab.task-runner.enabled=true \
--set gitlab.gitlab-shell.maxReplicas=3 \
--set redis.resources.requests.cpu=100m \
--set minio.resources.requests.cpu=100m \
......
......@@ -31,3 +31,4 @@ build/
CHANGELOG
changelogs/
bin/
spec/
source 'https://rubygems.org'
gem "rugged"
gem 'rspec'
gem 'aws-sdk-s3'
gem 'capybara'
gem 'selenium-webdriver'
gem 'chromedriver-helper'
gem "activesupport"
group :development, :test do
gem 'rspec', '~> 3.6.0'
group :non_test do
gem "rugged"
end
GEM
remote: https://rubygems.org/
specs:
activesupport (5.2.0)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
tzinfo (~> 1.1)
addressable (2.5.2)
public_suffix (>= 2.0.2, < 4.0)
archive-zip (0.11.0)
io-like (~> 0.3.0)
aws-eventstream (1.0.0)
aws-partitions (1.91.0)
aws-sdk-core (3.21.2)
aws-eventstream (~> 1.0)
aws-partitions (~> 1.0)
aws-sigv4 (~> 1.0)
jmespath (~> 1.0)
aws-sdk-kms (1.5.0)
aws-sdk-core (~> 3)
aws-sigv4 (~> 1.0)
aws-sdk-s3 (1.13.0)
aws-sdk-core (~> 3, >= 3.21.2)
aws-sdk-kms (~> 1)
aws-sigv4 (~> 1.0)
aws-sigv4 (1.0.2)
capybara (3.2.1)
addressable
mini_mime (>= 0.1.3)
nokogiri (~> 1.8)
rack (>= 1.6.0)
rack-test (>= 0.6.3)
xpath (~> 3.1)
childprocess (0.9.0)
ffi (~> 1.0, >= 1.0.11)
chromedriver-helper (1.2.0)
archive-zip (~> 0.10)
nokogiri (~> 1.8)
concurrent-ruby (1.0.5)
diff-lcs (1.3)
ffi (1.9.25)
i18n (1.0.1)
concurrent-ruby (~> 1.0)
io-like (0.3.0)
jmespath (1.4.0)
mini_mime (1.0.0)
mini_portile2 (2.3.0)
minitest (5.11.3)
nokogiri (1.8.2)
mini_portile2 (~> 2.3.0)
public_suffix (3.0.2)
rack (2.0.5)
rack-test (1.0.0)
rack (>= 1.0, < 3)
rspec (3.7.0)
rspec-core (~> 3.7.0)
rspec-expectations (~> 3.7.0)
rspec-mocks (~> 3.7.0)
rspec-core (3.7.1)
rspec-support (~> 3.7.0)
rspec-expectations (3.7.0)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.7.0)
rspec-mocks (3.7.0)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.7.0)
rspec-support (3.7.1)
rubyzip (1.2.1)
rugged (0.27.1)
selenium-webdriver (3.12.0)
childprocess (~> 0.5)
rubyzip (~> 1.2)
thread_safe (0.3.6)
tzinfo (1.2.5)
thread_safe (~> 0.1)
xpath (3.1.0)
nokogiri (~> 1.8)
PLATFORMS
ruby
DEPENDENCIES
activesupport
aws-sdk-s3
capybara
chromedriver-helper
rspec
rugged
selenium-webdriver
BUNDLED WITH
1.16.2
require 'spec_helper'
describe "Restoring a backup" do
before(:all) do
ensure_backups_on_object_storage
stdout, status = restore_from_backup
fail stdout unless status.success?
stdout, status = enforce_root_password(ENV['GITLAB_PASSWORD']) if ENV['GITLAB_PASSWORD']
fail stdout unless status.success?
end
describe 'Restored gitlab instance' do
before { sign_in }
it 'Home page should show projects' do
visit '/'
page.save_screenshot("/tmp/screenshots/1.png")
expect(page).to have_content 'Projects'
expect(page).to have_content 'Administrator / testproject1'
end
it 'Navigating to testproject1 repo should work' do
visit '/root/testproject1'
page.save_screenshot("/tmp/screenshots/2.png")
expect(page).to have_content 'Dockerfile'
end
it 'Should have runner registered' do
visit '/admin/runners'
expect(page.all('#content-body > div > div.runners-content > div > table > tbody > tr').count).to be > 0
end
it 'Issue attachments should load correctly' do
visit '/root/testproject1/issues/1'
image_selector = '#content-body > div.issue-details.issuable-details > div.detail-page-description.content-block > div:nth-child(2) > div > div.description.js-task-list-container.is-task-list-enabled > div.wiki > p > a > img'
expect(page).to have_selector(image_selector)
image_src = page.evaluate_script("$('#{image_selector}')[0].src")
open(image_src) do |f|
expect(f.status[0]).to eq '200'
end
end
it 'Could pull image from registry' do
stdout, status = Open3.capture2e("docker login #{registry_url} --username root --password #{ENV['GITLAB_PASSWORD']}")
expect(status.success?).to eq true
fail "Login failed: #{stdout}" unless status.success?
stdout, status = Open3.capture2e("docker pull #{registry_url}/root/testproject1/master:d88102fe7cf105b72643ecb9baf41a03070c9f1b")
fail "Pulling image failed: #{stdout}" unless status.success?
end
end
describe 'Backups' do
it 'Should be able to backup an identical tar' do
stdout, status = backup_instance
fail stdout unless status.success?
object_storage.get_object(
response_target: '/tmp/original_backup.tar',
bucket: 'gitlab-backups',
key: '0_11.0.0-pre_gitlab_backup.tar'
)
cmd = 'mkdir -p /tmp/original_backup && tar -xf /tmp/original_backup.tar -C /tmp/original_backup'
stdout, status = Open3.capture2e(cmd)
fail stdout unless status.success?
object_storage.get_object(
response_target: '/tmp/test_generated_backup.tar',
bucket: 'gitlab-backups',
key: 'test-backup_gitlab_backup.tar'
)
cmd = 'mkdir -p /tmp/test_backup && tar -xf /tmp/test_generated_backup.tar -C /tmp/test_backup'
stdout, status = Open3.capture2e(cmd)
fail stdout unless status.success?
Dir.glob("/tmp/original_backup/*") do |file|
expect(File.exist?("/tmp/test_backup/#{File.basename(file)}")).to be_truthy
# extract every tar file
if File.extname(file) == 'tar'
cmd = "tar -xf #{file} -C /tmp/original_backup"
stdout, status = Open3.capture2e(cmd)
fail stdout unless status.success?
cmd = "tar -xf #{file.gsub('original_backup', 'test_backup')} -C /tmp/test_backup"
stdout, status = Open3.capture2e(cmd)
fail stdout unless status.success?
end
end
Dir.glob("/tmp/original_backup/**/*") do |file|
next if ['tar', '.gz'].include? File.extname(file)
next if File.directory?(file)
next if File.basename(file) == 'backup_information.yml'
test_counterpart = file.gsub('original_backup', 'test_backup')
expect(File.exist?(test_counterpart)).to be_truthy, "Expected #{test_counterpart} to exist"
expect(Digest::MD5.hexdigest(File.read(file))).to eq(Digest::MD5.hexdigest(File.read(test_counterpart))),
"Expected #{file} to equal #{test_counterpart}"
end
end
end
end
module Gitlab
def self.included(klass)
klass.extend(TestHelper)
end
module TestHelper
def full_command(cmd)
"kubectl exec -it #{pod_name} -- #{cmd}"
end
def sign_in
visit '/users/sign_in'
fill_in 'Username or email', with: 'root'
fill_in 'Password', with: ENV['GITLAB_PASSWORD']
click_button 'Sign in'
page.save_screenshot("/tmp/screenshots/sign_in.png")
end
def enforce_root_password(password)
rails_dir = ENV['RAILS_DIR'] || '/home/git/gitlab'
cmd = full_command("#{rails_dir}/bin/rails runner \"user = User.find(1); user.password='#{password}'; user.password_confirmation='#{password}'; user.save!\"")
stdout, status = Open3.capture2e(cmd)
return [stdout, status]
end
def gitlab_url
protocol = ENV['PROTOCOL'] || 'https'
instance_url = ENV['GITLAB_URL'] || "gitlab.#{ENV['GITLAB_ROOT_DOMAIN']}"
"#{protocol}://#{instance_url}"
end
def registry_url
ENV['REGISTRY_URL'] || "registry.#{ENV['GITLAB_ROOT_DOMAIN']}"
end
def restore_from_backup
backup = ENV['BACKUP_TIMESTAMP'] || '0_11.0.0-pre'
cmd = full_command("backup-utility --restore -t #{backup}")
stdout, status = Open3.capture2e(cmd)
return [stdout, status]
end
def backup_instance
cmd = full_command("backup-utility --backup -t test-backup")
stdout, status = Open3.capture2e(cmd)
return [stdout, status]
end
def pod_name
filters = 'app=task-runner'
if ENV['RELEASE_NAME']
filters="#{filters},release=#{ENV['RELEASE_NAME']}"
end
@pod ||= `kubectl get pod -l #{filters} -o jsonpath="{.items[0].metadata.name}"`
end
def object_storage
return @object_storage if @object_storage
if ENV['S3_CONFIG_PATH']
s3_access_key = File.read("#{ENV['S3_CONFIG_PATH']}/accesskey")
s3_secret_key = File.read("#{ENV['S3_CONFIG_PATH']}/secretkey")
end
s3_access_key ||= ENV['S3_ACCESS_KEY']
s3_secret_key ||= ENV['S3_SECRET_KEY']
conf = {
region: ENV['S3_REGION'] || 'us-east-1',
access_key_id: s3_access_key,
secret_access_key: s3_secret_key,
endpoint: ENV['S3_ENDPOINT'],
force_path_style: true
}
@object_storage = Aws::S3::Client.new(conf)
end
def ensure_backups_on_object_storage
storage_url = 'https://storage.googleapis.com/gitlab-charts-ci/test-backups'
backup_file_names = ['11.0.0-pre_gitlab_backup.tar']
backup_file_names.each do |file_name|
file = open("#{storage_url}/#{file_name}").read
object_storage.put_object(
bucket: 'gitlab-backups',
key: "0_#{file_name}",
body: file
)
puts "Uploaded #{file_name}"
end
end
end
end
RSpec.configure do |config|
config.run_all_when_everything_filtered = true
config.filter_run :focus
config.order = 'random'
require 'aws-sdk-s3'
require 'open-uri'
require 'open3'
require 'capybara/rspec'
require 'selenium-webdriver'
require 'gitlab_test_helper'
config.mock_with :rspec do |mocks|
mocks.verify_partial_doubles = true
end
include Gitlab::TestHelper
Capybara.register_driver :headless_chrome do |app|
capabilities = Selenium::WebDriver::Remote::Capabilities.chrome(
chromeOptions: { args: %w(headless disable-gpu no-sandbox disable-dev-shm-usage) }
)
Capybara::Selenium::Driver.new app,
browser: :chrome,
desired_capabilities: capabilities
end
Capybara.configure do |config|
config.run_server = false
config.default_driver = :headless_chrome
config.app_host = gitlab_url
end
config.around(:example, :silence_stdout) do |example|
expect { example.run }.to output.to_stdout
RSpec.configure do |config|
config.include Capybara::DSL
config.expect_with :rspec do |expectations|
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
end
config.around(:example, :silence_stderr) do |example|
expect { example.run }.to output.to_stderr
config.mock_with :rspec do |mocks|
mocks.verify_partial_doubles = true
end
config.shared_context_metadata_behavior = :apply_to_host_groups
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment