...
 
Commits (56)
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:release-tools"
stages:
- test
- deploy
- automation
- chatops
......@@ -19,73 +18,8 @@ cache: &global-cache
# test ------------------------------------------------------------------------
dependency_scanning:
image: docker:stable
variables:
DOCKER_DRIVER: overlay2
allow_failure: true
tags: []
before_script: []
cache: {}
dependencies: []
services:
- docker:stable-dind
script:
- export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
- docker run
--env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}"
--volume "$PWD:/code"
--volume /var/run/docker.sock:/var/run/docker.sock
"registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code
artifacts:
paths: [gl-dependency-scanning-report.json]
except:
- schedules
- triggers
rubocop:
<<: *with-bundle
stage: test
except:
- schedules
- triggers
script:
- bundle exec rubocop
specs:
<<: *with-bundle
stage: test
except:
- schedules
- triggers
retry: 1
script:
- cp .env.example .env
- git config --global user.email "you@example.com"
- git config --global user.name "Your Name"
- bundle exec rspec
artifacts:
paths:
- coverage/assets
- coverage/index.html
# deploy ----------------------------------------------------------------------
pages:
stage: deploy
script:
- mkdir -p public/
- mv coverage/ public/
dependencies:
- specs
artifacts:
paths:
- public/
only:
- master
except:
- schedules
- triggers
# automation ------------------------------------------------------------------
......@@ -159,16 +93,35 @@ validate-security-merge-requests:
variables:
- $VALIDATE_SECURITY_MERGE_REQUESTS
create-auto-deploy-branches:
auto-deploy:prepare:
<<: *with-bundle
stage: automation
script:
- bundle exec rake auto_deploy:prepare
- bundle exec rake 'auto_deploy:prepare'
only:
refs:
- schedules
variables:
- $CREATE_AUTO_DEPLOY_BRANCHES
- $CREATE_AUTO_DEPLOY_BRANCH_SCHEDULE == "true"
auto-deploy:pick:
<<: *with-bundle
stage: automation
script:
- bundle exec rake 'auto_deploy:pick'
only:
variables:
- $CHERRY_PICK_AUTO_DEPLOY_BRANCH_SCHEDULE == "true"
auto-deploy:trigger:
<<: *with-bundle
stage: automation
script:
- bundle exec rake "passing_build:ee[$AUTO_DEPLOY_BRANCH,true]"
only:
variables:
- $PASSING_BUILD_AUTO_DEPLOY_SCHEDULE == "true"
# chatops ---------------------------------------------------------------------
......
......@@ -25,9 +25,57 @@ end
namespace :auto_deploy do
desc "Prepare for auto-deploy by creating branches from the latest green commit on gitlab-ee and omnibus-gitlab"
task :prepare do
pipeline_id = ENV['CI_PIPELINE_IID']
abort('CI_PIPELINE_IID must be set for this rake task'.colorize(:red)) unless pipeline_id
ReleaseTools::Services::AutoDeployBranchService.new(pipeline_id).create_auto_deploy_branches!
ReleaseTools::Services::AutoDeployBranchService
.new(ReleaseTools::AutoDeploy::Naming.branch)
.create_branches!
end
desc 'Pick commits into the auto deploy branches'
task :pick do
icon = ->(result) { result.success? ? "✓" : "✗" }
auto_deploy_branch = ENV.fetch('AUTO_DEPLOY_BRANCH') do |name|
abort("`#{name}` must be set for this rake task".colorize(:red))
end
version = auto_deploy_branch.sub(/\A(\d+)-(\d+)-auto-deploy.*/, '\1.\2')
version = ReleaseTools::Version.new(version).to_ee
target_branch = ReleaseTools::AutoDeployBranch.new(version, auto_deploy_branch)
$stdout.puts "--> Picking for #{version}..."
ee_results = ReleaseTools::CherryPick::Service
.new(ReleaseTools::Project::GitlabEe, version, target_branch)
.execute
ee_results.each do |result|
$stdout.puts " #{icon.call(result)} #{result.url}"
end
version = version.to_ce
$stdout.puts "--> Picking for #{version}..."
ce_results = ReleaseTools::CherryPick::Service
.new(ReleaseTools::Project::GitlabCe, version, target_branch)
.execute
ce_results.each do |result|
$stdout.puts " #{icon.call(result)} #{result.url}"
end
return if ReleaseTools::SharedStatus.dry_run?
if ee_results.any?(&:success?) || ce_results.any?(&:success?)
ReleaseTools::GitlabOpsClient.run_trigger(
ReleaseTools::Project::MergeTrain,
ENV.fetch('MERGE_TRAIN_TRIGGER_TOKEN'),
'master',
{
CE_BRANCH: auto_deploy_branch,
EE_BRANCH: auto_deploy_branch,
MERGE_MANUAL: '1'
}
)
end
end
end
......
......@@ -32,6 +32,8 @@ Dotenv.load
$LOAD_PATH.unshift(__dir__)
require 'release_tools/auto_deploy/naming'
require 'release_tools/auto_deploy_branch'
require 'release_tools/branch'
require 'release_tools/branch_creation'
require 'release_tools/version'
......@@ -74,6 +76,7 @@ require 'release_tools/project/gitlab_ce'
require 'release_tools/project/gitlab_ee'
require 'release_tools/project/gitlab_provisioner'
require 'release_tools/project/helm_gitlab'
require 'release_tools/project/merge_train'
require 'release_tools/project/omnibus_gitlab'
require 'release_tools/project/release/tasks'
require 'release_tools/project/release_tools'
......
# frozen_string_literal: true
module ReleaseTools
module AutoDeploy
class Naming
BRANCH_FORMAT = '%<major>d-%<minor>d-auto-deploy-%<pipeline_id>07d-ee'
TAG_FORMAT = '%<major>d.%<minor>d.%<pipeline_id>d+%<ee_ref>s.%<omnibus_ref>s'
def self.branch
new.branch
end
def self.tag(ee_ref:, omnibus_ref:)
new.tag(ee_ref: ee_ref, omnibus_ref: omnibus_ref)
end
def initialize
@pipeline_id = ENV.fetch('CI_PIPELINE_IID') do |key|
raise ArgumentError, "`#{key}` must be set in order to proceed"
end
end
def branch
format(
BRANCH_FORMAT,
major: version.first,
minor: version.last,
pipeline_id: @pipeline_id
)
end
def tag(ee_ref:, omnibus_ref:)
format(
TAG_FORMAT,
major: version.first,
minor: version.last,
pipeline_id: @pipeline_id,
ee_ref: ee_ref,
omnibus_ref: omnibus_ref
)
end
def version
@version ||=
begin
milestone = ReleaseTools::GitlabClient
.current_milestone
.title
unless milestone.match?(/\A\d+\.\d+\z/)
raise ArgumentError, "Invalid version from milestone: #{milestone}"
end
milestone.split('.')
end
end
end
end
end
# frozen_string_literal: true
module ReleaseTools
# Represents an auto-deploy branch for purposes of cherry-picking
class AutoDeployBranch
attr_reader :version
attr_reader :branch_name
def initialize(version, branch_name)
@version = version
@branch_name = branch_name
end
def exists?
true
end
# Included in cherry-pick summary messages
def pick_destination
"`#{branch_name}`"
end
def release_issue
MonthlyIssue.new(version: version)
end
end
end
......@@ -4,11 +4,11 @@ module ReleaseTools
module CherryPick
class CommentNotifier
attr_reader :version
attr_reader :prep_mr
attr_reader :target
def initialize(version, prep_mr)
def initialize(version, target:)
@version = version
@prep_mr = prep_mr
@target = target
end
def comment(pick_result)
......@@ -45,22 +45,22 @@ module ReleaseTools
MSG
end
create_merge_request_comment(prep_mr, message.join("\n"))
create_merge_request_comment(target, message.join("\n"))
end
def blog_post_summary(picked)
return if version.rc?
return if version.rc? || version.monthly?
return if picked.empty?
message = <<~MSG
The following merge requests were picked into #{prep_mr.url}:
The following merge requests were picked into #{target.url}:
```
#{markdown_list(picked.collect(&:to_markdown))}
```
MSG
create_issue_comment(prep_mr.release_issue, message)
create_issue_comment(target.release_issue, message)
end
private
......@@ -71,7 +71,7 @@ module ReleaseTools
def successful_comment(pick_result)
comment = <<~MSG
Automatically picked into #{prep_mr.url}, will merge into
Automatically picked into #{target.pick_destination}, will merge into
`#{version.stable_branch}` ready for `#{version}`.
/unlabel #{PickIntoLabel.reference(version)}
......@@ -89,7 +89,9 @@ module ReleaseTools
comment = <<~MSG
@#{author} This merge request could not automatically be picked into
`#{version.stable_branch}` for `#{version}` and will need manual
intervention.
intervention. Please create a new MR targeting the source branch
of #{target.pick_destination}, and assign to release managers.
MSG
create_merge_request_comment(pick_result.merge_request, comment)
......@@ -108,6 +110,9 @@ module ReleaseTools
end
def create_merge_request_comment(merge_request, comment)
return unless merge_request.respond_to?(:project_id)
return unless merge_request.respond_to?(:iid)
client.create_merge_request_comment(
merge_request.project_id,
merge_request.iid,
......
......@@ -2,35 +2,33 @@
module ReleaseTools
module CherryPick
# Performs automated cherry picking to a preparation branch for the specified
# Performs automated cherry picking to a target branch for the specified
# version.
#
# For the given project, this service will look for merged merge requests on
# that project labeled `Pick into X.Y` and attempt to cherry-pick their merge
# commits into the preparation merge request for the specified version.
# commits into the target merge request for the specified version.
#
# It will post a comment to each merge request with the status of the pick,
# and then a final summary message to the preparation merge request with the
# list of picked and unpicked merge requests for the release managers to
# perform any further manual actions.
# and a final summary message with the list of picked and unpicked merge
# requests for the release managers to perform any further manual actions.
class Service
# TODO (rspeicher): Support `SharedStatus.security_release?`
REMOTE = :gitlab
attr_reader :project
attr_reader :version
attr_reader :target
def initialize(project, version)
def initialize(project, version, target)
@project = project
@version = version
@target = target
assert_version!
assert_target!
@prep_mr = PreparationMergeRequest.new(version: version)
@target_branch = @target.branch_name
assert_prep_mr!
@prep_branch = @prep_mr.preparation_branch_name
@results = []
end
......@@ -59,17 +57,15 @@ module ReleaseTools
raise "Invalid version provided: `#{version}`" unless version.valid?
end
def assert_prep_mr!
unless @prep_mr.exists?
raise "Preparation merge request not found for `#{version}`"
end
def assert_target!
raise 'Invalid cherry-pick target provided' unless target.exists?
end
def notifier
if SharedStatus.dry_run?
@notifier ||= ConsoleNotifier.new(version, @prep_mr)
@notifier ||= ConsoleNotifier.new(version, target: target)
else
@notifier ||= CommentNotifier.new(version, @prep_mr)
@notifier ||= CommentNotifier.new(version, target: target)
end
end
......@@ -80,7 +76,7 @@ module ReleaseTools
GitlabClient.cherry_pick(
project,
ref: merge_request.merge_commit_sha,
target: @prep_branch
target: @target_branch
)
end
......
......@@ -19,6 +19,7 @@ module ReleaseTools
def_delegator :client, :update_variable
def_delegator :client, :create_commit
def_delegator :client, :create_tag
end
class MissingMilestone
......
# frozen_string_literal: true
require 'active_support/core_ext/string/access'
module ReleaseTools
class PassingBuild
attr_reader :project, :ref
......@@ -23,17 +25,36 @@ module ReleaseTools
$stdout.puts "#{component}: #{version}".indent(4)
end
trigger_build(versions) if args.trigger_build
# TODO: Move up to Rake task so it's explicit, and isolate $stdout
if args.trigger_build
commit = update_omnibus(versions)
tag_omnibus(commit, versions)
end
end
def trigger_build(version_map)
commit = ReleaseTools::ComponentVersions.update_omnibus(ref, version_map)
def update_omnibus(version_map)
ReleaseTools::ComponentVersions.update_omnibus(ref, version_map).tap do |commit|
url = commit_url(ReleaseTools::Project::OmnibusGitlab, commit.short_id)
$stdout.puts "Updated Omnibus versions at #{url}".indent(4)
end
end
def tag_omnibus(commit, versions)
prod_client = ReleaseTools::GitlabClient
project = ReleaseTools::Project::OmnibusGitlab
pipeline_id = ENV.fetch('CI_PIPELINE_IID')
url = commit_url(ReleaseTools::Project::OmnibusGitlab, commit.short_id)
ob_ref = commit.short_id
ee_ref = versions['VERSION'].first(ob_ref.length)
$stdout.puts "Updated Omnibus versions at #{url}".indent(4)
# NOTE: The tag name includes the pipeline ID in order to approximate
# semantic versioning for packages.
# TODO (rspeicher): Use ReleaseTools::AutoDeploy::Naming.tag
tag_name = "1.1.#{pipeline_id}+#{ee_ref}.#{ob_ref}"
# TODO: Tagging
prod_client.create_tag(project, tag_name, commit.id)
end
private
......
......@@ -11,7 +11,7 @@ module ReleaseTools
end
def source_branch
preparation_branch_name
branch_name
end
def target_branch
......@@ -38,7 +38,7 @@ module ReleaseTools
end
end
def preparation_branch_name
def branch_name
if version.rc?
"#{version.stable_branch}-prepare-rc#{version.rc}"
else
......@@ -46,6 +46,10 @@ module ReleaseTools
end
end
def pick_destination
url
end
def ee?
version.ee?
end
......
# frozen_string_literal: true
module ReleaseTools
module Project
class MergeTrain < BaseProject
REMOTES = {
gitlab: 'git@ops.gitlab.net:gitlab-org/merge-train.git'
}.freeze
end
end
end
......@@ -4,14 +4,16 @@ module ReleaseTools
module Services
class AutoDeployBranchService
include BranchCreation
PIPELINE_ID_PADDING = 7
CI_VAR_AUTO_DEPLOY = 'AUTO_DEPLOY_BRANCH'
def initialize(pipeline_id)
@pipeline_id = pipeline_id.to_s.rjust(PIPELINE_ID_PADDING, '0')
attr_reader :branch_name
def initialize(branch_name)
@branch_name = branch_name
end
def create_auto_deploy_branches!
def create_branches!
# Find passing commits before creating branches
ref_deployer = latest_successful_ref(Project::Deployer, gitlab_ops_client)
ref_ce = latest_successful_ref(Project::GitlabCe)
......@@ -28,14 +30,6 @@ module ReleaseTools
private
def version
@version ||= gitlab_client.current_milestone.title.tr('.', '-')
end
def branch_name
"#{version}-auto-deploy-#{@pipeline_id}"
end
def update_auto_deploy_ci
gitlab_client.update_variable(Project::ReleaseTools.path, CI_VAR_AUTO_DEPLOY, branch_name)
rescue Gitlab::Error::NotFound
......
......@@ -18,9 +18,10 @@ namespace :release do
# CE
version = get_version(args).to_ce
target = ReleaseTools::PreparationMergeRequest.new(version: version)
$stdout.puts "--> Picking for #{version}..."
results = ReleaseTools::CherryPick::Service
.new(ReleaseTools::Project::GitlabCe, version)
.new(ReleaseTools::Project::GitlabCe, version, target)
.execute
results.each do |result|
......@@ -31,7 +32,7 @@ namespace :release do
version = version.to_ee
$stdout.puts "--> Picking for #{version}..."
results = ReleaseTools::CherryPick::Service
.new(ReleaseTools::Project::GitlabEe, version)
.new(ReleaseTools::Project::GitlabEe, version, target)
.execute
results.each do |result|
......
# frozen_string_literal: true
require 'spec_helper'
describe ReleaseTools::AutoDeploy::Naming do
describe 'initialize' do
it 'fails when CI_PIPELINE_IID is unset' do
ClimateControl.modify(CI_PIPELINE_IID: nil) do
expect { described_class.new }
.to raise_error(/must be set in order to proceed/)
end
end
end
describe '.branch' do
it 'returns a branch name in the appropriate format' do
allow(ReleaseTools::GitlabClient).to receive(:current_milestone)
.and_return(double(title: '4.2'))
ClimateControl.modify(CI_PIPELINE_IID: '1234') do
expect(described_class.branch).to eq('4-2-auto-deploy-0001234-ee')
end
end
end
describe '.tag' do
it 'returns a tag name in the appropriate format' do
allow(ReleaseTools::GitlabClient).to receive(:current_milestone)
.and_return(double(title: '4.2'))
ClimateControl.modify(CI_PIPELINE_IID: '1234') do
expect(described_class.tag(ee_ref: 'abcdef', omnibus_ref: 'fedcba'))
.to eq('4.2.1234+abcdef.fedcba')
end
end
end
describe '#version' do
it 'raises an error when the milestone format is unexpected' do
allow(ReleaseTools::GitlabClient).to receive(:current_milestone)
.and_return(double(title: 'Backlog'))
ClimateControl.modify(CI_PIPELINE_IID: '1234') do
expect { described_class.new.version }
.to raise_error(/Invalid version from milestone/)
end
end
end
end
......@@ -4,14 +4,15 @@ require 'spec_helper'
describe ReleaseTools::CherryPick::CommentNotifier do
let(:client) { spy('GitlabClient') }
let(:version) { ReleaseTools::Version.new('11.4.0') }
let(:version) { ReleaseTools::Version.new('11.4.1') }
let(:prep_mr) do
double(
iid: 1,
project_id: 2,
url: 'https://example.com',
release_issue: double(project: spy, iid: 4)
release_issue: double(project: spy, iid: 4),
pick_destination: 'https://example.com'
)
end
......@@ -34,7 +35,7 @@ describe ReleaseTools::CherryPick::CommentNotifier do
end
subject do
described_class.new(version, prep_mr)
described_class.new(version, target: prep_mr)
end
before do
......
......@@ -4,20 +4,25 @@ require 'spec_helper'
describe ReleaseTools::CherryPick::Service do
let(:version) { ReleaseTools::Version.new('11.4.0-rc8') }
let(:target) { double(branch_name: 'branch-name', exists?: true) }
subject { described_class.new(ReleaseTools::Project::GitlabCe, version) }
subject do
described_class.new(ReleaseTools::Project::GitlabCe, version, target)
end
describe 'initialize' do
it 'validates version argument' do
expect { described_class.new(double, double(valid?: false)) }
version = double(valid?: false)
expect { described_class.new(double, version, target) }
.to raise_error(RuntimeError, /Invalid version provided/)
end
it 'validates preparation MR' do
stub_const('ReleaseTools::PreparationMergeRequest', spy(exists?: false))
it 'validates target exists' do
target = double(exists?: false)
expect { described_class.new(double, version) }
.to raise_error(RuntimeError, /Preparation merge request not found/)
expect { described_class.new(double, version, target) }
.to raise_error(RuntimeError, /Invalid cherry-pick target provided/)
end
end
......@@ -43,7 +48,6 @@ describe ReleaseTools::CherryPick::Service do
end
end
let(:target) { '11-4-stable-prepare-rc8' }
let(:notifier) { spy }
let(:picks) do
Gitlab::PaginatedResponse.new(
......
......@@ -24,7 +24,7 @@ describe ReleaseTools::PassingBuild do
.to raise_error(/Unable to find a passing/)
end
it 'fetches component versions', :silence_stdout do
it 'updates and tags Omnibus when `trigger_build` is true', :silence_stdout do
expect(fake_commits).to receive(:latest_dev_green_build_commit)
.and_return(fake_commit)
......@@ -32,26 +32,17 @@ describe ReleaseTools::PassingBuild do
.to receive(:get).with(project, fake_commit.id)
.and_return(version_map)
expect(service).not_to receive(:trigger_build)
service.execute(double(trigger_build: false))
end
it 'triggers a build when specified', :silence_stdout do
expect(fake_commits).to receive(:latest_dev_green_build_commit)
expect(service).to receive(:update_omnibus)
.with(version_map)
.and_return(fake_commit)
expect(ReleaseTools::ComponentVersions)
.to receive(:get).with(project, fake_commit.id)
.and_return(version_map)
expect(service).to receive(:trigger_build).with(version_map)
expect(service).to receive(:tag_omnibus)
.with(fake_commit, version_map)
service.execute(double(trigger_build: true))
end
end
describe '#trigger_build' do
describe '#update_omnibus' do
let(:fake_client) { spy }
it 'updates Omnibus versions', :silence_stdout do
......@@ -59,7 +50,11 @@ describe ReleaseTools::PassingBuild do
.to receive(:update_omnibus).with('master', version_map)
.and_return(double('Commit', short_id: 'abcdefg'))
service.trigger_build(version_map)
service.update_omnibus(version_map)
end
end
describe '#tag_omnibus' do
# TODO (rspeicher): All of it!
end
end
......@@ -30,22 +30,22 @@ describe ReleaseTools::PreparationMergeRequest do
expect(merge_request.milestone).to eq '9.4'
end
describe '#preparation_branch_name' do
describe '#branch_name' do
it 'appends the stable branch with patch number' do
expect(merge_request.preparation_branch_name).to eq '9-4-stable-patch-1'
expect(ee_merge_request.preparation_branch_name).to eq '9-4-stable-ee-patch-1'
expect(merge_request.branch_name).to eq '9-4-stable-patch-1'
expect(ee_merge_request.branch_name).to eq '9-4-stable-ee-patch-1'
end
context 'release candidate' do
it 'appends the stable branch with rc number' do
expect(rc_merge_request.preparation_branch_name).to eq '9-4-stable-prepare-rc2'
expect(ee_rc_merge_request.preparation_branch_name).to eq '9-4-stable-ee-prepare-rc2'
expect(rc_merge_request.branch_name).to eq '9-4-stable-prepare-rc2'
expect(ee_rc_merge_request.branch_name).to eq '9-4-stable-ee-prepare-rc2'
end
end
end
it 'sets source_branch to the new branch' do
expect(merge_request.source_branch).to eq merge_request.preparation_branch_name
expect(merge_request.source_branch).to eq merge_request.branch_name
end
it 'sets target_branch to the stable branch' do
......
......@@ -7,16 +7,17 @@ describe ReleaseTools::Services::AutoDeployBranchService do
let(:internal_client_ops) { spy('ReleaseTools::GitlabOpsClient') }
let(:branch_commit) { double(latest_successful: double(id: '1234')) }
subject(:service) { described_class.new('9000') }
subject(:service) { described_class.new('branch-name') }
before do
allow(service).to receive(:gitlab_client).and_return(internal_client)
allow(service).to receive(:gitlab_ops_client).and_return(internal_client_ops)
end
describe '#create_auto_deploy_branches!', :silence_stdout do
describe '#create_branches!', :silence_stdout do
it 'creates auto-deploy branches for gitlab-ee and gitlab-ce' do
branch_name = '11-10-auto-deploy-0009000'
branch_name = 'branch-name'
expect(ReleaseTools::Commits).to receive(:new).and_return(branch_commit).exactly(4).times
expect(internal_client_ops).to receive(:create_branch).with(
branch_name,
......@@ -43,8 +44,9 @@ describe ReleaseTools::Services::AutoDeployBranchService do
'AUTO_DEPLOY_BRANCH',
branch_name
)
without_dry_run do
service.create_auto_deploy_branches!
service.create_branches!
end
end
end
......