Commit 79ba6ee3 authored by DJ Mountney's avatar DJ Mountney Committed by DJ Mountney

Merge branch 'publish-images-manual' into 'master'

Add job to release CNG images to gitlab.com registry from dev registry

See merge request charts/gitlab!699
parent 44071040
Pipeline #56438661 failed with stages
in 4 minutes and 38 seconds
......@@ -64,18 +64,6 @@ lint_package:
except:
- tags
release_package:
stage: package
when: always
script:
- curl --request POST --form "token=$CI_JOB_TOKEN" --form ref=master
--form "variables[CHART_NAME]=$CI_PROJECT_NAME"
--form "variables[RELEASE_REF]=$CI_COMMIT_REF_NAME"
https://gitlab.com/api/v4/projects/2860651/trigger/pipeline
only:
- tags@charts/gitlab
review:
stage: review
script:
......@@ -667,5 +655,32 @@ qa:
retry: 1
allow_failure: true
sync_images:
image: registry.gitlab.com/gitlab-org/gitlab-omnibus-builder:ruby_docker-0.0.7
stage: prepare
services:
- docker:dind
before_script: []
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375
script:
- bundle install
- bundle exec rake images:sync
only:
- tags@gitlab/charts/gitlab
release_package:
stage: package
script:
- curl --request POST --form "token=${COM_CHARTS_TRIGGER_TOKEN}" --form ref=master
--form "variables[CHART_NAME]=$CI_PROJECT_NAME"
--form "variables[RELEASE_REF]=$CI_COMMIT_REF_NAME"
https://gitlab.com/api/v4/projects/2860651/trigger/pipeline
only:
- tags@gitlab/charts/gitlab
dependencies:
- sync_images
before_script:
- *auto_devops
......@@ -5,3 +5,5 @@ gem 'aws-sdk-s3'
gem 'capybara'
gem 'selenium-webdriver'
gem 'capybara-screenshot'
gem 'docker-api'
gem 'rake'
......@@ -31,18 +31,24 @@ GEM
childprocess (0.9.0)
ffi (~> 1.0, >= 1.0.11)
diff-lcs (1.3)
docker-api (1.34.2)
excon (>= 0.47.0)
multi_json
excon (0.62.0)
ffi (1.9.25)
jmespath (1.4.0)
launchy (2.4.3)
addressable (~> 2.3)
mini_mime (1.0.0)
mini_portile2 (2.3.0)
multi_json (1.13.1)
nokogiri (1.8.2)
mini_portile2 (~> 2.3.0)
public_suffix (3.0.2)
rack (2.0.5)
rack-test (1.0.0)
rack (>= 1.0, < 3)
rake (12.3.2)
rspec (3.7.0)
rspec-core (~> 3.7.0)
rspec-expectations (~> 3.7.0)
......@@ -70,6 +76,8 @@ DEPENDENCIES
aws-sdk-s3
capybara
capybara-screenshot
docker-api
rake
rspec
selenium-webdriver
......
require_relative 'scripts/update-docker-images.rb'
namespace :images do
desc 'Sync images between dev and com registries'
task :sync do
CNGImageSync.execute
end
end
#!/usr/bin/env ruby
require 'docker'
require 'yaml'
require 'net/http'
require 'json'
require 'cgi'
require 'zip'
class CNGImageSync
CI_API_V4_URL = ENV['CI_API_V4_URL'] || "https://dev.gitlab.org/api/v4".freeze
DEV_REGISTRY_URL = "dev.gitlab.org:5005".freeze
COM_REGISTRY_URL = "registry.gitlab.com".freeze
DEV_PROJECT_PATH = ENV['DEV_CNG_PROJECT'] || "gitlab/charts/components/images".freeze
COM_PROJECT_PATH = ENV['COM_CNG_PROJECT'] || "gitlab-org/build/cng".freeze
DEV_PROJECT_REGISTRY = ENV['DEV_CNG_REGISTRY'] || "#{DEV_REGISTRY_URL}/#{DEV_PROJECT_PATH}".freeze
COM_PROJECT_REGISTRY = ENV['COM_CNG_REGISTRY'] || "#{COM_REGISTRY_URL}/#{COM_PROJECT_PATH}".freeze
DEV_REGISTRY_PASSWORD = ENV['FETCH_DEV_ARTIFACTS_PAT'] || ENV['CI_JOB_TOKEN']
COM_REGISTRY_PASSWORD = ENV['PUSH_IMAGES_PAT']
DEV_API_TOKEN = ENV['FETCH_DEV_ARTIFACTS_PAT']
GITLAB_VERSION = YAML.load_file('Chart.yaml')['appVersion'].strip.freeze
class << self
def get_api(uri, token = DEV_API_TOKEN)
req = Net::HTTP::Get.new(uri)
req['PRIVATE-TOKEN'] = token
res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
http.request(req)
end
res
end
def get_components(version)
artifact_uri = URI("#{CI_API_V4_URL}/projects/#{CGI.escape(DEV_PROJECT_PATH)}/jobs/artifacts/v#{version}-ee/raw/artifacts/image_versions.txt?job=component-details")
puts "Fetching component list from #{artifact_uri}"
res = get_api(artifact_uri)
components = res.body.split("\n")
components.map { |c| c.split(":") }.to_h
end
def authenticate_registry(registry, password)
# `CI_REGISTRY_USER` variable translates to `gitlab-ci-token` internally,
# which will work with GitLab's container registry authentication as long
# as the password is a valid PAT. In other words, as long as the password
# is a valid PAT, we need not bother about a username and can simply use
# `gitlab-ci-token`, which is what we are doing here.
Docker.authenticate!(username: ENV['CI_REGISTRY_USER'], password: password, serveraddress: registry)
end
def pull_and_tag_images(initial_registry, new_registry, components)
components.each do |component, version|
initial_ref = "#{initial_registry}/#{component}:#{version}".downcase
target_repo = "#{new_registry}/#{component}".downcase
puts "Pulling #{initial_ref}"
STDOUT.flush
image = Docker::Image.create(fromImage: initial_ref)
image.tag(repo: target_repo, tag: version)
end
end
def push_images(registry, components)
components.each do |component, version|
ref = "#{registry}/#{component}:#{version}".downcase
puts "Pushing #{ref}"
STDOUT.flush
image = Docker::Image.get(ref)
image.push(nil, repo_tag: ref)
end
end
def check_auth
message = <<~MESSAGE
Login credentials for registries are missing. Make sure the following environment variables are set
PUSH_IMAGES_PAT - PAT with access to gitlab.com API
MESSAGE
raise message if COM_REGISTRY_PASSWORD.nil?
end
def execute
check_auth
puts "Syncing images for version #{GITLAB_VERSION}"
components = get_components(GITLAB_VERSION)
authenticate_registry(DEV_REGISTRY_URL, DEV_REGISTRY_PASSWORD)
pull_and_tag_images(DEV_PROJECT_REGISTRY, COM_PROJECT_REGISTRY, components)
authenticate_registry(COM_REGISTRY_URL, COM_REGISTRY_PASSWORD)
push_images(COM_PROJECT_REGISTRY, components)
puts "Sync completed"
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment