Improve more feature flags - iteration 3
What does this MR do?
Third iteration after: !47208 (merged).
This iterates all YAMLs, and fetches milestone associated with the MR or Issue.
This nows tries to find common ancestor introducing a given feature flag and finding associated MR.
Result
$ grep -hr "^milestone:" config/feature_flags ee/config/feature_flags| sort | uniq -c | sort -n -r
59 milestone: '13.4'
53 milestone:
43 milestone: '13.5'
36 milestone: '13.6'
27 milestone: '13.2'
24 milestone: '13.3'
21 milestone: '13.0'
17 milestone: '12.10'
10 milestone: '13.1'
9 milestone: '12.4'
8 milestone: '12.8'
6 milestone: '12.9'
5 milestone: '12.0'
4 milestone: '12.6'
4 milestone: '12.5'
4 milestone: '11.10'
3 milestone: '12.2'
3 milestone: '11.6'
3 milestone: '11.11'
2 milestone: '12.1'
2 milestone: '11.9'
1 milestone: '13.7'
1 milestone: '12.7'
1 milestone: '12.3'
1 milestone: '11.8'
1 milestone: '11.7'
1 milestone: '11.3'
1 milestone: '10.8'
1 milestone: '10.6'
1 milestone: '10.3'
1 milestone: '10.0'Script
#!/usr/bin/env ruby
require 'yaml'
require 'json'
require 'net/http'
require_relative 'lib/feature/shared.rb'
paths = [
'config/feature_flags/**/*.yml',
'ee/config/feature_flags/**/*.yml'
]
@cached = {}
def get_json(url)
resp = Net::HTTP.get_response(URI(url + ".json"))
return unless resp.is_a?(Net::HTTPSuccess)
JSON.parse(resp.body)
end
def get_project(url)
if url.to_s.include?("/gitlab-org/gitlab/")
"gitlab-org%2Fgitlab"
elsif url.to_s.include?("/gitlab-org/gitlab-foss/")
"gitlab-org%2Fgitlab-foss"
end
end
def get_mr_json(url)
return unless url.to_s.include?("/merge_requests/")
project = get_project(url)
return unless project
id = url.to_s.split('/').last
@cached[url] ||= get_json("https://gitlab.com/api/v4/projects/#{project}/merge_requests/#{id}")
end
def get_issue_json(url)
return unless url.to_s.include?("/issues/")
project = get_project(url)
return unless project
id = url.to_s.split('/').last
@cached[url] ||= get_json("https://gitlab.com/api/v4/projects/#{project}/issues/#{id}")
end
def get_milestone(config)
get_mr_json(config['introduced_by_url'])&.dig("milestone", "title") ||
get_issue_json(config['rollout_issue_url'])&.dig("milestone", "title")
end
def find_group(labels)
return unless labels
labels.find { |label| label.start_with?('group::') }
end
def get_group(config)
find_group(get_mr_json(config['introduced_by_url'])&.dig("labels")) ||
find_group(get_mr_json(config['rollout_issue_url'])&.dig("labels")) ||
config['group']
end
OLD_TAGS = %w[
v7.10.0-ee
v7.11.0-ee
v7.12.0-ee
v7.13.0-ee
v7.14.0-ee
v8.0.0-ee
v8.1.0-ee
v8.2.0-ee
v8.3.0-ee
v8.4.0-ee
v8.5.0-ee
v8.6.0-ee
v8.7.0-ee
v8.8.0-ee
v8.9.0-ee
v8.10.0-ee
v8.11.0-ee
v8.12.0-ee
v8.13.0-ee
v8.14.0-ee
v8.15.0-ee
v8.16.0-ee
v8.17.0-ee
v9.0.0-ee
v9.1.0-ee
v9.2.0-ee
v9.3.0-ee
v9.4.0-ee
v9.5.0-ee
]
CE_TAGS = %w[
v10.0.0
v10.1.0
v10.2.0
v10.3.0
v10.4.0
v10.5.0
v10.6.0
v10.7.0
v10.8.0
v11.0.0
v11.1.0
v11.2.0
v11.3.0
v11.4.0
v11.5.0
v11.6.0
v11.7.0
v11.8.0
v11.9.0
v11.10.0
v11.11.0
v12.0.0
v12.1.0
v12.2.0
v12.3.0
v12.4.0
v12.5.0
v12.6.0
v12.7.0
v12.8.0
v12.9.0
v13.0.0
v13.1.0
]
EE_TAGS = %w[
v10.0.7-ee
v10.1.7-ee
v10.2.8-ee
v10.3.9-ee
v10.4.7-ee
v10.5.8-ee
v10.6.6-ee
v10.7.7-ee
v10.8.7-ee
v11.0.6-ee
v11.1.8-ee
v11.2.8-ee
v11.3.14-ee
v11.4.14-ee
v11.5.11-ee
v11.6.11-ee
v11.7.12-ee
v11.8.10-ee
v11.9.12-ee
v11.10.8-ee
v11.11.8-ee
v12.0.12-ee
v12.1.17-ee
v12.2.12-ee
v12.3.9-ee
v12.4.8-ee
v12.5.10-ee
v12.6.8-ee
v12.7.9-ee
v12.8.10-ee
v12.9.10-ee
v12.10.14-ee
v13.0.14-ee
v13.1.11-ee
v13.2.10-ee
v13.3.9-ee
v13.4.6-ee
v13.5.3-ee
master
]
# ~/.gitconfig
# [alias]
# find-merge = "!sh -c 'commit=$0 && branch=${1:-HEAD} && (git rev-list $commit..$branch --ancestry-path | cat -n; git rev-list $commit..$branch --first-parent | cat -n) | sort -k2 -s | uniq -f1 -d | sort -n | tail -1 | cut -f2'"
# show-merge = "!sh -c 'merge=$(git find-merge $0 $1) && [ -n \"$merge\" ] && git show $merge'"
def process_commit_id(id)
commit_message = `git log -1 #{id.strip}`
puts commit_message
commit_message = commit_message.lines
commit_message = commit_message.find { |cm| cm.include?('See merge request') }.to_s.strip
if commit_message.include?('gitlab-org/gitlab-ce!')
return commit_message.sub(
'See merge request gitlab-org/gitlab-ce!',
'https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/')
elsif commit_message.include?('gitlab-org/gitlab-foss!')
return commit_message.sub(
'See merge request gitlab-org/gitlab-foss!',
'https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/')
elsif commit_message.include?('gitlab-org/gitlab-ee!')
return commit_message.sub(
'See merge request gitlab-org/gitlab-ee!',
'https://gitlab.com/gitlab-org/gitlab/-/merge_requests/')
elsif commit_message.include?('gitlab-org/gitlab!')
return commit_message.sub(
'See merge request gitlab-org/gitlab!',
'https://gitlab.com/gitlab-org/gitlab/-/merge_requests/')
end
end
def get_introduced_by(config, tags)
prior_tag = tags.first
tags.each do |tag|
next if tag == prior_tag
puts "Searching #{config['name']} to #{prior_tag}..#{tag}..."
commit_id = `git log --pretty=oneline -G ':#{config['name']}[,)]' #{prior_tag}..#{tag} | tail -n 1 | awk '{print $1}'`.strip
unless commit_id.to_s.length > 0
prior_tag = tag
next
end
url = process_commit_id(commit_id)
return url if url
id = `git find-merge #{commit_id} #{tag}`.strip
url = process_commit_id(id)
return url if url
# merges = `git log #{commit_id}..#{tag} --merges --oneline --reverse | awk '{print $1}'`.lines.map(&:strip)
# merges.each do |id|
# next unless system("git merge-base --is-ancestor #{commit_id} #{id}")
# url = process_commit_id(id)
# return url if url
# end
return true
end
nil
end
def process_flag(path)
config = YAML.load_file(path)
unless config['introduced_by_url']
url = get_introduced_by(config, CE_TAGS) || get_introduced_by(config, EE_TAGS)
config['introduced_by_url'] = url if url.is_a?(String)
puts "#{config['name']}... INTRODUCED BY: #{config['introduced_by_url']}"
end
unless config['milestone']
config['milestone'] = get_milestone(config)
puts "#{config['name']}... MILESTONE: #{config['milestone']}... #{config['introduced_by_url']} #{config['rollout_issue_url']}"
end
unless config['group']
config['group'] = get_group(config)
puts "#{config['name']}... GROUP: #{config['group']}... #{config['introduced_by_url']} #{config['rollout_issue_url']}"
end
config_yaml = config.slice(
*::Feature::Shared::PARAMS.map(&:to_s)
).to_yaml
File.write(path, config_yaml)
end
flags_paths = ARGV
if flags_paths.empty?
flags_paths = paths.map do |path|
Dir.glob(path)
end.flatten.sort
end
queue = Queue.new
threads = 5.times.map do
Thread.new do
while flag_path = queue.pop
process_flag(flag_path)
end
end
end
flags_paths.each do |flag_path|
queue.push(flag_path)
end
queue.close
threads.each(&:join)Does this MR meet the acceptance criteria?
Conformity
Edited by Kamil Trzciński