Commit 5d63e91a authored by Douwe Maan's avatar Douwe Maan Committed by Mayra Cabrera

Merge branch 'backport-rd-5566-improve-performance-of-repository-size-limit-check' into 'master'

Backport some changes from gitlab-ee!5476

See merge request gitlab-org/gitlab-ce!18760
parent 9cf92f40
Pipeline #21996129 failed with stages
in 25 minutes and 50 seconds
......@@ -62,6 +62,12 @@ module Gitlab
end
end
# Returns an array of Blob instances just with the metadata, that means
# the data attribute has no content.
def batch_metadata(repository, blob_references)
batch(repository, blob_references, blob_size_limit: 0)
end
# Find LFS blobs given an array of sha ids
# Returns array of Gitlab::Git::Blob
# Does not guarantee blob data will be set
......
......@@ -28,13 +28,14 @@ module Gitlab
# 85bc2f9753afd5f4fc5d7c75f74f8d526f26b4f3 107 R060\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee
def parse(raw_change)
@blob_id, @blob_size, @raw_operation, raw_paths = raw_change.split(' ', 4)
@blob_size = @blob_size.to_i
@operation = extract_operation
@old_path, @new_path = extract_paths(raw_paths)
end
def extract_paths(file_path)
case operation
when :renamed
when :copied, :renamed
file_path.split(/\t/)
when :deleted
[file_path, nil]
......
......@@ -579,29 +579,40 @@ module Gitlab
count_commits(from: from, to: to, **options)
end
# Counts the amount of commits between `from` and `to`.
def count_commits_between(from, to, options = {})
count_commits(from: from, to: to, **options)
end
# old_rev and new_rev are commit ID's
# the result of this method is an array of Gitlab::Git::RawDiffChange
def raw_changes_between(old_rev, new_rev)
gitaly_migrate(:raw_changes_between) do |is_enabled|
if is_enabled
gitaly_repository_client.raw_changes_between(old_rev, new_rev)
.each_with_object([]) do |msg, arr|
msg.raw_changes.each { |change| arr << ::Gitlab::Git::RawDiffChange.new(change) }
end
else
result = []
@raw_changes_between ||= {}
circuit_breaker.perform do
Open3.pipeline_r(git_diff_cmd(old_rev, new_rev), format_git_cat_file_script, git_cat_file_cmd) do |last_stdout, wait_threads|
last_stdout.each_line { |line| result << ::Gitlab::Git::RawDiffChange.new(line.chomp!) }
@raw_changes_between[[old_rev, new_rev]] ||= begin
return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA
if wait_threads.any? { |waiter| !waiter.value&.success? }
raise ::Gitlab::Git::Repository::GitError, "Unabled to obtain changes between #{old_rev} and #{new_rev}"
gitaly_migrate(:raw_changes_between) do |is_enabled|
if is_enabled
gitaly_repository_client.raw_changes_between(old_rev, new_rev)
.each_with_object([]) do |msg, arr|
msg.raw_changes.each { |change| arr << ::Gitlab::Git::RawDiffChange.new(change) }
end
else
result = []
circuit_breaker.perform do
Open3.pipeline_r(git_diff_cmd(old_rev, new_rev), format_git_cat_file_script, git_cat_file_cmd) do |last_stdout, wait_threads|
last_stdout.each_line { |line| result << ::Gitlab::Git::RawDiffChange.new(line.chomp!) }
if wait_threads.any? { |waiter| !waiter.value&.success? }
raise ::Gitlab::Git::Repository::GitError, "Unabled to obtain changes between #{old_rev} and #{new_rev}"
end
end
end
end
result
result
end
end
end
rescue ArgumentError => e
......
......@@ -251,6 +251,26 @@ describe Gitlab::Git::Blob, seed_helper: true do
end
end
describe '.batch_metadata' do
let(:blob_references) do
[
[SeedRepo::Commit::ID, "files/ruby/popen.rb"],
[SeedRepo::Commit::ID, 'six']
]
end
subject { described_class.batch_metadata(repository, blob_references) }
it 'returns an empty data attribute' do
first_blob, last_blob = subject
expect(first_blob.data).to be_blank
expect(first_blob.path).to eq("files/ruby/popen.rb")
expect(last_blob.data).to be_blank
expect(last_blob.path).to eq("six")
end
end
describe '.batch_lfs_pointers' do
let(:tree_object) { repository.rugged.rev_parse('master^{tree}') }
......
......@@ -12,7 +12,7 @@ describe Gitlab::Git::RawDiffChange do
expect(change.operation).to eq(:unknown)
expect(change.old_path).to be_blank
expect(change.new_path).to be_blank
expect(change.blob_size).to be_blank
expect(change.blob_size).to eq(0)
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment