Commit 7eba8537 authored by Sean McGivern's avatar Sean McGivern

Merge branch 'zj-remove-git-rake-tasks' into 'master'

Remove git rake tasks

Closes #41699 and gitaly#768

See merge request gitlab-org/gitlab-ce!20806
parents 5a3948a5 4600265e
Pipeline #26876881 passed with stages
in 41 minutes and 5 seconds
---
title: Remove gitlab:user:check_repos, gitlab:check_repo, gitlab:git:prune, gitlab:git:gc, and gitlab:git:repack
merge_request: 20806
author:
type: removed
# Check Rake Tasks
# Integrity Check Rake Task
## Repository Integrity
......@@ -28,14 +28,8 @@ exactly which repositories are causing the trouble.
### Check all GitLab repositories
>**Note:**
>
> - `gitlab:repo:check` has been deprecated in favor of `gitlab:git:fsck`
> - [Deprecated][ce-15931] in GitLab 10.4.
> - `gitlab:repo:check` will be removed in the future. [Removal issue][ce-41699]
This task loops through all repositories on the GitLab server and runs the
3 integrity checks described previously.
integrity check described previously.
**Omnibus Installation**
......@@ -49,33 +43,6 @@ sudo gitlab-rake gitlab:git:fsck
sudo -u git -H bundle exec rake gitlab:git:fsck RAILS_ENV=production
```
### Check repositories for a specific user
This task checks all repositories that a specific user has access to. This is important
because sometimes you know which user is experiencing trouble but you don't know
which project might be the cause.
If the rake task is executed without brackets at the end, you will be prompted
to enter a username.
**Omnibus Installation**
```bash
sudo gitlab-rake gitlab:user:check_repos
sudo gitlab-rake gitlab:user:check_repos[<username>]
```
**Source Installation**
```bash
sudo -u git -H bundle exec rake gitlab:user:check_repos RAILS_ENV=production
sudo -u git -H bundle exec rake gitlab:user:check_repos[<username>] RAILS_ENV=production
```
Example output:
![gitlab:user:check_repos output](../img/raketasks/check_repos_output.png)
## Uploaded Files Integrity
Various types of files can be uploaded to a GitLab installation by users.
......@@ -167,5 +134,4 @@ The LDAP check Rake task will test the bind_dn and password credentials
executed as part of the `gitlab:check` task, but can run independently.
See [LDAP Rake Tasks - LDAP Check](ldap.md#check) for details.
[ce-15931]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15931
[ce-41699]: https://gitlab.com/gitlab-org/gitlab-ce/issues/41699
[git-fsck]: https://git-scm.com/docs/git-fsck
......@@ -385,14 +385,6 @@ namespace :gitlab do
end
end
namespace :repo do
desc "GitLab | Check the integrity of the repositories managed by GitLab"
task check: :gitlab_environment do
puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red)
Rake::Task["gitlab:git:fsck"].execute
end
end
namespace :orphans do
desc 'Gitlab | Check for orphaned namespaces and repositories'
task check: :gitlab_environment do
......@@ -422,23 +414,6 @@ namespace :gitlab do
end
end
namespace :user do
desc "GitLab | Check the integrity of a specific user's repositories"
task :check_repos, [:username] => :gitlab_environment do |t, args|
username = args[:username] || prompt("Check repository integrity for username? ".color(:blue))
user = User.find_by(username: username)
if user
repo_dirs = user.authorized_projects.map do |p|
p.repository.path_to_repo
end
repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
else
puts "\nUser '#{username}' not found".color(:red)
end
end
end
# Helper methods
##########################
......
namespace :gitlab do
namespace :git do
desc "GitLab | Git | Repack"
task repack: :gitlab_environment do
failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo")
if failures.empty?
puts "Done".color(:green)
else
output_failures(failures)
end
end
desc "GitLab | Git | Run garbage collection on all repos"
task gc: :gitlab_environment do
failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting")
if failures.empty?
puts "Done".color(:green)
else
output_failures(failures)
end
end
desc "GitLab | Git | Prune all repos"
task prune: :gitlab_environment do
failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune")
if failures.empty?
puts "Done".color(:green)
else
output_failures(failures)
end
end
desc 'GitLab | Git | Check all repos integrity'
task fsck: :gitlab_environment do
failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo|
check_config_lock(repo)
check_ref_locks(repo)
end
if failures.empty?
puts "Done".color(:green)
else
output_failures(failures)
end
end
def perform_git_cmd(cmd, message)
puts "Starting #{message} on all repositories"
failures = []
all_repos do |repo|
if system(*cmd, chdir: repo)
puts "Performed #{message} at #{repo}"
else
failures << repo
Project.find_each(batch_size: 100) do |project|
begin
project.repository.fsck
rescue => e
failures << "#{project.full_path} on #{project.repository_storage}: #{e}"
end
yield(repo) if block_given?
puts "Performed integrity check for #{project.repository.full_path}"
end
failures
end
def output_failures(failures)
puts "The following repositories reported errors:".color(:red)
failures.each { |f| puts "- #{f}" }
end
def check_config_lock(repo_dir)
config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
end
def check_ref_locks(repo_dir)
lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
if lock_files.present?
puts "Ref lock files exist:".color(:red)
lock_files.each { |lock_file| puts " #{lock_file}" }
if failures.empty?
puts "Done".color(:green)
else
puts "No ref lock files exist".color(:green)
puts "The following repositories reported errors:".color(:red)
failures.each { |f| puts "- #{f}" }
end
end
end
......
......@@ -2,45 +2,19 @@ require 'rake_helper'
describe 'gitlab:git rake tasks' do
let(:base_path) { 'tmp/tests/default_storage' }
before(:all) do
@default_storage_hash = Gitlab.config.repositories.storages.default.to_h
end
let!(:project) { create(:project, :repository) }
before do
Rake.application.rake_require 'tasks/gitlab/git'
storages = { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => base_path)) }
path = Settings.absolute("#{base_path}/@hashed/1/2/test.git")
FileUtils.mkdir_p(path)
Gitlab::Popen.popen(%W[git -C #{path} init --bare])
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
allow_any_instance_of(String).to receive(:color) { |string, _color| string }
stub_warn_user_is_not_gitlab
end
after do
FileUtils.rm_rf(Settings.absolute(base_path))
end
describe 'fsck' do
it 'outputs the integrity check for a repo' do
expect { run_rake_task('gitlab:git:fsck') }.to output(%r{Performed Checking integrity at .*@hashed/1/2/test.git}).to_stdout
end
it 'errors out about config.lock issues' do
FileUtils.touch(Settings.absolute("#{base_path}/@hashed/1/2/test.git/config.lock"))
expect { run_rake_task('gitlab:git:fsck') }.to output(/file exists\? ... yes/).to_stdout
end
it 'errors out about ref lock issues' do
FileUtils.mkdir_p(Settings.absolute("#{base_path}/@hashed/1/2/test.git/refs/heads"))
FileUtils.touch(Settings.absolute("#{base_path}/@hashed/1/2/test.git/refs/heads/blah.lock"))
expect { run_rake_task('gitlab:git:fsck') }.to output(/Ref lock files exist:/).to_stdout
expect { run_rake_task('gitlab:git:fsck') }.to output(/Performed integrity check for/).to_stdout
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment