Commit 3fab8dac authored by Robert Speicher's avatar Robert Speicher

Merge branch 'mk/fix-move-upload-files-on-group-transfer' into 'master'

Fix moving local, unhashed upload or pages directories during group transfer

Closes #43993

See merge request gitlab-org/gitlab-ce!17658
parents 31012697 040e01a5
Pipeline #18924194 passed with stages
in 35 minutes and 23 seconds
...@@ -7,29 +7,24 @@ module Storage ...@@ -7,29 +7,24 @@ module Storage
raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry') raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry')
end end
expires_full_path_cache parent_was = if parent_changed? && parent_id_was.present?
Namespace.find(parent_id_was) # raise NotFound early if needed
# Move the namespace directory in all storage paths used by member projects end
repository_storage_paths.each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, full_path_was)
# Ensure new directory exists before moving it (if there's a parent)
gitlab_shell.add_namespace(repository_storage_path, parent.full_path) if parent
unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path) expires_full_path_cache
Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}" move_repositories
# if we cannot move namespace directory we should rollback if parent_changed?
# db changes in order to prevent out of sync between db and fs former_parent_full_path = parent_was&.full_path
raise Gitlab::UpdatePathError.new('namespace directory cannot be moved') parent_full_path = parent&.full_path
end Gitlab::UploadsTransfer.new.move_namespace(path, former_parent_full_path, parent_full_path)
Gitlab::PagesTransfer.new.move_namespace(path, former_parent_full_path, parent_full_path)
else
Gitlab::UploadsTransfer.new.rename_namespace(full_path_was, full_path)
Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
end end
Gitlab::UploadsTransfer.new.rename_namespace(full_path_was, full_path)
Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
remove_exports! remove_exports!
# If repositories moved successfully we need to # If repositories moved successfully we need to
...@@ -57,6 +52,26 @@ module Storage ...@@ -57,6 +52,26 @@ module Storage
private private
def move_repositories
# Move the namespace directory in all storage paths used by member projects
repository_storage_paths.each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, full_path_was)
# Ensure new directory exists before moving it (if there's a parent)
gitlab_shell.add_namespace(repository_storage_path, parent.full_path) if parent
unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path)
Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Gitlab::UpdatePathError.new('namespace directory cannot be moved')
end
end
end
def old_repository_storage_paths def old_repository_storage_paths
@old_repository_storage_paths ||= repository_storage_paths @old_repository_storage_paths ||= repository_storage_paths
end end
......
---
title: Fix missing uploads after group transfer
merge_request: 17658
author:
type: fixed
module Gitlab module Gitlab
# This class is used to move local, unhashed files owned by projects to their new location
class ProjectTransfer class ProjectTransfer
def move_project(project_path, namespace_path_was, namespace_path) # nil parent_path (or parent_path_was) represents a root namespace
new_namespace_folder = File.join(root_dir, namespace_path) def move_namespace(path, parent_path_was, parent_path)
FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder) parent_path_was ||= ''
from = File.join(root_dir, namespace_path_was, project_path) parent_path ||= ''
to = File.join(root_dir, namespace_path, project_path) new_parent_folder = File.join(root_dir, parent_path)
FileUtils.mkdir_p(new_parent_folder)
from = File.join(root_dir, parent_path_was, path)
to = File.join(root_dir, parent_path, path)
move(from, to, "") move(from, to, "")
end end
alias_method :move_project, :move_namespace
def rename_project(path_was, path, namespace_path) def rename_project(path_was, path, namespace_path)
base_dir = File.join(root_dir, namespace_path) base_dir = File.join(root_dir, namespace_path)
move(path_was, path, base_dir) move(path_was, path, base_dir)
......
...@@ -21,30 +21,77 @@ describe Gitlab::ProjectTransfer do ...@@ -21,30 +21,77 @@ describe Gitlab::ProjectTransfer do
describe '#move_project' do describe '#move_project' do
it "moves project upload to another namespace" do it "moves project upload to another namespace" do
FileUtils.mkdir_p(File.join(@root_dir, @namespace_path_was, @project_path)) path_to_be_moved = File.join(@root_dir, @namespace_path_was, @project_path)
expected_path = File.join(@root_dir, @namespace_path, @project_path)
FileUtils.mkdir_p(path_to_be_moved)
@project_transfer.move_project(@project_path, @namespace_path_was, @namespace_path) @project_transfer.move_project(@project_path, @namespace_path_was, @namespace_path)
expected_path = File.join(@root_dir, @namespace_path, @project_path)
expect(Dir.exist?(expected_path)).to be_truthy expect(Dir.exist?(expected_path)).to be_truthy
end end
end end
describe '#move_namespace' do
context 'when moving namespace from root into another namespace' do
it "moves namespace projects' upload" do
child_namespace = 'test_child_namespace'
path_to_be_moved = File.join(@root_dir, child_namespace, @project_path)
expected_path = File.join(@root_dir, @namespace_path, child_namespace, @project_path)
FileUtils.mkdir_p(path_to_be_moved)
@project_transfer.move_namespace(child_namespace, nil, @namespace_path)
expect(Dir.exist?(expected_path)).to be_truthy
end
end
context 'when moving namespace from one parent to another' do
it "moves namespace projects' upload" do
child_namespace = 'test_child_namespace'
path_to_be_moved = File.join(@root_dir, @namespace_path_was, child_namespace, @project_path)
expected_path = File.join(@root_dir, @namespace_path, child_namespace, @project_path)
FileUtils.mkdir_p(path_to_be_moved)
@project_transfer.move_namespace(child_namespace, @namespace_path_was, @namespace_path)
expect(Dir.exist?(expected_path)).to be_truthy
end
end
context 'when moving namespace from having a parent to root' do
it "moves namespace projects' upload" do
child_namespace = 'test_child_namespace'
path_to_be_moved = File.join(@root_dir, @namespace_path_was, child_namespace, @project_path)
expected_path = File.join(@root_dir, child_namespace, @project_path)
FileUtils.mkdir_p(path_to_be_moved)
@project_transfer.move_namespace(child_namespace, @namespace_path_was, nil)
expect(Dir.exist?(expected_path)).to be_truthy
end
end
end
describe '#rename_project' do describe '#rename_project' do
it "renames project" do it "renames project" do
FileUtils.mkdir_p(File.join(@root_dir, @namespace_path, @project_path_was)) path_to_be_moved = File.join(@root_dir, @namespace_path, @project_path_was)
expected_path = File.join(@root_dir, @namespace_path, @project_path)
FileUtils.mkdir_p(path_to_be_moved)
@project_transfer.rename_project(@project_path_was, @project_path, @namespace_path) @project_transfer.rename_project(@project_path_was, @project_path, @namespace_path)
expected_path = File.join(@root_dir, @namespace_path, @project_path)
expect(Dir.exist?(expected_path)).to be_truthy expect(Dir.exist?(expected_path)).to be_truthy
end end
end end
describe '#rename_namespace' do describe '#rename_namespace' do
it "renames namespace" do it "renames namespace" do
FileUtils.mkdir_p(File.join(@root_dir, @namespace_path_was, @project_path)) path_to_be_moved = File.join(@root_dir, @namespace_path_was, @project_path)
expected_path = File.join(@root_dir, @namespace_path, @project_path)
FileUtils.mkdir_p(path_to_be_moved)
@project_transfer.rename_namespace(@namespace_path_was, @namespace_path) @project_transfer.rename_namespace(@namespace_path_was, @namespace_path)
expected_path = File.join(@root_dir, @namespace_path, @project_path)
expect(Dir.exist?(expected_path)).to be_truthy expect(Dir.exist?(expected_path)).to be_truthy
end end
end end
......
...@@ -204,43 +204,67 @@ describe Namespace do ...@@ -204,43 +204,67 @@ describe Namespace do
expect(gitlab_shell.exists?(project.repository_storage_path, "#{namespace.path}/#{project.path}.git")).to be_truthy expect(gitlab_shell.exists?(project.repository_storage_path, "#{namespace.path}/#{project.path}.git")).to be_truthy
end end
context 'with subgroups' do context 'with subgroups', :nested_groups do
let(:parent) { create(:group, name: 'parent', path: 'parent') } let(:parent) { create(:group, name: 'parent', path: 'parent') }
let(:new_parent) { create(:group, name: 'new_parent', path: 'new_parent') }
let(:child) { create(:group, name: 'child', path: 'child', parent: parent) } let(:child) { create(:group, name: 'child', path: 'child', parent: parent) }
let!(:project) { create(:project_empty_repo, :legacy_storage, path: 'the-project', namespace: child, skip_disk_validation: true) } let!(:project) { create(:project_empty_repo, :legacy_storage, path: 'the-project', namespace: child, skip_disk_validation: true) }
let(:uploads_dir) { FileUploader.root } let(:uploads_dir) { FileUploader.root }
let(:pages_dir) { File.join(TestEnv.pages_path) } let(:pages_dir) { File.join(TestEnv.pages_path) }
def expect_project_directories_at(namespace_path)
expected_repository_path = File.join(TestEnv.repos_path, namespace_path, 'the-project.git')
expected_upload_path = File.join(uploads_dir, namespace_path, 'the-project')
expected_pages_path = File.join(pages_dir, namespace_path, 'the-project')
expect(File.directory?(expected_repository_path)).to be_truthy
expect(File.directory?(expected_upload_path)).to be_truthy
expect(File.directory?(expected_pages_path)).to be_truthy
end
before do before do
FileUtils.mkdir_p(File.join(TestEnv.repos_path, "#{project.full_path}.git"))
FileUtils.mkdir_p(File.join(uploads_dir, project.full_path)) FileUtils.mkdir_p(File.join(uploads_dir, project.full_path))
FileUtils.mkdir_p(File.join(pages_dir, project.full_path)) FileUtils.mkdir_p(File.join(pages_dir, project.full_path))
end end
context 'renaming child' do context 'renaming child' do
it 'correctly moves the repository, uploads and pages' do it 'correctly moves the repository, uploads and pages' do
expected_repository_path = File.join(TestEnv.repos_path, 'parent', 'renamed', 'the-project.git') child.update!(path: 'renamed')
expected_upload_path = File.join(uploads_dir, 'parent', 'renamed', 'the-project')
expected_pages_path = File.join(pages_dir, 'parent', 'renamed', 'the-project')
child.update_attributes!(path: 'renamed') expect_project_directories_at('parent/renamed')
expect(File.directory?(expected_repository_path)).to be(true)
expect(File.directory?(expected_upload_path)).to be(true)
expect(File.directory?(expected_pages_path)).to be(true)
end end
end end
context 'renaming parent' do context 'renaming parent' do
it 'correctly moves the repository, uploads and pages' do it 'correctly moves the repository, uploads and pages' do
expected_repository_path = File.join(TestEnv.repos_path, 'renamed', 'child', 'the-project.git') parent.update!(path: 'renamed')
expected_upload_path = File.join(uploads_dir, 'renamed', 'child', 'the-project')
expected_pages_path = File.join(pages_dir, 'renamed', 'child', 'the-project') expect_project_directories_at('renamed/child')
end
end
context 'moving from one parent to another' do
it 'correctly moves the repository, uploads and pages' do
child.update!(parent: new_parent)
parent.update_attributes!(path: 'renamed') expect_project_directories_at('new_parent/child')
end
end
context 'moving from having a parent to root' do
it 'correctly moves the repository, uploads and pages' do
child.update!(parent: nil)
expect_project_directories_at('child')
end
end
context 'moving from root to having a parent' do
it 'correctly moves the repository, uploads and pages' do
parent.update!(parent: new_parent)
expect(File.directory?(expected_repository_path)).to be(true) expect_project_directories_at('new_parent/parent/child')
expect(File.directory?(expected_upload_path)).to be(true)
expect(File.directory?(expected_pages_path)).to be(true)
end end
end end
end end
...@@ -525,7 +549,6 @@ describe Namespace do ...@@ -525,7 +549,6 @@ describe Namespace do
end end
end end
# Note: Group transfers are not yet implemented
context 'when a group is transferred into a root group' do context 'when a group is transferred into a root group' do
context 'when the root group "Share with group lock" is enabled' do context 'when the root group "Share with group lock" is enabled' do
let(:root_group) { create(:group, share_with_group_lock: true) } let(:root_group) { create(:group, share_with_group_lock: true) }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment