Commit 29953ced authored by James Lopez's avatar James Lopez 🔴

Refactor code to remove object storage flag from Import/Export

Updated docs, refactor import/export code
Fix AvatarUploader path issue
Fix project export upload webhook error
parent 2037dfd0
......@@ -193,10 +193,8 @@ class ProjectsController < Projects::ApplicationController
end
def download_export
if export_project_object_storage?
if @project.export_project_object_exists?
send_upload(@project.import_export_upload.export_file)
elsif export_project_path
send_file export_project_path, disposition: 'attachment'
else
redirect_to(
edit_project_path(@project, anchor: 'js-export-project'),
......@@ -434,12 +432,4 @@ class ProjectsController < Projects::ApplicationController
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
end
def export_project_path
@export_project_path ||= @project.export_project_path
end
def export_project_object_storage?
@project.export_project_object_exists?
end
end
......@@ -25,8 +25,6 @@ module Storage
Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
end
remove_exports!
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
......@@ -101,8 +99,6 @@ module Storage
end
end
end
remove_exports!
end
def remove_legacy_exports!
......
......@@ -254,18 +254,6 @@ class Namespace < ActiveRecord::Base
end
end
# Exports belonging to projects with legacy storage are placed in a common
# subdirectory of the namespace, so a simple `rm -rf` is sufficient to remove
# them.
#
# Exports of projects using hashed storage are placed in a location defined
# only by the project ID, so each must be removed individually.
def remove_exports!
remove_legacy_exports!
all_projects.with_storage_feature(:repository).find_each(&:remove_exports)
end
def refresh_project_authorizations
owner.refresh_authorized_projects
end
......
......@@ -1751,16 +1751,12 @@ class Project < ActiveRecord::Base
import_export_shared.archive_path
end
def export_project_path
Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) }
end
def export_status
if export_in_progress?
:started
elsif after_export_in_progress?
:after_export_action
elsif export_project_path || export_project_object_exists?
elsif export_project_object_exists?
:finished
else
:none
......@@ -1775,21 +1771,15 @@ class Project < ActiveRecord::Base
import_export_shared.after_export_in_progress?
end
def remove_exports(path = export_path)
if path.present?
FileUtils.rm_rf(path)
elsif export_project_object_exists?
import_export_upload.remove_export_file!
import_export_upload.save
end
end
def remove_exports
return unless export_project_object_exists?
def remove_exported_project_file
remove_exports(export_project_path)
import_export_upload.remove_export_file!
import_export_upload.save
end
def export_project_object_exists?
Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file
import_export_upload&.export_file&.file
end
def full_path_slug
......
......@@ -18,6 +18,10 @@ class AvatarUploader < GitlabUploader
false
end
def absolute_path
self.class.absolute_path(model.avatar)
end
private
def dynamic_segment
......
---
title: Update Import/Export to only use new storage uploaders logic
merge_request: 21409
author:
type: added
......@@ -9,6 +9,7 @@
> application settings (`/admin/application_settings`) under 'Import sources'.
> - The exports are stored in a temporary [shared directory][tmp] and are deleted
> every 24 hours by a specific worker.
> - ImportExport can use object storage automatically starting from GitLab 11.3
The GitLab Import/Export version can be checked by using:
......@@ -30,12 +31,6 @@ sudo gitlab-rake gitlab:import_export:data
bundle exec rake gitlab:import_export:data RAILS_ENV=production
```
In order to enable Object Storage on the Export, you can use the [feature flag][feature-flags]:
```
import_export_object_storage
```
[ce-3050]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
[feature-flags]: https://docs.gitlab.com/ee/api/features.html
[tmp]: ../../development/shared_files.md
......@@ -21,11 +21,7 @@ module API
detail 'This feature was introduced in GitLab 10.6.'
end
get ':id/export/download' do
path = user_project.export_project_path
if path
present_disk_file!(path, File.basename(path), 'application/gzip')
elsif user_project.export_project_object_exists?
if user_project.export_project_object_exists?
present_carrierwave_file!(user_project.import_export_upload.export_file)
else
render_api_error!('404 Not found or has expired', 404)
......
......@@ -40,10 +40,6 @@ module Gitlab
"#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
end
def object_storage?
Feature.enabled?(:import_export_object_storage)
end
def version
VERSION
end
......
......@@ -23,7 +23,7 @@ module Gitlab
def strategy_execute
handle_response_error(send_file)
project.remove_exported_project_file
project.remove_exports
end
def handle_response_error(response)
......@@ -40,15 +40,11 @@ module Gitlab
def send_file
Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend
ensure
export_file.close if export_file && !object_storage?
export_file.close if export_file
end
def export_file
if object_storage?
project.import_export_upload.export_file.file.open
else
File.open(project.export_project_path)
end
project.import_export_upload.export_file.open
end
def send_file_options
......@@ -63,11 +59,7 @@ module Gitlab
end
def export_size
if object_storage?
project.import_export_upload.export_file.file.size
else
File.size(project.export_project_path)
end
project.import_export_upload.export_file.file.size
end
end
end
......
......@@ -19,7 +19,7 @@ module Gitlab
private
def avatar_export_file
@avatar_export_file ||= Dir["#{avatar_export_path}/*"].first
@avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].first
end
def avatar_export_path
......
module Gitlab
module ImportExport
class AvatarSaver
include Gitlab::ImportExport::CommandLineUtil
def initialize(project:, shared:)
@project = project
@shared = shared
......@@ -14,19 +12,12 @@ module Gitlab
Gitlab::ImportExport::UploadsManager.new(
project: @project,
shared: @shared,
relative_export_path: 'avatar',
from: avatar_path
relative_export_path: 'avatar'
).save
rescue => e
@shared.error(e)
false
end
private
def avatar_path
@project.avatar.path
end
end
end
end
......@@ -92,8 +92,6 @@ module Gitlab
end
def remove_import_file
return unless Gitlab::ImportExport.object_storage?
upload = @project.import_export_upload
return unless upload&.import_file&.file
......
......@@ -199,7 +199,7 @@ module Gitlab
end
def excluded_keys_for_relation(relation)
@reader.attributes_finder.find_excluded_keys(relation)
reader.attributes_finder.find_excluded_keys(relation)
end
end
end
......
......@@ -18,7 +18,7 @@ module Gitlab
Rails.logger.info("Saved project export #{archive_file}")
save_on_object_storage if use_object_storage?
save_on_object_storage
else
@shared.error(Gitlab::ImportExport::Error.new(error_message))
false
......@@ -27,10 +27,8 @@ module Gitlab
@shared.error(e)
false
ensure
if use_object_storage?
remove_archive
remove_export_path
end
remove_archive
remove_export_path
end
private
......@@ -59,12 +57,8 @@ module Gitlab
upload.save!
end
def use_object_storage?
Gitlab::ImportExport.object_storage?
end
def error_message
"Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}"
"Unable to save #{archive_file} into #{@shared.export_path}."
end
end
end
......
......@@ -5,18 +5,13 @@ module Gitlab
UPLOADS_BATCH_SIZE = 100
def initialize(project:, shared:, relative_export_path: 'uploads', from: nil)
def initialize(project:, shared:, relative_export_path: 'uploads')
@project = project
@shared = shared
@relative_export_path = relative_export_path
@from = from || default_uploads_path
end
def save
if File.file?(@from) && @relative_export_path == 'avatar'
copy_files(@from, File.join(uploads_export_path, @project.avatar.filename))
end
copy_project_uploads
true
......@@ -55,17 +50,11 @@ module Gitlab
copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path))
else
next unless Gitlab::ImportExport.object_storage?
download_and_copy(uploader)
end
end
end
def default_uploads_path
FileUploader.absolute_base_dir(@project)
end
def uploads_export_path
@uploads_export_path ||= File.join(@shared.export_path, @relative_export_path)
end
......
......@@ -2,30 +2,14 @@ module Gitlab
module ImportExport
class UploadsRestorer < UploadsSaver
def restore
if Gitlab::ImportExport.object_storage?
Gitlab::ImportExport::UploadsManager.new(
project: @project,
shared: @shared
).restore
elsif File.directory?(uploads_export_path)
copy_files(uploads_export_path, uploads_path)
true
else
true # Proceed without uploads
end
Gitlab::ImportExport::UploadsManager.new(
project: @project,
shared: @shared
).restore
rescue => e
@shared.error(e)
false
end
def uploads_path
FileUploader.absolute_base_dir(@project)
end
def uploads_export_path
@uploads_export_path ||= File.join(@shared.export_path, 'uploads')
end
end
end
end
module Gitlab
module ImportExport
class UploadsSaver
include Gitlab::ImportExport::CommandLineUtil
def initialize(project:, shared:)
@project = project
@shared = shared
......
module Gitlab
module TemplateHelper
include Gitlab::Utils::StrongMemoize
def prepare_template_environment(file)
return unless file
if Gitlab::ImportExport.object_storage?
params[:import_export_upload] = ImportExportUpload.new(import_file: file)
else
FileUtils.mkdir_p(File.dirname(import_upload_path))
FileUtils.copy_entry(file.path, import_upload_path)
params[:import_source] = import_upload_path
end
end
def import_upload_path
strong_memoize(:import_upload_path) do
Gitlab::ImportExport.import_upload_path(filename: tmp_filename)
end
params[:import_export_upload] = ImportExportUpload.new(import_file: file)
end
def tmp_filename
......
......@@ -6,6 +6,8 @@ namespace :gitlab do
desc "GitLab | Update project templates"
task :update_project_templates do
include Gitlab::ImportExport::CommandLineUtil
if Rails.env.production?
puts "This rake task is not meant fo production instances".red
exit(1)
......@@ -52,7 +54,7 @@ namespace :gitlab do
end
Projects::ImportExport::ExportService.new(project, admin).execute
FileUtils.cp(project.export_project_path, template.archive_path)
download_or_copy_upload(project.import_export_upload.export_file, template.archive_path)
Projects::DestroyService.new(admin, project).execute
puts "Exported #{template.name}".green
end
......
......@@ -848,37 +848,7 @@ describe ProjectsController do
project.add_maintainer(user)
end
context 'object storage disabled' do
before do
stub_feature_flags(import_export_object_storage: false)
end
context 'when project export is enabled' do
it 'returns 302' do
get :download_export, namespace_id: project.namespace, id: project
expect(response).to have_gitlab_http_status(302)
end
end
context 'when project export is disabled' do
before do
stub_application_setting(project_export_enabled?: false)
end
it 'returns 404' do
get :download_export, namespace_id: project.namespace, id: project
expect(response).to have_gitlab_http_status(404)
end
end
end
context 'object storage enabled' do
before do
stub_feature_flags(import_export_object_storage: true)
end
context 'when project export is enabled' do
it 'returns 302' do
get :download_export, namespace_id: project.namespace, id: project
......
......@@ -141,27 +141,11 @@ FactoryBot.define do
end
trait :with_export do
before(:create) do |_project, _evaluator|
allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { false }
allow(Feature).to receive(:enabled?).with('import_export_object_storage') { false }
end
after(:create) do |project, _evaluator|
ProjectExportWorker.new.perform(project.creator.id, project.id)
end
end
trait :with_object_export do
before(:create) do |_project, _evaluator|
allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { true }
allow(Feature).to receive(:enabled?).with('import_export_object_storage') { true }
end
after(:create) do |project, evaluator|
ProjectExportWorker.new.perform(project.creator.id, project.id)
end
end
trait :broken_storage do
after(:create) do |project|
project.update_column(:repository_storage, 'broken')
......
......@@ -25,7 +25,6 @@ describe 'Import/Export - project export integration test', :js do
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
stub_feature_flags(import_export_object_storage: false)
end
after do
......
require 'spec_helper'
describe 'Import/Export - project import integration test', :js do
include Select2Helper
let(:user) { create(:user) }
let(:file) { File.join(Rails.root, 'spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') }
let(:export_path) { "#{Dir.tmpdir}/import_file_spec" }
before do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
gitlab_sign_in(user)
end
after do
FileUtils.rm_rf(export_path, secure: true)
end
context 'when selecting the namespace' do
let(:user) { create(:admin) }
let!(:namespace) { user.namespace }
let(:project_path) { 'test-project-path' + SecureRandom.hex }
context 'prefilled the path' do
it 'user imports an exported project successfully' do
visit new_project_path
select2(namespace.id, from: '#project_namespace_id')
fill_in :project_path, with: project_path, visible: true
click_import_project_tab
click_link 'GitLab export'
expect(page).to have_content('Import an exported GitLab project')
expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}")
attach_file('file', file)
click_on 'Import project'
expect(Project.count).to eq(1)
project = Project.last
expect(project).not_to be_nil
expect(project.description).to eq("Foo Bar")
expect(project.issues).not_to be_empty
expect(project.merge_requests).not_to be_empty
expect(project_hook_exists?(project)).to be true
expect(wiki_exists?(project)).to be true
expect(project.import_state.status).to eq('finished')
end
end
context 'path is not prefilled' do
it 'user imports an exported project successfully' do
visit new_project_path
click_import_project_tab
click_link 'GitLab export'
fill_in :path, with: 'test-project-path', visible: true
attach_file('file', file)
expect { click_on 'Import project' }.to change { Project.count }.by(1)
project = Project.last
expect(project).not_to be_nil
expect(page).to have_content("Project 'test-project-path' is being imported")
end
end
end
it 'invalid project' do
project = create(:project, namespace: user.namespace)
visit new_project_path
select2(user.namespace.id, from: '#project_namespace_id')
fill_in :project_path, with: project.name, visible: true
click_import_project_tab
click_link 'GitLab export'
attach_file('file', file)
click_on 'Import project'
page.within('.flash-container') do
expect(page).to have_content('Project could not be imported')
end
end
def wiki_exists?(project)
wiki = ProjectWiki.new(project)
wiki.repository.exists? && !wiki.repository.empty?
end
def project_hook_exists?(project)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab::Git::Hook.new('post-receive', project.repository.raw_repository).exists?
end
end
def click_import_project_tab
find('#import-project-tab').click
end
end
......@@ -8,7 +8,7 @@ describe 'Import/Export - project import integration test', :js do
let(:export_path) { "#{Dir.tmpdir}/import_file_spec" }
before do
stub_feature_flags(import_export_object_storage: false)
stub_uploads_object_storage(FileUploader)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
gitlab_sign_in(user)
end
......@@ -33,7 +33,6 @@ describe 'Import/Export - project import integration test', :js do
expect(page).to have_content('Import an exported GitLab project')
expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}")
expect(Gitlab::ImportExport).to receive(:import_upload_path).with(filename: /\A\h{32}\z/).and_call_original
attach_file('file', file)
click_on 'Import project'
......
require 'spec_helper'
describe 'Import/Export - Namespace export file cleanup', :js do
let(:export_path) { Dir.mktmpdir('namespace_export_file_spec') }
before do
allow(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
stub_feature_flags(import_export_object_storage: false)
end
after do
FileUtils.rm_rf(export_path, secure: true)
end
shared_examples_for 'handling project exports on namespace change' do
let!(:old_export_path) { project.export_path }
before do
sign_in(create(:admin))
setup_export_project
end
context 'moving the namespace' do
it 'removes the export file' do
expect(File).to exist(old_export_path)
project.namespace.update!(path: build(:namespace).path)
expect(File).not_to exist(old_export_path)
end
end
context 'deleting the namespace' do
it 'removes the export file' do
expect(File).to exist(old_export_path)
project.namespace.destroy
expect(File).not_to exist(old_export_path)
end
end
end
describe 'legacy storage' do
let(:project) { create(:project, :legacy_storage) }
it_behaves_like 'handling project exports on namespace change'
end
describe 'hashed storage' do
let(:project) { create(:project) }
it_behaves_like 'handling project exports on namespace change'
end
def setup_export_project
visit edit_project_path(project)
expect(page).to have_content('Export project')
find(:link, 'Export project').send_keys(:return)
visit edit_project_path(project)
expect(page).to have_content('Download export')
end