Commit 0c3f1214 authored by 🤖 GitLab Bot 🤖's avatar 🤖 GitLab Bot 🤖

Add latest changes from gitlab-org/[email protected]

parent ff67e3ed
......@@ -23,10 +23,16 @@ class TodosFinder
NONE = '0'
TODO_TYPES = Set.new(%w(Issue MergeRequest Epic)).freeze
TODO_TYPES = Set.new(%w(Issue MergeRequest)).freeze
attr_accessor :current_user, :params
class << self
def todo_types
TODO_TYPES
end
end
def initialize(current_user, params = {})
@current_user = current_user
@params = params
......@@ -124,7 +130,7 @@ class TodosFinder
end
def type?
type.present? && TODO_TYPES.include?(type)
type.present? && self.class.todo_types.include?(type)
end
def type
......@@ -201,3 +207,5 @@ class TodosFinder
end
end
end
TodosFinder.prepend_if_ee('EE::TodosFinder')
......@@ -55,6 +55,8 @@ class Group < Namespace
has_many :todos
has_one :import_export_upload
accepts_nested_attributes_for :variables, allow_destroy: true
validate :visibility_level_allowed_by_projects
......
......@@ -5,6 +5,7 @@ class ImportExportUpload < ApplicationRecord
include ObjectStorage::BackgroundMove
belongs_to :project
belongs_to :group
# These hold the project Import/Export archives (.tar.gz files)
mount_uploader :import_file, ImportExportUploader
......
# frozen_string_literal: true
module Groups
module ImportExport
class ExportService
def initialize(group:, user:, params: {})
@group = group
@current_user = user
@params = params
@shared = @params[:shared] || Gitlab::ImportExport::Shared.new(@group)
end
def execute
save!
end
private
attr_accessor :shared
def save!
if savers.all?(&:save)
notify_success
else
cleanup_and_notify_error!
end
end
def savers
[tree_exporter, file_saver]
end
def tree_exporter
Gitlab::ImportExport::GroupTreeSaver.new(group: @group, current_user: @current_user, shared: @shared, params: @params)
end
def file_saver
Gitlab::ImportExport::Saver.new(exportable: @group, shared: @shared)
end
def cleanup_and_notify_error
FileUtils.rm_rf(shared.export_path)
notify_error
end
def cleanup_and_notify_error!
cleanup_and_notify_error
raise Gitlab::ImportExport::Error.new(shared.errors.to_sentence)
end
def notify_success
@shared.logger.info(
group_id: @group.id,
group_name: @group.name,
message: 'Group Import/Export: Export succeeded'
)
end
def notify_error
@shared.logger.error(
group_id: @group.id,
group_name: @group.name,
error: @shared.errors.join(', '),
message: 'Group Import/Export: Export failed'
)
end
end
end
end
......@@ -24,7 +24,7 @@ module Projects
def save_all!
if save_exporters
Gitlab::ImportExport::Saver.save(project: project, shared: shared)
Gitlab::ImportExport::Saver.save(exportable: project, shared: shared)
notify_success
else
cleanup_and_notify_error!
......
......@@ -179,3 +179,4 @@
- import_issues_csv
- project_daily_statistics
- create_evidence
- group_export
# frozen_string_literal: true
class GroupExportWorker
include ApplicationWorker
include ExceptionBacktrace
feature_category :source_code_management
def perform(current_user_id, group_id, params = {})
current_user = User.find(current_user_id)
group = Group.find(group_id)
::Groups::ImportExport::ExportService.new(group: group, user: current_user, params: params).execute
end
end
......@@ -98,6 +98,7 @@
- [update_namespace_statistics, 1]
- [chaos, 2]
- [create_evidence, 2]
- [group_export, 1]
# EE-specific queues
- [analytics, 1]
......
# frozen_string_literal: true
class AddGroupIdToImportExportUploads < ActiveRecord::Migration[5.2]
DOWNTIME = false
def change
add_column :import_export_uploads, :group_id, :bigint
end
end
# frozen_string_literal: true
class AddGroupFkToImportExportUploads < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_foreign_key :import_export_uploads, :namespaces, column: :group_id, on_delete: :cascade
add_concurrent_index :import_export_uploads, :group_id, unique: true, where: 'group_id IS NOT NULL'
end
def down
remove_foreign_key_without_error(:import_export_uploads, column: :group_id)
remove_concurrent_index(:import_export_uploads, :group_id)
end
end
......@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2019_11_05_140942) do
ActiveRecord::Schema.define(version: 2019_11_11_115431) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_trgm"
......@@ -1874,6 +1874,8 @@ ActiveRecord::Schema.define(version: 2019_11_05_140942) do
t.integer "project_id"
t.text "import_file"
t.text "export_file"
t.bigint "group_id"
t.index ["group_id"], name: "index_import_export_uploads_on_group_id", unique: true, where: "(group_id IS NOT NULL)"
t.index ["project_id"], name: "index_import_export_uploads_on_project_id"
t.index ["updated_at"], name: "index_import_export_uploads_on_updated_at"
end
......@@ -4288,6 +4290,7 @@ ActiveRecord::Schema.define(version: 2019_11_05_140942) do
add_foreign_key "group_group_links", "namespaces", column: "shared_group_id", on_delete: :cascade
add_foreign_key "group_group_links", "namespaces", column: "shared_with_group_id", on_delete: :cascade
add_foreign_key "identities", "saml_providers", name: "fk_aade90f0fc", on_delete: :cascade
add_foreign_key "import_export_uploads", "namespaces", column: "group_id", name: "fk_83319d9721", on_delete: :cascade
add_foreign_key "import_export_uploads", "projects", on_delete: :cascade
add_foreign_key "index_statuses", "projects", name: "fk_74b2492545", on_delete: :cascade
add_foreign_key "insights", "namespaces", on_delete: :cascade
......
......@@ -15,7 +15,7 @@ module Gitlab
end
def storage_path
File.join(Settings.shared['path'], 'tmp/project_exports')
File.join(Settings.shared['path'], 'tmp/gitlab_exports')
end
def import_upload_path(filename:)
......@@ -50,8 +50,8 @@ module Gitlab
'VERSION'
end
def export_filename(project:)
basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.full_path.tr('/', '_')}"
def export_filename(exportable:)
basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{exportable.full_path.tr('/', '_')}"
"#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
end
......@@ -63,6 +63,14 @@ module Gitlab
def reset_tokens?
true
end
def group_filename
'group.json'
end
def group_config_file
Rails.root.join('lib/gitlab/import_export/group_import_export.yml')
end
end
end
......
......@@ -3,7 +3,8 @@
module Gitlab
module ImportExport
class Config
def initialize
def initialize(config: Gitlab::ImportExport.config_file)
@config = config
@hash = parse_yaml
@hash.deep_symbolize_keys!
@ee_hash = @hash.delete(:ee) || {}
......@@ -50,7 +51,7 @@ module Gitlab
end
def parse_yaml
YAML.load_file(Gitlab::ImportExport.config_file)
YAML.load_file(@config)
end
end
end
......
......@@ -60,7 +60,7 @@ module Gitlab
def copy_archive
return if @archive_file
@archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
@archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project))
download_or_copy_upload(@project.import_export_upload.import_file, @archive_file)
end
......
# Model relationships to be included in the group import/export
#
# This list _must_ only contain relationships that are available to both FOSS and
# Enterprise editions. EE specific relationships must be defined in the `ee` section further
# down below.
tree:
group:
- :milestones
- :badges
- labels:
- :priorities
- :boards
- members:
- :user
included_attributes:
excluded_attributes:
group:
- :runners_token
- :runners_token_encrypted
methods:
labels:
- :type
badges:
- :type
preloads:
# EE specific relationships and settings to include. All of this will be merged
# into the previous structures if EE is used.
ee:
tree:
group:
- :epics
# frozen_string_literal: true
module Gitlab
module ImportExport
class GroupTreeSaver
attr_reader :full_path
def initialize(group:, current_user:, shared:, params: {})
@params = params
@current_user = current_user
@shared = shared
@group = group
@full_path = File.join(@shared.export_path, ImportExport.group_filename)
end
def save
group_tree = serialize(@group, reader.group_tree)
tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename)
true
rescue => e
@shared.error(e)
false
end
private
def serialize(group, relations_tree)
group_tree = tree_saver.serialize(group, relations_tree)
group.descendants.each do |descendant|
group_tree['descendants'] = [] unless group_tree['descendants']
group_tree['descendants'] << serialize(descendant, relations_tree)
end
group_tree
rescue => e
@shared.error(e)
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(
shared: @shared,
config: Gitlab::ImportExport::Config.new(
config: Gitlab::ImportExport.group_config_file
).to_h
)
end
def tree_saver
@tree_saver ||= RelationTreeSaver.new
end
end
end
end
......@@ -3,25 +3,20 @@
module Gitlab
module ImportExport
class ProjectTreeSaver
include Gitlab::ImportExport::CommandLineUtil
attr_reader :full_path
def initialize(project:, current_user:, shared:, params: {})
@params = params
@project = project
@params = params
@project = project
@current_user = current_user
@shared = shared
@full_path = File.join(@shared.export_path, ImportExport.project_filename)
@shared = shared
@full_path = File.join(@shared.export_path, ImportExport.project_filename)
end
def save
mkdir_p(@shared.export_path)
project_tree = serialize_project_tree
project_tree = tree_saver.serialize(@project, reader.project_tree)
fix_project_tree(project_tree)
project_tree_json = JSON.generate(project_tree)
File.write(full_path, project_tree_json)
tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
true
rescue => e
......@@ -43,16 +38,6 @@ module Gitlab
RelationRenameService.add_new_associations(project_tree)
end
def serialize_project_tree
if Feature.enabled?(:export_fast_serialize, default_enabled: true)
Gitlab::ImportExport::FastHashSerializer
.new(@project, reader.project_tree)
.execute
else
@project.as_json(reader.project_tree)
end
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
end
......@@ -74,6 +59,10 @@ module Gitlab
GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
end
def tree_saver
@tree_saver ||= RelationTreeSaver.new
end
end
end
end
......@@ -5,24 +5,31 @@ module Gitlab
class Reader
attr_reader :tree, :attributes_finder
def initialize(shared:)
@shared = shared
@attributes_finder = Gitlab::ImportExport::AttributesFinder.new(
config: ImportExport::Config.new.to_h)
def initialize(shared:, config: ImportExport::Config.new.to_h)
@shared = shared
@config = config
@attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config)
end
# Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
# for outputting a project in JSON format, including its relations and sub relations.
def project_tree
attributes_finder.find_root(:project)
rescue => e
@shared.error(e)
false
tree_by_key(:project)
end
def group_tree
tree_by_key(:group)
end
def group_members_tree
attributes_finder.find_root(:group_members)
tree_by_key(:group_members)
end
def tree_by_key(key)
attributes_finder.find_root(key)
rescue => e
@shared.error(e)
false
end
end
end
......
......@@ -8,7 +8,7 @@
# The behavior of these renamed relationships should be transient and it should
# only last one release until you completely remove the renaming from the list.
#
# When importing, this class will check the project hash and:
# When importing, this class will check the hash and:
# - if only the old relationship name is found, it will rename it with the new one
# - if only the new relationship name is found, it will do nothing
# - if it finds both, it will use the new relationship data
......
# frozen_string_literal: true
module Gitlab
module ImportExport
class RelationTreeSaver
include Gitlab::ImportExport::CommandLineUtil
def serialize(exportable, relations_tree)
if Feature.enabled?(:export_fast_serialize, default_enabled: true)
Gitlab::ImportExport::FastHashSerializer
.new(exportable, relations_tree)
.execute
else
exportable.as_json(relations_tree)
end
end
def save(tree, dir_path, filename)
mkdir_p(dir_path)
tree_json = JSON.generate(tree)
File.write(File.join(dir_path, filename), tree_json)
end
end
end
end
......@@ -9,16 +9,16 @@ module Gitlab
new(*args).save
end
def initialize(project:, shared:)
@project = project
@shared = shared
def initialize(exportable:, shared:)
@exportable = exportable
@shared = shared
end
def save
if compress_and_save
remove_export_path
Rails.logger.info("Saved project export #{archive_file}") # rubocop:disable Gitlab/RailsLogger
Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger
save_upload
else
......@@ -48,11 +48,11 @@ module Gitlab
end
def archive_file
@archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
@archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @exportable))
end
def save_upload
upload = ImportExportUpload.find_or_initialize_by(project: @project)
upload = initialize_upload
File.open(archive_file) { |file| upload.export_file = file }
......@@ -62,6 +62,12 @@ module Gitlab
def error_message
"Unable to save #{archive_file} into #{@shared.export_path}."
end
def initialize_upload
exportable_kind = @exportable.class.name.downcase
ImportExportUpload.find_or_initialize_by(Hash[exportable_kind, @exportable])
end
end
end
end
......@@ -23,21 +23,21 @@
module Gitlab
module ImportExport
class Shared
attr_reader :errors, :project
attr_reader :errors, :exportable, :logger
LOCKS_DIRECTORY = 'locks'
def initialize(project)
@project = project
@errors = []
@logger = Gitlab::Import::Logger.build
def initialize(exportable)
@exportable = exportable
@errors = []
@logger = Gitlab::Import::Logger.build
end
def active_export_count
Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) }
end
# The path where the project metadata and repository bundle is saved
# The path where the exportable metadata and repository bundle (in case of project) is saved
def export_path
@export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path)
end
......@@ -84,11 +84,18 @@ module Gitlab
end
def relative_archive_path
@relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex)
@relative_archive_path ||= File.join(relative_base_path, SecureRandom.hex)
end
def relative_base_path
@project.disk_path
case exportable_type
when 'Project'
@exportable.disk_path
when 'Group'
@exportable.full_path
else
raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}")
end
end
def log_error(details)
......@@ -100,17 +107,24 @@ module Gitlab
end
def log_base_data
{
importer: 'Import/Export',
import_jid: @project&.import_state&.jid,
project_id: @project&.id,
project_path: @project&.full_path
log = {
importer: 'Import/Export',
exportable_id: @exportable&.id,
exportable_path: @exportable&.full_path
}
log[:import_jid] = @exportable&.import_state&.jid if exportable_type == 'Project'
log
end
def filtered_error_message(message)
Projects::ImportErrorFilter.filter_message(message)
end
def exportable_type
@exportable.class.name
end
end
end
end
......@@ -234,6 +234,19 @@ describe TodosFinder do
end
end
describe '.todo_types' do
it 'returns the expected types' do
expected_result =
if Gitlab.ee?
%w[Epic Issue MergeRequest]
else
%w[Issue MergeRequest]
end
expect(described_class.todo_types).to contain_exactly(*expected_result)
end
end
describe '#any_for_target?' do
it 'returns true if there are any todos for the given target' do