Commit 9b7fd748 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'feature/project-export' into 'master'

Export project functionality

This is a MR for the export functionality of https://gitlab.com/gitlab-org/gitlab-ce/issues/3050, which adds the ability to export single projects.

- [x] members
- DB data
  - [x] issues
  - [x] issue comments
  - [x] merge requests
  - [x] merge request diff
  - [x] merge request comments
  - [x] labels
  - [x] milestones
  - [x] snippets
  - [x] releases
  - [x] events
  - [x] commit statuses
  - [x] CI builds
- File system data
  - [x] Git repository
  - [x] wiki
  - [x] uploads
  - [ ] ~~CI build traces~~
  - [ ] ~~CI build artifacts~~
  - [ ] ~~LFS objects~~
- DB configuration
  - [x] services
  - [x] web hooks
  - [x] protected branches
  - [x] deploy keys
  - [x] CI variables
  - [x] CI triggers

See merge request !3114
parents ae4491b4 8891bef2
......@@ -100,6 +100,7 @@ v 8.9.0 (unreleased)
- Improved UX of date pickers on issue & milestone forms
- Cache on the database if a project has an active external issue tracker.
- Put project Labels and Milestones pages links under Issues and Merge Requests tabs as subnav
- GitLab project import and export functionality
- All classes in the Banzai::ReferenceParser namespace are now instrumented
- Remove deprecated issues_tracker and issues_tracker_id from project model
- Allow users to create confidential issues in private projects
......
......@@ -221,7 +221,7 @@ gem 'jquery-turbolinks', '~> 2.1.0'
gem 'addressable', '~> 2.3.8'
gem 'bootstrap-sass', '~> 3.3.0'
gem 'font-awesome-rails', '~> 4.2'
gem 'font-awesome-rails', '~> 4.6.1'
gem 'gitlab_emoji', '~> 0.3.0'
gem 'gon', '~> 6.0.1'
gem 'jquery-atwho-rails', '~> 1.3.2'
......
......@@ -246,7 +246,7 @@ GEM
fog-xml (0.1.2)
fog-core
nokogiri (~> 1.5, >= 1.5.11)
font-awesome-rails (4.5.0.1)
font-awesome-rails (4.6.1.0)
railties (>= 3.2, < 5.1)
foreman (0.78.0)
thor (~> 0.19.1)
......@@ -866,7 +866,7 @@ DEPENDENCIES
fog-google (~> 0.3)
fog-local (~> 0.3)
fog-openstack (~> 0.1)
font-awesome-rails (~> 4.2)
font-awesome-rails (~> 4.6.1)
foreman
fuubar (~> 2.0.0)
gemnasium-gitlab-service (~> 0.2)
......
......@@ -24,7 +24,7 @@ class ApplicationController < ActionController::Base
protect_from_forgery with: :exception
helper_method :abilities, :can?, :current_application_settings
helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :gitorious_import_enabled?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?
helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :gitorious_import_enabled?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?
rescue_from Encoding::CompatibilityError do |exception|
log_exception(exception)
......@@ -319,6 +319,10 @@ class ApplicationController < ActionController::Base
current_application_settings.import_sources.include?('git')
end
def gitlab_project_import_enabled?
current_application_settings.import_sources.include?('gitlab_project')
end
def two_factor_authentication_required?
current_application_settings.require_two_factor_authentication
end
......
class Import::GitlabProjectsController < Import::BaseController
before_action :verify_gitlab_project_import_enabled
def new
@namespace_id = project_params[:namespace_id]
@namespace_name = Namespace.find(project_params[:namespace_id]).name
@path = project_params[:path]
end
def create
unless file_is_valid?
return redirect_back_or_default(options: { alert: "You need to upload a GitLab project export archive." })
end
@project = Gitlab::ImportExport::ProjectCreator.new(project_params[:namespace_id],
current_user,
File.expand_path(project_params[:file].path),
project_params[:path]).execute
if @project.saved?
redirect_to(
project_path(@project),
notice: "Project '#{@project.name}' is being imported."
)
else
redirect_to(
new_import_gitlab_project_path,
alert: "Project could not be imported: #{@project.errors.full_messages.join(', ')}"
)
end
end
private
def file_is_valid?
project_params[:file] && project_params[:file].respond_to?(:read)
end
def verify_gitlab_project_import_enabled
render_404 unless gitlab_project_import_enabled?
end
def project_params
params.permit(
:path, :namespace_id, :file
)
end
end
......@@ -7,7 +7,7 @@ class ProjectsController < Projects::ApplicationController
before_action :assign_ref_vars, :tree, only: [:show], if: :repo_exists?
# Authorize
before_action :authorize_admin_project!, only: [:edit, :update, :housekeeping]
before_action :authorize_admin_project!, only: [:edit, :update, :housekeeping, :download_export, :export, :remove_export, :generate_new_export]
before_action :event_filter, only: [:show, :activity]
layout :determine_layout
......@@ -186,6 +186,48 @@ class ProjectsController < Projects::ApplicationController
)
end
def export
@project.add_export_job(current_user: current_user)
redirect_to(
edit_project_path(@project),
notice: "Project export started. A download link will be sent by email."
)
end
def download_export
export_project_path = @project.export_project_path
if export_project_path
send_file export_project_path, disposition: 'attachment'
else
redirect_to(
edit_project_path(@project),
alert: "Project export link has expired. Please generate a new export from your project settings."
)
end
end
def remove_export
if @project.remove_exports
flash[:notice] = "Project export has been deleted."
else
flash[:alert] = "Project export could not be deleted."
end
redirect_to(edit_project_path(@project))
end
def generate_new_export
if @project.remove_exports
export
else
redirect_to(
edit_project_path(@project),
alert: "Project export could not be deleted."
)
end
end
def toggle_star
current_user.toggle_star(@project)
@project.reload
......
......@@ -9,6 +9,19 @@ module Emails
subject: subject("Project was moved"))
end
def project_was_exported_email(current_user, project)
@project = project
mail(to: current_user.notification_email,
subject: subject("Project was exported"))
end
def project_was_not_exported_email(current_user, project, errors)
@project = project
@errors = errors
mail(to: current_user.notification_email,
subject: subject("Project export error"))
end
def repository_push_email(project_id, opts = {})
@message =
Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts)
......
......@@ -123,7 +123,7 @@ class ApplicationSetting < ActiveRecord::Base
default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
max_artifacts_size: Settings.artifacts['max_size'],
require_two_factor_authentication: false,
......
......@@ -170,6 +170,10 @@ module Ci
builds.where.not(environment: nil).success.pluck(:environment).uniq
end
def notes
Note.for_commit_id(sha)
end
private
def build_builds_for_stages(stages, user, status, trigger_request)
......
class CommitStatus < ActiveRecord::Base
include Statuseable
include Importable
self.table_name = 'ci_builds'
......@@ -7,7 +8,7 @@ class CommitStatus < ActiveRecord::Base
belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id, touch: true
belongs_to :user
validates :pipeline, presence: true
validates :pipeline, presence: true, unless: :importing?
validates_presence_of :name
......
module Importable
extend ActiveSupport::Concern
attr_accessor :importing
alias_method :importing?, :importing
end
class Member < ActiveRecord::Base
include Sortable
include Importable
include Gitlab::Access
attr_accessor :raw_invite_token
......@@ -41,11 +42,11 @@ class Member < ActiveRecord::Base
before_validation :generate_invite_token, on: :create, if: -> (member) { member.invite_email.present? }
after_create :send_invite, if: :invite?
after_create :send_request, if: :request?
after_create :create_notification_setting, unless: :pending?
after_create :post_create_hook, unless: :pending?
after_update :post_update_hook, unless: :pending?
after_create :send_invite, if: :invite?, unless: :importing?
after_create :send_request, if: :request?, unless: :importing?
after_create :create_notification_setting, unless: [:pending?, :importing?]
after_create :post_create_hook, unless: [:pending?, :importing?]
after_update :post_update_hook, unless: [:pending?, :importing?]
after_destroy :post_destroy_hook, unless: :pending?
after_destroy :post_decline_request, if: :request?
......
......@@ -4,6 +4,7 @@ class MergeRequest < ActiveRecord::Base
include Referable
include Sortable
include Taskable
include Importable
belongs_to :target_project, foreign_key: :target_project_id, class_name: "Project"
belongs_to :source_project, foreign_key: :source_project_id, class_name: "Project"
......@@ -13,7 +14,7 @@ class MergeRequest < ActiveRecord::Base
serialize :merge_params, Hash
after_create :create_merge_request_diff
after_create :create_merge_request_diff, unless: :importing
after_update :update_merge_request_diff
delegate :commits, :diffs, :real_size, to: :merge_request_diff, prefix: nil
......@@ -95,12 +96,12 @@ class MergeRequest < ActiveRecord::Base
end
end
validates :source_project, presence: true, unless: :allow_broken
validates :source_project, presence: true, unless: [:allow_broken, :importing?]
validates :source_branch, presence: true
validates :target_project, presence: true
validates :target_branch, presence: true
validates :merge_user, presence: true, if: :merge_when_build_succeeds?
validate :validate_branches, unless: :allow_broken
validate :validate_branches, unless: [:allow_broken, :importing?]
validate :validate_fork
scope :by_branch, ->(branch_name) { where("(source_branch LIKE :branch) OR (target_branch LIKE :branch)", branch: branch_name) }
......
class MergeRequestDiff < ActiveRecord::Base
include Sortable
include Importable
# Prevent store of diff if commits amount more then 500
COMMITS_SAFE_SIZE = 100
......@@ -22,7 +23,7 @@ class MergeRequestDiff < ActiveRecord::Base
serialize :st_commits
serialize :st_diffs
after_create :reload_content
after_create :reload_content, unless: :importing?
def reload_content
reload_commits
......
......@@ -4,6 +4,7 @@ class Note < ActiveRecord::Base
include Participable
include Mentionable
include Awardable
include Importable
default_value_for :system, false
......@@ -28,11 +29,11 @@ class Note < ActiveRecord::Base
validates :attachment, file_size: { maximum: :max_attachment_size }
validates :noteable_type, presence: true
validates :noteable_id, presence: true, unless: :for_commit?
validates :noteable_id, presence: true, unless: [:for_commit?, :importing?]
validates :commit_id, presence: true, if: :for_commit?
validates :author, presence: true
validate unless: :for_commit? do |note|
validate unless: [:for_commit?, :importing?] do |note|
unless note.noteable.try(:project) == note.project
errors.add(:invalid_project, 'Note and noteable project mismatch')
end
......
......@@ -367,6 +367,11 @@ class Project < ActiveRecord::Base
joins(join_body).reorder('join_note_counts.amount DESC')
end
# Deletes gitlab project export files older than 24 hours
def remove_gitlab_exports!
Gitlab::Popen.popen(%W(find #{Gitlab::ImportExport.storage_path} -not -path #{Gitlab::ImportExport.storage_path} -mmin +1440 -delete))
end
end
def team
......@@ -470,7 +475,7 @@ class Project < ActiveRecord::Base
end
def import?
external_import? || forked?
external_import? || forked? || gitlab_project_import?
end
def no_import?
......@@ -501,6 +506,10 @@ class Project < ActiveRecord::Base
Gitlab::UrlSanitizer.new(import_url).masked_url
end
def gitlab_project_import?
import_type == 'gitlab_project'
end
def check_limit
unless creator.can_create_project? or namespace.kind == 'group'
projects_limit = creator.projects_limit
......@@ -1096,4 +1105,27 @@ class Project < ActiveRecord::Base
ensure
@errors = original_errors
end
def add_export_job(current_user:)
job_id = ProjectExportWorker.perform_async(current_user.id, self.id)
if job_id
Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}"
else
Rails.logger.error "Export job failed to start for project ID #{self.id}"
end
end
def export_path
File.join(Gitlab::ImportExport.storage_path, path_with_namespace)
end
def export_project_path
Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) }
end
def remove_exports
_, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
status.zero?
end
end
......@@ -266,6 +266,14 @@ class NotificationService
end
end
def project_exported(project, current_user)
mailer.project_was_exported_email(current_user, project).deliver_later
end
def project_not_exported(project, current_user, errors)
mailer.project_was_not_exported_email(current_user, project, errors).deliver_later
end
protected
# Get project users with WATCH notification level
......
......@@ -80,16 +80,18 @@ module Projects
def after_create_actions
log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"")
@project.create_wiki if @project.wiki_enabled?
unless @project.gitlab_project_import?
@project.create_wiki if @project.wiki_enabled?
@project.build_missing_services
@project.build_missing_services
@project.create_labels
@project.create_labels
end
event_service.create_project(@project, current_user)
system_hook_service.execute_hooks_for(@project, :create)
unless @project.group
unless @project.group || @project.gitlab_project_import?
@project.team << [current_user, :master, current_user]
end
end
......
module Projects
module ImportExport
class ExportService < BaseService
def execute(_options = {})
@shared = Gitlab::ImportExport::Shared.new(relative_path: File.join(project.path_with_namespace, 'work'))
save_all
end
private
def save_all
if [version_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save)
Gitlab::ImportExport::Saver.save(shared: @shared)
notify_success
else
cleanup_and_notify
end
end
def version_saver
Gitlab::ImportExport::VersionSaver.new(shared: @shared)
end
def project_tree_saver
Gitlab::ImportExport::ProjectTreeSaver.new(project: project, shared: @shared)
end
def uploads_saver
Gitlab::ImportExport::UploadsSaver.new(project: project, shared: @shared)
end
def repo_saver
Gitlab::ImportExport::RepoSaver.new(project: project, shared: @shared)
end
def wiki_repo_saver
Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared)
end
def cleanup_and_notify
FileUtils.rm_rf(@shared.export_path)
notify_error
raise Gitlab::ImportExport::Error.new(@shared.errors.join(', '))
end
def notify_success
notification_service.project_exported(@project, @current_user)
end
def notify_error
notification_service.project_not_exported(@project, @current_user, @shared.errors.join(', '))
end
end
end
end
......@@ -9,26 +9,31 @@ module Projects
'fogbugz',
'gitlab',
'github',
'google_code'
'google_code',
'gitlab_project'
]
def execute
if unknown_url?
# In this case, we only want to import issues, not a repository.
create_repository
else
import_repository
end
add_repository_to_project unless project.gitlab_project_import?
import_data
success
rescue Error => e
rescue => e
error(e.message)
end
private
def add_repository_to_project
if unknown_url?
# In this case, we only want to import issues, not a repository.
create_repository
else
import_repository
end
end
def create_repository
unless project.create_repository
raise Error, 'The repository could not be created.'
......@@ -46,7 +51,7 @@ module Projects
def import_data
return unless has_importer?
project.repository.before_import
project.repository.before_import unless project.gitlab_project_import?
unless importer.execute
raise Error, 'The remote data could not be imported.'
......@@ -58,6 +63,8 @@ module Projects
end
def importer
return Gitlab::ImportExport::Importer.new(project) if @project.gitlab_project_import?
class_name = "Gitlab::#{project.import_type.camelize}Import::Importer"
class_name.constantize.new(project)
end
......
- page_title "GitLab Import"
- header_title "Projects", root_path
%h3.page-title
= icon('gitlab')
Import an exported GitLab project
%hr
= form_tag import_gitlab_project_path, class: 'form-horizontal', multipart: true do
%p
Project will be imported as
%strong
#{@namespace_name}/#{@path}
%p
To move or copy an entire GitLab project from another GitLab installation to this one, navigate to the original project's settings page, generate an export file, and upload it here.
.form-group
= hidden_field_tag :namespace_id, @namespace_id
= hidden_field_tag :path, @path
= label_tag :file, class: 'control-label' do
%span GitLab project export
.col-sm-10
= file_field_tag :file, class: ''
.form-actions
= submit_tag 'Import project', class: 'btn btn-create'
%p
Project #{@project.name} was exported successfully.
%p
The project export can be downloaded from:
= link_to download_export_namespace_project_url(@project.namespace, @project) do
= @project.name_with_namespace + " export"
%p
The download link will expire in 24 hours.
Project <%= @project.name %> was exported successfully.
The project export can be downloaded from:
<%= download_export_namespace_project_url(@project.namespace, @project) %>
The download link will expire in 24 hours.
%p
Project #{@project.name} couldn't be exported.
%p
The errors we encountered were:
%ul
- @errors.each do |error|
%li
error
Project <%= @project.name %> couldn't be exported.
The errors we encountered were:
- @errors.each do |error|
<%= error %>
\ No newline at end of file
......@@ -120,6 +120,42 @@
= link_to 'Housekeeping', housekeeping_namespace_project_path(@project.namespace, @project),
method: :post, class: "btn btn-save"
%hr
.row.prepend-top-default
.col-lg-3
%h4.prepend-top-0
Export project
%p.append-bottom-0
%p
Export this project with all its related data in order to move your project to a new GitLab instance. Once the export is finished, you can import the file from the "New Project" page.
%p
Once the exported file is ready, you will receive a notification email with a download link.
.col-lg-9
- if @project.export_project_path
= link_to 'Download export', download_export_namespace_project_path(@project.namespace, @project),
method: :get, class: "btn btn-default"
= link_to 'Generate new export', generate_new_export_namespace_project_path(@project.namespace, @project),
method: :post, class: "btn btn-default"
- else
= link_to 'Export project', export_namespace_project_path(@project.namespace, @project),
method: :post, class: "btn btn-default"
.bs-callout.bs-callout-info
%p.append-bottom-0
%p
The following items will be exported:
%ul
%li Project and wiki repositories
%li Project uploads
%li Project configuration including web hooks and services
%li Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities
%p
The following items will NOT be exported:
%ul
%li Build traces and artifacts
%li LFS objects
%hr
- if can? current_user, :archive_project, @project
.row.prepend-top-default
.col-lg-3
......
......@@ -7,7 +7,7 @@
Forking in progress.
- else
Import in progress.
- unless @project.forked?
- if @project.external_import?
%p.monospace git clone --bare #{@project.safe_import_url}
%p Please wait while we import the repository for you. Refresh at will.
:javascript
......
......@@ -84,7 +84,12 @@
- if git_import_enabled?
= link_to "#", class: 'btn js-toggle-button import_git' do
%i.fa.fa-git
%span Any repo by URL
%span Repo by URL
- if gitlab_project_import_enabled?
= link_to new_import_gitlab_project_path, class: 'btn import_gitlab_project project-submit' do
%i.fa.fa-gitlab
%span GitLab export
.js-toggle-content.hide
= render "shared/import_form", f: f
......@@ -115,6 +120,33 @@
e.preventDefault();
var import_modal = $(this).next(".modal").show();
});
$('.modal-header .close').bind('click', function() {
$(".modal").hide();
});
$('.import_gitlab_project').bind('click', function() {
var _href = $("a.import_gitlab_project").attr("href");
$(".import_gitlab_project").attr("href", _href + '?namespace_id=' + $("#project_namespace_id").val() + '&path=' + $("#project_path").val());
});
$('.import_gitlab_project').attr('disabled',true)
$('.import_gitlab_project').attr('title', 'Project path required.');