Commit d737abc5 authored by Tim Zallmann's avatar Tim Zallmann

Merge branch 'sh-support-bitbucket-server-import' into 'master'

Add support for Bitbucket Server imports

Closes #25393

See merge request gitlab-org/gitlab-ce!20164
parents 0ffd7931 ee851e58
......@@ -36,6 +36,8 @@ class ImporterStatus {
const $targetField = $tr.find('.import-target');
const $namespaceInput = $targetField.find('.js-select-namespace option:selected');
const id = $tr.attr('id').replace('repo_', '');
const repoData = $tr.data();
let targetNamespace;
let newName;
if ($namespaceInput.length > 0) {
......@@ -45,12 +47,20 @@ class ImporterStatus {
}
$btn.disable().addClass('is-loading');
return axios.post(this.importUrl, {
this.id = id;
let attributes = {
repo_id: id,
target_namespace: targetNamespace,
new_name: newName,
ci_cd_only: this.ciCdOnly,
})
};
if (repoData) {
attributes = Object.assign(repoData, attributes);
}
return axios.post(this.importUrl, attributes)
.then(({ data }) => {
const job = $(`tr#repo_${id}`);
job.attr('id', `project_${data.id}`);
......@@ -70,6 +80,9 @@ class ImporterStatus {
.catch((error) => {
let details = error;
const $statusField = $(`#repo_${this.id} .job-status`);
$statusField.text(__('Failed'));
if (error.response && error.response.data && error.response.data.errors) {
details = error.response.data.errors;
}
......
......@@ -35,6 +35,7 @@ class ApplicationController < ActionController::Base
:gitea_import_enabled?, :github_import_configured?,
:gitlab_import_enabled?, :gitlab_import_configured?,
:bitbucket_import_enabled?, :bitbucket_import_configured?,
:bitbucket_server_import_enabled?,
:google_code_import_enabled?, :fogbugz_import_enabled?,
:git_import_enabled?, :gitlab_project_import_enabled?,
:manifest_import_enabled?
......@@ -337,6 +338,10 @@ class ApplicationController < ActionController::Base
!Gitlab::CurrentSettings.import_sources.empty?
end
def bitbucket_server_import_enabled?
Gitlab::CurrentSettings.import_sources.include?('bitbucket_server')
end
def github_import_enabled?
Gitlab::CurrentSettings.import_sources.include?('github')
end
......
# frozen_string_literal: true
class Import::BitbucketServerController < Import::BaseController
before_action :verify_bitbucket_server_import_enabled
before_action :bitbucket_auth, except: [:new, :configure]
before_action :validate_import_params, only: [:create]
# As a basic sanity check to prevent URL injection, restrict project
# repository input and repository slugs to allowed characters. For Bitbucket:
#
# Project keys must start with a letter and may only consist of ASCII letters, numbers and underscores (A-Z, a-z, 0-9, _).
#
# Repository names are limited to 128 characters. They must start with a
# letter or number and may contain spaces, hyphens, underscores, and periods.
# (https://community.atlassian.com/t5/Answers-Developer-Questions/stash-repository-names/qaq-p/499054)
VALID_BITBUCKET_CHARS = /\A[\w\-_\.\s]+\z/
def new
end
def create
repo = bitbucket_client.repo(@project_key, @repo_slug)
unless repo
return render json: { errors: "Project #{@project_key}/#{@repo_slug} could not be found" }, status: :unprocessable_entity
end
project_name = params[:new_name].presence || repo.name
namespace_path = params[:new_namespace].presence || current_user.username
target_namespace = find_or_create_namespace(namespace_path, current_user)
if current_user.can?(:create_projects, target_namespace)
project = Gitlab::BitbucketServerImport::ProjectCreator.new(@project_key, @repo_slug, repo, project_name, target_namespace, current_user, credentials).execute
if project.persisted?
render json: ProjectSerializer.new.represent(project)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
else
render json: { errors: 'This namespace has already been taken! Please choose another one.' }, status: :unprocessable_entity
end
rescue BitbucketServer::Client::ServerError => e
render json: { errors: "Unable to connect to server: #{e}" }, status: :unprocessable_entity
end
def configure
session[personal_access_token_key] = params[:personal_access_token]
session[bitbucket_server_username_key] = params[:bitbucket_username]
session[bitbucket_server_url_key] = params[:bitbucket_server_url]
redirect_to status_import_bitbucket_server_path
end
def status
repos = bitbucket_client.repos
@repos, @incompatible_repos = repos.partition { |repo| repo.valid? }
@already_added_projects = find_already_added_projects('bitbucket_server')
already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.to_a.reject! { |repo| already_added_projects_names.include?(repo.browse_url) }
rescue BitbucketServer::Connection::ConnectionError, BitbucketServer::Client::ServerError => e
flash[:alert] = "Unable to connect to server: #{e}"
clear_session_data
redirect_to new_import_bitbucket_server_path
end
def jobs
render json: find_jobs('bitbucket_server')
end
private
def bitbucket_client
@bitbucket_client ||= BitbucketServer::Client.new(credentials)
end
def validate_import_params
@project_key = params[:project]
@repo_slug = params[:repository]
return render_validation_error('Missing project key') unless @project_key.present? && @repo_slug.present?
return render_validation_error('Missing repository slug') unless @repo_slug.present?
return render_validation_error('Invalid project key') unless @project_key =~ VALID_BITBUCKET_CHARS
return render_validation_error('Invalid repository slug') unless @repo_slug =~ VALID_BITBUCKET_CHARS
end
def render_validation_error(message)
render json: { errors: message }, status: :unprocessable_entity
end
def bitbucket_auth
unless session[bitbucket_server_url_key].present? &&
session[bitbucket_server_username_key].present? &&
session[personal_access_token_key].present?
redirect_to new_import_bitbucket_server_path
end
end
def verify_bitbucket_server_import_enabled
render_404 unless bitbucket_server_import_enabled?
end
def bitbucket_server_url_key
:bitbucket_server_url
end
def bitbucket_server_username_key
:bitbucket_server_username
end
def personal_access_token_key
:bitbucket_server_personal_access_token
end
def clear_session_data
session[bitbucket_server_url_key] = nil
session[bitbucket_server_username_key] = nil
session[personal_access_token_key] = nil
end
def credentials
{
base_uri: session[bitbucket_server_url_key],
user: session[bitbucket_server_username_key],
password: session[personal_access_token_key]
}
end
end
......@@ -9,13 +9,23 @@ module NamespacesHelper
.includes(:route)
.order('routes.path')
users = [current_user.namespace]
selected_id = selected
unless extra_group.nil? || extra_group.is_a?(Group)
extra_group = Group.find(extra_group) if Namespace.find(extra_group).kind == 'group'
end
if extra_group && extra_group.is_a?(Group) && (!Group.exists?(name: extra_group.name) || Ability.allowed?(current_user, :read_group, extra_group))
groups |= [extra_group]
if extra_group && extra_group.is_a?(Group)
extra_group = dedup_extra_group(extra_group)
if Ability.allowed?(current_user, :read_group, extra_group)
# Assign the value to an invalid primary ID so that the select box works
extra_group.id = -1 unless extra_group.persisted?
selected_id = extra_group.id if selected == :extra_group
groups |= [extra_group]
else
selected_id = current_user.namespace.id
end
end
options = []
......@@ -25,11 +35,11 @@ module NamespacesHelper
options << options_for_group(users, display_path: display_path, type: 'user')
if selected == :current_user && current_user.namespace
selected = current_user.namespace.id
selected_id = current_user.namespace.id
end
end
grouped_options_for_select(options, selected)
grouped_options_for_select(options, selected_id)
end
def namespace_icon(namespace, size = 40)
......@@ -42,6 +52,17 @@ module NamespacesHelper
private
# Many importers create a temporary Group, so use the real
# group if one exists by that name to prevent duplicates.
def dedup_extra_group(extra_group)
unless extra_group.persisted?
existing_group = Group.find_by(name: extra_group.name)
extra_group = existing_group if existing_group&.persisted?
end
extra_group
end
def options_for_group(namespaces, display_path:, type:)
group_label = type.pluralize
elements = namespaces.sort_by(&:human_name).map! do |n|
......
......@@ -654,6 +654,8 @@ class Project < ActiveRecord::Base
project_import_data.credentials ||= {}
project_import_data.credentials = project_import_data.credentials.merge(credentials)
end
project_import_data
end
def import?
......
- title = _('Bitbucket Server Import')
- page_title title
- breadcrumb_title title
- header_title "Projects", root_path
%h3.page-title
= icon 'bitbucket-square', text: _('Import repositories from Bitbucket Server')
%p
= _('Enter in your Bitbucket Server URL and personal access token below')
= form_tag configure_import_bitbucket_server_path, method: :post do
.form-group.row
= label_tag :bitbucket_server_url, 'Bitbucket Server URL', class: 'col-form-label col-md-2'
.col-md-4
= text_field_tag :bitbucket_server_url, '', class: 'form-control append-right-8', placeholder: _('https://your-bitbucket-server'), size: 40
.form-group.row
= label_tag :bitbucket_server_url, 'Username', class: 'col-form-label col-md-2'
.col-md-4
= text_field_tag :bitbucket_username, '', class: 'form-control append-right-8', placeholder: _('username'), size: 40
.form-group.row
= label_tag :personal_access_token, 'Password/Personal Access Token', class: 'col-form-label col-md-2'
.col-md-4
= password_field_tag :personal_access_token, '', class: 'form-control append-right-8', placeholder: _('Personal Access Token'), size: 40
.form-actions
= submit_tag _('List your Bitbucket Server repositories'), class: 'btn btn-success'
- page_title 'Bitbucket Server import'
- header_title 'Projects', root_path
%h3.page-title
%i.fa.fa-bitbucket-square
= _('Import projects from Bitbucket Server')
- if @repos.any?
%p.light
= _('Select projects you want to import.')
.btn-group
- if @incompatible_repos.any?
= button_tag class: 'btn btn-import btn-success js-import-all' do
= _('Import all compatible projects')
= icon('spinner spin', class: 'loading-icon')
- else
= button_tag class: 'btn btn-import btn-success js-import-all' do
= _('Import all projects')
= icon('spinner spin', class: 'loading-icon')
.btn-group
= link_to('Reconfigure', configure_import_bitbucket_server_path, class: 'btn btn-primary', method: :post)
.table-responsive.prepend-top-10
%table.table.import-jobs
%colgroup.import-jobs-from-col
%colgroup.import-jobs-to-col
%colgroup.import-jobs-status-col
%thead
%tr
%th= _('From Bitbucket Server')
%th= _('To GitLab')
%th= _(' Status')
%tbody
- @already_added_projects.each do |project|
%tr{ id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}" }
%td
= link_to project.import_source, project.import_source, target: '_blank', rel: 'noopener noreferrer'
%td
= link_to project.full_path, [project.namespace.becomes(Namespace), project]
%td.job-status
- if project.import_status == 'finished'
= icon('check', text: 'Done')
- elsif project.import_status == 'started'
= icon('spin', text: 'started')
- else
= project.human_import_status_name
- @repos.each do |repo|
%tr{ id: "repo_#{repo.project_key}___#{repo.slug}", data: { project: repo.project_key, repository: repo.slug } }
%td
= link_to repo.browse_url, repo.browse_url, target: '_blank', rel: 'noopener noreferrer'
%td.import-target
%fieldset.row
.input-group
.project-path.input-group-prepend
- if current_user.can_select_namespace?
- selected = params[:namespace_id] || :extra_group
- opts = current_user.can_create_group? ? { extra_group: Group.new(name: repo.project_key, path: repo.project_key) } : {}
= select_tag :namespace_id, namespaces_options(selected, opts.merge({ display_path: true })), { class: 'input-group-text select2 js-select-namespace', tabindex: 1 }
- else
= text_field_tag :path, current_user.namespace_path, class: "input-group-text input-large form-control", tabindex: 1, disabled: true
%span.input-group-prepend
.input-group-text /
= text_field_tag :path, repo.name, class: "input-mini form-control", tabindex: 2, autofocus: true, required: true
%td.import-actions.job-status
= button_tag class: 'btn btn-import js-add-to-import' do
Import
= icon('spinner spin', class: 'loading-icon')
- @incompatible_repos.each do |repo|
%tr{ id: "repo_#{repo.project_key}___#{repo.slug}" }
%td
= link_to repo.browse_url, repo.browse_url, target: '_blank', rel: 'noopener noreferrer'
%td.import-target
%td.import-actions-job-status
= label_tag 'Incompatible Project', nil, class: 'label badge-danger'
- if @incompatible_repos.any?
%p
One or more of your Bitbucket Server projects cannot be imported into GitLab
directly because they use Subversion or Mercurial for version control,
rather than Git. Please convert
= link_to 'them to Git,', 'https://www.atlassian.com/git/tutorials/migrating-overview'
and go through the
= link_to 'import flow', status_import_bitbucket_server_path
again.
.js-importer-status{ data: { jobs_import_path: "#{jobs_import_bitbucket_server_path}", import_path: "#{import_bitbucket_server_path}" } }
......@@ -18,10 +18,14 @@
- if bitbucket_import_enabled?
%div
= link_to status_import_bitbucket_path, class: "btn import_bitbucket #{'how_to_import_link' unless bitbucket_import_configured?}" do
= icon('bitbucket', text: 'Bitbucket')
= icon('bitbucket', text: 'Bitbucket Cloud')
- unless bitbucket_import_configured?
= render 'bitbucket_import_modal'
- if bitbucket_server_import_enabled?
%div
= link_to status_import_bitbucket_server_path, class: "btn import_bitbucket" do
= icon('bitbucket-square', text: 'Bitbucket Server')
%div
- if gitlab_import_enabled?
%div
= link_to status_import_gitlab_path, class: "btn import_gitlab #{'how_to_import_link' unless gitlab_import_configured?}" do
......
......@@ -24,6 +24,13 @@ namespace :import do
get :jobs
end
resource :bitbucket_server, only: [:create, :new], controller: :bitbucket_server do
post :configure
get :status
get :callback
get :jobs
end
resource :google_code, only: [:create, :new], controller: :google_code do
get :status
post :callback
......
# frozen_string_literal: true
module BitbucketServer
class Client
attr_reader :connection
ServerError = Class.new(StandardError)
SERVER_ERRORS = [SocketError,
OpenSSL::SSL::SSLError,
Errno::ECONNRESET,
Errno::ECONNREFUSED,
Errno::EHOSTUNREACH,
Net::OpenTimeout,
Net::ReadTimeout,
Gitlab::HTTP::BlockedUrlError,
BitbucketServer::Connection::ConnectionError].freeze
def initialize(options = {})
@connection = Connection.new(options)
end
def pull_requests(project_key, repo)
path = "/projects/#{project_key}/repos/#{repo}/pull-requests?state=ALL"
get_collection(path, :pull_request)
end
def activities(project_key, repo, pull_request_id)
path = "/projects/#{project_key}/repos/#{repo}/pull-requests/#{pull_request_id}/activities"
get_collection(path, :activity)
end
def repo(project, repo_name)
parsed_response = connection.get("/projects/#{project}/repos/#{repo_name}")
BitbucketServer::Representation::Repo.new(parsed_response)
end
def repos
path = "/repos"
get_collection(path, :repo)
end
def create_branch(project_key, repo, branch_name, sha)
payload = {
name: branch_name,
startPoint: sha,
message: 'GitLab temporary branch for import'
}
connection.post("/projects/#{project_key}/repos/#{repo}/branches", payload.to_json)
end
def delete_branch(project_key, repo, branch_name, sha)
payload = {
name: Gitlab::Git::BRANCH_REF_PREFIX + branch_name,
dryRun: false
}
connection.delete(:branches, "/projects/#{project_key}/repos/#{repo}/branches", payload.to_json)
end
private
def get_collection(path, type)
paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type)
BitbucketServer::Collection.new(paginator)
rescue *SERVER_ERRORS => e
raise ServerError, e
end
end
end
# frozen_string_literal: true
module BitbucketServer
class Collection < Enumerator
def initialize(paginator)
super() do |yielder|
loop do
paginator.items.each { |item| yielder << item }
end
end
lazy
end
def method_missing(method, *args)
return super unless self.respond_to?(method)
self.__send__(method, *args) do |item| # rubocop:disable GitlabSecurity/PublicSend
block_given? ? yield(item) : item
end
end
end
end
# frozen_string_literal: true
module BitbucketServer
class Connection
include ActionView::Helpers::SanitizeHelper
DEFAULT_API_VERSION = '1.0'
SEPARATOR = '/'
attr_reader :api_version, :base_uri, :username, :token
ConnectionError = Class.new(StandardError)
def initialize(options = {})
@api_version = options.fetch(:api_version, DEFAULT_API_VERSION)
@base_uri = options[:base_uri]
@username = options[:user]
@token = options[:password]
end
def get(path, extra_query = {})
response = Gitlab::HTTP.get(build_url(path),
basic_auth: auth,
headers: accept_headers,
query: extra_query)
check_errors!(response)
response.parsed_response
end
def post(path, body)
response = Gitlab::HTTP.post(build_url(path),
basic_auth: auth,
headers: post_headers,
body: body)
check_errors!(response)
response.parsed_response
end
# We need to support two different APIs for deletion:
#
# /rest/api/1.0/projects/{projectKey}/repos/{repositorySlug}/branches/default
# /rest/branch-utils/1.0/projects/{projectKey}/repos/{repositorySlug}/branches
def delete(resource, path, body)
url = delete_url(resource, path)
response = Gitlab::HTTP.delete(url,
basic_auth: auth,
headers: post_headers,
body: body)
check_errors!(response)
response.parsed_response
end
private
def check_errors!(response)
raise ConnectionError, "Response is not valid JSON" unless response.parsed_response.is_a?(Hash)
return if response.code >= 200 && response.code < 300
details = sanitize(response.parsed_response.dig('errors', 0, 'message'))
message = "Error #{response.code}"
message += ": #{details}" if details
raise ConnectionError, message
rescue JSON::ParserError
raise ConnectionError, "Unable to parse the server response as JSON"
end
def auth
@auth ||= { username: username, password: token }
end
def accept_headers
@accept_headers ||= { 'Accept' => 'application/json' }
end
def post_headers
@post_headers ||= accept_headers.merge({ 'Content-Type' => 'application/json' })
end
def build_url(path)
return path if path.starts_with?(root_url)
url_join_paths(root_url, path)
end
def root_url
url_join_paths(base_uri, "/rest/api/#{api_version}")
end
def delete_url(resource, path)
if resource == :branches
url_join_paths(base_uri, "/rest/branch-utils/#{api_version}#{path}")
else
build_url(path)
end
end
# URI.join is stupid in that slashes are important:
#
# # URI.join('http://example.com/subpath', 'hello')
# => http://example.com/hello
#
# We really want http://example.com/subpath/hello
#
def url_join_paths(*paths)
paths.map { |path| strip_slashes(path) }.join(SEPARATOR)
end
def strip_slashes(path)
path = path[1..-1] if path.starts_with?(SEPARATOR)
path.chomp(SEPARATOR)
end
end
end
# frozen_string_literal: true
module BitbucketServer
class Page
attr_reader :attrs, :items
def initialize(raw, type)
@attrs = parse_attrs(raw)
@items = parse_values(raw, representation_class(type))
end
def next?
!attrs.fetch(:isLastPage, true)
end
def next
attrs.fetch(:nextPageStart)
end
private
def parse_attrs(raw)
raw.slice('size', 'nextPageStart', 'isLastPage').symbolize_keys
end
def parse_values(raw, bitbucket_rep_class)
return [] unless raw['values'] && raw['values'].is_a?(Array)
bitbucket_rep_class.decorate(raw['values'])
end
def representation_class(type)
BitbucketServer::Representation.const_get(type.to_s.camelize)
end
end
end
# frozen_string_literal: true
module BitbucketServer
class Paginator
PAGE_LENGTH = 25
def initialize(connection, url, type)
@connection = connection
@type = type
@url = url
@page = nil
end
def items
raise StopIteration unless has_next_page?
@page = fetch_next_page
@page.items
end
private
attr_reader :connection, :page, :url, :type
def has_next_page?
page.nil? || page.next?
end
def next_offset
page.nil? ? 0 : page.next
end
def fetch_next_page
parsed_response = connection.get(@url, start: next_offset, limit: PAGE_LENGTH)