Skip to content
Snippets Groups Projects
Commit b46754d9 authored by Rodrigo Tomonari's avatar Rodrigo Tomonari :two:
Browse files

Merge branch '412614-import-issues' into 'master'

Import Bitbucket issues

See merge request !131138



Merged-by: default avatarRodrigo Tomonari <rtomonari@gitlab.com>
Approved-by: default avatarCarla Drago <cdrago@gitlab.com>
Approved-by: default avatarRodrigo Tomonari <rtomonari@gitlab.com>
Co-authored-by: default avatarmaddievn <mvanniekerk@gitlab.com>
parents 34db6e61 732209bc
No related branches found
No related tags found
1 merge request!131138Import Bitbucket issues
Pipeline #1011478858 failed
Showing
with 462 additions and 23 deletions
......@@ -2352,6 +2352,15 @@
:weight: 1
:idempotent: false
:tags: []
- :name: bitbucket_import_import_issue
:worker_name: Gitlab::BitbucketImport::ImportIssueWorker
:feature_category: :importers
:has_external_dependencies: true
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: bitbucket_import_import_pull_request
:worker_name: Gitlab::BitbucketImport::ImportPullRequestWorker
:feature_category: :importers
......@@ -2370,6 +2379,15 @@
:weight: 1
:idempotent: false
:tags: []
- :name: bitbucket_import_stage_import_issues
:worker_name: Gitlab::BitbucketImport::Stage::ImportIssuesWorker
:feature_category: :importers
:has_external_dependencies: true
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: false
:tags: []
- :name: bitbucket_import_stage_import_pull_requests
:worker_name: Gitlab::BitbucketImport::Stage::ImportPullRequestsWorker
:feature_category: :importers
......
......@@ -20,6 +20,7 @@ class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker
# The known importer stages and their corresponding Sidekiq workers.
STAGES = {
issues: Stage::ImportIssuesWorker,
finish: Stage::FinishImportWorker
}.freeze
......
# frozen_string_literal: true
module Gitlab
module BitbucketImport
class ImportIssueWorker # rubocop:disable Scalability/IdempotentWorker
include ObjectImporter
def importer_class
Importers::IssueImporter
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BitbucketImport
module Stage
class ImportIssuesWorker # rubocop:disable Scalability/IdempotentWorker
include StageMethods
private
# project - An instance of Project.
def import(project)
waiter = importer_class.new(project).execute
project.import_state.refresh_jid_expiration
AdvanceStageWorker.perform_async(
project.id,
{ waiter.key => waiter.jobs_remaining },
:finish
)
end
def importer_class
Importers::IssuesImporter
end
end
end
end
end
......@@ -17,7 +17,7 @@ def import(project)
AdvanceStageWorker.perform_async(
project.id,
{ waiter.key => waiter.jobs_remaining },
:finish
:issues
)
end
......
......@@ -81,10 +81,14 @@
- 1
- - bitbucket_import_advance_stage
- 1
- - bitbucket_import_import_issue
- 1
- - bitbucket_import_import_pull_request
- 1
- - bitbucket_import_stage_finish_import
- 1
- - bitbucket_import_stage_import_issues
- 1
- - bitbucket_import_stage_import_pull_requests
- 1
- - bitbucket_import_stage_import_repository
......
......@@ -45,6 +45,19 @@ def to_s
iid
end
def to_hash
{
iid: iid,
title: title,
description: description,
state: state,
author: author,
milestone: milestone,
created_at: created_at,
updated_at: updated_at
}
end
private
def closed?
......
# frozen_string_literal: true
module Gitlab
module BitbucketImport
module ErrorTracking
def track_import_failure!(project, exception:, **args)
Gitlab::Import::ImportFailureService.track(
project_id: project.id,
error_source: self.class.name,
exception: exception,
**args
)
end
end
end
end
......@@ -31,6 +31,17 @@ def execute
true
end
def create_labels
LABELS.each do |label_params|
label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true)
if label.valid?
@labels[label_params[:title]] = label
else
raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.full_name}\""
end
end
end
private
def already_imported?(collection, iid)
......@@ -182,17 +193,6 @@ def import_issue_comments(issue, gitlab_issue)
end
end
def create_labels
LABELS.each do |label_params|
label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true)
if label.valid?
@labels[label_params[:title]] = label
else
raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.full_name}\""
end
end
end
def import_pull_requests
pull_requests = client.pull_requests(repo)
......
# frozen_string_literal: true
module Gitlab
module BitbucketImport
module Importers
class IssueImporter
include Loggable
include ErrorTracking
def initialize(project, hash)
@project = project
@formatter = Gitlab::ImportFormatter.new
@user_finder = UserFinder.new(project)
@object = hash.with_indifferent_access
end
def execute
log_info(import_stage: 'import_issue', message: 'starting', iid: object[:iid])
description = ''
description += author_line
description += object[:description] if object[:description]
milestone = object[:milestone] ? project.milestones.find_or_create_by(title: object[:milestone]) : nil # rubocop: disable CodeReuse/ActiveRecord
attributes = {
iid: object[:iid],
title: object[:title],
description: description,
state_id: Issue.available_states[object[:state]],
author_id: author_id,
assignee_ids: [author_id],
namespace_id: project.project_namespace_id,
milestone: milestone,
work_item_type_id: object[:issue_type_id],
label_ids: [object[:label_id]].compact,
created_at: object[:created_at],
updated_at: object[:updated_at]
}
project.issues.create!(attributes)
log_info(import_stage: 'import_issue', message: 'finished', iid: object[:iid])
rescue StandardError => e
track_import_failure!(project, exception: e)
end
private
attr_reader :object, :project, :formatter, :user_finder
def author_line
return '' if find_user_id
formatter.author_line(object[:author])
end
def find_user_id
user_finder.find_user_id(object[:author])
end
def author_id
user_finder.gitlab_user_id(project, object[:author])
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BitbucketImport
module Importers
class IssuesImporter
include ParallelScheduling
def execute
log_info(import_stage: 'import_issues', message: 'importing issues')
issues = client.issues(project.import_source)
labels = build_labels_hash
issues.each do |issue|
job_waiter.jobs_remaining += 1
next if already_enqueued?(issue)
job_delay = calculate_job_delay(job_waiter.jobs_remaining)
issue_hash = issue.to_hash.merge({ issue_type_id: default_issue_type_id, label_id: labels[issue.kind] })
sidekiq_worker_class.perform_in(job_delay, project.id, issue_hash, job_waiter.key)
mark_as_enqueued(issue)
end
job_waiter
rescue StandardError => e
track_import_failure!(project, exception: e)
end
private
def sidekiq_worker_class
ImportIssueWorker
end
def collection_method
:issues
end
def id_for_already_enqueued_cache(object)
object.iid
end
def default_issue_type_id
::WorkItems::Type.default_issue_type.id
end
def build_labels_hash
labels = {}
project.labels.each { |l| labels[l.title.to_s] = l.id }
labels
end
end
end
end
end
......@@ -5,6 +5,7 @@ module BitbucketImport
module Importers
class PullRequestImporter
include Loggable
include ErrorTracking
def initialize(project, hash)
@project = project
......@@ -48,7 +49,7 @@ def execute
log_info(import_stage: 'import_pull_request', message: 'finished', iid: object[:iid])
rescue StandardError => e
Gitlab::Import::ImportFailureService.track(project_id: project.id, exception: e)
track_import_failure!(project, exception: e)
end
private
......
......@@ -23,6 +23,7 @@ def execute
end
import_wiki
create_labels
log_info(import_stage: 'import_repository', message: 'finished import')
......@@ -59,6 +60,11 @@ def import_wiki
)
end
def create_labels
importer = Gitlab::BitbucketImport::Importer.new(project)
importer.create_labels
end
def wiki
WikiFormatter.new(project)
end
......
......@@ -4,6 +4,7 @@ module Gitlab
module BitbucketImport
module ParallelScheduling
include Loggable
include ErrorTracking
attr_reader :project, :already_enqueued_cache_key, :job_waiter_cache_key
......@@ -79,15 +80,6 @@ def calculate_job_delay(job_index)
(multiplier * 1.minute) + 1.second
end
def track_import_failure!(project, exception:, **args)
Gitlab::Import::ImportFailureService.track(
project_id: project.id,
error_source: self.class.name,
exception: exception,
**args
)
end
end
end
end
......@@ -2,7 +2,7 @@
require 'fast_spec_helper'
RSpec.describe Bitbucket::Representation::Issue do
RSpec.describe Bitbucket::Representation::Issue, feature_category: :importers do
describe '#iid' do
it { expect(described_class.new('id' => 1).iid).to eq(1) }
end
......@@ -46,4 +46,32 @@
describe '#updated_at' do
it { expect(described_class.new('edited_on' => Date.today).updated_at).to eq(Date.today) }
end
describe '#to_hash' do
it do
raw = {
'id' => 111,
'title' => 'title',
'content' => { 'raw' => 'description' },
'state' => 'resolved',
'reporter' => { 'nickname' => 'User1' },
'milestone' => { 'name' => 1 },
'created_on' => 'created_at',
'edited_on' => 'updated_at'
}
expected_hash = {
iid: 111,
title: 'title',
description: 'description',
state: 'closed',
author: 'User1',
milestone: 1,
created_at: 'created_at',
updated_at: 'updated_at'
}
expect(described_class.new(raw).to_hash).to eq(expected_hash)
end
end
end
......@@ -384,6 +384,12 @@
expect(label_after_import.attributes).to eq(existing_label.attributes)
end
end
it 'raises an error if a label is not valid' do
stub_const("#{described_class}::LABELS", [{ title: nil, color: nil }])
expect { importer.create_labels }.to raise_error(StandardError, /Failed to create label/)
end
end
it 'maps statuses to open or closed' do
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BitbucketImport::Importers::IssueImporter, :clean_gitlab_redis_cache, feature_category: :importers do
include AfterNextHelpers
let_it_be(:project) { create(:project, :repository) }
let_it_be(:bitbucket_user) { create(:user) }
let_it_be(:identity) { create(:identity, user: bitbucket_user, extern_uid: 'bitbucket_user', provider: :bitbucket) }
let_it_be(:default_work_item_type) { create(:work_item_type) }
let_it_be(:label) { create(:label, project: project) }
let(:hash) do
{
iid: 111,
title: 'title',
description: 'description',
state: 'closed',
author: 'bitbucket_user',
milestone: 'my milestone',
issue_type_id: default_work_item_type.id,
label_id: label.id,
created_at: Date.today,
updated_at: Date.today
}
end
subject(:importer) { described_class.new(project, hash) }
before do
allow(Gitlab::Git).to receive(:ref_name).and_return('refname')
end
describe '#execute' do
it 'creates an issue' do
expect { importer.execute }.to change { project.issues.count }.from(0).to(1)
issue = project.issues.first
expect(issue.description).to eq('description')
expect(issue.author).to eq(bitbucket_user)
expect(issue.closed?).to be_truthy
expect(issue.milestone).to eq(project.milestones.first)
expect(issue.work_item_type).to eq(default_work_item_type)
expect(issue.labels).to eq([label])
expect(issue.created_at).to eq(Date.today)
expect(issue.updated_at).to eq(Date.today)
end
context 'when the author does not have a bitbucket identity' do
before do
identity.update!(provider: :github)
end
it 'sets the author to the project creator and adds the author to the description' do
importer.execute
issue = project.issues.first
expect(issue.author).to eq(project.creator)
expect(issue.description).to eq("*Created by: bitbucket_user*\n\ndescription")
end
end
context 'when a milestone with the same title exists' do
let_it_be(:milestone) { create(:milestone, project: project, title: 'my milestone') }
it 'assigns the milestone and does not create a new milestone' do
expect { importer.execute }.not_to change { project.milestones.count }
expect(project.issues.first.milestone).to eq(milestone)
end
end
context 'when a milestone with the same title does not exist' do
it 'creates a new milestone and assigns it' do
expect { importer.execute }.to change { project.milestones.count }.from(0).to(1)
expect(project.issues.first.milestone).to eq(project.milestones.first)
end
end
context 'when an error is raised' do
it 'tracks the failure and does not fail' do
expect(Gitlab::Import::ImportFailureService).to receive(:track).once
described_class.new(project, hash.except(:title)).execute
end
end
it 'logs its progress' do
allow(Gitlab::Import::MergeRequestCreator).to receive_message_chain(:new, :execute)
expect(Gitlab::BitbucketImport::Logger)
.to receive(:info).with(include(message: 'starting', iid: anything)).and_call_original
expect(Gitlab::BitbucketImport::Logger)
.to receive(:info).with(include(message: 'finished', iid: anything)).and_call_original
importer.execute
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BitbucketImport::Importers::IssuesImporter, feature_category: :importers do
let_it_be(:project) do
create(:project, :import_started,
import_data_attributes: {
data: { 'project_key' => 'key', 'repo_slug' => 'slug' },
credentials: { 'base_uri' => 'http://bitbucket.org/', 'user' => 'bitbucket', 'password' => 'password' }
}
)
end
subject(:importer) { described_class.new(project) }
describe '#execute', :clean_gitlab_redis_cache do
before do
allow_next_instance_of(Bitbucket::Client) do |client|
allow(client).to receive(:issues).and_return(
[
Bitbucket::Representation::Issue.new({ 'id' => 1 }),
Bitbucket::Representation::Issue.new({ 'id' => 2 })
],
[]
)
end
end
it 'imports each issue in parallel', :aggregate_failures do
expect(Gitlab::BitbucketImport::ImportIssueWorker).to receive(:perform_in).twice
waiter = importer.execute
expect(waiter).to be_an_instance_of(Gitlab::JobWaiter)
expect(waiter.jobs_remaining).to eq(2)
expect(Gitlab::Cache::Import::Caching.values_from_set(importer.already_enqueued_cache_key))
.to match_array(%w[1 2])
end
context 'when the client raises an error' do
before do
allow_next_instance_of(Bitbucket::Client) do |client|
allow(client).to receive(:issues).and_raise(StandardError)
end
end
it 'tracks the failure and does not fail' do
expect(Gitlab::Import::ImportFailureService).to receive(:track).once
importer.execute
end
end
context 'when issue was already enqueued' do
before do
Gitlab::Cache::Import::Caching.set_add(importer.already_enqueued_cache_key, 1)
end
it 'does not schedule job for enqueued issues', :aggregate_failures do
expect(Gitlab::BitbucketImport::ImportIssueWorker).to receive(:perform_in).once
waiter = importer.execute
expect(waiter).to be_an_instance_of(Gitlab::JobWaiter)
expect(waiter.jobs_remaining).to eq(2)
end
end
end
end
......@@ -258,6 +258,7 @@
'GeoRepositoryDestroyWorker' => 3,
'Gitlab::BitbucketImport::AdvanceStageWorker' => 3,
'Gitlab::BitbucketImport::Stage::FinishImportWorker' => 3,
'Gitlab::BitbucketImport::Stage::ImportIssuesWorker' => 3,
'Gitlab::BitbucketImport::Stage::ImportPullRequestsWorker' => 3,
'Gitlab::BitbucketImport::Stage::ImportRepositoryWorker' => 3,
'Gitlab::BitbucketServerImport::AdvanceStageWorker' => 3,
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BitbucketImport::ImportIssueWorker, feature_category: :importers do
subject(:worker) { described_class.new }
it_behaves_like Gitlab::BitbucketImport::ObjectImporter
end
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment