Update bulk_imports api and docs

The client now sends a migrate_membership param
so this commit modifies the endpoint to receive it
and pass to create service. Also updates docs.

app/controllers/import/bulk_imports_controller.rb

@@ -148,6 +148,7 @@ def bulk_import_params
       destination_slug
       destination_namespace
       migrate_projects
+      migrate_memberships
     ]
   end
 

app/services/bulk_imports/create_service.rb

@@ -105,7 +105,8 @@ def create_bulk_import
             source_full_path: entity_params[:source_full_path],
             destination_slug: entity_params[:destination_slug] || entity_params[:destination_name],
             destination_namespace: entity_params[:destination_namespace],
-            migrate_projects: Gitlab::Utils.to_boolean(entity_params[:migrate_projects], default: true)
+            migrate_projects: Gitlab::Utils.to_boolean(entity_params[:migrate_projects], default: true),
+            migrate_memberships: Gitlab::Utils.to_boolean(entity_params[:migrate_memberships], default: true)
           )
         end
         bulk_import

doc/api/bulk_imports.md

@@ -49,6 +49,7 @@ POST /bulk_imports
 | `entities[destination_name]`      | String | no       | Deprecated: Use `destination_slug` instead. Destination slug for the entity. |
 | `entities[destination_namespace]` | String | yes      | Full path of the destination group [namespace](../user/namespace/index.md) for the entity. Must be an existing group in the destination instance. |
 | `entities[migrate_projects]`      | Boolean | no      | Also import all nested projects of the group (if `source_type` is `group_entity`). Defaults to `true`. |
+| `entities[migrate_memberships]`   | Boolean | no      | Import user memberships. Defaults to `true`. |
 
 ```shell
 curl --request POST --header "PRIVATE-TOKEN: <your_access_token_for_destination_gitlab_instance>" "https://destination-gitlab-instance.example.com/api/v4/bulk_imports" \
@@ -171,6 +172,7 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab
         "updated_at": "2021-06-18T09:47:51.867Z",
         "failures": [],
         "migrate_projects": true,
+        "migrate_memberships": true,
         "has_failures": false,
         "stats": {
             "labels": {
@@ -213,6 +215,7 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab
             }
         ],
         "migrate_projects": true,
+        "migrate_memberships": true,
         "has_failures": false,
         "stats": { }
     }
@@ -294,6 +297,7 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab
             }
         ],
         "migrate_projects": true,
+        "migrate_memberships": true,
         "has_failures": true,
         "stats": {
             "labels": {
@@ -350,6 +354,7 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab
         }
     ],
     "migrate_projects": true,
+    "migrate_memberships": true,
     "has_failures": true,
     "stats": {
         "labels": {

lib/api/bulk_imports.rb

@@ -86,6 +86,10 @@ def bulk_import_entity
             type: Boolean,
             default: true,
             desc: 'Indicates group migration should include nested projects'
+          optional :migrate_memberships,
+            type: Boolean,
+            default: true,
+            desc: 'The option to migrate memberships or not'
 
           mutually_exclusive :destination_slug, :destination_name
           at_least_one_of :destination_slug, :destination_name

lib/api/entities/bulk_imports/entity.rb

@@ -24,6 +24,7 @@ class Entity < Grape::Entity
         expose :updated_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' }
         expose :failures, using: EntityFailure, documentation: { is_array: true }
         expose :migrate_projects, documentation: { type: 'boolean', example: true }
+        expose :migrate_memberships, documentation: { type: 'boolean', example: true }
         expose :has_failures, documentation: { type: 'boolean', example: false }
         expose :checksums, as: :stats, documentation: { type: 'object' }
       end

lib/bulk_imports/groups/stage.rb

@@ -23,7 +23,7 @@ class Stage < ::BulkImports::Stage
       # SubGroup Entities must be imported in later stage than Project Entities to avoid `full_path` naming conflicts.
 
       def config
-        {
+        base_config = {
           group: {
             pipeline: BulkImports::Groups::Pipelines::GroupPipeline,
             stage: 0
@@ -37,10 +37,6 @@ def config
             stage: 1,
             minimum_source_version: '15.0.0'
           },
-          members: {
-            pipeline: BulkImports::Common::Pipelines::MembersPipeline,
-            stage: 1
-          },
           labels: {
             pipeline: BulkImports::Common::Pipelines::LabelsPipeline,
             stage: 1
@@ -70,7 +66,11 @@ def config
             pipeline: BulkImports::Common::Pipelines::EntityFinisher,
             stage: 4
           }
-        }.merge(project_entities_pipeline)
+        }
+
+        base_config
+          .merge(project_entities_pipeline)
+          .merge(members_pipeline)
       end
 
       def project_entities_pipeline
@@ -86,10 +86,25 @@ def project_entities_pipeline
         end
       end
 
+      def members_pipeline
+        return {} unless migrate_memberships?
+
+        {
+          members: {
+            pipeline: BulkImports::Common::Pipelines::MembersPipeline,
+            stage: 1
+          }
+        }
+      end
+
       def migrate_projects?
         bulk_import_entity.migrate_projects
       end
 
+      def migrate_memberships?
+        bulk_import_entity.migrate_memberships
+      end
+
       def project_pipeline_available?
         @bulk_import.source_version_info >= BulkImport.min_gl_version_for_project_migration
       end

lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb

@@ -11,7 +11,8 @@ def transform(context, entry)
             destination_name: entry['path'],
             destination_namespace: context.entity.group.full_path,
             parent_id: context.entity.id,
-            migrate_projects: context.entity.migrate_projects
+            migrate_projects: context.entity.migrate_projects,
+            migrate_memberships: context.entity.migrate_memberships
           }
         end
       end

lib/bulk_imports/projects/stage.rb

@@ -21,7 +21,7 @@ class Stage < ::BulkImports::Stage
       # instance version is 15.2.0, 15.2.1, 16.0.0, etc.
 
       def config
-        {
+        base_config = {
           project: {
             pipeline: BulkImports::Projects::Pipelines::ProjectPipeline,
             stage: 0
@@ -40,10 +40,6 @@ def config
             pipeline: BulkImports::Projects::Pipelines::ProjectAttributesPipeline,
             stage: 1
           },
-          members: {
-            pipeline: BulkImports::Common::Pipelines::MembersPipeline,
-            stage: 1
-          },
           labels: {
             pipeline: BulkImports::Common::Pipelines::LabelsPipeline,
             stage: 2
@@ -143,6 +139,23 @@ def config
             stage: 6
           }
         }
+
+        base_config.merge(members_pipeline)
+      end
+
+      def members_pipeline
+        return {} unless migrate_memberships?
+
+        {
+          members: {
+            pipeline: BulkImports::Common::Pipelines::MembersPipeline,
+            stage: 1
+          }
+        }
+      end
+
+      def migrate_memberships?
+        bulk_import_entity.migrate_memberships
       end
     end
   end

spec/factories/bulk_import/entities.rb

@@ -11,6 +11,7 @@
     destination_slug { 'imported-entity' }
     sequence(:source_xid)
     migrate_projects { true }
+    migrate_memberships { true }
 
     trait(:group_entity) do
       source_type { :group_entity }

spec/lib/api/entities/bulk_imports/entity_spec.rb

@@ -23,6 +23,7 @@
       :updated_at,
       :failures,
       :migrate_projects,
+      :migrate_memberships,
       :has_failures,
       :stats
     )

spec/lib/bulk_imports/groups/stage_spec.rb

@@ -105,5 +105,27 @@
         )
       end
     end
+
+    describe 'migrate memberships flag' do
+      context 'when true' do
+        it 'includes members pipeline' do
+          entity.update!(migrate_memberships: true)
+
+          expect(described_class.new(entity).pipelines).to include(
+            hash_including({ pipeline: BulkImports::Common::Pipelines::MembersPipeline })
+          )
+        end
+      end
+
+      context 'when false' do
+        it 'does not include members pipeline' do
+          entity.update!(migrate_memberships: false)
+
+          expect(described_class.new(entity).pipelines).not_to include(
+            hash_including({ pipeline: BulkImports::Common::Pipelines::MembersPipeline })
+          )
+        end
+      end
+    end
   end
 end

spec/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer_spec.rb

@@ -6,7 +6,8 @@
   describe "#transform" do
     it "transforms subgroups data in entity params" do
       parent = create(:group)
-      parent_entity = instance_double(BulkImports::Entity, group: parent, id: 1, migrate_projects: false)
+      parent_entity = instance_double(BulkImports::Entity, group: parent, id: 1,
+        migrate_projects: false, migrate_memberships: false)
       context = instance_double(BulkImports::Pipeline::Context, entity: parent_entity)
       subgroup_data = {
         "path" => "sub-group",
@@ -19,7 +20,8 @@
         destination_name: "sub-group",
         destination_namespace: parent.full_path,
         parent_id: 1,
-        migrate_projects: false
+        migrate_projects: false,
+        migrate_memberships: false
       )
     end
   end

spec/lib/bulk_imports/projects/stage_spec.rb

@@ -3,11 +3,9 @@
 require 'spec_helper'
 
 RSpec.describe BulkImports::Projects::Stage, feature_category: :importers do
-  subject do
-    entity = build(:bulk_import_entity, :project_entity)
+  let(:entity) { build(:bulk_import_entity, :project_entity) }
 
-    described_class.new(entity)
-  end
+  subject { described_class.new(entity) }
 
   describe '#pipelines' do
     it 'list all the pipelines' do
@@ -55,5 +53,27 @@
         expect(expected_stages).to eq([0, 1, 2, 2])
       end
     end
+
+    describe 'migrate memberships flag' do
+      context 'when true' do
+        it 'includes memberships pipeline' do
+          entity.update!(migrate_memberships: true)
+
+          expect(described_class.new(entity).pipelines).to include(
+            hash_including({ pipeline: BulkImports::Common::Pipelines::MembersPipeline })
+          )
+        end
+      end
+
+      context 'when false' do
+        it 'does not include memberships pipeline' do
+          entity.update!(migrate_memberships: false)
+
+          expect(described_class.new(entity).pipelines).not_to include(
+            hash_including({ pipeline: BulkImports::Common::Pipelines::MembersPipeline })
+          )
+        end
+      end
+    end
   end
 end

spec/requests/api/bulk_imports_spec.rb

@@ -210,6 +210,36 @@
           end
         end
       end
+
+      describe 'migrate memberships flag' do
+        context 'when true' do
+          it 'sets true' do
+            params[:entities][0][:migrate_memberships] = true
+
+            request
+
+            expect(user.bulk_imports.last.entities.pluck(:migrate_memberships)).to contain_exactly(true)
+          end
+        end
+
+        context 'when false' do
+          it 'sets false' do
+            params[:entities][0][:migrate_memberships] = false
+
+            request
+
+            expect(user.bulk_imports.last.entities.pluck(:migrate_memberships)).to contain_exactly(false)
+          end
+        end
+
+        context 'when unspecified' do
+          it 'sets true' do
+            request
+
+            expect(user.bulk_imports.last.entities.pluck(:migrate_memberships)).to contain_exactly(true)
+          end
+        end
+      end
     end
 
     include_examples 'starting a new migration' do

spec/services/bulk_imports/create_service_spec.rb

@@ -9,6 +9,7 @@
   let(:credentials) { { url: 'http://gitlab.example', access_token: 'token' } }
   let(:destination_group) { create(:group, path: 'destination1') }
   let(:migrate_projects) { true }
+  let(:migrate_memberships) { true }
   let_it_be(:parent_group) { create(:group, path: 'parent-group') }
   # note: destination_name and destination_slug are currently interchangable so we need to test for both possibilities
   let(:params) do
@@ -18,21 +19,24 @@
         source_full_path: 'full/path/to/group1',
         destination_slug: 'destination-group-1',
         destination_namespace: 'parent-group',
-        migrate_projects: migrate_projects
+        migrate_projects: migrate_projects,
+        migrate_memberships: migrate_memberships
       },
       {
         source_type: 'group_entity',
         source_full_path: 'full/path/to/group2',
         destination_name: 'destination-group-2',
         destination_namespace: 'parent-group',
-        migrate_projects: migrate_projects
+        migrate_projects: migrate_projects,
+        migrate_memberships: migrate_memberships
       },
       {
         source_type: 'project_entity',
         source_full_path: 'full/path/to/project1',
         destination_slug: 'destination-project-1',
         destination_namespace: 'parent-group',
-        migrate_projects: migrate_projects
+        migrate_projects: migrate_projects,
+        migrate_memberships: migrate_memberships
       }
     ]
   end
@@ -293,6 +297,40 @@
             end
           end
         end
+
+        describe 'memberships migration flag' do
+          let(:import) { BulkImport.last }
+
+          context 'when false' do
+            let(:migrate_memberships) { false }
+
+            it 'sets false' do
+              subject.execute
+
+              expect(import.entities.pluck(:migrate_memberships)).to contain_exactly(false, false, false)
+            end
+          end
+
+          context 'when true' do
+            let(:migrate_memberships) { true }
+
+            it 'sets true' do
+              subject.execute
+
+              expect(import.entities.pluck(:migrate_memberships)).to contain_exactly(true, true, true)
+            end
+          end
+
+          context 'when nil' do
+            let(:migrate_memberships) { nil }
+
+            it 'sets true' do
+              subject.execute
+
+              expect(import.entities.pluck(:migrate_memberships)).to contain_exactly(true, true, true)
+            end
+          end
+        end
       end
     end
 
@@ -670,7 +708,8 @@
                 source_full_path: 'full/path/to/source',
                 destination_slug: 'destination-slug',
                 destination_namespace: 'destination-namespace',
-                migrate_projects: migrate_projects
+                migrate_projects: migrate_projects,
+                migrate_memberships: migrate_memberships
               }
             ]
           end
@@ -694,7 +733,8 @@
                 source_full_path: 'full/path/to/source',
                 destination_slug: 'destination-slug',
                 destination_namespace: parent_group.path,
-                migrate_projects: migrate_projects
+                migrate_projects: migrate_projects,
+                migrate_memberships: migrate_memberships
               }
             ]
           end
@@ -720,7 +760,8 @@
                 source_full_path: 'full/path/to/source',
                 destination_slug: 'destination-slug',
                 destination_namespace: parent_group.path,
-                migrate_projects: migrate_projects
+                migrate_projects: migrate_projects,
+                migrate_memberships: migrate_memberships
               }
             ]
           end
@@ -748,7 +789,8 @@
                 source_full_path: 'full/path/to/source',
                 destination_slug: 'destin-*-ation-slug',
                 destination_namespace: parent_group.path,
-                migrate_projects: migrate_projects
+                migrate_projects: migrate_projects,
+                migrate_memberships: migrate_memberships
               }
             ]
           end
@@ -784,7 +826,8 @@
                   source_full_path: 'full/path/to/source',
                   destination_slug: existing_subgroup.path,
                   destination_namespace: parent_group.path,
-                  migrate_projects: migrate_projects
+                  migrate_projects: migrate_projects,
+                  migrate_memberships: migrate_memberships
                 }
               ]
             end
@@ -811,7 +854,8 @@
                   source_full_path: 'full/path/to/source',
                   destination_slug: existing_top_level_group.path,
                   destination_namespace: '',
-                  migrate_projects: migrate_projects
+                  migrate_projects: migrate_projects,
+                  migrate_memberships: migrate_memberships
                 }
               ]
             end
@@ -837,7 +881,8 @@
                   source_full_path: 'full/path/to/source',
                   destination_slug: 'new-group',
                   destination_namespace: parent_group.path,
-                  migrate_projects: migrate_projects
+                  migrate_projects: migrate_projects,
+                  migrate_memberships: migrate_memberships
                 }
               ]
             end
@@ -862,7 +907,8 @@
                   source_full_path: 'full/path/to/source',
                   destination_slug: existing_project.path,
                   destination_namespace: existing_group.path,
-                  migrate_projects: migrate_projects
+                  migrate_projects: migrate_projects,
+                  migrate_memberships: migrate_memberships
                 }
               ]
             end
@@ -891,7 +937,8 @@
                   source_full_path: 'full/path/to/source',
                   destination_slug: 'new-project',
                   destination_namespace: 'existing-group',
-                  migrate_projects: migrate_projects
+                  migrate_projects: migrate_projects,
+                  migrate_memberships: migrate_memberships
                 }
               ]
             end