Merge branch '509600-use-project_id_from-and-project_id_to-2' into 'master'

Use project_id_from and project_id_to in the InitialIndexingEventWorker

See merge request !176224



Merged-by: Ravi Kumar <rkumar@gitlab.com>
Approved-by: Dmitry Gruzd <dgruzd@gitlab.com>
Approved-by: John Mason <9717668-johnmason@users.noreply.gitlab.com>
Reviewed-by: Dmitry Gruzd <dgruzd@gitlab.com>
Reviewed-by: GitLab Duo <gitlab-duo@gitlab.com>

ee/app/workers/search/zoekt/initial_indexing_event_worker.rb

@@ -11,18 +11,58 @@ class InitialIndexingEventWorker
 
       # Create the pending zoekt_repositories and move the index to initializing
       def handle_event(event)
-        index = Index.find_by_id(event.data[:index_id])
+        index = find_index(event.data[:index_id])
         return if index.nil? || !index.pending?
 
-        namespace = ::Namespace.find_by_id(index.namespace_id)
+        namespace = find_namespace(index.namespace_id)
         return if namespace.nil?
 
+        create_repositories(namespace: namespace, index: index)
+        index.initializing!
+      end
+
+      private
+
+      def find_index(index_id)
+        Index.find_by_id(index_id)
+      end
+
+      def find_namespace(namespace_id)
+        ::Namespace.find_by_id(namespace_id)
+      end
+
+      def create_repositories(namespace:, index:)
+        if index.metadata['project_id_from'].blank?
+          create_repositories_with_scope(namespace: namespace, index: index)
+        else
+          create_repositories_for_project_range(namespace: namespace, index: index)
+        end
+      end
+
+      def create_repositories_for_project_range(namespace:, index:)
+        range_ids = determine_project_id_range(index)
+        create_repositories_with_scope(namespace: namespace, index: index) { |scope| scope.id_in(range_ids) }
+      end
+
+      def create_repositories_with_scope(namespace:, index:)
         ::Namespace.by_root_id(namespace.id).project_namespaces.each_batch do |project_namespaces_batch|
-          project_ids = ::Project.by_project_namespace(project_namespaces_batch.select(:id)).pluck_primary_key
-          data = project_ids.map { |p_id| { zoekt_index_id: index.id, project_id: p_id, project_identifier: p_id } }
-          Repository.insert_all(data)
+          scope = ::Project.by_project_namespace(project_namespaces_batch.select(:id))
+          scope = yield(scope) if block_given?
+          insert_repositories!(index: index, project_ids: scope.pluck_primary_key)
         end
-        index.initializing!
+      end
+
+      def determine_project_id_range(index)
+        return (index.metadata['project_id_from']..) if index.metadata['project_id_to'].blank?
+
+        index.metadata['project_id_from']..index.metadata['project_id_to']
+      end
+
+      def insert_repositories!(index:, project_ids:)
+        data = project_ids.map do |p_id|
+          { zoekt_index_id: index.id, project_id: p_id, project_identifier: p_id }
+        end
+        Repository.insert_all(data)
       end
     end
   end

ee/spec/workers/search/zoekt/initial_indexing_event_worker_spec.rb

@@ -15,18 +15,59 @@
   end
 
   before do
-    [namespace, namespace.children.first].each { |n| create(:project, namespace: n) }
+    [namespace, namespace.children.first, namespace.children.first.children.first].each do |n|
+      create(:project, namespace: n)
+      # Create project whose id falls in between the project_id_from and project_id_to that don't belong to namespace
+      create(:project)
+    end
   end
 
   it_behaves_like 'subscribes to event'
 
   it_behaves_like 'an idempotent worker' do
-    it 'creates pending zoekt_repositories for each project move the index to initializing' do
-      expect(zoekt_repositories_for_index(zoekt_index)).to be_empty
-      expect { consume_event(subscriber: described_class, event: event) }
-        .to change { zoekt_index.reload.state }.from('pending').to('initializing')
-      expect(zoekt_repositories_for_index(zoekt_index).count).to eq namespace.all_project_ids.count
-      expect(zoekt_repositories_for_index(zoekt_index).pluck(:state).uniq).to contain_exactly 'pending'
+    context 'when metadata does not have project_id_from and project_id_to' do
+      it 'creates pending zoekt_repositories for each project move the index to initializing' do
+        expect(zoekt_repositories_for_index(zoekt_index)).to be_empty
+        expect { consume_event(subscriber: described_class, event: event) }
+          .to change { zoekt_index.reload.state }.from('pending').to('initializing')
+        expect(zoekt_repositories_for_index(zoekt_index).count).to eq namespace.all_project_ids.count
+        expect(zoekt_repositories_for_index(zoekt_index).all?(&:pending?)).to be true
+      end
+    end
+
+    context 'when metadata has project_id_from and project_id_to' do
+      let(:project_id_from) { namespace.all_project_ids.first.id }
+      let(:project_id_to) { namespace.all_project_ids.second.id }
+      let(:expected_project_ids) { namespace.all_project_ids.where(id: project_id_from..project_id_to).pluck(:id) }
+
+      before do
+        zoekt_index.update!(metadata: { project_id_from: project_id_from, project_id_to: project_id_to })
+      end
+
+      it 'creates pending zoekt_repositories for project_ids of range project_id_from and project_id_to' do
+        expect(zoekt_repositories_for_index(zoekt_index)).to be_empty
+        expect { consume_event(subscriber: described_class, event: event) }
+          .to change { zoekt_index.reload.state }.from('pending').to('initializing')
+        expect(zoekt_repositories_for_index(zoekt_index).pluck(:project_id)).to match_array(expected_project_ids)
+        expect(zoekt_repositories_for_index(zoekt_index).all?(&:pending?)).to be true
+      end
+    end
+
+    context 'when metadata has only project_id_from' do
+      let(:project_id_from) { namespace.all_project_ids.second.id }
+      let(:expected_project_ids) { namespace.all_project_ids.where(id: project_id_from..).pluck(:id) }
+
+      before do
+        zoekt_index.update!(metadata: { project_id_from: project_id_from })
+      end
+
+      it 'creates pending zoekt_repositories for all project_ids from project_id_from' do
+        expect(zoekt_repositories_for_index(zoekt_index)).to be_empty
+        expect { consume_event(subscriber: described_class, event: event) }
+          .to change { zoekt_index.reload.state }.from('pending').to('initializing')
+        expect(zoekt_repositories_for_index(zoekt_index).pluck(:project_id)).to match_array(expected_project_ids)
+        expect(zoekt_repositories_for_index(zoekt_index).all?(&:pending?)).to be true
+      end
     end
 
     context 'when index is not in pending' do