Merge branch '404534-zoekt-admin-apis' into 'master'

Add /admin/search/zoekt APIs for controlling Zoekt rollout

See merge request !116650



Merged-by: Peter Leitzen <pleitzen@gitlab.com>
Approved-by: Peter Leitzen <pleitzen@gitlab.com>
Approved-by: Siddharth Dungarwal <sdungarwal@gitlab.com>
Reviewed-by: Peter Leitzen <pleitzen@gitlab.com>
Co-authored-by: Dylan Griffith <dyl.griffith@gmail.com>

ee/app/models/zoekt/indexed_namespace.rb

@@ -11,6 +11,17 @@ def self.table_name_prefix
 
     validate :only_root_namespaces_can_be_indexed
 
+    scope :recent, -> { order(id: :desc) }
+    scope :with_limit, ->(maximum) { limit(maximum) }
+
+    def self.for_shard_and_namespace!(shard:, namespace:)
+      find_by!(shard: shard, namespace: namespace)
+    end
+
+    def self.find_or_create_for_shard_and_namespace!(shard:, namespace:)
+      find_or_create_by!(shard: shard, namespace: namespace)
+    end
+
     def self.enabled_for_project?(project)
       where(namespace: project.root_namespace).exists?
     end

ee/lib/api/admin/search/zoekt.rb

+# frozen_string_literal: true
+
+module API
+  module Admin
+    module Search
+      class Zoekt < ::API::Base # rubocop:disable Search/NamespacedClass
+        MAX_RESULTS = 20
+
+        feature_category :global_search
+        urgency :low
+        before do
+          authenticated_as_admin!
+        end
+
+        namespace 'admin' do
+          resources 'zoekt/projects/:project_id/index' do
+            desc 'Triggers indexing for the specified project' do
+              success ::API::Entities::Search::Zoekt::ProjectIndexSuccess
+              failure [
+                { code: 401, message: '401 Unauthorized' },
+                { code: 403, message: '403 Forbidden' },
+                { code: 404, message: '404 Not found' }
+              ]
+            end
+            params do
+              requires :project_id,
+                type: Integer,
+                desc: 'The id of the project you want to index'
+            end
+            put do
+              project = Project.find(params[:project_id])
+
+              job_id = project.repository.async_update_zoekt_index
+
+              present({ job_id: job_id }, with: ::API::Entities::Search::Zoekt::ProjectIndexSuccess)
+            end
+          end
+
+          resources 'zoekt/shards' do
+            desc 'Get all the Zoekt shards' do
+              success ::API::Entities::Search::Zoekt::Shard
+              failure [
+                { code: 401, message: '401 Unauthorized' },
+                { code: 403, message: '403 Forbidden' },
+                { code: 404, message: '404 Not found' }
+              ]
+            end
+            get do
+              present ::Zoekt::Shard.all, with: ::API::Entities::Search::Zoekt::Shard
+            end
+
+            resources ':shard_id/indexed_namespaces' do
+              desc 'Get all the indexed namespaces for this shard' do
+                success ::API::Entities::Search::Zoekt::IndexedNamespace
+                failure [
+                  { code: 401, message: '401 Unauthorized' },
+                  { code: 403, message: '403 Forbidden' },
+                  { code: 404, message: '404 Not found' }
+                ]
+              end
+              params do
+                requires :shard_id,
+                  type: Integer,
+                  desc: 'The id of the Zoekt::Shard'
+              end
+              get do
+                shard = ::Zoekt::Shard.find(params[:shard_id])
+                indexed_namespaces = shard.indexed_namespaces.recent.with_limit(MAX_RESULTS)
+
+                present indexed_namespaces, with: ::API::Entities::Search::Zoekt::IndexedNamespace
+              end
+
+              resources ':namespace_id' do
+                desc 'Add a namespace to a shard for Zoekt indexing' do
+                  success ::API::Entities::Search::Zoekt::IndexedNamespace
+                  failure [
+                    { code: 401, message: '401 Unauthorized' },
+                    { code: 403, message: '403 Forbidden' },
+                    { code: 404, message: '404 Not found' }
+                  ]
+                end
+                params do
+                  requires :shard_id,
+                    type: Integer,
+                    desc: 'The id of the Zoekt::Shard'
+                  requires :namespace_id,
+                    type: Integer,
+                    desc: 'The id of the namespace you want to index in this shard'
+                end
+                put do
+                  shard = ::Zoekt::Shard.find(params[:shard_id])
+                  namespace = Namespace.find(params[:namespace_id])
+
+                  indexed_namespace = ::Zoekt::IndexedNamespace
+                    .find_or_create_for_shard_and_namespace!(shard: shard, namespace: namespace)
+                  present indexed_namespace, with: ::API::Entities::Search::Zoekt::IndexedNamespace
+                end
+
+                desc 'Remove a namespace from a shard for Zoekt indexing' do
+                  failure [
+                    { code: 401, message: '401 Unauthorized' },
+                    { code: 403, message: '403 Forbidden' },
+                    { code: 404, message: '404 Not found' }
+                  ]
+                end
+                params do
+                  requires :shard_id,
+                    type: Integer,
+                    desc: 'The id of the Zoekt::Shard'
+                  requires :namespace_id,
+                    type: Integer,
+                    desc: 'The id of the namespace you want to index in this shard'
+                end
+                delete do
+                  shard = ::Zoekt::Shard.find(params[:shard_id])
+                  namespace = Namespace.find(params[:namespace_id])
+
+                  indexed_namespace = ::Zoekt::IndexedNamespace
+                    .for_shard_and_namespace!(shard: shard, namespace: namespace)
+                  indexed_namespace.destroy!
+
+                  ''
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/api/entities/search/zoekt.rb

+# frozen_string_literal: true
+
+module API
+  module Entities
+    module Search
+      module Zoekt # rubocop:disable Search/NamespacedClass
+        class IndexedNamespace < Grape::Entity # rubocop:disable Search/NamespacedClass
+          expose :id, documentation: { type: :int, example: 1234 }
+          expose :zoekt_shard_id, documentation: { type: :int, example: 1234 }
+          expose :namespace_id, documentation: { type: :int, example: 1234 }
+        end
+
+        class Shard < Grape::Entity # rubocop:disable Search/NamespacedClass
+          expose :id, documentation: { type: :int, example: 1234 }
+          expose :index_base_url, documentation: { type: :string, example: 'http://127.0.0.1:6060/' }
+          expose :search_base_url, documentation: { type: :string, example: 'http://127.0.0.1:6070/' }
+        end
+
+        class ProjectIndexSuccess < Grape::Entity # rubocop:disable Search/NamespacedClass
+          expose :job_id do |item|
+            item[:job_id]
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/ee/api/api.rb

@@ -10,6 +10,7 @@ module API
 
         mount ::EE::API::GroupBoards
 
+        mount ::API::Admin::Search::Zoekt
         mount ::API::Ai::Experimentation::OpenAi
         mount ::API::AuditEvents
         mount ::API::ProjectApprovalRules

ee/spec/requests/api/admin/search/zoekt_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe API::Admin::Search::Zoekt, :zoekt, feature_category: :global_search do
+  let(:admin) { create(:admin) }
+  let_it_be(:namespace) { create(:group) }
+  let_it_be(:unindexed_namespace) { create(:group) }
+  let_it_be(:project) { create(:project) }
+  let(:project_id) { project.id }
+  let(:namespace_id) { namespace.id }
+  let(:params) { {} }
+  let(:shard) { ::Zoekt::Shard.first }
+  let(:shard_id) { shard.id }
+
+  shared_examples 'an API that returns 404 for missing ids' do |verb|
+    it 'returns not_found status' do
+      send(verb, api(path, admin, admin_mode: true))
+
+      expect(response).to have_gitlab_http_status(:not_found)
+    end
+  end
+
+  shared_examples 'an API that returns 401 for unauthenticated requests' do |verb|
+    it 'returns not_found status' do
+      send(verb, api(path, nil))
+
+      expect(response).to have_gitlab_http_status(:unauthorized)
+    end
+  end
+
+  describe 'PUT /admin/zoekt/projects/:projects/index' do
+    let(:path) { "/admin/zoekt/projects/#{project_id}/index" }
+
+    it_behaves_like "PUT request permissions for admin mode"
+    it_behaves_like "an API that returns 401 for unauthenticated requests", :put
+
+    it 'triggers indexing for the project' do
+      expect(::Zoekt::IndexerWorker).to receive(:perform_async).with(project.id).and_return('the-job-id')
+
+      put api(path, admin, admin_mode: true)
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(json_response['job_id']).to eq('the-job-id')
+    end
+
+    it_behaves_like 'an API that returns 404 for missing ids', :put do
+      let(:project_id) { Project.maximum(:id) + 100 }
+    end
+  end
+
+  describe 'GET /admin/zoekt/shards' do
+    let(:path) { '/admin/zoekt/shards' }
+    let!(:another_shard) { ::Zoekt::Shard.create!(index_base_url: 'http://111.111.111.111/', search_base_url: 'http://111.111.111.112/') }
+
+    it_behaves_like "GET request permissions for admin mode"
+    it_behaves_like "an API that returns 401 for unauthenticated requests", :get
+
+    it 'returns all shards' do
+      get api(path, admin, admin_mode: true)
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(json_response).to match_array([
+        hash_including(
+          'id' => shard.id,
+          'index_base_url' => shard.index_base_url,
+          'search_base_url' => shard.search_base_url
+        ),
+        hash_including(
+          'id' => another_shard.id,
+          'index_base_url' => 'http://111.111.111.111/',
+          'search_base_url' => 'http://111.111.111.112/'
+        )
+      ])
+    end
+  end
+
+  describe 'GET /admin/zoekt/shards/:shard_id/indexed_namespaces' do
+    let(:path) { "/admin/zoekt/shards/#{shard_id}/indexed_namespaces" }
+
+    let!(:indexed_namespace) { ::Zoekt::IndexedNamespace.create!(shard: shard, namespace: namespace) }
+    let!(:another_indexed_namespace) { ::Zoekt::IndexedNamespace.create!(shard: shard, namespace: create(:namespace)) }
+
+    let!(:another_shard) { ::Zoekt::Shard.create!(index_base_url: 'http://111.111.111.198/', search_base_url: 'http://111.111.111.199/') }
+    let!(:indexed_namespace_for_another_shard) do
+      ::Zoekt::IndexedNamespace.create!(shard: another_shard, namespace: create(:namespace))
+    end
+
+    it_behaves_like "GET request permissions for admin mode"
+    it_behaves_like "an API that returns 401 for unauthenticated requests", :get
+
+    it 'returns all indexed namespaces for this shard' do
+      get api(path, admin, admin_mode: true)
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(json_response).to match_array([
+        hash_including(
+          'id' => indexed_namespace.id,
+          'zoekt_shard_id' => shard.id,
+          'namespace_id' => namespace.id
+        ),
+        hash_including(
+          'id' => another_indexed_namespace.id,
+          'zoekt_shard_id' => shard.id,
+          'namespace_id' => another_indexed_namespace.namespace_id
+        )
+      ])
+    end
+
+    it 'returns at most MAX_RESULTS most recent rows' do
+      stub_const("#{described_class}::MAX_RESULTS", 1)
+
+      get api(path, admin, admin_mode: true)
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(json_response).to match_array([
+        hash_including(
+          'id' => another_indexed_namespace.id,
+          'zoekt_shard_id' => shard.id,
+          'namespace_id' => another_indexed_namespace.namespace_id
+        )
+      ])
+    end
+
+    it_behaves_like 'an API that returns 404 for missing ids', :get do
+      let(:shard_id) { ::Zoekt::Shard.maximum(:id) + 100 }
+    end
+  end
+
+  describe 'PUT /admin/zoekt/shards/:shard_id/indexed_namespaces/:namespace_id' do
+    let(:path) { "/admin/zoekt/shards/#{shard_id}/indexed_namespaces/#{namespace_id}" }
+
+    it_behaves_like "PUT request permissions for admin mode"
+    it_behaves_like "an API that returns 401 for unauthenticated requests", :put
+
+    it 'creates a Zoekt::IndexedNamespace for this shard and namespace pair' do
+      expect do
+        put api(path, admin, admin_mode: true)
+      end.to change { ::Zoekt::IndexedNamespace.count }.from(0).to(1)
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(json_response['id']).to eq(::Zoekt::IndexedNamespace.find_by(shard: shard, namespace: namespace).id)
+    end
+
+    context 'when it already exists' do
+      it 'returns the existing one' do
+        id = ::Zoekt::IndexedNamespace.create!(shard: shard, namespace: namespace).id
+
+        put api(path, admin, admin_mode: true)
+
+        expect(json_response['id']).to eq(id)
+      end
+    end
+
+    context 'with missing shard_id' do
+      it_behaves_like 'an API that returns 404 for missing ids', :put do
+        let(:shard_id) { ::Zoekt::Shard.maximum(:id) + 100 }
+      end
+    end
+
+    context 'with missing namespace_id' do
+      it_behaves_like 'an API that returns 404 for missing ids', :put do
+        let(:namespace_id) { ::Namespace.maximum(:id) + 100 }
+      end
+    end
+  end
+
+  describe 'DELETE /admin/zoekt/shards/:shard_id/indexed_namespaces/:namespace_id' do
+    let(:path) { "/admin/zoekt/shards/#{shard_id}/indexed_namespaces/#{namespace_id}" }
+
+    before do
+      ::Zoekt::IndexedNamespace.create!(shard: shard, namespace: namespace)
+    end
+
+    it_behaves_like "DELETE request permissions for admin mode"
+    it_behaves_like "an API that returns 401 for unauthenticated requests", :delete
+
+    it 'removes the Zoekt::IndexedNamespace for this shard and namespace pair' do
+      expect do
+        delete api(path, admin, admin_mode: true)
+      end.to change { ::Zoekt::IndexedNamespace.count }.from(1).to(0)
+
+      expect(response).to have_gitlab_http_status(:no_content)
+    end
+
+    context 'with missing shard_id' do
+      it_behaves_like 'an API that returns 404 for missing ids', :delete do
+        let(:shard_id) { ::Zoekt::Shard.maximum(:id) + 100 }
+      end
+    end
+
+    context 'with missing namespace_id' do
+      it_behaves_like 'an API that returns 404 for missing ids', :delete do
+        let(:namespace_id) { ::Namespace.maximum(:id) + 100 }
+      end
+    end
+  end
+end