From 1b8da271d259b3493776a26f3e0e464374de52c4 Mon Sep 17 00:00:00 2001
From: Dylan Griffith <dyl.griffith@gmail.com>
Date: Tue, 4 Apr 2023 17:57:44 +1000
Subject: [PATCH] Add /admin/search/zoekt APIs for controlling Zoekt rollout
As part of our rollout of Zoekt we're going to be enabling it 1
namespace at a time. This means that we'll need tooling to simplify this
process. Originally I was planning on doing this via the Rails console
but I've since learnt any rails console commands require a C2 change
request. This requires lots of approvals and someone with access to
perform the steps. This seemed quite wasteful so I've subsequently
realised this would be better to roll out via chatops. Chatops requires
an API to execute the steps so this adds some admin APIs we'd need to
run from chatops.
Also these same APIs might end up being useful for other administrators
managing Zoekt for their instances or possibly to support a future UI.
Changelog: added
MR: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/116650
EE: true
---
ee/app/models/zoekt/indexed_namespace.rb | 11 +
ee/lib/api/admin/search/zoekt.rb | 131 ++++++++++++
ee/lib/api/entities/search/zoekt.rb | 27 +++
ee/lib/ee/api/api.rb | 1 +
.../requests/api/admin/search/zoekt_spec.rb | 198 ++++++++++++++++++
5 files changed, 368 insertions(+)
create mode 100644 ee/lib/api/admin/search/zoekt.rb
create mode 100644 ee/lib/api/entities/search/zoekt.rb
create mode 100644 ee/spec/requests/api/admin/search/zoekt_spec.rb
diff --git a/ee/app/models/zoekt/indexed_namespace.rb b/ee/app/models/zoekt/indexed_namespace.rb
index c8498784a43cdc..75912043777d0b 100644
--- a/ee/app/models/zoekt/indexed_namespace.rb
+++ b/ee/app/models/zoekt/indexed_namespace.rb
@@ -11,6 +11,17 @@ def self.table_name_prefix
validate :only_root_namespaces_can_be_indexed
+ scope :recent, -> { order(id: :desc) }
+ scope :with_limit, ->(maximum) { limit(maximum) }
+
+ def self.for_shard_and_namespace!(shard:, namespace:)
+ find_by!(shard: shard, namespace: namespace)
+ end
+
+ def self.find_or_create_for_shard_and_namespace!(shard:, namespace:)
+ find_or_create_by!(shard: shard, namespace: namespace)
+ end
+
def self.enabled_for_project?(project)
where(namespace: project.root_namespace).exists?
end
diff --git a/ee/lib/api/admin/search/zoekt.rb b/ee/lib/api/admin/search/zoekt.rb
new file mode 100644
index 00000000000000..5f60bdd98b746f
--- /dev/null
+++ b/ee/lib/api/admin/search/zoekt.rb
@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module API
+ module Admin
+ module Search
+ class Zoekt < ::API::Base # rubocop:disable Search/NamespacedClass
+ MAX_RESULTS = 20
+
+ feature_category :global_search
+ urgency :low
+ before do
+ authenticated_as_admin!
+ end
+
+ namespace 'admin' do
+ resources 'zoekt/projects/:project_id/index' do
+ desc 'Triggers indexing for the specified project' do
+ success ::API::Entities::Search::Zoekt::ProjectIndexSuccess
+ failure [
+ { code: 401, message: '401 Unauthorized' },
+ { code: 403, message: '403 Forbidden' },
+ { code: 404, message: '404 Not found' }
+ ]
+ end
+ params do
+ requires :project_id,
+ type: Integer,
+ desc: 'The id of the project you want to index'
+ end
+ put do
+ project = Project.find(params[:project_id])
+
+ job_id = project.repository.async_update_zoekt_index
+
+ present({ job_id: job_id }, with: ::API::Entities::Search::Zoekt::ProjectIndexSuccess)
+ end
+ end
+
+ resources 'zoekt/shards' do
+ desc 'Get all the Zoekt shards' do
+ success ::API::Entities::Search::Zoekt::Shard
+ failure [
+ { code: 401, message: '401 Unauthorized' },
+ { code: 403, message: '403 Forbidden' },
+ { code: 404, message: '404 Not found' }
+ ]
+ end
+ get do
+ present ::Zoekt::Shard.all, with: ::API::Entities::Search::Zoekt::Shard
+ end
+
+ resources ':shard_id/indexed_namespaces' do
+ desc 'Get all the indexed namespaces for this shard' do
+ success ::API::Entities::Search::Zoekt::IndexedNamespace
+ failure [
+ { code: 401, message: '401 Unauthorized' },
+ { code: 403, message: '403 Forbidden' },
+ { code: 404, message: '404 Not found' }
+ ]
+ end
+ params do
+ requires :shard_id,
+ type: Integer,
+ desc: 'The id of the Zoekt::Shard'
+ end
+ get do
+ shard = ::Zoekt::Shard.find(params[:shard_id])
+ indexed_namespaces = shard.indexed_namespaces.recent.with_limit(MAX_RESULTS)
+
+ present indexed_namespaces, with: ::API::Entities::Search::Zoekt::IndexedNamespace
+ end
+
+ resources ':namespace_id' do
+ desc 'Add a namespace to a shard for Zoekt indexing' do
+ success ::API::Entities::Search::Zoekt::IndexedNamespace
+ failure [
+ { code: 401, message: '401 Unauthorized' },
+ { code: 403, message: '403 Forbidden' },
+ { code: 404, message: '404 Not found' }
+ ]
+ end
+ params do
+ requires :shard_id,
+ type: Integer,
+ desc: 'The id of the Zoekt::Shard'
+ requires :namespace_id,
+ type: Integer,
+ desc: 'The id of the namespace you want to index in this shard'
+ end
+ put do
+ shard = ::Zoekt::Shard.find(params[:shard_id])
+ namespace = Namespace.find(params[:namespace_id])
+
+ indexed_namespace = ::Zoekt::IndexedNamespace
+ .find_or_create_for_shard_and_namespace!(shard: shard, namespace: namespace)
+ present indexed_namespace, with: ::API::Entities::Search::Zoekt::IndexedNamespace
+ end
+
+ desc 'Remove a namespace from a shard for Zoekt indexing' do
+ failure [
+ { code: 401, message: '401 Unauthorized' },
+ { code: 403, message: '403 Forbidden' },
+ { code: 404, message: '404 Not found' }
+ ]
+ end
+ params do
+ requires :shard_id,
+ type: Integer,
+ desc: 'The id of the Zoekt::Shard'
+ requires :namespace_id,
+ type: Integer,
+ desc: 'The id of the namespace you want to index in this shard'
+ end
+ delete do
+ shard = ::Zoekt::Shard.find(params[:shard_id])
+ namespace = Namespace.find(params[:namespace_id])
+
+ indexed_namespace = ::Zoekt::IndexedNamespace
+ .for_shard_and_namespace!(shard: shard, namespace: namespace)
+ indexed_namespace.destroy!
+
+ ''
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/ee/lib/api/entities/search/zoekt.rb b/ee/lib/api/entities/search/zoekt.rb
new file mode 100644
index 00000000000000..2a67d6576c264c
--- /dev/null
+++ b/ee/lib/api/entities/search/zoekt.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Search
+ module Zoekt # rubocop:disable Search/NamespacedClass
+ class IndexedNamespace < Grape::Entity # rubocop:disable Search/NamespacedClass
+ expose :id, documentation: { type: :int, example: 1234 }
+ expose :zoekt_shard_id, documentation: { type: :int, example: 1234 }
+ expose :namespace_id, documentation: { type: :int, example: 1234 }
+ end
+
+ class Shard < Grape::Entity # rubocop:disable Search/NamespacedClass
+ expose :id, documentation: { type: :int, example: 1234 }
+ expose :index_base_url, documentation: { type: :string, example: 'http://127.0.0.1:6060/' }
+ expose :search_base_url, documentation: { type: :string, example: 'http://127.0.0.1:6070/' }
+ end
+
+ class ProjectIndexSuccess < Grape::Entity # rubocop:disable Search/NamespacedClass
+ expose :job_id do |item|
+ item[:job_id]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/ee/lib/ee/api/api.rb b/ee/lib/ee/api/api.rb
index 0e8bb0c70e5326..95d17aaadfe8be 100644
--- a/ee/lib/ee/api/api.rb
+++ b/ee/lib/ee/api/api.rb
@@ -10,6 +10,7 @@ module API
mount ::EE::API::GroupBoards
+ mount ::API::Admin::Search::Zoekt
mount ::API::Ai::Experimentation::OpenAi
mount ::API::AuditEvents
mount ::API::ProjectApprovalRules
diff --git a/ee/spec/requests/api/admin/search/zoekt_spec.rb b/ee/spec/requests/api/admin/search/zoekt_spec.rb
new file mode 100644
index 00000000000000..6e610c3d8c293e
--- /dev/null
+++ b/ee/spec/requests/api/admin/search/zoekt_spec.rb
@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe API::Admin::Search::Zoekt, :zoekt, feature_category: :global_search do
+ let(:admin) { create(:admin) }
+ let_it_be(:namespace) { create(:group) }
+ let_it_be(:unindexed_namespace) { create(:group) }
+ let_it_be(:project) { create(:project) }
+ let(:project_id) { project.id }
+ let(:namespace_id) { namespace.id }
+ let(:params) { {} }
+ let(:shard) { ::Zoekt::Shard.first }
+ let(:shard_id) { shard.id }
+
+ shared_examples 'an API that returns 404 for missing ids' do |verb|
+ it 'returns not_found status' do
+ send(verb, api(path, admin, admin_mode: true))
+
+ expect(response).to have_gitlab_http_status(:not_found)
+ end
+ end
+
+ shared_examples 'an API that returns 401 for unauthenticated requests' do |verb|
+ it 'returns not_found status' do
+ send(verb, api(path, nil))
+
+ expect(response).to have_gitlab_http_status(:unauthorized)
+ end
+ end
+
+ describe 'PUT /admin/zoekt/projects/:projects/index' do
+ let(:path) { "/admin/zoekt/projects/#{project_id}/index" }
+
+ it_behaves_like "PUT request permissions for admin mode"
+ it_behaves_like "an API that returns 401 for unauthenticated requests", :put
+
+ it 'triggers indexing for the project' do
+ expect(::Zoekt::IndexerWorker).to receive(:perform_async).with(project.id).and_return('the-job-id')
+
+ put api(path, admin, admin_mode: true)
+
+ expect(response).to have_gitlab_http_status(:ok)
+ expect(json_response['job_id']).to eq('the-job-id')
+ end
+
+ it_behaves_like 'an API that returns 404 for missing ids', :put do
+ let(:project_id) { Project.maximum(:id) + 100 }
+ end
+ end
+
+ describe 'GET /admin/zoekt/shards' do
+ let(:path) { '/admin/zoekt/shards' }
+ let!(:another_shard) { ::Zoekt::Shard.create!(index_base_url: 'http://111.111.111.111/', search_base_url: 'http://111.111.111.112/') }
+
+ it_behaves_like "GET request permissions for admin mode"
+ it_behaves_like "an API that returns 401 for unauthenticated requests", :get
+
+ it 'returns all shards' do
+ get api(path, admin, admin_mode: true)
+
+ expect(response).to have_gitlab_http_status(:ok)
+ expect(json_response).to match_array([
+ hash_including(
+ 'id' => shard.id,
+ 'index_base_url' => shard.index_base_url,
+ 'search_base_url' => shard.search_base_url
+ ),
+ hash_including(
+ 'id' => another_shard.id,
+ 'index_base_url' => 'http://111.111.111.111/',
+ 'search_base_url' => 'http://111.111.111.112/'
+ )
+ ])
+ end
+ end
+
+ describe 'GET /admin/zoekt/shards/:shard_id/indexed_namespaces' do
+ let(:path) { "/admin/zoekt/shards/#{shard_id}/indexed_namespaces" }
+
+ let!(:indexed_namespace) { ::Zoekt::IndexedNamespace.create!(shard: shard, namespace: namespace) }
+ let!(:another_indexed_namespace) { ::Zoekt::IndexedNamespace.create!(shard: shard, namespace: create(:namespace)) }
+
+ let!(:another_shard) { ::Zoekt::Shard.create!(index_base_url: 'http://111.111.111.198/', search_base_url: 'http://111.111.111.199/') }
+ let!(:indexed_namespace_for_another_shard) do
+ ::Zoekt::IndexedNamespace.create!(shard: another_shard, namespace: create(:namespace))
+ end
+
+ it_behaves_like "GET request permissions for admin mode"
+ it_behaves_like "an API that returns 401 for unauthenticated requests", :get
+
+ it 'returns all indexed namespaces for this shard' do
+ get api(path, admin, admin_mode: true)
+
+ expect(response).to have_gitlab_http_status(:ok)
+ expect(json_response).to match_array([
+ hash_including(
+ 'id' => indexed_namespace.id,
+ 'zoekt_shard_id' => shard.id,
+ 'namespace_id' => namespace.id
+ ),
+ hash_including(
+ 'id' => another_indexed_namespace.id,
+ 'zoekt_shard_id' => shard.id,
+ 'namespace_id' => another_indexed_namespace.namespace_id
+ )
+ ])
+ end
+
+ it 'returns at most MAX_RESULTS most recent rows' do
+ stub_const("#{described_class}::MAX_RESULTS", 1)
+
+ get api(path, admin, admin_mode: true)
+
+ expect(response).to have_gitlab_http_status(:ok)
+ expect(json_response).to match_array([
+ hash_including(
+ 'id' => another_indexed_namespace.id,
+ 'zoekt_shard_id' => shard.id,
+ 'namespace_id' => another_indexed_namespace.namespace_id
+ )
+ ])
+ end
+
+ it_behaves_like 'an API that returns 404 for missing ids', :get do
+ let(:shard_id) { ::Zoekt::Shard.maximum(:id) + 100 }
+ end
+ end
+
+ describe 'PUT /admin/zoekt/shards/:shard_id/indexed_namespaces/:namespace_id' do
+ let(:path) { "/admin/zoekt/shards/#{shard_id}/indexed_namespaces/#{namespace_id}" }
+
+ it_behaves_like "PUT request permissions for admin mode"
+ it_behaves_like "an API that returns 401 for unauthenticated requests", :put
+
+ it 'creates a Zoekt::IndexedNamespace for this shard and namespace pair' do
+ expect do
+ put api(path, admin, admin_mode: true)
+ end.to change { ::Zoekt::IndexedNamespace.count }.from(0).to(1)
+
+ expect(response).to have_gitlab_http_status(:ok)
+ expect(json_response['id']).to eq(::Zoekt::IndexedNamespace.find_by(shard: shard, namespace: namespace).id)
+ end
+
+ context 'when it already exists' do
+ it 'returns the existing one' do
+ id = ::Zoekt::IndexedNamespace.create!(shard: shard, namespace: namespace).id
+
+ put api(path, admin, admin_mode: true)
+
+ expect(json_response['id']).to eq(id)
+ end
+ end
+
+ context 'with missing shard_id' do
+ it_behaves_like 'an API that returns 404 for missing ids', :put do
+ let(:shard_id) { ::Zoekt::Shard.maximum(:id) + 100 }
+ end
+ end
+
+ context 'with missing namespace_id' do
+ it_behaves_like 'an API that returns 404 for missing ids', :put do
+ let(:namespace_id) { ::Namespace.maximum(:id) + 100 }
+ end
+ end
+ end
+
+ describe 'DELETE /admin/zoekt/shards/:shard_id/indexed_namespaces/:namespace_id' do
+ let(:path) { "/admin/zoekt/shards/#{shard_id}/indexed_namespaces/#{namespace_id}" }
+
+ before do
+ ::Zoekt::IndexedNamespace.create!(shard: shard, namespace: namespace)
+ end
+
+ it_behaves_like "DELETE request permissions for admin mode"
+ it_behaves_like "an API that returns 401 for unauthenticated requests", :delete
+
+ it 'removes the Zoekt::IndexedNamespace for this shard and namespace pair' do
+ expect do
+ delete api(path, admin, admin_mode: true)
+ end.to change { ::Zoekt::IndexedNamespace.count }.from(1).to(0)
+
+ expect(response).to have_gitlab_http_status(:no_content)
+ end
+
+ context 'with missing shard_id' do
+ it_behaves_like 'an API that returns 404 for missing ids', :delete do
+ let(:shard_id) { ::Zoekt::Shard.maximum(:id) + 100 }
+ end
+ end
+
+ context 'with missing namespace_id' do
+ it_behaves_like 'an API that returns 404 for missing ids', :delete do
+ let(:namespace_id) { ::Namespace.maximum(:id) + 100 }
+ end
+ end
+ end
+end
--
GitLab