Merge branch 'jm-zoekt-watermark-callback' into 'master'

Add zoekt index watermark level update callback

See merge request !181056



Merged-by: Dmitry Gruzd <dgruzd@gitlab.com>
Approved-by: Dmitry Gruzd <dgruzd@gitlab.com>
Co-authored-by: John Mason <9717668-johnmason@users.noreply.gitlab.com>
Co-authored-by: John Mason <jmason@gitlab.com>

ee/app/models/search/zoekt/index.rb

@@ -27,6 +27,7 @@ class Index < ApplicationRecord
 
       validates :metadata, json_schema: { filename: 'zoekt_indices_metadata' }
 
+      before_save :set_watermark_level, if: :storage_bytes_changed?
       after_commit :delete_from_index, on: :destroy
 
       enum state: {
@@ -133,7 +134,6 @@ def update_reserved_storage_bytes!
         return if new_reserved_bytes == reserved_storage_bytes
 
         self.reserved_storage_bytes = new_reserved_bytes
-        self.watermark_level = appropriate_watermark_level
         save!
       rescue ActiveRecord::ActiveRecordError => err
         logger.error(build_structured_payload(
@@ -176,6 +176,14 @@ def appropriate_watermark_level
         end
       end
 
+      def set_watermark_level
+        self.watermark_level = appropriate_watermark_level
+      end
+
+      def storage_bytes_changed?
+        reserved_storage_bytes_changed? || used_storage_bytes_changed?
+      end
+
       def delete_from_index
         ::Search::Zoekt::NamespaceIndexerWorker.perform_async(namespace_id, 'delete', zoekt_node_id)
       end

ee/spec/models/search/zoekt/index_spec.rb

@@ -295,13 +295,15 @@
 
       it 'returns indices where watermark_level is mismatched (healthy)' do
         # Setup a healthy record but with incorrect watermark_level
-        create(
+        idx = create(
           :zoekt_index,
           used_storage_bytes: 40,
-          reserved_storage_bytes: 100,
-          watermark_level: :low_watermark_exceeded # Incorrect level
+          reserved_storage_bytes: 100
         )
 
+        # Skip active record callback: set_watermark_level
+        idx.update!(watermark_level: :low_watermark_exceeded) # Incorrect level
+
         expect(mismatched_indices.count).to eq(1)
         expect(mismatched_indices.first.watermark_level).to eq('low_watermark_exceeded')
       end
@@ -320,24 +322,30 @@
 
       it 'detects overprovisioned mismatches' do
         # Setup an overprovisioned record with incorrect watermark_level
-        create(
+        idx = create(
           :zoekt_index,
           used_storage_bytes: 10,
-          reserved_storage_bytes: 100,
-          watermark_level: :healthy # Incorrect level
+          reserved_storage_bytes: 100
         )
 
+        # Skip active record callback: set_watermark_level
+        idx.update!(watermark_level: :healthy) # Incorrect level
+
         expect(mismatched_indices.count).to eq(1)
         expect(mismatched_indices.first.watermark_level).to eq('healthy')
       end
 
       it 'handles edge cases at the exact boundary' do
         # Setup a record exactly at the STORAGE_LOW_WATERMARK
-        create(
+        idx = create(
           :zoekt_index,
           used_storage_bytes: (low_watermark * 100).to_i,
-          reserved_storage_bytes: 100,
-          watermark_level: :healthy
+          reserved_storage_bytes: 100
+        )
+
+        # Skip active record callback
+        idx.update!(
+          watermark_level: :healthy # Incorrect level
         )
 
         expect(mismatched_indices.count).to eq(1)
@@ -358,13 +366,14 @@
 
       it 'returns indices where watermark_level is mismatched (critical)' do
         # Setup a record that should be critical but has incorrect watermark_level
-        create(
+        idx = create(
           :zoekt_index,
           used_storage_bytes: (critical_watermark * 100) + 1,
-          reserved_storage_bytes: 100,
-          watermark_level: :high_watermark_exceeded # Incorrect level
+          reserved_storage_bytes: 100
         )
 
+        idx.update!(watermark_level: :high_watermark_exceeded) # Incorrect level
+
         expect(mismatched_indices.count).to eq(1)
         expect(mismatched_indices.first.watermark_level).to eq('high_watermark_exceeded')
       end

ee/spec/workers/search/zoekt/index_watermark_changed_event_worker_spec.rb

@@ -32,7 +32,8 @@
       let_it_be_with_reload(:mismatched_index) { create(:zoekt_index, :critical_watermark_exceeded) }
 
       before do
-        mismatched_index.healthy!
+        mismatched_index.update!(watermark_level: :healthy) # skip active record callback
+        negative_bytes_index.update!(watermark_level: :healthy)
       end
 
       it 'calls update_reserved_storage_bytes! on the indices' do