Merge branch 'backfill-desired-sharding-key-small-table-resource_weight_events' into 'master'

Add and backfill namespace_id for resource_weight_events

See merge request !174405



Merged-by: Omar Qunsul <oqunsul@gitlab.com>
Approved-by: Omar Qunsul <oqunsul@gitlab.com>
Co-authored-by: Shubham Kumar <shukumar@gitlab.com>

db/docs/batched_background_migrations/backfill_resource_weight_events_namespace_id.yml

+---
+migration_job_name: BackfillResourceWeightEventsNamespaceId
+description: Backfills sharding key `resource_weight_events.namespace_id` from `issues`.
+feature_category: team_planning
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/174405
+milestone: '17.7'
+queued_migration_version: 20241202142254
+finalized_by: # version of the migration that finalized this BBM

db/docs/resource_weight_events.yml

@@ -18,3 +18,4 @@ desired_sharding_key:
         sharding_key: namespace_id
         belongs_to: issue
 table_size: small
+desired_sharding_key_migration_job_name: BackfillResourceWeightEventsNamespaceId

db/migrate/20241202142250_add_namespace_id_to_resource_weight_events.rb

+# frozen_string_literal: true
+
+class AddNamespaceIdToResourceWeightEvents < Gitlab::Database::Migration[2.2]
+  milestone '17.7'
+
+  def change
+    add_column :resource_weight_events, :namespace_id, :bigint
+  end
+end

db/post_migrate/20241202142251_index_resource_weight_events_on_namespace_id.rb

+# frozen_string_literal: true
+
+class IndexResourceWeightEventsOnNamespaceId < Gitlab::Database::Migration[2.2]
+  milestone '17.7'
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_resource_weight_events_on_namespace_id'
+
+  def up
+    add_concurrent_index :resource_weight_events, :namespace_id, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :resource_weight_events, INDEX_NAME
+  end
+end

db/post_migrate/20241202142252_add_resource_weight_events_namespace_id_fk.rb

+# frozen_string_literal: true
+
+class AddResourceWeightEventsNamespaceIdFk < Gitlab::Database::Migration[2.2]
+  milestone '17.7'
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :resource_weight_events, :namespaces, column: :namespace_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :resource_weight_events, column: :namespace_id
+    end
+  end
+end

db/post_migrate/20241202142253_add_resource_weight_events_namespace_id_trigger.rb

+# frozen_string_literal: true
+
+class AddResourceWeightEventsNamespaceIdTrigger < Gitlab::Database::Migration[2.2]
+  milestone '17.7'
+
+  def up
+    install_sharding_key_assignment_trigger(
+      table: :resource_weight_events,
+      sharding_key: :namespace_id,
+      parent_table: :issues,
+      parent_sharding_key: :namespace_id,
+      foreign_key: :issue_id
+    )
+  end
+
+  def down
+    remove_sharding_key_assignment_trigger(
+      table: :resource_weight_events,
+      sharding_key: :namespace_id,
+      parent_table: :issues,
+      parent_sharding_key: :namespace_id,
+      foreign_key: :issue_id
+    )
+  end
+end

db/post_migrate/20241202142254_queue_backfill_resource_weight_events_namespace_id.rb

+# frozen_string_literal: true
+
+class QueueBackfillResourceWeightEventsNamespaceId < Gitlab::Database::Migration[2.2]
+  milestone '17.7'
+  restrict_gitlab_migration gitlab_schema: :gitlab_main_cell
+
+  MIGRATION = "BackfillResourceWeightEventsNamespaceId"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :resource_weight_events,
+      :id,
+      :namespace_id,
+      :issues,
+      :namespace_id,
+      :issue_id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(
+      MIGRATION,
+      :resource_weight_events,
+      :id,
+      [
+        :namespace_id,
+        :issues,
+        :namespace_id,
+        :issue_id
+      ]
+    )
+  end
+end

db/schema_migrations/20241202142250

+9424cfaf0ef1901c77d2be035812764811d78ea903975370ef55d58005af0c7b
\ No newline at end of file

db/schema_migrations/20241202142251

+f0a1dbd97945d7a6302de20b5603812aacff11f063d0be57f289adee15f75e9b
\ No newline at end of file

db/schema_migrations/20241202142252

+b2b931e7b644a07bb468b5b4cf8f822094c08f44696044360db83e16bdee5c51
\ No newline at end of file

db/schema_migrations/20241202142253

+30f26b3bea60cb0a03e0623cc825c6641a3155ab327b2aaafb330e823b256536
\ No newline at end of file

db/schema_migrations/20241202142254

+fb67017012babaa58f32b3ea301123cf46a606b96967f4dca21e97c6660a3679
\ No newline at end of file

db/structure.sql

@@ -2945,6 +2945,22 @@ RETURN NEW;
 END
 $$;
+CREATE FUNCTION trigger_e4a6cde57b42() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+IF NEW."namespace_id" IS NULL THEN
+  SELECT "namespace_id"
+  INTO NEW."namespace_id"
+  FROM "issues"
+  WHERE "issues"."id" = NEW."issue_id";
+END IF;
+
+RETURN NEW;
+
+END
+$$;
+
 CREATE FUNCTION trigger_e815625b59fa() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -19207,7 +19223,8 @@ CREATE TABLE resource_weight_events (
     issue_id bigint NOT NULL,
     weight integer,
     created_at timestamp with time zone NOT NULL,
-    previous_weight integer
+    previous_weight integer,
+    namespace_id bigint
 );
 CREATE SEQUENCE resource_weight_events_id_seq
@@ -32362,6 +32379,8 @@ CREATE INDEX index_resource_weight_events_on_issue_id_and_created_at ON resource
 CREATE INDEX index_resource_weight_events_on_issue_id_and_weight ON resource_weight_events USING btree (issue_id, weight);
+CREATE INDEX index_resource_weight_events_on_namespace_id ON resource_weight_events USING btree (namespace_id);
+
 CREATE INDEX index_resource_weight_events_on_user_id ON resource_weight_events USING btree (user_id);
 CREATE INDEX index_reviews_on_author_id ON reviews USING btree (author_id);
@@ -35748,6 +35767,8 @@ CREATE TRIGGER trigger_e1da4a738230 BEFORE INSERT OR UPDATE ON vulnerability_ext
 CREATE TRIGGER trigger_e49ab4d904a0 BEFORE INSERT OR UPDATE ON vulnerability_finding_links FOR EACH ROW EXECUTE FUNCTION trigger_e49ab4d904a0();
+CREATE TRIGGER trigger_e4a6cde57b42 BEFORE INSERT OR UPDATE ON resource_weight_events FOR EACH ROW EXECUTE FUNCTION trigger_e4a6cde57b42();
+
 CREATE TRIGGER trigger_e815625b59fa BEFORE INSERT OR UPDATE ON resource_link_events FOR EACH ROW EXECUTE FUNCTION trigger_e815625b59fa();
 CREATE TRIGGER trigger_ebab34f83f1d BEFORE INSERT OR UPDATE ON packages_debian_publications FOR EACH ROW EXECUTE FUNCTION trigger_ebab34f83f1d();
@@ -36794,6 +36815,9 @@ ALTER TABLE ONLY dependency_proxy_blob_states
 ALTER TABLE ONLY issues
     ADD CONSTRAINT fk_96b1dd429c FOREIGN KEY (milestone_id) REFERENCES milestones(id) ON DELETE SET NULL;
+ALTER TABLE ONLY resource_weight_events
+    ADD CONSTRAINT fk_97c7849ca4 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY agent_user_access_group_authorizations
     ADD CONSTRAINT fk_97ce8e8284 FOREIGN KEY (agent_id) REFERENCES cluster_agents(id) ON DELETE CASCADE;

lib/gitlab/background_migration/backfill_resource_weight_events_namespace_id.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class BackfillResourceWeightEventsNamespaceId < BackfillDesiredShardingKeyJob
+      operation_name :backfill_resource_weight_events_namespace_id
+      feature_category :team_planning
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_resource_weight_events_namespace_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillResourceWeightEventsNamespaceId,
+  feature_category: :team_planning,
+  schema: 20241202142250 do
+  include_examples 'desired sharding key backfill job' do
+    let(:batch_table) { :resource_weight_events }
+    let(:backfill_column) { :namespace_id }
+    let(:backfill_via_table) { :issues }
+    let(:backfill_via_column) { :namespace_id }
+    let(:backfill_via_foreign_key) { :issue_id }
+  end
+end

spec/migrations/20241202142254_queue_backfill_resource_weight_events_namespace_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueBackfillResourceWeightEventsNamespaceId, feature_category: :team_planning do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :resource_weight_events,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE,
+          gitlab_schema: :gitlab_main_cell,
+          job_arguments: [
+            :namespace_id,
+            :issues,
+            :namespace_id,
+            :issue_id
+          ]
+        )
+      }
+    end
+  end
+end