Merge branch 'backfill-desired-sharding-key-small-table-issuable_severities' into 'master'

Add and backfill namespace_id for issuable_severities

See merge request !173728



Merged-by: Omar Qunsul <oqunsul@gitlab.com>
Approved-by: Omar Qunsul <oqunsul@gitlab.com>
Co-authored-by: Shubham Kumar <shukumar@gitlab.com>

db/docs/batched_background_migrations/backfill_issuable_severities_namespace_id.yml

+---
+migration_job_name: BackfillIssuableSeveritiesNamespaceId
+description: Backfills sharding key `issuable_severities.namespace_id` from `issues`.
+feature_category: team_planning
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/173728
+milestone: '17.7'
+queued_migration_version: 20241125145009
+finalized_by: # version of the migration that finalized this BBM

db/docs/issuable_severities.yml

@@ -18,3 +18,4 @@ desired_sharding_key:
         sharding_key: namespace_id
         belongs_to: issue
 table_size: small
+desired_sharding_key_migration_job_name: BackfillIssuableSeveritiesNamespaceId

db/migrate/20241125145005_add_namespace_id_to_issuable_severities.rb

+# frozen_string_literal: true
+
+class AddNamespaceIdToIssuableSeverities < Gitlab::Database::Migration[2.2]
+  milestone '17.7'
+
+  def change
+    add_column :issuable_severities, :namespace_id, :bigint
+  end
+end

db/post_migrate/20241125145006_index_issuable_severities_on_namespace_id.rb

+# frozen_string_literal: true
+
+class IndexIssuableSeveritiesOnNamespaceId < Gitlab::Database::Migration[2.2]
+  milestone '17.7'
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_issuable_severities_on_namespace_id'
+
+  def up
+    add_concurrent_index :issuable_severities, :namespace_id, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :issuable_severities, INDEX_NAME
+  end
+end

db/post_migrate/20241125145007_add_issuable_severities_namespace_id_fk.rb

+# frozen_string_literal: true
+
+class AddIssuableSeveritiesNamespaceIdFk < Gitlab::Database::Migration[2.2]
+  milestone '17.7'
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :issuable_severities, :namespaces, column: :namespace_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :issuable_severities, column: :namespace_id
+    end
+  end
+end

db/post_migrate/20241125145008_add_issuable_severities_namespace_id_trigger.rb

+# frozen_string_literal: true
+
+class AddIssuableSeveritiesNamespaceIdTrigger < Gitlab::Database::Migration[2.2]
+  milestone '17.7'
+
+  def up
+    install_sharding_key_assignment_trigger(
+      table: :issuable_severities,
+      sharding_key: :namespace_id,
+      parent_table: :issues,
+      parent_sharding_key: :namespace_id,
+      foreign_key: :issue_id
+    )
+  end
+
+  def down
+    remove_sharding_key_assignment_trigger(
+      table: :issuable_severities,
+      sharding_key: :namespace_id,
+      parent_table: :issues,
+      parent_sharding_key: :namespace_id,
+      foreign_key: :issue_id
+    )
+  end
+end

db/post_migrate/20241125145009_queue_backfill_issuable_severities_namespace_id.rb

+# frozen_string_literal: true
+
+class QueueBackfillIssuableSeveritiesNamespaceId < Gitlab::Database::Migration[2.2]
+  milestone '17.7'
+  restrict_gitlab_migration gitlab_schema: :gitlab_main_cell
+
+  MIGRATION = "BackfillIssuableSeveritiesNamespaceId"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :issuable_severities,
+      :id,
+      :namespace_id,
+      :issues,
+      :namespace_id,
+      :issue_id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(
+      MIGRATION,
+      :issuable_severities,
+      :id,
+      [
+        :namespace_id,
+        :issues,
+        :namespace_id,
+        :issue_id
+      ]
+    )
+  end
+end

db/schema_migrations/20241125145005

+0ab0371d24515c2fcb804e47fa7526edfb681204fed2137db4e9498f4b7fe116
\ No newline at end of file

db/schema_migrations/20241125145006

+a3632c565094b3a1672ff9c5c3f7ec04b993555ad3d1eb633b1b033c9741729f
\ No newline at end of file

db/schema_migrations/20241125145007

+c2ce13c3186404aec3981a847dd971c3a3a9be1cedd7aad0554821e83c8fe1b5
\ No newline at end of file

db/schema_migrations/20241125145008

+f27f4eacc953b45d95559b141ec42e6a312c44c9496a8283dc61dcb2443ddc47
\ No newline at end of file

db/schema_migrations/20241125145009

+98452a102d7cafe648ad028a41f2d71a42a9bd429b1df7356f66cd78bad370ed
\ No newline at end of file

db/structure.sql

@@ -2721,6 +2721,22 @@ RETURN NEW;
 END
 $$;
+CREATE FUNCTION trigger_dfad97659d5f() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+IF NEW."namespace_id" IS NULL THEN
+  SELECT "namespace_id"
+  INTO NEW."namespace_id"
+  FROM "issues"
+  WHERE "issues"."id" = NEW."issue_id";
+END IF;
+
+RETURN NEW;
+
+END
+$$;
+
 CREATE FUNCTION trigger_e0864d1cff37() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -13514,7 +13530,8 @@ ALTER SEQUENCE issuable_resource_links_id_seq OWNED BY issuable_resource_links.i
 CREATE TABLE issuable_severities (
     id bigint NOT NULL,
     issue_id bigint NOT NULL,
-    severity smallint DEFAULT 0 NOT NULL
+    severity smallint DEFAULT 0 NOT NULL,
+    namespace_id bigint
 );
 CREATE SEQUENCE issuable_severities_id_seq
@@ -30616,6 +30633,8 @@ CREATE INDEX index_issuable_resource_links_on_issue_id ON issuable_resource_link
 CREATE UNIQUE INDEX index_issuable_severities_on_issue_id ON issuable_severities USING btree (issue_id);
+CREATE INDEX index_issuable_severities_on_namespace_id ON issuable_severities USING btree (namespace_id);
+
 CREATE INDEX index_issuable_slas_on_due_at_id_label_applied_issuable_closed ON issuable_slas USING btree (due_at, id) WHERE ((label_applied = false) AND (issuable_closed = false));
 CREATE UNIQUE INDEX index_issuable_slas_on_issue_id ON issuable_slas USING btree (issue_id);
@@ -35428,6 +35447,8 @@ CREATE TRIGGER trigger_dc13168b8025 BEFORE INSERT OR UPDATE ON vulnerability_fla
 CREATE TRIGGER trigger_delete_project_namespace_on_project_delete AFTER DELETE ON projects FOR EACH ROW WHEN ((old.project_namespace_id IS NOT NULL)) EXECUTE FUNCTION delete_associated_project_namespace();
+CREATE TRIGGER trigger_dfad97659d5f BEFORE INSERT OR UPDATE ON issuable_severities FOR EACH ROW EXECUTE FUNCTION trigger_dfad97659d5f();
+
 CREATE TRIGGER trigger_e0864d1cff37 BEFORE INSERT OR UPDATE ON packages_debian_group_architectures FOR EACH ROW EXECUTE FUNCTION trigger_e0864d1cff37();
 CREATE TRIGGER trigger_e1da4a738230 BEFORE INSERT OR UPDATE ON vulnerability_external_issue_links FOR EACH ROW EXECUTE FUNCTION trigger_e1da4a738230();
@@ -37127,6 +37148,9 @@ ALTER TABLE ONLY protected_tag_create_access_levels
 ALTER TABLE ONLY application_settings
     ADD CONSTRAINT fk_f9867b3540 FOREIGN KEY (web_ide_oauth_application_id) REFERENCES oauth_applications(id) ON DELETE SET NULL;
+ALTER TABLE ONLY issuable_severities
+    ADD CONSTRAINT fk_f9df19ecb6 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
+
 ALTER TABLE p_ci_stages
     ADD CONSTRAINT fk_fb57e6cc56_p FOREIGN KEY (partition_id, pipeline_id) REFERENCES p_ci_pipelines(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE;

lib/gitlab/background_migration/backfill_issuable_severities_namespace_id.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class BackfillIssuableSeveritiesNamespaceId < BackfillDesiredShardingKeyJob
+      operation_name :backfill_issuable_severities_namespace_id
+      feature_category :team_planning
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_issuable_severities_namespace_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillIssuableSeveritiesNamespaceId,
+  feature_category: :team_planning,
+  schema: 20241125145005 do
+  include_examples 'desired sharding key backfill job' do
+    let(:batch_table) { :issuable_severities }
+    let(:backfill_column) { :namespace_id }
+    let(:backfill_via_table) { :issues }
+    let(:backfill_via_column) { :namespace_id }
+    let(:backfill_via_foreign_key) { :issue_id }
+  end
+end

spec/migrations/20241125145009_queue_backfill_issuable_severities_namespace_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueBackfillIssuableSeveritiesNamespaceId, feature_category: :team_planning do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :issuable_severities,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE,
+          gitlab_schema: :gitlab_main_cell,
+          job_arguments: [
+            :namespace_id,
+            :issues,
+            :namespace_id,
+            :issue_id
+          ]
+        )
+      }
+    end
+  end
+end