Skip to content
Snippets Groups Projects
Commit 1adcebd6 authored by Marius Bobin's avatar Marius Bobin :two:
Browse files

Merge branch 're-enqueue-partitioned-fk-creation' into 'master'

Replace foreign keys to use the partitioned table - V2

See merge request !123355



Merged-by: default avatarMarius Bobin <mbobin@gitlab.com>
Approved-by: default avatarMax Orefice <morefice@gitlab.com>
Reviewed-by: default avatarMarius Bobin <mbobin@gitlab.com>
Reviewed-by: default avatarMax Orefice <morefice@gitlab.com>
parents c4720833 c79e00ba
No related branches found
No related tags found
No related merge requests found
Pipeline #899979777 passed
Pipeline: E2E GDK

#900017546

    Pipeline: GitLab

    #899990359

      Showing
      with 293 additions and 0 deletions
      # frozen_string_literal: true
      class ReplaceCiJobArtifactsForeignKeyV2 < Gitlab::Database::Migration[2.1]
      include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum
      disable_ddl_transaction!
      def up
      return unless should_run?
      add_concurrent_foreign_key :ci_job_artifacts, :p_ci_builds,
      name: 'temp_fk_rails_c5137cb2c1_p',
      column: [:partition_id, :job_id],
      target_column: [:partition_id, :id],
      on_update: :cascade,
      on_delete: :cascade,
      validate: false,
      reverse_lock_order: true
      prepare_async_foreign_key_validation :ci_job_artifacts,
      name: 'temp_fk_rails_c5137cb2c1_p'
      end
      def down
      return unless should_run?
      unprepare_async_foreign_key_validation :ci_job_artifacts, name: 'temp_fk_rails_c5137cb2c1_p'
      remove_foreign_key_if_exists :ci_job_artifacts, name: 'temp_fk_rails_c5137cb2c1_p'
      end
      private
      def should_run?
      can_execute_on?(:ci_job_artifacts, :ci_builds)
      end
      end
      # frozen_string_literal: true
      class ReplaceCiRunningBuildsForeignKeyV2 < Gitlab::Database::Migration[2.1]
      include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum
      disable_ddl_transaction!
      def up
      return unless should_run?
      add_concurrent_foreign_key :ci_running_builds, :p_ci_builds,
      name: 'temp_fk_rails_da45cfa165_p',
      column: [:partition_id, :build_id],
      target_column: [:partition_id, :id],
      on_update: :cascade,
      on_delete: :cascade,
      validate: false,
      reverse_lock_order: true
      prepare_async_foreign_key_validation :ci_running_builds,
      name: 'temp_fk_rails_da45cfa165_p'
      end
      def down
      return unless should_run?
      unprepare_async_foreign_key_validation :ci_running_builds, name: 'temp_fk_rails_da45cfa165_p'
      remove_foreign_key_if_exists :ci_running_builds, name: 'temp_fk_rails_da45cfa165_p'
      end
      private
      def should_run?
      can_execute_on?(:ci_running_builds, :ci_builds)
      end
      end
      # frozen_string_literal: true
      class ReplaceCiJobVariablesForeignKeyV2 < Gitlab::Database::Migration[2.1]
      include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum
      disable_ddl_transaction!
      def up
      return unless should_run?
      add_concurrent_foreign_key :ci_job_variables, :p_ci_builds,
      name: 'temp_fk_rails_fbf3b34792_p',
      column: [:partition_id, :job_id],
      target_column: [:partition_id, :id],
      on_update: :cascade,
      on_delete: :cascade,
      validate: false,
      reverse_lock_order: true
      prepare_async_foreign_key_validation :ci_job_variables,
      name: 'temp_fk_rails_fbf3b34792_p'
      end
      def down
      return unless should_run?
      unprepare_async_foreign_key_validation :ci_job_variables, name: 'temp_fk_rails_fbf3b34792_p'
      remove_foreign_key_if_exists :ci_job_variables, name: 'temp_fk_rails_fbf3b34792_p'
      end
      private
      def should_run?
      can_execute_on?(:ci_job_variables, :ci_builds)
      end
      end
      # frozen_string_literal: true
      class ReplacePCiBuildsMetadataForeignKeyV2 < Gitlab::Database::Migration[2.1]
      include Gitlab::Database::PartitioningMigrationHelpers
      include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum
      disable_ddl_transaction!
      def up
      return unless should_run?
      add_concurrent_partitioned_foreign_key :p_ci_builds_metadata, :p_ci_builds,
      name: 'temp_fk_e20479742e_p',
      column: [:partition_id, :build_id],
      target_column: [:partition_id, :id],
      on_update: :cascade,
      on_delete: :cascade,
      validate: false,
      reverse_lock_order: true
      prepare_partitioned_async_foreign_key_validation :p_ci_builds_metadata,
      name: 'temp_fk_e20479742e_p'
      end
      def down
      return unless should_run?
      unprepare_partitioned_async_foreign_key_validation :p_ci_builds_metadata, name: 'temp_fk_e20479742e_p'
      Gitlab::Database::PostgresPartitionedTable.each_partition(:p_ci_builds_metadata) do |partition|
      execute "ALTER TABLE #{partition.identifier} DROP CONSTRAINT IF EXISTS temp_fk_e20479742e_p"
      end
      end
      private
      def should_run?
      can_execute_on?(:ci_builds_metadata, :ci_builds)
      end
      end
      # frozen_string_literal: true
      class ReplacePCiRunnerMachineBuildsForeignKeyV2 < Gitlab::Database::Migration[2.1]
      include Gitlab::Database::PartitioningMigrationHelpers
      include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum
      disable_ddl_transaction!
      def up
      return unless should_run?
      add_concurrent_partitioned_foreign_key :p_ci_runner_machine_builds, :p_ci_builds,
      name: 'temp_fk_bb490f12fe_p',
      column: [:partition_id, :build_id],
      target_column: [:partition_id, :id],
      on_update: :cascade,
      on_delete: :cascade,
      validate: false,
      reverse_lock_order: true
      prepare_partitioned_async_foreign_key_validation :p_ci_runner_machine_builds,
      name: 'temp_fk_bb490f12fe_p'
      end
      def down
      return unless should_run?
      unprepare_partitioned_async_foreign_key_validation :p_ci_runner_machine_builds, name: 'temp_fk_bb490f12fe_p'
      Gitlab::Database::PostgresPartitionedTable.each_partition(:p_ci_runner_machine_builds) do |partition|
      execute "ALTER TABLE #{partition.identifier} DROP CONSTRAINT IF EXISTS temp_fk_bb490f12fe_p"
      end
      end
      private
      def should_run?
      can_execute_on?(:ci_builds)
      end
      end
      c1bd88e757c7c08d340066201e3cba4fe23c1d0e78b7a4fbca0e7c9d87df802e
      \ No newline at end of file
      21ad4cb522f5a7bb8f4d5883be0d195968206af01051bc4288d287dc15e87572
      \ No newline at end of file
      dbe7656846b46a8e4e7c4b4d04ae034d08eb35b7bad1b530e428c2725542fe43
      \ No newline at end of file
      fe353050a5b4e9650e5ba4985668958032ac105c9cff86d75c974e567bfe9074
      \ No newline at end of file
      3c03bbb4082c8b0e42197a4bc9ad80aa3dcfa09d8e183fdaa503278ec5db79be
      \ No newline at end of file
      ...@@ -37973,6 +37973,9 @@ ALTER TABLE ONLY ci_sources_pipelines ...@@ -37973,6 +37973,9 @@ ALTER TABLE ONLY ci_sources_pipelines
      ALTER TABLE ONLY ci_resources ALTER TABLE ONLY ci_resources
      ADD CONSTRAINT temp_fk_e169a8e3d5_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE SET NULL; ADD CONSTRAINT temp_fk_e169a8e3d5_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE SET NULL;
         
      ALTER TABLE ONLY ci_builds_metadata
      ADD CONSTRAINT temp_fk_e20479742e_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE NOT VALID;
      ALTER TABLE ONLY ci_build_report_results ALTER TABLE ONLY ci_build_report_results
      ADD CONSTRAINT temp_fk_rails_16cb1ff064_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE; ADD CONSTRAINT temp_fk_rails_16cb1ff064_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE;
         
      ...@@ -37988,6 +37991,15 @@ ALTER TABLE ONLY ci_pending_builds ...@@ -37988,6 +37991,15 @@ ALTER TABLE ONLY ci_pending_builds
      ALTER TABLE ONLY ci_build_trace_metadata ALTER TABLE ONLY ci_build_trace_metadata
      ADD CONSTRAINT temp_fk_rails_aebc78111f_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE; ADD CONSTRAINT temp_fk_rails_aebc78111f_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE;
         
      ALTER TABLE ONLY ci_job_artifacts
      ADD CONSTRAINT temp_fk_rails_c5137cb2c1_p FOREIGN KEY (partition_id, job_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE NOT VALID;
      ALTER TABLE ONLY ci_running_builds
      ADD CONSTRAINT temp_fk_rails_da45cfa165_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE NOT VALID;
      ALTER TABLE ONLY ci_job_variables
      ADD CONSTRAINT temp_fk_rails_fbf3b34792_p FOREIGN KEY (partition_id, job_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE NOT VALID;
      ALTER TABLE ONLY user_follow_users ALTER TABLE ONLY user_follow_users
      ADD CONSTRAINT user_follow_users_followee_id_fkey FOREIGN KEY (followee_id) REFERENCES users(id) ON DELETE CASCADE; ADD CONSTRAINT user_follow_users_followee_id_fkey FOREIGN KEY (followee_id) REFERENCES users(id) ON DELETE CASCADE;
         
      # frozen_string_literal: true
      module Gitlab
      module Database
      module MigrationHelpers
      module WraparoundAutovacuum
      # This is used for partitioning CI tables because the autovacuum for wraparound
      # prevention can take many hours to complete on some of the tables and this in
      # turn blocks the post deployment migrations pipeline.
      # Intended workflow for this helper:
      # 1. Introduce a migration that is guarded with this helper
      # 2. Check that the migration was successfully executed on .com
      # 3. Introduce the migration again for self-managed.
      #
      def can_execute_on?(*tables)
      return false unless Gitlab.com? || Gitlab.dev_or_test_env?
      if wraparound_prevention_on_tables?(tables)
      Gitlab::AppLogger.info(message: "Wraparound prevention vacuum detected", class: self.class)
      say "Wraparound prevention vacuum detected, skipping migration"
      return false
      end
      true
      end
      def wraparound_prevention_on_tables?(tables)
      Gitlab::Database::PostgresAutovacuumActivity.reset_column_information
      Gitlab::Database::PostgresAutovacuumActivity
      .wraparound_prevention
      .for_tables(tables)
      .any?
      end
      end
      end
      end
      end
      # frozen_string_literal: true
      require 'spec_helper'
      RSpec.describe Gitlab::Database::MigrationHelpers::WraparoundAutovacuum, feature_category: :database do
      include Database::DatabaseHelpers
      let(:migration) do
      Class.new(Gitlab::Database::Migration[2.1])
      .include(described_class)
      .new
      end
      describe '#can_execute_on?' do
      using RSpec::Parameterized::TableSyntax
      where(:dot_com, :dev_or_test, :wraparound_prevention, :expectation) do
      true | true | true | false
      true | false | true | false
      false | true | true | false
      false | false | true | false
      true | true | false | true
      true | false | false | true
      false | true | false | true
      false | false | false | false
      end
      with_them do
      it 'returns true for GitLab.com, dev, or test' do
      allow(Gitlab).to receive(:com?).and_return(dot_com)
      allow(Gitlab).to receive(:dev_or_test_env?).and_return(dev_or_test)
      allow(migration).to receive(:wraparound_prevention_on_tables?).with([:table]).and_return(wraparound_prevention)
      expect(migration.can_execute_on?(:table)).to eq(expectation)
      end
      end
      end
      describe '#wraparound_prevention_on_tables?' do
      before do
      swapout_view_for_table(:postgres_autovacuum_activity, connection: ApplicationRecord.connection)
      create(:postgres_autovacuum_activity, table: 'foo', wraparound_prevention: false)
      create(:postgres_autovacuum_activity, table: 'bar', wraparound_prevention: true)
      end
      it { expect(migration.wraparound_prevention_on_tables?([:foo])).to be_falsey }
      it { expect(migration.wraparound_prevention_on_tables?([:bar])).to be_truthy }
      it { expect(migration.wraparound_prevention_on_tables?([:foo, :bar])).to be_truthy }
      end
      end
      0% Loading or .
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment