Skip to content

Add generator for partitioning FKs

Marius Bobin requested to merge mb-ci-partitioning-fk-generator into master

What does this MR do and why?

For partitioning CI resources we will have a lot of migrations to generate and some of them can be automated.

marius@rocket-sled ~/W/g/gitlab (mb-ci-partitioning-fk-generator)> rails generate gitlab:partitioning:foreign_keys --target ci_builds --source ci_build_trace_chunks
      create  db/post_migrate/20230118112350_add_fk_index_to_ci_build_trace_chunks_on_partition_id_and_build_id.rb
      create  db/post_migrate/20230118112351_add_fk_to_ci_build_trace_chunks_on_partition_id_and_build_id.rb
      create  db/post_migrate/20230118112352_validate_fk_on_ci_build_trace_chunks_partition_id_and_build_id.rb
      create  db/post_migrate/20230118112353_remove_fk_to_ci_builds_ci_build_trace_chunks_on_build_id.rb
marius@rocket-sled ~/W/g/gitlab (mb-ci-partitioning-fk-generator)> rails d gitlab:partitioning:foreign_keys --target ci_builds --source ci_build_trace_chunks
      remove  db/post_migrate/20230118112350_add_fk_index_to_ci_build_trace_chunks_on_partition_id_and_build_id.rb
      remove  db/post_migrate/20230118112351_add_fk_to_ci_build_trace_chunks_on_partition_id_and_build_id.rb
      remove  db/post_migrate/20230118112352_validate_fk_on_ci_build_trace_chunks_partition_id_and_build_id.rb
      remove  db/post_migrate/20230118112353_remove_fk_to_ci_builds_ci_build_trace_chunks_on_build_id.rb
File contents:
# db/post_migrate/20230117171400_add_fk_index_to_ci_build_trace_chunks_on_partition_id_and_build_id.rb
# frozen_string_literal: true

class AddFkIndexToCiBuildTraceChunksOnPartitionIdAndBuildId < Gitlab::Database::Migration[2.1]
  disable_ddl_transaction!

  INDEX_NAME = :index_ci_build_trace_chunks_on_partition_id_build_id
  TABLE_NAME = :ci_build_trace_chunks
  COLUMNS = [:partition_id, :build_id]

  def up
    add_concurrent_index(TABLE_NAME, COLUMNS, name: INDEX_NAME)
  end

  def down
    remove_concurrent_index_by_name(TABLE_NAME, INDEX_NAME)
  end
end
# db/post_migrate/20230117171401_add_fk_to_ci_build_trace_chunks_on_partition_id_and_build_id.rb
# frozen_string_literal: true

class AddFkToCiBuildTraceChunksOnPartitionIdAndBuildId < Gitlab::Database::Migration[2.1]
  disable_ddl_transaction!

  SOURCE_TABLE_NAME = :ci_build_trace_chunks
  TARGET_TABLE_NAME = :ci_builds
  COLUMN = :build_id
  TARGET_COLUMN = :id
  FK_NAME = :fk_rails_1013b761f2_p
  PARTITION_COLUMN = :partition_id

  def up
    add_concurrent_foreign_key(
      SOURCE_TABLE_NAME,
      TARGET_TABLE_NAME,
      column: [PARTITION_COLUMN, COLUMN],
      target_column: [PARTITION_COLUMN, TARGET_COLUMN],
      validate: false,
      reverse_lock_order: true,
      on_update: :cascade,
      on_delete: :cascade,
      name: FK_NAME
    )
  end

  def down
    with_lock_retries do
      remove_foreign_key_if_exists(
        SOURCE_TABLE_NAME,
        TARGET_TABLE_NAME,
        name: FK_NAME,
        reverse_lock_order: true
      )
    end
  end
end
# db/post_migrate/20230117171402_validate_fk_on_ci_build_trace_chunks_partition_id_and_build_id.rb
# frozen_string_literal: true

class ValidateFkOnCiBuildTraceChunksPartitionIdAndBuildId < Gitlab::Database::Migration[2.1]
  disable_ddl_transaction!

  TABLE_NAME = :ci_build_trace_chunks
  FK_NAME = :fk_rails_1013b761f2_p
  COLUMNS = [:partition_id, :build_id]

  def up
    validate_foreign_key(TABLE_NAME, COLUMNS, name: FK_NAME)
  end

  def down
    # no-op
  end
end
#  db/post_migrate/20230118112353_remove_fk_to_ci_builds_ci_build_trace_chunks_on_build_id.rb
# frozen_string_literal: true

class RemoveFkToCiBuildsCiBuildTraceChunksOnBuildId < Gitlab::Database::Migration[2.1]
  disable_ddl_transaction!

  SOURCE_TABLE_NAME = :ci_build_trace_chunks
  TARGET_TABLE_NAME = :ci_builds
  COLUMN = :build_id
  TARGET_COLUMN = :id
  FK_NAME = :fk_rails_1013b761f2

  def up
    with_lock_retries do
      remove_foreign_key_if_exists(
        SOURCE_TABLE_NAME,
        TARGET_TABLE_NAME,
        name: FK_NAME,
        reverse_lock_order: true
      )
    end
  end

  def down
    add_concurrent_foreign_key(
      SOURCE_TABLE_NAME,
      TARGET_TABLE_NAME,
      column: COLUMN,
      target_column: TARGET_COLUMN,
      validate: true,
      reverse_lock_order: true,
      on_delete: :cascade,
      name: FK_NAME
    )
  end
end

The next step to this would be to accept only the target table and generate migrations for all possible source tables.

The migration templates were extracted from !106717 (merged)

I'm putting it to the test in !109316 (merged)

MR acceptance checklist

This checklist encourages us to confirm any changes have been analyzed to reduce risks in quality, performance, reliability, security, and maintainability.

Edited by Marius Bobin

Merge request reports