Skip to content
Snippets Groups Projects

Add rake task for copying 'main' database to 'ci' database

2 files
+ 74
15
Compare changes
  • Side-by-side
  • Inline
Files
2
+ 181
0
# frozen_string_literal: true
module Gitlab
module Database
module Decomposition
MigrateError = Class.new(RuntimeError)
class Migrate
TABLE_SIZE_QUERY = <<-SQL
select sum(pg_table_size(concat(table_schema,'.',table_name))) as total
from information_schema.tables
where table_catalog = :table_catalog and table_type = 'BASE TABLE'
SQL
TABLE_COUNT_QUERY = <<-SQL
select count(*) as total
from information_schema.tables
where table_catalog = :table_catalog and table_type = 'BASE TABLE'
and table_schema not in ('information_schema', 'pg_catalog')
SQL
DISKSPACE_HEADROOM_FACTOR = 1.25
attr_reader :backup_location
def initialize(backup_base_location: nil)
random_post_fix = SecureRandom.alphanumeric(10)
@backup_base_location = backup_base_location || Gitlab.config.backup.path
@backup_location = File.join(@backup_base_location, "migration_#{random_post_fix}")
end
def process!
return unless can_migrate?
dump_main_db
import_dump_to_ci_db
FileUtils.remove_entry_secure(@backup_location, true)
end
private
def valid_backup_location?
FileUtils.mkdir_p(@backup_base_location)
true
rescue StandardError => e
raise MigrateError, "Failed to create directory #{@backup_base_location}: #{e.message}"
end
def main_table_sizes
ApplicationRecord.connection.execute(
ApplicationRecord.sanitize_sql([
TABLE_SIZE_QUERY,
{ table_catalog: main_config.dig(:activerecord, :database) }
])
).first["total"].to_f
end
def diskspace_free
Sys::Filesystem.stat(
File.expand_path("#{@backup_location}/../")
).bytes_free
end
def required_diskspace_available?
needed = main_table_sizes * DISKSPACE_HEADROOM_FACTOR
available = diskspace_free
if needed > available
raise MigrateError,
"Not enough diskspace available on #{@backup_location}: " \
"Available: #{ActiveSupport::NumberHelper.number_to_human_size(available)}, " \
"Needed: #{ActiveSupport::NumberHelper.number_to_human_size(needed)}"
end
true
end
def single_database_setup?
if Gitlab::Database.database_mode == Gitlab::Database::MODE_MULTIPLE_DATABASES
raise MigrateError, "GitLab is already configured to run on multiple databases"
end
true
end
def ci_database_connect_ok?
_, status = with_transient_pg_env(ci_config[:pg_env]) do
psql_args = ["--dbname=#{ci_database_name}", "-tAc", "select 1"]
Open3.capture2e('psql', *psql_args)
end
unless status.success?
raise MigrateError,
"Can't connect to database '#{ci_database_name} on host '#{ci_config[:pg_env]['PGHOST']}'. " \
"Ensure the database has been created."
end
true
end
def ci_database_empty?
sql = ApplicationRecord.sanitize_sql([
TABLE_COUNT_QUERY,
{ table_catalog: ci_database_name }
])
output, status = with_transient_pg_env(ci_config[:pg_env]) do
psql_args = ["--dbname=#{ci_database_name}", "-tAc", sql]
Open3.capture2e('psql', *psql_args)
end
unless status.success? && output.chomp.to_i == 0
raise MigrateError,
"Database '#{ci_database_name}' is not empty"
end
true
end
def background_migrations_done?
unfinished_count = Gitlab::Database::BackgroundMigration::BatchedMigration.without_status(:finished).count
if unfinished_count > 0
raise MigrateError,
"Found #{unfinished_count} unfinished Background Migration(s). Please wait until they are finished."
end
true
end
def can_migrate?
valid_backup_location? &&
single_database_setup? &&
ci_database_connect_ok? &&
ci_database_empty? &&
required_diskspace_available? &&
background_migrations_done?
end
def with_transient_pg_env(extended_env)
ENV.merge!(extended_env)
result = yield
ENV.reject! { |k, _| extended_env.key?(k) }
result
end
def import_dump_to_ci_db
with_transient_pg_env(ci_config[:pg_env]) do
restore_args = ["--jobs=4", "--dbname=#{ci_database_name}"]
Open3.capture2e('pg_restore', *restore_args, @backup_location)
end
end
def dump_main_db
with_transient_pg_env(main_config[:pg_env]) do
args = ['--format=d', '--jobs=4', "--file=#{@backup_location}"]
Open3.capture2e('pg_dump', *args, main_config.dig(:activerecord, :database))
end
end
def main_config
@main_config ||= ::Backup::DatabaseModel.new('main').config
end
def ci_config
@ci_config ||= ::Backup::DatabaseModel.new('ci').config
end
def ci_database_name
@ci_database_name ||= "#{main_config.dig(:activerecord, :database)}_ci"
end
end
end
end
end
Loading