Skip to content
Snippets Groups Projects
Commit 94f1cb1d authored by Sean McGivern's avatar Sean McGivern :red_circle:
Browse files

Merge branch 'backup_logging' into 'master'

Improve backup logging

See merge request !83055
parents 2a4b6349 dd358a39
No related branches found
No related tags found
1 merge request!83055Improve backup logging
Pipeline #497215933 failed
Pipeline: GitLab

#497231071

    ......@@ -88,34 +88,33 @@ def create
    remove_old
    end
    progress.puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
    puts_time "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
    "and are not included in this backup. You will need these files to restore a backup.\n" \
    "Please back them up manually.".color(:red)
    progress.puts "Backup task is done."
    puts_time "Backup task is done."
    end
    def run_create_task(task_name)
    definition = @definitions[task_name]
    build_backup_information
    puts_time "Dumping #{definition.task.human_name} ... ".color(:blue)
    unless definition.task.enabled
    puts_time "[DISABLED]".color(:cyan)
    puts_time "Dumping #{definition.task.human_name} ... ".color(:blue) + "[DISABLED]".color(:cyan)
    return
    end
    if skipped?(task_name)
    puts_time "[SKIPPED]".color(:cyan)
    puts_time "Dumping #{definition.task.human_name} ... ".color(:blue) + "[SKIPPED]".color(:cyan)
    return
    end
    puts_time "Dumping #{definition.task.human_name} ... ".color(:blue)
    definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path))
    puts_time "done".color(:green)
    puts_time "Dumping #{definition.task.human_name} ... ".color(:blue) + "done".color(:green)
    rescue Backup::DatabaseBackupError, Backup::FileBackupError => e
    progress.puts "#{e.message}"
    puts_time "Dumping #{definition.task.human_name} failed: #{e.message}".color(:red)
    end
    def restore
    ......@@ -136,21 +135,21 @@ def restore
    remove_tmp
    puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
    "and are not included in this backup. You will need to restore these files manually.".color(:red)
    puts "Restore task is done."
    puts_time "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
    "and are not included in this backup. You will need to restore these files manually.".color(:red)
    puts_time "Restore task is done."
    end
    def run_restore_task(task_name)
    definition = @definitions[task_name]
    puts_time "Restoring #{definition.task.human_name} ... ".color(:blue)
    unless definition.task.enabled
    puts_time "[DISABLED]".color(:cyan)
    puts_time "Restoring #{definition.task.human_name} ... ".color(:blue) + "[DISABLED]".color(:cyan)
    return
    end
    puts_time "Restoring #{definition.task.human_name} ... ".color(:blue)
    warning = definition.task.pre_restore_warning
    if warning.present?
    puts_time warning.color(:red)
    ......@@ -159,7 +158,7 @@ def run_restore_task(task_name)
    definition.task.restore(File.join(Gitlab.config.backup.path, definition.destination_path))
    puts_time "done".color(:green)
    puts_time "Restoring #{definition.task.human_name} ... ".color(:blue) + "done".color(:green)
    warning = definition.task.post_restore_warning
    if warning.present?
    ......@@ -209,103 +208,104 @@ def backup_information
    def pack
    Dir.chdir(backup_path) do
    # create archive
    progress.print "Creating backup archive: #{tar_file} ... "
    puts_time "Creating backup archive: #{tar_file} ... ".color(:blue)
    # Set file permissions on open to prevent chmod races.
    tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] }
    if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options)
    progress.puts "done".color(:green)
    puts_time "Creating backup archive: #{tar_file} ... ".color(:blue) + 'done'.color(:green)
    else
    puts "creating archive #{tar_file} failed".color(:red)
    puts_time "Creating archive #{tar_file} failed".color(:red)
    raise Backup::Error, 'Backup failed'
    end
    end
    end
    def upload
    progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
    connection_settings = Gitlab.config.backup.upload.connection
    if connection_settings.blank?
    progress.puts "skipped".color(:yellow)
    puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + "[SKIPPED]".color(:cyan)
    return
    end
    puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue)
    directory = connect_to_remote_directory
    upload = directory.files.create(create_attributes)
    if upload
    if upload.respond_to?(:encryption) && upload.encryption
    progress.puts "done (encrypted with #{upload.encryption})".color(:green)
    puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + "done (encrypted with #{upload.encryption})".color(:green)
    else
    progress.puts "done".color(:green)
    puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + "done".color(:green)
    end
    else
    puts "uploading backup to #{remote_directory} failed".color(:red)
    puts_time "Uploading backup to #{remote_directory} failed".color(:red)
    raise Backup::Error, 'Backup failed'
    end
    end
    def cleanup
    progress.print "Deleting tmp directories ... "
    puts_time "Deleting tar staging files ... ".color(:blue)
    remove_backup_path(MANIFEST_NAME)
    @definitions.each do |_, definition|
    remove_backup_path(definition.cleanup_path || definition.destination_path)
    end
    puts_time "Deleting tar staging files ... ".color(:blue) + 'done'.color(:green)
    end
    def remove_backup_path(path)
    return unless File.exist?(File.join(backup_path, path))
    absolute_path = File.join(backup_path, path)
    return unless File.exist?(absolute_path)
    FileUtils.rm_rf(File.join(backup_path, path))
    progress.puts "done".color(:green)
    puts_time "Cleaning up #{absolute_path}"
    FileUtils.rm_rf(absolute_path)
    end
    def remove_tmp
    # delete tmp inside backups
    progress.print "Deleting backups/tmp ... "
    puts_time "Deleting backups/tmp ... ".color(:blue)
    if FileUtils.rm_rf(File.join(backup_path, "tmp"))
    progress.puts "done".color(:green)
    else
    puts "deleting backups/tmp failed".color(:red)
    end
    FileUtils.rm_rf(File.join(backup_path, "tmp"))
    puts_time "Deleting backups/tmp ... ".color(:blue) + "done".color(:green)
    end
    def remove_old
    # delete backups
    progress.print "Deleting old backups ... "
    keep_time = Gitlab.config.backup.keep_time.to_i
    if keep_time > 0
    removed = 0
    Dir.chdir(backup_path) do
    backup_file_list.each do |file|
    # For backward compatibility, there are 3 names the backups can have:
    # - 1495527122_gitlab_backup.tar
    # - 1495527068_2017_05_23_gitlab_backup.tar
    # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar
    matched = backup_file?(file)
    next unless matched
    timestamp = matched[1].to_i
    if Time.at(timestamp) < (Time.now - keep_time)
    begin
    FileUtils.rm(file)
    removed += 1
    rescue StandardError => e
    progress.puts "Deleting #{file} failed: #{e.message}".color(:red)
    end
    if keep_time <= 0
    puts_time "Deleting old backups ... ".color(:blue) + "[SKIPPED]".color(:cyan)
    return
    end
    puts_time "Deleting old backups ... ".color(:blue)
    removed = 0
    Dir.chdir(backup_path) do
    backup_file_list.each do |file|
    # For backward compatibility, there are 3 names the backups can have:
    # - 1495527122_gitlab_backup.tar
    # - 1495527068_2017_05_23_gitlab_backup.tar
    # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar
    matched = backup_file?(file)
    next unless matched
    timestamp = matched[1].to_i
    if Time.at(timestamp) < (Time.now - keep_time)
    begin
    FileUtils.rm(file)
    removed += 1
    rescue StandardError => e
    puts_time "Deleting #{file} failed: #{e.message}".color(:red)
    end
    end
    end
    progress.puts "done. (#{removed} removed)".color(:green)
    else
    progress.puts "skipping".color(:yellow)
    end
    puts_time "Deleting old backups ... ".color(:blue) + "done. (#{removed} removed)".color(:green)
    end
    def verify_backup_version
    ......@@ -327,7 +327,7 @@ def verify_backup_version
    def unpack
    if ENV['BACKUP'].blank? && non_tarred_backup?
    progress.puts "Non tarred backup found in #{backup_path}, using that"
    puts_time "Non tarred backup found in #{backup_path}, using that"
    return false
    end
    ......@@ -335,15 +335,15 @@ def unpack
    Dir.chdir(backup_path) do
    # check for existing backups in the backup dir
    if backup_file_list.empty?
    progress.puts "No backups found in #{backup_path}"
    progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
    puts_time "No backups found in #{backup_path}"
    puts_time "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
    exit 1
    elsif backup_file_list.many? && ENV["BACKUP"].nil?
    progress.puts 'Found more than one backup:'
    puts_time 'Found more than one backup:'
    # print list of available backups
    progress.puts " " + available_timestamps.join("\n ")
    progress.puts 'Please specify which one you want to restore:'
    progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
    puts_time " " + available_timestamps.join("\n ")
    puts_time 'Please specify which one you want to restore:'
    puts_time 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
    exit 1
    end
    ......@@ -354,16 +354,16 @@ def unpack
    end
    unless File.exist?(tar_file)
    progress.puts "The backup file #{tar_file} does not exist!"
    puts_time "The backup file #{tar_file} does not exist!"
    exit 1
    end
    progress.print 'Unpacking backup ... '
    puts_time 'Unpacking backup ... '.color(:blue)
    if Kernel.system(*%W(tar -xf #{tar_file}))
    progress.puts 'done'.color(:green)
    puts_time 'Unpacking backup ... '.color(:blue) + 'done'.color(:green)
    else
    progress.puts 'unpacking backup failed'.color(:red)
    puts_time 'Unpacking backup failed'.color(:red)
    exit 1
    end
    end
    ......
    ......@@ -28,7 +28,7 @@
    it 'calls the named task' do
    expect(task).to receive(:dump)
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... ')
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'done')
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... done')
    subject.run_create_task('my_task')
    end
    ......@@ -37,8 +37,7 @@
    let(:enabled) { false }
    it 'informs the user' do
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... ')
    expect(Gitlab::BackupLogger).to receive(:info).with(message: '[DISABLED]')
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... [DISABLED]')
    subject.run_create_task('my_task')
    end
    ......@@ -48,8 +47,7 @@
    it 'informs the user' do
    stub_env('SKIP', 'my_task')
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... ')
    expect(Gitlab::BackupLogger).to receive(:info).with(message: '[SKIPPED]')
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... [SKIPPED]')
    subject.run_create_task('my_task')
    end
    ......@@ -78,7 +76,7 @@
    it 'calls the named task' do
    expect(task).to receive(:restore)
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... ').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'done').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... done').ordered
    subject.run_restore_task('my_task')
    end
    ......@@ -87,8 +85,7 @@
    let(:enabled) { false }
    it 'informs the user' do
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... ').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: '[DISABLED]').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... [DISABLED]').ordered
    subject.run_restore_task('my_task')
    end
    ......@@ -100,7 +97,7 @@
    it 'displays and waits for the user' do
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... ').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Watch out!').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'done').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... done').ordered
    expect(Gitlab::TaskHelpers).to receive(:ask_to_continue)
    expect(task).to receive(:restore)
    ......@@ -124,7 +121,7 @@
    it 'displays and waits for the user' do
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... ').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'done').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... done').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Watch out!').ordered
    expect(Gitlab::TaskHelpers).to receive(:ask_to_continue)
    expect(task).to receive(:restore)
    ......@@ -134,7 +131,7 @@
    it 'does not continue when the user quits' do
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... ').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'done').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... done').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Watch out!').ordered
    expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Quitting...').ordered
    expect(task).to receive(:restore)
    ......@@ -170,7 +167,9 @@
    before do
    allow(ActiveRecord::Base.connection).to receive(:reconnect!)
    allow(Gitlab::BackupLogger).to receive(:info)
    allow(Kernel).to receive(:system).and_return(true)
    allow(YAML).to receive(:load_file).and_call_original
    allow(YAML).to receive(:load_file).with(File.join(Gitlab.config.backup.path, 'backup_information.yml'))
    .and_return(backup_information)
    ......@@ -185,6 +184,20 @@
    expect(Kernel).to have_received(:system).with(*tar_cmdline)
    end
    context 'tar fails' do
    before do
    expect(Kernel).to receive(:system).with(*tar_cmdline).and_return(false)
    end
    it 'logs a failure' do
    expect do
    subject.create # rubocop:disable Rails/SaveBang
    end.to raise_error(Backup::Error, 'Backup failed')
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: "Creating archive #{tar_file} failed")
    end
    end
    context 'when BACKUP is set' do
    let(:tar_file) { 'custom_gitlab_backup.tar' }
    ......@@ -248,6 +261,7 @@
    end
    before do
    allow(Gitlab::BackupLogger).to receive(:info)
    allow(Dir).to receive(:chdir).and_yield
    allow(Dir).to receive(:glob).and_return(files)
    allow(FileUtils).to receive(:rm)
    ......@@ -266,7 +280,7 @@
    end
    it 'prints a skipped message' do
    expect(progress).to have_received(:puts).with('skipping')
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting old backups ... [SKIPPED]')
    end
    end
    ......@@ -290,7 +304,7 @@
    end
    it 'prints a done message' do
    expect(progress).to have_received(:puts).with('done. (0 removed)')
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting old backups ... done. (0 removed)')
    end
    end
    ......@@ -307,7 +321,7 @@
    end
    it 'prints a done message' do
    expect(progress).to have_received(:puts).with('done. (0 removed)')
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting old backups ... done. (0 removed)')
    end
    end
    ......@@ -348,7 +362,7 @@
    end
    it 'prints a done message' do
    expect(progress).to have_received(:puts).with('done. (8 removed)')
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting old backups ... done. (8 removed)')
    end
    end
    ......@@ -372,11 +386,11 @@
    end
    it 'sets the correct removed count' do
    expect(progress).to have_received(:puts).with('done. (7 removed)')
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting old backups ... done. (7 removed)')
    end
    it 'prints the error from file that could not be removed' do
    expect(progress).to have_received(:puts).with(a_string_matching(message))
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: a_string_matching(message))
    end
    end
    end
    ......@@ -386,6 +400,7 @@
    let(:backup_filename) { File.basename(backup_file.path) }
    before do
    allow(Gitlab::BackupLogger).to receive(:info)
    allow(subject).to receive(:tar_file).and_return(backup_filename)
    stub_backup_setting(
    ......@@ -462,7 +477,7 @@
    it 'sets encryption attributes' do
    subject.create # rubocop:disable Rails/SaveBang
    expect(progress).to have_received(:puts).with("done (encrypted with AES256)")
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Uploading backup archive to remote storage directory ... done (encrypted with AES256)')
    end
    end
    ......@@ -473,7 +488,7 @@
    it 'sets encryption attributes' do
    subject.create # rubocop:disable Rails/SaveBang
    expect(progress).to have_received(:puts).with("done (encrypted with AES256)")
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Uploading backup archive to remote storage directory ... done (encrypted with AES256)')
    end
    end
    ......@@ -488,7 +503,7 @@
    it 'sets encryption attributes' do
    subject.create # rubocop:disable Rails/SaveBang
    expect(progress).to have_received(:puts).with("done (encrypted with aws:kms)")
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Uploading backup archive to remote storage directory ... done (encrypted with aws:kms)')
    end
    end
    end
    ......@@ -570,6 +585,7 @@
    Rake.application.rake_require 'tasks/gitlab/shell'
    Rake.application.rake_require 'tasks/cache'
    allow(Gitlab::BackupLogger).to receive(:info)
    allow(task1).to receive(:restore).with(File.join(Gitlab.config.backup.path, 'task1.tar.gz'))
    allow(task2).to receive(:restore).with(File.join(Gitlab.config.backup.path, 'task2.tar.gz'))
    allow(YAML).to receive(:load_file).with(File.join(Gitlab.config.backup.path, 'backup_information.yml'))
    ......@@ -634,7 +650,10 @@
    end
    context 'when BACKUP variable is set to a correct file' do
    let(:tar_cmdline) { %w{tar -xf 1451606400_2016_01_01_1.2.3_gitlab_backup.tar} }
    before do
    allow(Gitlab::BackupLogger).to receive(:info)
    allow(Dir).to receive(:glob).and_return(
    [
    '1451606400_2016_01_01_1.2.3_gitlab_backup.tar'
    ......@@ -649,8 +668,21 @@
    it 'unpacks the file' do
    subject.restore
    expect(Kernel).to have_received(:system)
    .with("tar", "-xf", "1451606400_2016_01_01_1.2.3_gitlab_backup.tar")
    expect(Kernel).to have_received(:system).with(*tar_cmdline)
    end
    context 'tar fails' do
    before do
    expect(Kernel).to receive(:system).with(*tar_cmdline).and_return(false)
    end
    it 'logs a failure' do
    expect do
    subject.restore
    end.to raise_error(SystemExit)
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Unpacking backup failed')
    end
    end
    context 'on version mismatch' do
    ......@@ -680,7 +712,7 @@
    subject.restore
    expect(progress).to have_received(:print).with('Deleting backups/tmp ... ')
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting backups/tmp ... ')
    end
    end
    end
    ......@@ -731,7 +763,7 @@
    subject.restore
    expect(progress).to have_received(:print).with('Deleting backups/tmp ... ')
    expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting backups/tmp ... ')
    end
    end
    end
    ......
    ......@@ -199,18 +199,25 @@ def reenable_backup_sub_tasks
    end
    it 'logs the progress to log file' do
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping database ... ")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "[SKIPPED]")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping database ... [SKIPPED]")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping repositories ... ")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping repositories ... done")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping uploads ... ")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping uploads ... done")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping builds ... ")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping builds ... done")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping artifacts ... ")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping artifacts ... done")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping pages ... ")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping pages ... done")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping lfs objects ... ")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping lfs objects ... done")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping terraform states ... ")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping terraform states ... done")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping container registry images ... ")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping container registry images ... done")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping packages ... ")
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "done").exactly(9).times
    expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping packages ... done")
    backup_tasks.each do |task|
    run_rake_task("gitlab:backup:#{task}:create")
    ......
    0% Loading or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment