Skip to content
Snippets Groups Projects
Commit edba4356 authored by Mehmet Emin INAC's avatar Mehmet Emin INAC :two: Committed by João Alexandre Cunha
Browse files

Mark stale `security_scans` as purged

Changelog: other
EE: true
parent 01f04fa0
No related branches found
No related tags found
1 merge request!82711Mark stale `security_scans` as `purged`
# frozen_string_literal: true
class SchedulePurgingStaleSecurityScans < Gitlab::Database::Migration[2.0]
MIGRATION = 'PurgeStaleSecurityScans'
BATCH_SIZE = 10_000
DELAY_INTERVAL = 2.minutes
restrict_gitlab_migration gitlab_schema: :gitlab_main
disable_ddl_transaction!
def up
return unless should_run?
queue_background_migration_jobs_by_range_at_intervals(
Gitlab::BackgroundMigration::PurgeStaleSecurityScans::SecurityScan.to_purge,
MIGRATION,
DELAY_INTERVAL,
batch_size: BATCH_SIZE,
track_jobs: true
)
end
def down
# no-op
end
private
def should_run?
Gitlab.dev_or_test_env? || Gitlab.com?
end
end
dabbd8b95ec49b4267d53768013b4e62ae1219a4575dc8b0fccb0e117e725885
\ No newline at end of file
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
# rubocop:disable Style/Documentation
module PurgeStaleSecurityScans
extend ::Gitlab::Utils::Override
extend ActiveSupport::Concern
override :perform
def perform(start_id, end_id)
updated_scans_count = 0
relation = ::Gitlab::BackgroundMigration::PurgeStaleSecurityScans::SecurityScan.by_range(start_id..end_id)
relation.each_batch do |batch|
updated_scans_count += batch.succeeded.update_all(status: :purged)
end
log_info(updated_scans_count)
mark_job_as_succeeded(start_id, end_id)
end
def log_info(updated_scans_count)
::Gitlab::BackgroundMigration::Logger.info(
migrator: self.class.name,
message: 'Records have been updated',
updated_scans_count: updated_scans_count
)
end
def mark_job_as_succeeded(*arguments)
::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
self.class.name.demodulize,
arguments
)
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::PurgeStaleSecurityScans, schema: 20220407163559 do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:pipelines) { table(:ci_pipelines) }
let(:builds) { table(:ci_builds) }
let(:security_scans) { table(:security_scans) }
let(:scanners) { table(:vulnerability_scanners) }
let(:succeded_status) { 1 }
let(:failed_status) { 2 }
let(:purged_status) { 6 }
let(:namespace) { namespaces.create!(name: 'foo', path: 'bar') }
let(:project) { projects.create!(namespace_id: namespace.id, project_namespace_id: namespace.id) }
let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a', status: 'success') }
let(:ci_build_1) { builds.create!(commit_id: pipeline.id, retried: false, type: 'Ci::Build', status: 'success') }
let(:ci_build_2) { builds.create!(commit_id: pipeline.id, retried: false, type: 'Ci::Build', status: 'failed') }
let!(:scanner) { scanners.create!(project_id: project.id, external_id: 'foo', name: 'Scanner', vendor: 'GitLab') }
let!(:security_scan_1) { security_scans.create!(build_id: ci_build_1.id, scan_type: 1, status: succeded_status) }
let!(:security_scan_2) { security_scans.create!(build_id: ci_build_2.id, scan_type: 1, status: failed_status) }
let!(:security_scan_3) { security_scans.create!(build_id: ci_build_2.id, scan_type: 2) }
describe '#perform' do
subject(:migrate) { described_class.new.perform(security_scan_1.id, security_scan_2.id) }
before do
allow(::Gitlab::BackgroundMigration::Logger).to receive(:info)
end
it 'changes the status of the security_scan records and writes the log message' do
expect { migrate }.to change { security_scan_1.reload.status }.from(succeded_status).to(purged_status)
.and not_change { security_scan_2.reload.status }.from(failed_status)
expect(::Gitlab::BackgroundMigration::Logger).to have_received(:info).with(migrator: described_class.name,
message: 'Records have been updated',
updated_scans_count: 1)
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop:disable Style/Documentation
class PurgeStaleSecurityScans # rubocop:disable Migration/BackgroundMigrationBaseClass
class SecurityScan < ::ApplicationRecord
include EachBatch
STALE_AFTER = 90.days
self.table_name = 'security_scans'
# Otherwise the schema_spec fails
validates :info, json_schema: { filename: 'security_scan_info', draft: 7 }
enum status: { succeeded: 1, purged: 6 }
scope :to_purge, -> { where('id <= ?', last_stale_record_id) }
scope :by_range, -> (range) { where(id: range) }
def self.last_stale_record_id
where('created_at < ?', STALE_AFTER.ago).order(created_at: :desc).first
end
end
def perform(_start_id, _end_id); end
end
end
end
Gitlab::BackgroundMigration::PurgeStaleSecurityScans.prepend_mod
# frozen_string_literal: true
require 'spec_helper'
require_migration!
RSpec.describe SchedulePurgingStaleSecurityScans do
let_it_be(:namespaces) { table(:namespaces) }
let_it_be(:projects) { table(:projects) }
let_it_be(:pipelines) { table(:ci_pipelines) }
let_it_be(:builds) { table(:ci_builds) }
let_it_be(:security_scans) { table(:security_scans) }
let_it_be(:namespace) { namespaces.create!(name: "foo", path: "bar") }
let_it_be(:project) { projects.create!(namespace_id: namespace.id, project_namespace_id: namespace.id) }
let_it_be(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a', status: 'success') }
let_it_be(:ci_build) { builds.create!(commit_id: pipeline.id, retried: false, type: 'Ci::Build') }
let!(:security_scan_1) { security_scans.create!(build_id: ci_build.id, scan_type: 1, created_at: 91.days.ago) }
let!(:security_scan_2) { security_scans.create!(build_id: ci_build.id, scan_type: 2, created_at: 91.days.ago) }
let(:com?) { false }
let(:dev_or_test_env?) { false }
before do
allow(::Gitlab).to receive(:com?).and_return(com?)
allow(::Gitlab).to receive(:dev_or_test_env?).and_return(dev_or_test_env?)
stub_const("#{described_class.name}::BATCH_SIZE", 1)
end
shared_examples_for 'schedules the background jobs' do
before do
# This will not be scheduled as it's not stale
security_scans.create!(build_id: ci_build.id, scan_type: 3)
end
around do |example|
freeze_time { Sidekiq::Testing.fake! { example.run } }
end
it 'creates 2 jobs', :aggregate_failures do
migrate!
expect(BackgroundMigrationWorker.jobs.size).to be(2)
expect(described_class::MIGRATION)
.to be_scheduled_delayed_migration(2.minutes, security_scan_1.id, security_scan_1.id)
expect(described_class::MIGRATION)
.to be_scheduled_delayed_migration(4.minutes, security_scan_2.id, security_scan_2.id)
end
end
context 'when the migration does not run on GitLab.com or `dev_or_test_env`' do
it 'does not run the migration' do
expect { migrate! }.not_to change { BackgroundMigrationWorker.jobs.size }
end
end
context 'when the migration runs on GitLab.com' do
let(:com?) { true }
it_behaves_like 'schedules the background jobs'
end
context 'when the migration runs on dev or test env' do
let(:dev_or_test_env?) { true }
it_behaves_like 'schedules the background jobs'
end
end
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment