Skip to content
Snippets Groups Projects
Verified Commit f032f04c authored by Gregory Havenga's avatar Gregory Havenga :two: Committed by Mehmet Emin INAC
Browse files

Backfill archived and traversal_ids for vulnerability_reads

Changelog: added
parent b38f82d3
No related branches found
No related tags found
1 merge request!144765Backfill traversal_ids and archived for existing `vulnerability_reads`
---
migration_job_name: BackfillArchivedAndTraversalIdsToVulnerabilityReads
description: Backfill project.archived and project.namespace.traversal_ids values to the denormalized columns of the same name on vulnerability_reads
feature_category: vulnerability_management
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/144765
milestone: '16.10'
queued_migration_version: 20240214163238
finalize_after: '2024-03-15'
finalized_by:
\ No newline at end of file
# frozen_string_literal: true
class QueueBackfillArchivedAndTraversalIdsToVulnerabilityReads < Gitlab::Database::Migration[2.2]
milestone '16.10'
restrict_gitlab_migration gitlab_schema: :gitlab_main
MIGRATION = "BackfillArchivedAndTraversalIdsToVulnerabilityReads"
DELAY_INTERVAL = 2.minutes
BATCH_SIZE = 10_000
SUB_BATCH_SIZE = 100
def up
queue_batched_background_migration(
MIGRATION,
:vulnerability_reads,
:id,
job_interval: DELAY_INTERVAL,
batch_size: BATCH_SIZE,
sub_batch_size: SUB_BATCH_SIZE
)
end
def down
delete_batched_background_migration(MIGRATION, :vulnerability_reads, :id, [])
end
end
3b6c1f5d12ab7c97d8bc72064b1ef66b64ba8131c8a37a368d0ea58472e60fc8
\ No newline at end of file
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
class BackfillArchivedAndTraversalIdsToVulnerabilityReads < BatchedMigrationJob
operation_name :backfill_archived_and_traversal_ids_in_vulnerability_reads_table
feature_category :vulnerability_management
def perform
each_sub_batch do |sub_batch|
connection.exec_update(update_sql(sub_batch))
end
end
private
def update_sql(sub_batch)
<<~SQL
UPDATE
vulnerability_reads
SET
traversal_ids = namespaces.traversal_ids,
archived = projects.archived
FROM
projects
INNER JOIN namespaces ON namespaces.id = projects.namespace_id
WHERE
vulnerability_reads.id IN (#{sub_batch.select(:id).to_sql}) AND
vulnerability_reads.project_id = projects.id
SQL
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillArchivedAndTraversalIdsToVulnerabilityReads,
feature_category: :vulnerability_management do
let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
let(:vulnerability_findings) { table(:vulnerability_occurrences) }
let(:vulnerabilities) { table(:vulnerabilities) }
let(:vulnerability_reads) { table(:vulnerability_reads) }
let(:projects) { table(:projects) }
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:scanners) { table(:vulnerability_scanners) }
let(:user) { users.create!(username: 'john_doe', email: 'johndoe@gitlab.com', projects_limit: 1) }
let(:args) do
min, max = vulnerability_reads.pick('MIN(id)', 'MAX(id)')
{
start_id: min,
end_id: max,
batch_table: 'vulnerability_reads',
batch_column: 'id',
sub_batch_size: 100,
pause_ms: 0,
connection: ApplicationRecord.connection
}
end
let!(:group_namespace) do
namespaces.create!(
name: 'gitlab-org',
path: 'gitlab-org',
type: 'Group'
).tap { |namespace| namespace.update!(traversal_ids: [namespace.id]) }
end
let!(:other_group_namespace) do
namespaces.create!(
name: 'gitlab-com',
path: 'gitlab-com',
type: 'Group'
).tap { |namespace| namespace.update!(traversal_ids: [namespace.id]) }
end
let!(:project) { create_project('gitlab', group_namespace) }
let!(:other_project) { create_project('www-gitlab-com', other_group_namespace) }
subject(:perform_migration) { described_class.new(**args).perform }
before do
[project, other_project].each do |p|
create_vulnerability_read(project_id: p.id)
end
end
it 'backfills traversal_ids and archived', :aggregate_failures do
perform_migration
vulnerability_reads.find_each do |vulnerability_read|
project = projects.find(vulnerability_read.project_id)
namespace = namespaces.find(project.namespace_id)
expect(vulnerability_read.traversal_ids).to eq(namespace.traversal_ids)
expect(vulnerability_read.archived).to eq(project.archived)
end
end
def create_vulnerability_read(project_id:)
scanner = scanners.create!(project_id: project_id, external_id: 'foo', name: 'bar')
identifier = vulnerability_identifiers.create!(
project_id: project_id,
external_id: "CVE-2018-1234",
external_type: "CVE",
name: "CVE-2018-1234",
fingerprint: SecureRandom.hex(20)
)
finding = vulnerability_findings.create!(
project_id: project_id,
scanner_id: scanner.id,
severity: 5, # medium
confidence: 2, # unknown,
report_type: 99, # generic
primary_identifier_id: identifier.id,
project_fingerprint: SecureRandom.hex(20),
location_fingerprint: SecureRandom.hex(20),
uuid: SecureRandom.uuid,
name: "CVE-2018-1234",
raw_metadata: "{}",
metadata_version: "test:1.0"
)
vulnerability = vulnerabilities.create!(
project_id: project_id,
author_id: user.id,
title: 'Vulnerability 1',
severity: 1,
confidence: 1,
report_type: 1,
state: 1,
finding_id: finding.id
)
vulnerability_reads.create!(
vulnerability_id: vulnerability.id,
namespace_id: project.namespace_id,
project_id: project_id,
scanner_id: scanner.id,
report_type: 1,
severity: 1,
state: 1,
uuid: SecureRandom.uuid,
archived: false,
traversal_ids: []
)
end
def create_project(name, group)
project_namespace = namespaces.create!(
name: name,
path: name,
type: 'Project'
)
projects.create!(
namespace_id: group.id,
project_namespace_id: project_namespace.id,
name: name,
path: name,
archived: true
)
end
end
# frozen_string_literal: true
require 'spec_helper'
require_migration!
RSpec.describe QueueBackfillArchivedAndTraversalIdsToVulnerabilityReads, feature_category: :vulnerability_management do
let!(:batched_migration) { described_class::MIGRATION }
it 'schedules a new batched migration' do
reversible_migration do |migration|
migration.before -> {
expect(batched_migration).not_to have_scheduled_batched_migration
}
migration.after -> {
expect(batched_migration).to have_scheduled_batched_migration(
table_name: :vulnerability_reads,
column_name: :id,
interval: described_class::DELAY_INTERVAL,
batch_size: described_class::BATCH_SIZE,
sub_batch_size: described_class::SUB_BATCH_SIZE
)
}
end
end
end
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment