Merge branch 'bwill/backfill-sbom-occurrences-traversal-ids-and-archived' into 'master'

Backfill sbom_occurrences traversal_ids and archived columns

See merge request !144802



Merged-by: Krasimir Angelov <kangelov@gitlab.com>
Approved-by: Malcolm Locke <mlocke@gitlab.com>
Approved-by: Mehmet Emin INAC <minac@gitlab.com>
Approved-by: Krasimir Angelov <kangelov@gitlab.com>
Reviewed-by: Krasimir Angelov <kangelov@gitlab.com>
Reviewed-by: Alper Akgun <aakgun@gitlab.com>
Reviewed-by: Brian Williams <bwilliams@gitlab.com>
Reviewed-by: Malcolm Locke <mlocke@gitlab.com>
Co-authored-by: Brian Williams <bwilliams@gitlab.com>

db/docs/batched_background_migrations/backfill_sbom_occurrences_traversal_ids_and_archived.yml

+---
+migration_job_name: BackfillSbomOccurrencesTraversalIdsAndArchived
+description: >
+  Backfills sbom_occurrences.traversal_ids and sbom_occurrences.archived columns with
+  values from sbom_occurrences.project.namespace.traversal_ids and
+  sbom_occurrences.project.archived.
+feature_category: dependency_management
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/144802
+milestone: '16.10'
+queued_migration_version: 20240214203242
+finalize_after: '2024-04-11'
+finalized_by: # Not yet implemented

db/post_migrate/20240214203242_queue_backfill_sbom_occurrences_traversal_ids_and_archived.rb

+# frozen_string_literal: true
+
+class QueueBackfillSbomOccurrencesTraversalIdsAndArchived < Gitlab::Database::Migration[2.2]
+  milestone '16.10'
+
+  MIGRATION = "BackfillSbomOccurrencesTraversalIdsAndArchived"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 10_000
+  SUB_BATCH_SIZE = 100
+
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :sbom_occurrences,
+      :id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(MIGRATION, :sbom_occurrences, :id, [])
+  end
+end

db/schema_migrations/20240214203242

+c8fb4783e43e1276a74b0890c35b51ed4dd2fc15ea9855be1def9fca221d7d5f
\ No newline at end of file

lib/gitlab/background_migration/backfill_sbom_occurrences_traversal_ids_and_archived.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class BackfillSbomOccurrencesTraversalIdsAndArchived < BatchedMigrationJob
+      feature_category :dependency_management
+      operation_name :backfill_sbom_occurrences_traversal_ids_and_archived
+
+      def perform
+        each_sub_batch do |relation|
+          batch_start_id, batch_end_id = relation.pick(Arel.sql("MIN(#{batch_column}), MAX(#{batch_column})"))
+          connection.exec_update(update_sql(batch_start_id, batch_end_id))
+        end
+      end
+
+      private
+
+      def update_sql(batch_start_id, batch_end_id)
+        <<~SQL
+        UPDATE
+          sbom_occurrences
+        SET
+          traversal_ids = namespaces.traversal_ids,
+          archived = projects.archived
+        FROM
+          projects JOIN namespaces ON namespaces.id = projects.namespace_id
+        WHERE
+          sbom_occurrences.project_id = projects.id AND
+          sbom_occurrences.id >= #{batch_start_id} AND
+          sbom_occurrences.id <= #{batch_end_id}
+        SQL
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_sbom_occurrences_traversal_ids_and_archived_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillSbomOccurrencesTraversalIdsAndArchived,
+  feature_category: :dependency_management do
+  let(:sbom_occurrences) { table(:sbom_occurrences) }
+  let(:sbom_components) { table(:sbom_components) }
+  let(:projects) { table(:projects) }
+  let(:namespaces) { table(:namespaces) }
+  let(:args) do
+    min, max = sbom_occurrences.pick('MIN(id)', 'MAX(id)')
+
+    {
+      start_id: min,
+      end_id: max,
+      batch_table: 'sbom_occurrences',
+      batch_column: 'id',
+      sub_batch_size: 1,
+      pause_ms: 0,
+      connection: ApplicationRecord.connection
+    }
+  end
+
+  let!(:group_namespace) do
+    namespaces.create!(
+      name: 'gitlab-org',
+      path: 'gitlab-org',
+      type: 'Group'
+    ).tap { |namespace| namespace.update!(traversal_ids: [namespace.id]) }
+  end
+
+  let!(:other_group_namespace) do
+    namespaces.create!(
+      name: 'gitlab-com',
+      path: 'gitlab-com',
+      type: 'Group'
+    ).tap { |namespace| namespace.update!(traversal_ids: [namespace.id]) }
+  end
+
+  let!(:project) { create_project('gitlab', group_namespace) }
+  let!(:other_project) { create_project('www-gitlab-com', other_group_namespace) }
+
+  let!(:sbom_component) { sbom_components.create!(name: 'component', component_type: 0) }
+
+  subject(:perform_migration) { described_class.new(**args).perform }
+
+  before do
+    [project, other_project].each do |p|
+      sbom_occurrences.create!(
+        project_id: p.id,
+        commit_sha: '3bc8e151d3c4d242d76897399b8716815556673a',
+        component_id: sbom_component.id,
+        uuid: SecureRandom.uuid,
+        traversal_ids: [],
+        archived: false
+      )
+    end
+  end
+
+  it 'backfills traversal_ids and archived', :aggregate_failures do
+    perform_migration
+
+    sbom_occurrences.find_each do |occurrence|
+      project = projects.find(occurrence.project_id)
+      namespace = namespaces.find(project.namespace_id)
+
+      expect(occurrence.traversal_ids).to eq(namespace.traversal_ids)
+      expect(occurrence.archived).to eq(project.archived)
+    end
+  end
+
+  def create_project(name, group)
+    project_namespace = namespaces.create!(
+      name: name,
+      path: name,
+      type: 'Project'
+    )
+
+    projects.create!(
+      namespace_id: group.id,
+      project_namespace_id: project_namespace.id,
+      name: name,
+      path: name,
+      archived: true
+    )
+  end
+end

spec/migrations/20240214203242_queue_backfill_sbom_occurrences_traversal_ids_and_archived_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueBackfillSbomOccurrencesTraversalIdsAndArchived, feature_category: :dependency_management do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :sbom_occurrences,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE
+        )
+      }
+    end
+  end
+end