Skip to content
Snippets Groups Projects
Verified Commit bde3c007 authored by Michael Becker's avatar Michael Becker Committed by GitLab
Browse files

Merge branch...

Merge branch '450797-remove-usages-of-the-deprecate_vulnerability_occurrence_pipelines-FF' into 'master' 

Remove the `deprecate_vulnerability_occurrence_pipelines` FF

See merge request !164890



Merged-by: default avatarMichael Becker <mbecker@gitlab.com>
Approved-by: default avatarAman Luthra <aluthra@gitlab.com>
Approved-by: Bob Van Landuyt's avatarBob Van Landuyt <bob@gitlab.com>
Approved-by: default avatarHarsha Muralidhar <hmuralidhar@gitlab.com>
Approved-by: default avatarAvielle Wolfe <awolfe@gitlab.com>
Reviewed-by: default avatarMichael Becker <mbecker@gitlab.com>
parents 8b062c51 4cd5220c
No related branches found
No related tags found
1 merge request!164890Remove the `deprecate_vulnerability_occurrence_pipelines` FF
Pipeline #1445788897 passed with warnings
Showing
with 271 additions and 511 deletions
......@@ -9,9 +9,6 @@ module Pipeline
prepended do
include UsageStatistics
has_many :vulnerabilities_finding_pipelines, class_name: 'Vulnerabilities::FindingPipeline', inverse_of: :pipeline
has_many :vulnerability_findings, source: :finding, through: :vulnerabilities_finding_pipelines, class_name: 'Vulnerabilities::Finding'
# Subscriptions to this pipeline
has_many :downstream_bridges, class_name: '::Ci::Bridge', foreign_key: :upstream_pipeline_id
has_many :security_scans, class_name: 'Security::Scan', inverse_of: :pipeline
......@@ -110,14 +107,6 @@ module Pipeline
end
end
def vulnerability_findings
unless ::Feature.enabled?(:deprecate_vulnerability_occurrence_pipelines, project)
return super.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/480363')
end
raise NotImplementedError, 'Use pipeline.project.vulnerability_findings instead'
end
def needs_touch?
updated_at < 5.minutes.ago
end
......
......@@ -505,19 +505,11 @@ def false_positive?
end
def first_finding_pipeline
if Feature.enabled?(:deprecate_vulnerability_occurrence_pipelines, project)
initial_finding_pipeline
else
finding_pipelines.first&.pipeline
end
initial_finding_pipeline
end
def last_finding_pipeline
if Feature.enabled?(:deprecate_vulnerability_occurrence_pipelines, project)
latest_finding_pipeline
else
finding_pipelines.last&.pipeline
end
latest_finding_pipeline
end
def vulnerable_lines
......
......@@ -33,7 +33,7 @@ def ingest_reports
end
def ingest_report(sbom_report)
IngestReportService.execute(pipeline, sbom_report, vulnerabilities_info)
IngestReportService.execute(pipeline, sbom_report)
end
def set_latest_ingested_sbom_pipeline_id
......@@ -51,12 +51,6 @@ def publish_ingested_sbom_event
Sbom::SbomIngestedEvent.new(data: { pipeline_id: pipeline.id })
)
end
def vulnerabilities_info
return {} if ::Feature.enabled?(:deprecate_vulnerability_occurrence_pipelines, project)
@vulnerabilities_info ||= Sbom::Ingestion::Vulnerabilities.new(pipeline)
end
end
end
end
......
......@@ -5,14 +5,13 @@ module Ingestion
class IngestReportService
BATCH_SIZE = 10
def self.execute(pipeline, sbom_report, vulnerabilities_info)
new(pipeline, sbom_report, vulnerabilities_info).execute
def self.execute(pipeline, sbom_report)
new(pipeline, sbom_report).execute
end
def initialize(pipeline, sbom_report, vulnerabilities_info)
def initialize(pipeline, sbom_report)
@pipeline = pipeline
@sbom_report = sbom_report
@vulnerabilities_info = vulnerabilities_info
end
def execute
......@@ -23,10 +22,10 @@ def execute
private
attr_reader :pipeline, :sbom_report, :vulnerabilities_info
attr_reader :pipeline, :sbom_report
def occurrence_map_collection
@occurrence_map_collection ||= OccurrenceMapCollection.new(sbom_report, vulnerabilities_info)
@occurrence_map_collection ||= OccurrenceMapCollection.new(sbom_report)
end
def ingest_slice(slice)
......
......@@ -5,15 +5,15 @@ module Ingestion
class OccurrenceMap
include Gitlab::Utils::StrongMemoize
attr_writer :vulnerability_ids
attr_reader :report_component, :report_source, :vulnerabilities
attr_accessor :component_id, :component_version_id, :source_id, :occurrence_id, :source_package_id, :uuid
attr_accessor :component_id, :component_version_id, :source_id, :occurrence_id, :source_package_id, :uuid,
:vulnerability_ids
def initialize(report_component, report_source, vulnerabilities)
def initialize(report_component, report_source)
@report_component = report_component
@report_source = report_source
@vulnerabilities = vulnerabilities
@vulnerability_ids = nil
@vulnerability_ids = []
end
def to_h
......@@ -36,29 +36,6 @@ def version_present?
version.present?
end
def vulnerability_count
original_vulnerability_ids.count
end
def highest_severity
vulnerabilities_info[:highest_severity]
end
# This weird code is here until we remove the
# `deprecate_vulnerability_occurrence_pipelines` FF. When that
# FF is removed, the original_* code branch will be deleted and
# this can be rolled up into a regular attr_accesor
def vulnerability_ids
return @vulnerability_ids unless @vulnerability_ids.nil?
original_vulnerability_ids
end
def original_vulnerability_ids
vulnerabilities_info[:vulnerability_ids]
end
strong_memoize_attr :original_vulnerability_ids
def purl_type
report_component.purl&.type
end
......@@ -78,11 +55,6 @@ def input_file_path
private
def vulnerabilities_info
@vulnerabilities.fetch(name, version, input_file_path)
end
strong_memoize_attr :vulnerabilities_info
def image_data_present?
image_name.present? && image_tag.present?
end
......
......@@ -5,22 +5,21 @@ module Ingestion
class OccurrenceMapCollection
include Enumerable
def initialize(sbom_report, vulnerabilities_info)
def initialize(sbom_report)
@sbom_report = sbom_report
@vulnerabilities_info = vulnerabilities_info
end
def each
return to_enum(:each) unless block_given?
sbom_report.components.sort.each do |report_component|
yield OccurrenceMap.new(report_component, sbom_report.source, vulnerabilities_info)
yield OccurrenceMap.new(report_component, sbom_report.source)
end
end
private
attr_reader :sbom_report, :vulnerabilities_info
attr_reader :sbom_report
end
end
end
......@@ -16,25 +16,18 @@ class IngestOccurrences < Base
'VulnerabilityData',
:vulnerabilities_info,
:occurrence_map,
:project,
:key) do
include Gitlab::Utils::StrongMemoize
def count
return occurrence_map.vulnerability_count unless deprecation_flag_enabled?
vulnerabilities_info.dig(key, :vulnerability_count) || 0
end
def highest_severity
return occurrence_map.highest_severity unless deprecation_flag_enabled?
vulnerabilities_info.dig(key, :highest_severity)
end
def vulnerability_ids
return unless deprecation_flag_enabled?
vulnerabilities_info.dig(key, :vulnerability_ids) || []
end
......@@ -47,11 +40,6 @@ def key
occurrence_map.input_file_path&.delete_prefix(CONTAINER_IMAGE_PREFIX)
]
end
def deprecation_flag_enabled?
::Feature.enabled?(:deprecate_vulnerability_occurrence_pipelines, project)
end
strong_memoize_attr :deprecation_flag_enabled?
end
private
......@@ -75,11 +63,7 @@ def attributes
occurrence_maps.filter_map do |occurrence_map|
uuid = occurrence_map.uuid
vulnerability_data = VulnerabilityData.new(
vulnerabilities_info,
occurrence_map,
project
)
vulnerability_data = VulnerabilityData.new(vulnerabilities_info, occurrence_map)
new_attributes = {
project_id: project.id,
......@@ -119,8 +103,6 @@ def vulnerabilities_info
end
def build_vulnerabilities_info
return {} unless ::Feature.enabled?(:deprecate_vulnerability_occurrence_pipelines, project)
# rubocop:disable CodeReuse/ActiveRecord -- highly customized query
occurrence_maps_values = occurrence_maps.map do |om|
[om.name, om.version, om.input_file_path&.delete_prefix(CONTAINER_IMAGE_PREFIX)]
......
# frozen_string_literal: true
# TODO: This is dead code and will be removed as part of
# https://gitlab.com/gitlab-org/gitlab/-/issues/450797
module Sbom
module Ingestion
class Vulnerabilities
......@@ -49,25 +51,11 @@ def build_vulnerabilities_info
end
def vulnerability_findings
if ::Feature.enabled?(:deprecate_vulnerability_occurrence_pipelines, project)
vulnerability_findings_from_project
else
vulnerability_findings_from_pipelines
end
end
strong_memoize_attr :vulnerability_findings
def vulnerability_findings_from_pipelines
pipeline
.vulnerability_findings
.by_report_types(%i[container_scanning dependency_scanning])
end
def vulnerability_findings_from_project
project
.vulnerability_findings
.by_report_types(%i[container_scanning dependency_scanning])
end
strong_memoize_attr :vulnerability_findings
def dependency_path(finding)
return finding.file if finding.dependency_scanning?
......
---
name: deprecate_vulnerability_occurrence_pipelines
feature_issue_url:
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/147412
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/450802
milestone: '16.11'
group: group::threat insights
type: wip
default_enabled: false
......@@ -4,7 +4,6 @@
factory :sbom_occurrence_map, class: '::Sbom::Ingestion::OccurrenceMap' do
report_component factory: :ci_reports_sbom_component
report_source factory: :ci_reports_sbom_source
vulnerabilities factory: :sbom_vulnerabilities
trait :with_component do
component factory: :sbom_component
......@@ -38,7 +37,7 @@
initialize_with do
::Sbom::Ingestion::OccurrenceMap.new(
*attributes.values_at(:report_component, :report_source, :vulnerabilities)
*attributes.values_at(:report_component, :report_source)
).tap do |object|
object.component_id = attributes[:component]&.id
object.component_version_id = attributes[:component_version]&.id
......
......@@ -3,61 +3,51 @@
require "spec_helper"
RSpec.describe "User sees dependency list", :js, feature_category: :vulnerability_management do
using RSpec::Parameterized::TableSyntax
let_it_be(:owner) { create(:user) }
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, group: group) }
let_it_be(:ci_pipeline) { create(:ee_ci_pipeline, :with_cyclonedx_pypi_only, project: project) }
where(:feature_flag, :feature_flag_stub) do
:deprecate_vulnerability_occurrence_pipelines | false
:deprecate_vulnerability_occurrence_pipelines | true
before do
stub_licensed_features(
security_dashboard: true,
dependency_scanning: true
)
end
with_them do
before do
stub_licensed_features(
security_dashboard: true,
dependency_scanning: true
)
stub_feature_flags(feature_flag => feature_flag_stub)
end
before_all do
Gitlab::ExclusiveLease.skipping_transaction_check do
# `before_all` runs in a transaction which triggers LeaseWithinTransactionError.
# Skip the check since it only happens in tests.
Sidekiq::Worker.skipping_transaction_check do
::Sbom::Ingestion::IngestReportsService.execute(ci_pipeline)
end
before_all do
Gitlab::ExclusiveLease.skipping_transaction_check do
# `before_all` runs in a transaction which triggers LeaseWithinTransactionError.
# Skip the check since it only happens in tests.
Sidekiq::Worker.skipping_transaction_check do
::Sbom::Ingestion::IngestReportsService.execute(ci_pipeline)
end
group.add_owner(owner)
sign_in(owner)
end
group.add_owner(owner)
sign_in(owner)
end
it "shows the dependency list" do
visit(group_dependencies_path(group))
# The default sort order of the page is by 'Severity' which orders by
# `sbom_occurrences.hightest_severity_level NULL LAST`. All of the test
# data is NULL for this field so the order is indeterminate.
# So just check for some content and the right number of elements.
within_testid("dependencies-table-content") do
expect(page).to have_content "beautifulsoup4"
expect(page).to have_content "soupsieve"
expect(page).to have_selector("tbody tr", count: 12)
end
it "shows the dependency list" do
visit(group_dependencies_path(group))
# The default sort order of the page is by 'Severity' which orders by
# `sbom_occurrences.hightest_severity_level NULL LAST`. All of the test
# data is NULL for this field so the order is indeterminate.
# So just check for some content and the right number of elements.
within_testid("dependencies-table-content") do
expect(page).to have_content "beautifulsoup4"
expect(page).to have_content "soupsieve"
expect(page).to have_selector("tbody tr", count: 12)
end
within(".gl-sorting") do
click_on "Severity"
find("li", text: "Component name").click
end
within(".gl-sorting") do
click_on "Severity"
find("li", text: "Component name").click
end
within_testid("dependencies-table-content") do
expect(find("tbody tr:first-child")).to have_content "beautifulsoup4"
expect(find("tbody tr:last-child")).to have_content "soupsieve"
end
within_testid("dependencies-table-content") do
expect(find("tbody tr:first-child")).to have_content "beautifulsoup4"
expect(find("tbody tr:last-child")).to have_content "soupsieve"
end
end
end
......@@ -17,8 +17,6 @@
it { is_expected.to have_many(:security_scans).class_name('Security::Scan') }
it { is_expected.to have_many(:security_findings).through(:security_scans).class_name('Security::Finding').source(:findings) }
it { is_expected.to have_many(:downstream_bridges) }
it { is_expected.to have_many(:vulnerability_findings).through(:vulnerabilities_finding_pipelines).class_name('Vulnerabilities::Finding') }
it { is_expected.to have_many(:vulnerabilities_finding_pipelines).class_name('Vulnerabilities::FindingPipeline') }
it { is_expected.to have_one(:dast_profiles_pipeline).class_name('Dast::ProfilesPipeline').with_foreign_key(:ci_pipeline_id) }
it { is_expected.to have_one(:dast_profile).class_name('Dast::Profile').through(:dast_profiles_pipeline) }
end
......@@ -1075,43 +1073,6 @@
end
end
describe '#vulnerability_findings' do
subject(:vulnerability_findings) { pipeline.vulnerability_findings }
let_it_be(:other_pipeline) { create(:ci_pipeline) }
let_it_be(:initial_pipeline_finding) do
create(
:vulnerabilities_finding,
pipeline: pipeline,
latest_pipeline_id: other_pipeline.id
).tap { |finding| create(:vulnerabilities_finding_pipeline, finding: finding, pipeline: other_pipeline) }
end
let_it_be(:latest_pipeline_finding) do
create(
:vulnerabilities_finding,
pipeline: other_pipeline,
latest_pipeline_id: pipeline.id
).tap { |finding| create(:vulnerabilities_finding_pipeline, finding: finding, pipeline: pipeline) }
end
# When the FF is on, we simulate having dropped the table by
# raising an exception
it 'raises an exception when the FF is on' do
expect { vulnerability_findings }.to raise_error NotImplementedError
end
context 'with deprecate_vulnerability_occurrence_pipelines FF disabled' do
let(:expected_findings) { [initial_pipeline_finding, latest_pipeline_finding] }
before do
stub_feature_flags(deprecate_vulnerability_occurrence_pipelines: false)
end
it { is_expected.to match_array expected_findings }
end
end
describe '#has_security_report_ingestion_warnings?' do
subject { pipeline.has_security_report_ingestion_warnings? }
......
......@@ -1538,35 +1538,19 @@
:vulnerabilities_finding,
pipeline: pipelines.first,
latest_pipeline_id: pipelines.last.id
).tap { |finding| create(:vulnerabilities_finding_pipeline, finding: finding, pipeline: pipelines.last) }
)
end
describe '#first_finding_pipeline' do
subject { finding.first_finding_pipeline }
it { is_expected.to eq pipelines.first }
context 'with deprecate_vulnerability_occurrence_pipelines FF disabled' do
before do
stub_feature_flags(deprecate_vulnerability_occurrence_pipelines: false)
end
it { is_expected.to eq pipelines.first }
end
end
describe '#last_finding_pipeline' do
subject { finding.last_finding_pipeline }
it { is_expected.to eq pipelines.last }
context 'with deprecate_vulnerability_occurrence_pipelines FF disabled' do
before do
stub_feature_flags(deprecate_vulnerability_occurrence_pipelines: false)
end
it { is_expected.to eq pipelines.last }
end
end
end
end
......
......@@ -16,7 +16,7 @@
before do
reports.zip(report_ingested_ids) do |report, ingested_ids|
allow(Sbom::Ingestion::IngestReportService)
.to receive(:execute).with(pipeline, report, {})
.to receive(:execute).with(pipeline, report)
.and_return(ingested_ids)
end
......@@ -28,7 +28,7 @@
reports.each do |report|
expect(Sbom::Ingestion::IngestReportService).to have_received(:execute)
.with(pipeline, report, {})
.with(pipeline, report)
end
end
......
......@@ -23,7 +23,7 @@
setup_ingest_report_service
end
it 'ingests the reports with vulnerability info' do
it 'ingests the reports' do
strategy.execute
expect_ingest_report_service_calls
......@@ -42,19 +42,6 @@
ingested_ids)
end
context 'when feature flag deprecate_vulnerability_occurrence_pipelines is disabled' do
before do
stub_feature_flags(deprecate_vulnerability_occurrence_pipelines: false)
setup_ingest_report_service(vulnerability: true)
end
it 'ingests the reports with empty vulnerability info' do
strategy.execute
expect_ingest_report_service_calls(vulnerability: true)
end
end
context 'when reports are ingested' do
it 'publishes the ingested SBOM event' do
strategy.execute
......@@ -90,20 +77,18 @@
private
def setup_ingest_report_service(vulnerability: false)
def setup_ingest_report_service
reports.zip(report_ingested_ids) do |report, ingested_ids|
args = vulnerability ? instance_of(Sbom::Ingestion::Vulnerabilities) : {}
allow(Sbom::Ingestion::IngestReportService)
.to receive(:execute).with(pipeline, report, args)
.to receive(:execute).with(pipeline, report)
.and_return(ingested_ids)
end
end
def expect_ingest_report_service_calls(vulnerability: false)
def expect_ingest_report_service_calls
reports.each do |report|
args = vulnerability ? instance_of(Sbom::Ingestion::Vulnerabilities) : {}
expect(Sbom::Ingestion::IngestReportService).to have_received(:execute)
.with(pipeline, report, args)
.with(pipeline, report)
end
end
end
......@@ -9,7 +9,7 @@
let(:sequencer) { ::Ingestion::Sequencer.new }
subject(:execute) { described_class.execute(pipeline, sbom_report, {}) }
subject(:execute) { described_class.execute(pipeline, sbom_report) }
describe '#execute' do
before do
......
......@@ -62,11 +62,11 @@
ref: "pkg:deb/debian/readline-common@8.1-1?distro=debian-11.4" }
].map do |attributes|
component = Gitlab::Ci::Reports::Sbom::Component.new(**component_attributes(attributes))
an_occurrence_map(Sbom::Ingestion::OccurrenceMap.new(component, sbom_report.source, {}))
an_occurrence_map(Sbom::Ingestion::OccurrenceMap.new(component, sbom_report.source))
end
end
subject(:occurrence_map_collection) { described_class.new(sbom_report, {}) }
subject(:occurrence_map_collection) { described_class.new(sbom_report) }
RSpec::Matchers.define :an_occurrence_map do |expected|
attributes = %i[
......
......@@ -6,7 +6,6 @@
let_it_be(:report_component) { build_stubbed(:ci_reports_sbom_component, source_package_name: 'source-package-name') }
let_it_be(:report_source) { build_stubbed(:ci_reports_sbom_source) }
let(:vulnerability_info) { create(:sbom_vulnerabilities) }
let(:base_data) do
{
component_id: nil,
......@@ -24,7 +23,7 @@
}
end
subject(:occurrence_map) { described_class.new(report_component, report_source, vulnerability_info) }
subject(:occurrence_map) { described_class.new(report_component, report_source) }
describe '#to_h' do
it 'returns a hash with base data without ids assigned' do
......@@ -260,56 +259,6 @@
end
context 'without vulnerability data' do
it { expect(occurrence_map.vulnerability_ids).to be_empty }
it { expect(occurrence_map.vulnerability_count).to be_zero }
it { expect(occurrence_map.highest_severity).to be_nil }
end
context 'with vulnerability data' do
let(:pipeline) { vulnerability_info.pipeline }
let!(:finding) do
create(
:vulnerabilities_finding,
:detected,
:with_dependency_scanning_metadata,
project: pipeline.project,
file: occurrence_map.input_file_path,
package: occurrence_map.name,
version: occurrence_map.version,
pipeline: pipeline
)
end
it { expect(occurrence_map.vulnerability_ids).to eq([finding.vulnerability_id]) }
it { expect(occurrence_map.vulnerability_count).to be 1 }
it { expect(occurrence_map.highest_severity).to eq 'high' }
context 'when component was found by trivy' do
let_it_be(:report_component_properties) do
build_stubbed(:ci_reports_sbom_source, type: :trivy, data: { 'PkgType' => 'alpine' })
end
let_it_be(:report_component) do
build_stubbed(:ci_reports_sbom_component, properties: report_component_properties,
purl_type: 'apk', namespace: 'alpine', name: 'alpine-baselayout')
end
let!(:finding) do
create(
:vulnerabilities_finding,
:detected,
:with_dependency_scanning_metadata,
project: pipeline.project,
file: occurrence_map.input_file_path,
package: report_component.purl.name,
version: occurrence_map.version,
pipeline: pipeline
)
end
it { expect(occurrence_map.vulnerability_ids).to eq([finding.vulnerability_id]) }
it { expect(occurrence_map.vulnerability_count).to be 1 }
it { expect(occurrence_map.highest_severity).to eq 'high' }
end
it { expect(occurrence_map.vulnerability_ids).to eq [] }
end
end
......@@ -17,7 +17,6 @@
end
let_it_be(:project) { vulnerability.project }
let_it_be(:vulnerability_info) { Sbom::Ingestion::Vulnerabilities.new(pipeline) }
let_it_be(:dependency_scanning_finding) do
create(
:vulnerabilities_finding,
......@@ -42,283 +41,265 @@
describe '#execute' do
subject(:task) { described_class.execute(pipeline, occurrence_maps) }
using RSpec::Parameterized::TableSyntax
where(:case_name, :feature_flag, :feature_flag_stub, :passes_correct_vulnerability_ids_value) do
[
[
'We are running all original code paths that depend on the vulnerability_occurrence_pipelines table',
:deprecate_vulnerability_occurrence_pipelines,
false,
'passes nil into the occurrence_map'
],
[
'We are simulating dropping the vulnerability_occurrence_pipelines with a FF',
:deprecate_vulnerability_occurrence_pipelines,
true,
'passes an array of vulnerability ids into the occurrence_map'
]
]
end
with_them do
before do
stub_feature_flags(feature_flag => feature_flag_stub)
end
let(:finding) { dependency_scanning_finding }
let(:occurrence_maps) { create_list(:sbom_occurrence_map, 4, :for_occurrence_ingestion, vulnerabilities: nil) }
let(:finding) { dependency_scanning_finding }
let(:occurrence_maps) { create_list(:sbom_occurrence_map, 4, :for_occurrence_ingestion) }
it_behaves_like 'bulk insertable task'
it_behaves_like 'bulk insertable task'
it 'is idempotent' do
expect { task }.to change(Sbom::Occurrence, :count).by(4)
expect { task }.not_to change(Sbom::Occurrence, :count)
end
it 'is idempotent' do
expect { task }.to change(Sbom::Occurrence, :count).by(4)
expect { task }.not_to change(Sbom::Occurrence, :count)
describe 'attributes' do
let(:occurrence_maps) { [occurrence_map] }
let(:vulnerability_component) { create(:ci_reports_sbom_component, name: package_name, version: version) }
let(:dependency) { finding.location['dependency'] }
let(:package_name) { dependency['package']['name'] }
let(:version) { dependency['version'] }
let(:path) { finding.file }
let(:occurrence_map) do
create(
:sbom_occurrence_map,
:for_occurrence_ingestion,
report_component: vulnerability_component,
report_source: vulnerability_source,
vulnerabilities: nil
)
end
describe 'attributes' do
let(:occurrence_maps) { [occurrence_map] }
let(:vulnerability_component) { create(:ci_reports_sbom_component, name: package_name, version: version) }
let(:dependency) { finding.location['dependency'] }
let(:package_name) { dependency['package']['name'] }
let(:version) { dependency['version'] }
let(:path) { finding.file }
let(:occurrence_map) do
before do
occurrence_maps.map(&:report_component).each do |component|
create(
:sbom_occurrence_map,
:for_occurrence_ingestion,
report_component: vulnerability_component,
report_source: vulnerability_source,
vulnerabilities: vulnerability_info
:pm_package,
name: component.name,
purl_type: component.purl&.type,
lowest_version: component.version,
highest_version: component.version,
default_license_names: default_license_names
)
end
end
before do
occurrence_maps.map(&:report_component).each do |component|
create(
:pm_package,
name: component.name,
purl_type: component.purl&.type,
lowest_version: component.version,
highest_version: component.version,
default_license_names: default_license_names
)
end
context 'for a dependency scanning occurrence' do
let(:finding) { dependency_scanning_finding }
let(:vulnerability_source) { create(:ci_reports_sbom_source, :dependency_scanning, input_file_path: path) }
let(:expected_attrs) do
expected_attributes_for(occurrence_map)
.merge('vulnerability_count' => 1, 'highest_severity' => finding.severity)
.then { |attrs| hash_including(attrs) }
end
context 'for a dependency scanning occurrence' do
let(:finding) { dependency_scanning_finding }
let(:vulnerability_source) { create(:ci_reports_sbom_source, :dependency_scanning, input_file_path: path) }
let(:expected_attrs) { hash_including(expected_attributes_for(occurrence_map)) }
it 'sets the correct attributes for the occurrence' do
task
expect(Sbom::Occurrence.last&.attributes).to match(expected_attrs)
end
context "for each occurrence_map" do
let(:expected_value) { feature_flag_stub ? [finding.vulnerability_id] : nil }
before do
allow(occurrence_map).to receive(:vulnerability_ids=).once
end
it params[:passes_correct_vulnerability_ids_value] do
task
it 'sets the correct attributes for the occurrence' do
task
expect(occurrence_map).to have_received(:vulnerability_ids=).with(expected_value)
end
end
expect(Sbom::Occurrence.last&.attributes).to match(expected_attrs)
end
context 'for a container scanning occurrence' do
let(:finding) { container_scanning_finding }
let(:vulnerability_component) do
create(:ci_reports_sbom_component, :with_trivy_properties, name: package_name, version: version)
end
let(:vulnerability_source) do
create(
:ci_reports_sbom_source, :container_scanning,
image_name: 'docker.io/library/alpine',
image_tag: '3.12',
operating_system_name: 'Alpine',
operating_system_version: '3.12'
)
end
let(:expected_attrs) do
hash_including(
expected_attributes_for(occurrence_map).merge(
'package_manager' => vulnerability_component.properties.packager,
'input_file_path' => 'container-image:docker.io/library/alpine:3.12'
))
context "for each occurrence_map" do
before do
allow(occurrence_map).to receive(:vulnerability_ids=).once
end
it 'sets the correct attributes for the occurrence' do
it 'passes an array of vulnerability ids into the occurrence_map' do
task
expect(Sbom::Occurrence.last&.attributes).to match(expected_attrs)
end
context "for each occurrence_map" do
let(:expected_value) { feature_flag_stub ? [finding.vulnerability_id] : nil }
before do
allow(occurrence_map).to receive(:vulnerability_ids=).once
end
it params[:passes_correct_vulnerability_ids_value] do
task
expect(occurrence_map).to have_received(:vulnerability_ids=).with(expected_value)
end
expect(occurrence_map).to have_received(:vulnerability_ids=).with([finding.vulnerability_id])
end
end
end
context 'when there is an existing occurrence' do
let(:occurrence_map) { occurrence_maps.first }
let!(:existing_occurrence) do
attributes = expected_attributes_for(occurrence_map).symbolize_keys!
create(:sbom_occurrence, **attributes)
context 'for a container scanning occurrence' do
let(:finding) { container_scanning_finding }
let(:vulnerability_component) do
create(:ci_reports_sbom_component, :with_trivy_properties, name: package_name, version: version)
end
before do
component = occurrence_map.report_component
let(:vulnerability_source) do
create(
:pm_package,
name: component.name,
purl_type: component.purl&.type,
lowest_version: component.version,
highest_version: component.version,
default_license_names: default_license_names
:ci_reports_sbom_source, :container_scanning,
image_name: 'docker.io/library/alpine',
image_tag: '3.12',
operating_system_name: 'Alpine',
operating_system_version: '3.12'
)
end
it 'does not create a new record for the existing version' do
expect { task }.to change(Sbom::Occurrence, :count).by(3)
expect(occurrence_maps.map(&:occurrence_id)).to match_array([Integer, Integer, Integer,
existing_occurrence.id])
let(:expected_attrs) do
expected_attributes_for(occurrence_map)
.merge(
'package_manager' => vulnerability_component.properties.packager,
'input_file_path' => 'container-image:docker.io/library/alpine:3.12',
'vulnerability_count' => 1,
'highest_severity' => finding.severity
).then { |attrs| hash_including(attrs) }
end
context 'when only attributes related to the pipeline have been changed' do
subject(:task) { described_class.execute(other_pipeline, occurrence_maps) }
it 'sets the correct attributes for the occurrence' do
task
let_it_be(:other_pipeline) do
create(:ci_pipeline, sha: '5716ca5987cbf97d6bb54920bea6adde242d87e6', project: pipeline.project)
end
expect(Sbom::Occurrence.last&.attributes).to match(expected_attrs)
end
context "for each occurrence_map" do
let(:expected_value) { feature_flag_stub ? [finding.vulnerability_id] : nil }
before do
existing_occurrence.update!(pipeline: other_pipeline, commit_sha: other_pipeline.sha)
allow(occurrence_map).to receive(:vulnerability_ids=).once
end
it 'does not update existing records' do
expect { task }.not_to change { existing_occurrence.reload.updated_at }
it 'passes an array of vulnerability ids into the occurrence_map' do
task
expect(occurrence_map).to have_received(:vulnerability_ids=).with([finding.vulnerability_id])
end
end
end
end
context 'when attributes not related to the pipeline have been changed' do
let_it_be(:start_project) { create(:project) }
let(:start_traversals) { start_project.namespace.traversal_ids }
let(:traversals) { project.namespace.traversal_ids }
context 'when there is an existing occurrence' do
let(:occurrence_map) { occurrence_maps.first }
let!(:existing_occurrence) do
attributes = expected_attributes_for(occurrence_map).symbolize_keys!
create(:sbom_occurrence, **attributes)
end
before do
existing_occurrence.update!(project: start_project, traversal_ids: start_traversals)
end
before do
component = occurrence_map.report_component
create(
:pm_package,
name: component.name,
purl_type: component.purl&.type,
lowest_version: component.version,
highest_version: component.version,
default_license_names: default_license_names
)
end
it 'updates the record' do
expect { task }.to change { existing_occurrence.reload.project }.from(start_project).to(project)
.and change {
existing_occurrence.reload.traversal_ids
}.from(start_traversals).to(traversals)
end
end
it 'does not create a new record for the existing version' do
expect { task }.to change(Sbom::Occurrence, :count).by(3)
expect(occurrence_maps.map(&:occurrence_id)).to match_array([Integer, Integer, Integer,
existing_occurrence.id])
end
context 'when there is no component version' do
let(:occurrence_maps) do
create_list(:sbom_occurrence_map, 4, :for_occurrence_ingestion, component_version: nil)
end
context 'when only attributes related to the pipeline have been changed' do
subject(:task) { described_class.execute(other_pipeline, occurrence_maps) }
it 'inserts records without the version' do
expect { task }.to change(Sbom::Occurrence, :count).by(4)
expect(occurrence_maps).to all(have_attributes(occurrence_id: Integer))
let_it_be(:other_pipeline) do
create(:ci_pipeline, sha: '5716ca5987cbf97d6bb54920bea6adde242d87e6', project: pipeline.project)
end
it 'does not include licenses' do
task
before do
existing_occurrence.update!(pipeline: other_pipeline, commit_sha: other_pipeline.sha)
end
expect(Sbom::Occurrence.pluck(:licenses)).to all(be_empty)
it 'does not update existing records' do
expect { task }.not_to change { existing_occurrence.reload.updated_at }
end
end
context 'when there is no source package' do
let(:occurrence_maps) { create_list(:sbom_occurrence_map, 4, :for_occurrence_ingestion, source_package: nil) }
context 'when attributes not related to the pipeline have been changed' do
let_it_be(:start_project) { create(:project) }
let(:start_traversals) { start_project.namespace.traversal_ids }
let(:traversals) { project.namespace.traversal_ids }
it 'inserts records without the source package' do
expect { task }.to change(Sbom::Occurrence, :count).by(4)
expect(occurrence_maps).to all(have_attributes(occurrence_id: Integer))
before do
existing_occurrence.update!(project: start_project, traversal_ids: start_traversals)
end
it 'updates the record' do
expect { task }.to change { existing_occurrence.reload.project }.from(start_project).to(project)
.and change {
existing_occurrence.reload.traversal_ids
}.from(start_traversals).to(traversals)
end
end
end
context 'when there is no component version' do
let(:occurrence_maps) do
create_list(:sbom_occurrence_map, 4, :for_occurrence_ingestion, component_version: nil, vulnerabilities: nil)
end
context 'when there is no purl' do
let(:component) { create(:ci_reports_sbom_component, purl: nil) }
let(:occurrence_map) { create(:sbom_occurrence_map, :for_occurrence_ingestion, report_component: component) }
let(:occurrence_maps) { [occurrence_map] }
it 'inserts records without the version' do
expect { task }.to change(Sbom::Occurrence, :count).by(4)
expect(occurrence_maps).to all(have_attributes(occurrence_id: Integer))
end
it 'skips licenses for components without a purl' do
expect { task }.to change(Sbom::Occurrence, :count).by(1)
it 'does not include licenses' do
task
expect(Sbom::Occurrence.pluck(:licenses)).to all(be_empty)
end
expect(Sbom::Occurrence.pluck(:licenses)).to all(be_empty)
end
end
context 'when there are two duplicate occurrences' do
let(:occurrence_maps) do
map1 = create(:sbom_occurrence_map, :for_occurrence_ingestion)
map2 = create(:sbom_occurrence_map)
map2.component_id = map1.component_id
map2.component_version_id = map1.component_version_id
map2.source_id = map1.source_id
context 'when there is no source package' do
let(:occurrence_maps) do
create_list(:sbom_occurrence_map, 4, :for_occurrence_ingestion, source_package: nil, vulnerabilities: nil)
end
[map1, map2]
end
it 'inserts records without the source package' do
expect { task }.to change(Sbom::Occurrence, :count).by(4)
expect(occurrence_maps).to all(have_attributes(occurrence_id: Integer))
end
end
it 'discards duplicates' do
expect { task }.to change { ::Sbom::Occurrence.count }.by(1)
expect(occurrence_maps.size).to eq(1)
expect(occurrence_maps).to all(have_attributes(occurrence_id: Integer))
end
context 'when there is no purl' do
let(:component) { create(:ci_reports_sbom_component, purl: nil) }
let(:occurrence_map) do
create(:sbom_occurrence_map, :for_occurrence_ingestion, report_component: component, vulnerabilities: nil)
end
let(:occurrence_maps) { [occurrence_map] }
it 'skips licenses for components without a purl' do
expect { task }.to change(Sbom::Occurrence, :count).by(1)
expect(Sbom::Occurrence.pluck(:licenses)).to all(be_empty)
end
end
def expected_attributes_for(occurrence_map)
Gitlab::Database.allow_cross_joins_across_databases(
url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/480165'
) do
{
ancestors: occurrence_map.ancestors,
archived: project.archived,
commit_sha: pipeline.sha,
component_id: occurrence_map.component_id,
component_name: occurrence_map.name,
component_version_id: occurrence_map.component_version_id,
highest_severity: occurrence_map.highest_severity,
input_file_path: occurrence_map.input_file_path,
licenses: default_licenses,
package_manager: occurrence_map.packager,
pipeline_id: pipeline.id,
project_id: project.id,
source_id: occurrence_map.source_id,
source_package_id: occurrence_map.source_package_id,
traversal_ids: project.namespace.traversal_ids,
vulnerability_count: occurrence_map.vulnerability_count
}.deep_stringify_keys!
context 'when there are two duplicate occurrences' do
let(:occurrence_maps) do
map1 = create(:sbom_occurrence_map, :for_occurrence_ingestion, vulnerabilities: nil)
map2 = create(:sbom_occurrence_map, vulnerabilities: nil)
map2.component_id = map1.component_id
map2.component_version_id = map1.component_version_id
map2.source_id = map1.source_id
[map1, map2]
end
it 'discards duplicates' do
expect { task }.to change { ::Sbom::Occurrence.count }.by(1)
expect(occurrence_maps.size).to eq(1)
expect(occurrence_maps).to all(have_attributes(occurrence_id: Integer))
end
end
end
def expected_attributes_for(occurrence_map)
Gitlab::Database.allow_cross_joins_across_databases(
url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/480165'
) do
{
ancestors: occurrence_map.ancestors,
archived: project.archived,
commit_sha: pipeline.sha,
component_id: occurrence_map.component_id,
component_name: occurrence_map.name,
component_version_id: occurrence_map.component_version_id,
input_file_path: occurrence_map.input_file_path,
licenses: default_licenses,
package_manager: occurrence_map.packager,
pipeline_id: pipeline.id,
project_id: project.id,
source_id: occurrence_map.source_id,
source_package_id: occurrence_map.source_package_id,
traversal_ids: project.namespace.traversal_ids
}.deep_stringify_keys!
end
end
end
......@@ -33,20 +33,24 @@
end
let(:occurrence_map_1) do
create(:sbom_occurrence_map, :for_occurrence_ingestion, :with_occurrence, vulnerabilities: vulnerability_info)
create(:sbom_occurrence_map, :for_occurrence_ingestion, :with_occurrence)
end
let(:occurrence_map_2) do
create(:sbom_occurrence_map, :for_occurrence_ingestion, :with_occurrence, vulnerabilities: vulnerability_info)
create(:sbom_occurrence_map, :for_occurrence_ingestion, :with_occurrence)
end
let(:occurrence_maps) { [occurrence_map_1, occurrence_map_2] }
let(:vulnerability_info) { create(:sbom_vulnerabilities, pipeline: pipeline) }
subject(:ingest_occurrences_vulnerabilities) do
described_class.execute(pipeline, occurrence_maps)
end
before do
occurrence_map_1.vulnerability_ids = [finding_1.vulnerability_id]
occurrence_map_2.vulnerability_ids = [finding_2.vulnerability_id]
end
it_behaves_like 'bulk insertable task'
it 'is idempotent' do
......@@ -83,7 +87,7 @@
context 'when there is more than one vulnerability per occurrence' do
before do
create(
finding = create(
:vulnerabilities_finding,
:detected,
:with_dependency_scanning_metadata,
......@@ -93,6 +97,7 @@
version: occurrence_map_1.version,
pipeline: pipeline
)
occurrence_map_1.vulnerability_ids << finding.vulnerability_id
end
it 'creates all related occurrences_vulnerabilities' do
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment