Merge branch 'bwill/remove-vulnerability-occurrence-pipelines' into 'master'

Remove `vulnerability_occurrence_pipelines` table and code

See merge request !170153



Merged-by: Gregory Havenga <11164960-ghavenga@users.noreply.gitlab.com>
Approved-by: Sincheol (David) Kim <dkim@gitlab.com>
Approved-by: Gregory Havenga <11164960-ghavenga@users.noreply.gitlab.com>
Approved-by: Daniel P. <dprause@gitlab.com>
Approved-by: Mehmet Emin INAC <minac@gitlab.com>
Reviewed-by: Sincheol (David) Kim <dkim@gitlab.com>
Co-authored-by: Brian Williams <bwilliams@gitlab.com>

.rubocop_todo/gitlab/strong_memoize_attr.yml

@@ -366,7 +366,6 @@ Gitlab/StrongMemoizeAttr:
     - 'ee/lib/ee/gitlab/analytics/cycle_analytics/stage_events.rb'
     - 'ee/lib/ee/gitlab/auth/o_auth/auth_hash.rb'
     - 'ee/lib/ee/gitlab/background_migration/migrate_approver_to_approval_rules.rb'
-    - 'ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb'
     - 'ee/lib/ee/gitlab/checks/base_checker.rb'
     - 'ee/lib/ee/gitlab/checks/diff_check.rb'
     - 'ee/lib/ee/gitlab/ci/matching/runner_matcher.rb'

.rubocop_todo/layout/line_length.yml

@@ -805,7 +805,6 @@ Layout/LineLength:
     - 'ee/app/models/vulnerabilities/finding.rb'
     - 'ee/app/models/vulnerabilities/finding_identifier.rb'
     - 'ee/app/models/vulnerabilities/finding_link.rb'
-    - 'ee/app/models/vulnerabilities/finding_pipeline.rb'
     - 'ee/app/models/vulnerabilities/finding_remediation.rb'
     - 'ee/app/models/vulnerabilities/flag.rb'
     - 'ee/app/models/vulnerabilities/historical_statistic.rb'
@@ -1041,7 +1040,6 @@ Layout/LineLength:
     - 'ee/lib/ee/gitlab/background_migration/create_security_setting.rb'
     - 'ee/lib/ee/gitlab/background_migration/migrate_approver_to_approval_rules.rb'
     - 'ee/lib/ee/gitlab/background_migration/populate_latest_pipeline_ids.rb'
-    - 'ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb'
     - 'ee/lib/ee/gitlab/checks/push_rules/commit_check.rb'
     - 'ee/lib/ee/gitlab/checks/push_rules/file_size_check.rb'
     - 'ee/lib/ee/gitlab/ci/pipeline/chain/create_cross_database_associations.rb'
@@ -2266,7 +2264,6 @@ Layout/LineLength:
     - 'lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb'
     - 'lib/gitlab/background_migration/migrate_job_artifact_registry_to_ssf.rb'
     - 'lib/gitlab/background_migration/migrate_requirements_to_work_items.rb'
-    - 'lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb'
     - 'lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb'
     - 'lib/gitlab/bullet/exclusions.rb'
     - 'lib/gitlab/cache/helpers.rb'

.rubocop_todo/lint/redundant_cop_disable_directive.yml

@@ -291,7 +291,6 @@ Lint/RedundantCopDisableDirective:
     - 'lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb'
     - 'lib/gitlab/background_migration/populate_denormalized_columns_for_sbom_occurrences.rb'
     - 'lib/gitlab/background_migration/populate_latest_pipeline_ids.rb'
-    - 'lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb'
     - 'lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb'
     - 'lib/gitlab/background_migration/purge_stale_security_scans.rb'
     - 'lib/gitlab/background_migration/resolve_vulnerabilities_for_removed_analyzers.rb'

.rubocop_todo/migration/background_migration_record.yml

@@ -7,7 +7,6 @@ Migration/BackgroundMigrationRecord:
     - 'ee/lib/ee/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb'
     - 'ee/lib/ee/gitlab/background_migration/migrate_requirements_to_work_items.rb'
     - 'ee/lib/ee/gitlab/background_migration/populate_latest_pipeline_ids.rb'
-    - 'ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb'
     - 'lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb'
     - 'lib/gitlab/background_migration/backfill_project_repositories.rb'
     - 'lib/gitlab/background_migration/backfill_topics_title.rb'

.rubocop_todo/migration/batched_migration_base_class.yml

@@ -26,7 +26,6 @@ Migration/BatchedMigrationBaseClass:
     - 'lib/gitlab/background_migration/migrate_null_private_profile_to_false.rb'
     - 'lib/gitlab/background_migration/migrate_requirements_to_work_items.rb'
     - 'lib/gitlab/background_migration/populate_latest_pipeline_ids.rb'
-    - 'lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb'
     - 'lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb'
     - 'lib/gitlab/background_migration/project_namespaces/models/namespace.rb'
     - 'lib/gitlab/background_migration/project_namespaces/models/project.rb'

.rubocop_todo/rspec/context_wording.yml

@@ -461,7 +461,6 @@ RSpec/ContextWording:
     - 'ee/spec/models/saml_provider_spec.rb'
     - 'ee/spec/models/status_page/project_setting_spec.rb'
     - 'ee/spec/models/vulnerabilities/feedback_spec.rb'
-    - 'ee/spec/models/vulnerabilities/finding_pipeline_spec.rb'
     - 'ee/spec/models/vulnerabilities/finding_spec.rb'
     - 'ee/spec/models/vulnerabilities/statistic_spec.rb'
     - 'ee/spec/policies/app_sec/fuzzing/coverage/corpus_policy_spec.rb'

.rubocop_todo/style/inline_disable_annotation.yml

@@ -2017,7 +2017,6 @@ Style/InlineDisableAnnotation:
     - 'lib/gitlab/background_migration/migrate_shared_vulnerability_scanners.rb'
     - 'lib/gitlab/background_migration/populate_denormalized_columns_for_sbom_occurrences.rb'
     - 'lib/gitlab/background_migration/populate_latest_pipeline_ids.rb'
-    - 'lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb'
     - 'lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb'
     - 'lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb'
     - 'lib/gitlab/background_migration/purge_stale_security_scans.rb'

.rubocop_todo/style/string_concatenation.yml

@@ -43,7 +43,6 @@ Style/StringConcatenation:
     - 'ee/lib/api/project_mirror.rb'
     - 'ee/lib/ee/gitlab/auth/ldap/person.rb'
     - 'ee/lib/ee/gitlab/background_migration/populate_latest_pipeline_ids.rb'
-    - 'ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb'
     - 'ee/lib/gitlab/geo/git_ssh_proxy.rb'
     - 'ee/lib/omni_auth/strategies/kerberos.rb'
     - 'ee/lib/tasks/gitlab/license.rake'

config/gitlab_loose_foreign_keys.yml

@@ -657,10 +657,6 @@ vulnerability_namespace_historical_statistics:
   - table: namespaces
     column: namespace_id
     on_delete: async_delete
-vulnerability_occurrence_pipelines:
-  - table: ci_pipelines
-    column: pipeline_id
-    on_delete: async_delete
 vulnerability_occurrences:
   - table: ci_pipelines
     column: initial_pipeline_id

db/docs/batched_background_migrations/backfill_vulnerability_occurrence_pipelines_project_id.yml

----
-migration_job_name: BackfillVulnerabilityOccurrencePipelinesProjectId
-description: Backfills sharding key `vulnerability_occurrence_pipelines.project_id` from `vulnerability_occurrences`.
-feature_category: vulnerability_management
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/161221
-milestone: '17.3'
-queued_migration_version: 20240730172818
-finalized_by: # version of the migration that finalized this BBM

db/docs/vulnerability_occurrence_pipelines.yml → db/docs/deleted_tables/vulnerability_occurrence_pipelines.yml

@@ -8,13 +8,5 @@ description: Join table between Findings and Pipelines
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7578
 milestone: '11.5'
 gitlab_schema: gitlab_sec
-desired_sharding_key:
-  project_id:
-    references: projects
-    backfill_via:
-      parent:
-        foreign_key: occurrence_id
-        table: vulnerability_occurrences
-        sharding_key: project_id
-        belongs_to: finding
-desired_sharding_key_migration_job_name: BackfillVulnerabilityOccurrencePipelinesProjectId
+removed_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/170153
+removed_in_milestone: '17.6'

db/gitlab_schemas/gitlab_sec.yaml

@@ -25,7 +25,6 @@ allow_cross_joins:
       - vulnerability_issue_links
       - vulnerability_merge_request_links
       - vulnerability_occurrence_identifiers
-      - vulnerability_occurrence_pipelines
       - vulnerability_occurrences
       - vulnerability_reads
       - vulnerability_remediations
@@ -77,7 +76,6 @@ allow_cross_transactions:
       - vulnerability_issue_links
       - vulnerability_merge_request_links
       - vulnerability_occurrence_identifiers
-      - vulnerability_occurrence_pipelines
       - vulnerability_occurrences
       - vulnerability_reads
       - vulnerability_remediations
@@ -127,7 +125,6 @@ allow_cross_foreign_keys:
       - vulnerability_issue_links
       - vulnerability_merge_request_links
       - vulnerability_occurrence_identifiers
-      - vulnerability_occurrence_pipelines
       - vulnerability_occurrences
       - vulnerability_reads
       - vulnerability_remediations

db/post_migrate/20240730172818_queue_backfill_vulnerability_occurrence_pipelines_project_id.rb

@@ -5,23 +5,9 @@ class QueueBackfillVulnerabilityOccurrencePipelinesProjectId < Gitlab::Database:
   restrict_gitlab_migration gitlab_schema: :gitlab_sec
 
   MIGRATION = "BackfillVulnerabilityOccurrencePipelinesProjectId"
-  DELAY_INTERVAL = 2.minutes
-  BATCH_SIZE = 1000
-  SUB_BATCH_SIZE = 100
 
   def up
-    queue_batched_background_migration(
-      MIGRATION,
-      :vulnerability_occurrence_pipelines,
-      :id,
-      :project_id,
-      :vulnerability_occurrences,
-      :project_id,
-      :occurrence_id,
-      job_interval: DELAY_INTERVAL,
-      batch_size: BATCH_SIZE,
-      sub_batch_size: SUB_BATCH_SIZE
-    )
+    # This migration has been deleted because the table has been dropped.
   end
 
   def down

db/post_migrate/20241022181600_remove_backfill_vulnerability_occurrence_pipelines_project_id.rb

+# frozen_string_literal: true
+
+class RemoveBackfillVulnerabilityOccurrencePipelinesProjectId < Gitlab::Database::Migration[2.2]
+  milestone '17.6'
+  restrict_gitlab_migration gitlab_schema: :gitlab_sec
+
+  MIGRATION = "BackfillVulnerabilityOccurrencePipelinesProjectId"
+
+  def up
+    delete_batched_background_migration(
+      MIGRATION,
+      :vulnerability_occurrence_pipelines,
+      :id,
+      [
+        :project_id,
+        :vulnerability_occurrences,
+        :project_id,
+        :occurrence_id
+      ]
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20241022181639_drop_vulnerability_occurrence_pipelines.rb

+# frozen_string_literal: true
+
+class DropVulnerabilityOccurrencePipelines < Gitlab::Database::Migration[2.2]
+  include Gitlab::Database::SchemaHelpers
+
+  TABLE_NAME = :vulnerability_occurrence_pipelines
+  TRIGGER_NAME = 'trigger_8ba31bddd655'
+
+  milestone '17.6'
+
+  disable_ddl_transaction!
+
+  def up
+    drop_table TABLE_NAME
+    drop_function TRIGGER_NAME
+  end
+
+  def down
+    create_table TABLE_NAME do |t|
+      t.timestamps_with_timezone null: false
+      t.bigint :occurrence_id, null: false
+      t.bigint :pipeline_id, null: false
+      t.bigint :project_id
+    end
+
+    add_concurrent_index TABLE_NAME, [:occurrence_id, :pipeline_id],
+      unique: true,
+      name: 'vulnerability_occurrence_pipelines_on_unique_keys'
+    add_concurrent_index TABLE_NAME, [:occurrence_id, :id],
+      order: { occurrence_id: :asc, id: :desc },
+      name: 'index_vulnerability_occurrence_pipelines_occurrence_id_and_id'
+    add_concurrent_index TABLE_NAME, :pipeline_id,
+      name: 'index_vulnerability_occurrence_pipelines_on_pipeline_id'
+
+    add_concurrent_foreign_key TABLE_NAME, :vulnerability_occurrences,
+      column: :occurrence_id, on_delete: :cascade, name: 'fk_rails_dc3ae04693'
+
+    install_sharding_key_assignment_trigger(
+      table: TABLE_NAME,
+      sharding_key: :project_id,
+      parent_table: :vulnerability_occurrences,
+      parent_sharding_key: :project_id,
+      foreign_key: :occurrence_id,
+      trigger_name: TRIGGER_NAME
+    )
+  end
+end

db/schema_migrations/20241022181600

+50de83ee644af67af6dcac2dd533fa03e20249fd9056418b038d1889599b6bd8
\ No newline at end of file

db/schema_migrations/20241022181639

+6b16fc7572186fdf3ef1fe1d3161d49e9ff46c2ffa927e1a6fdf07834080bc2c
\ No newline at end of file

db/structure.sql

@@ -1888,22 +1888,6 @@ RETURN NEW;
 END
 $$;
-CREATE FUNCTION trigger_8ba31bddd655() RETURNS trigger
-    LANGUAGE plpgsql
-    AS $$
-BEGIN
-IF NEW."project_id" IS NULL THEN
-  SELECT "project_id"
-  INTO NEW."project_id"
-  FROM "vulnerability_occurrences"
-  WHERE "vulnerability_occurrences"."id" = NEW."occurrence_id";
-END IF;
-
-RETURN NEW;
-
-END
-$$;
-
 CREATE FUNCTION trigger_8d002f38bdef() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -20730,24 +20714,6 @@ CREATE SEQUENCE vulnerability_occurrence_identifiers_id_seq
 ALTER SEQUENCE vulnerability_occurrence_identifiers_id_seq OWNED BY vulnerability_occurrence_identifiers.id;
-CREATE TABLE vulnerability_occurrence_pipelines (
-    id bigint NOT NULL,
-    created_at timestamp with time zone NOT NULL,
-    updated_at timestamp with time zone NOT NULL,
-    occurrence_id bigint NOT NULL,
-    pipeline_id bigint NOT NULL,
-    project_id bigint
-);
-
-CREATE SEQUENCE vulnerability_occurrence_pipelines_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-ALTER SEQUENCE vulnerability_occurrence_pipelines_id_seq OWNED BY vulnerability_occurrence_pipelines.id;
-
 CREATE TABLE vulnerability_occurrences (
     id bigint NOT NULL,
     created_at timestamp with time zone NOT NULL,
@@ -23192,8 +23158,6 @@ ALTER TABLE ONLY vulnerability_namespace_historical_statistics ALTER COLUMN id S
 ALTER TABLE ONLY vulnerability_occurrence_identifiers ALTER COLUMN id SET DEFAULT nextval('vulnerability_occurrence_identifiers_id_seq'::regclass);
-ALTER TABLE ONLY vulnerability_occurrence_pipelines ALTER COLUMN id SET DEFAULT nextval('vulnerability_occurrence_pipelines_id_seq'::regclass);
-
 ALTER TABLE ONLY vulnerability_occurrences ALTER COLUMN id SET DEFAULT nextval('vulnerability_occurrences_id_seq'::regclass);
 ALTER TABLE ONLY vulnerability_reads ALTER COLUMN id SET DEFAULT nextval('vulnerability_reads_id_seq'::regclass);
@@ -25986,9 +25950,6 @@ ALTER TABLE ONLY vulnerability_namespace_historical_statistics
 ALTER TABLE ONLY vulnerability_occurrence_identifiers
     ADD CONSTRAINT vulnerability_occurrence_identifiers_pkey PRIMARY KEY (id);
-ALTER TABLE ONLY vulnerability_occurrence_pipelines
-    ADD CONSTRAINT vulnerability_occurrence_pipelines_pkey PRIMARY KEY (id);
-
 ALTER TABLE ONLY vulnerability_occurrences
     ADD CONSTRAINT vulnerability_occurrences_pkey PRIMARY KEY (id);
@@ -31612,10 +31573,6 @@ CREATE INDEX index_vulnerability_occurrence_identifiers_on_identifier_id ON vuln
 CREATE UNIQUE INDEX index_vulnerability_occurrence_identifiers_on_unique_keys ON vulnerability_occurrence_identifiers USING btree (occurrence_id, identifier_id);
-CREATE INDEX index_vulnerability_occurrence_pipelines_occurrence_id_and_id ON vulnerability_occurrence_pipelines USING btree (occurrence_id, id DESC);
-
-CREATE INDEX index_vulnerability_occurrence_pipelines_on_pipeline_id ON vulnerability_occurrence_pipelines USING btree (pipeline_id);
-
 CREATE INDEX index_vulnerability_occurrences_for_override_uuids_logic ON vulnerability_occurrences USING btree (project_id, report_type, location_fingerprint);
 CREATE INDEX index_vulnerability_occurrences_on_initial_pipeline_id ON vulnerability_occurrences USING btree (initial_pipeline_id);
@@ -32090,8 +32047,6 @@ CREATE UNIQUE INDEX virtual_reg_packages_maven_reg_upstreams_on_unique_upstream_
 CREATE UNIQUE INDEX virtual_registries_pkgs_maven_registries_on_unique_group_ids ON virtual_registries_packages_maven_registries USING btree (group_id);
-CREATE UNIQUE INDEX vulnerability_occurrence_pipelines_on_unique_keys ON vulnerability_occurrence_pipelines USING btree (occurrence_id, pipeline_id);
-
 CREATE INDEX wi_colors_namespace_id_index ON work_item_colors USING btree (namespace_id);
 CREATE INDEX wi_datessources_due_date_sourcing_milestone_id_index ON work_item_dates_sources USING btree (due_date_sourcing_milestone_id);
@@ -33866,8 +33821,6 @@ CREATE TRIGGER trigger_8ac78f164b2d BEFORE INSERT OR UPDATE ON design_management
 CREATE TRIGGER trigger_8b39d532224c BEFORE INSERT OR UPDATE ON ci_secure_file_states FOR EACH ROW EXECUTE FUNCTION trigger_8b39d532224c();
-CREATE TRIGGER trigger_8ba31bddd655 BEFORE INSERT OR UPDATE ON vulnerability_occurrence_pipelines FOR EACH ROW EXECUTE FUNCTION trigger_8ba31bddd655();
-
 CREATE TRIGGER trigger_8d002f38bdef BEFORE INSERT OR UPDATE ON packages_debian_group_components FOR EACH ROW EXECUTE FUNCTION trigger_8d002f38bdef();
 CREATE TRIGGER trigger_8d17725116fe BEFORE INSERT OR UPDATE ON merge_request_reviewers FOR EACH ROW EXECUTE FUNCTION trigger_8d17725116fe();
@@ -37237,9 +37190,6 @@ ALTER TABLE ONLY dependency_proxy_blobs
 ALTER TABLE ONLY board_user_preferences
     ADD CONSTRAINT fk_rails_dbebdaa8fe FOREIGN KEY (board_id) REFERENCES boards(id) ON DELETE CASCADE;
-ALTER TABLE ONLY vulnerability_occurrence_pipelines
-    ADD CONSTRAINT fk_rails_dc3ae04693 FOREIGN KEY (occurrence_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE;
-
 ALTER TABLE ONLY instance_audit_events_streaming_headers
     ADD CONSTRAINT fk_rails_dc933c1f3c FOREIGN KEY (instance_external_audit_event_destination_id) REFERENCES audit_events_instance_external_audit_event_destinations(id) ON DELETE CASCADE;

ee/app/models/vulnerabilities/finding.rb

@@ -93,8 +93,6 @@ class Finding < Gitlab::Database::SecApplicationRecord
     has_many :finding_remediations, class_name: 'Vulnerabilities::FindingRemediation', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id'
     has_many :remediations, through: :finding_remediations
 
-    has_many :finding_pipelines, class_name: 'Vulnerabilities::FindingPipeline', inverse_of: :finding, foreign_key: 'occurrence_id'
-
     # rubocop: disable Rails/InverseOf -- these relations are not present on Ci::Pipeline
     belongs_to :initial_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'initial_pipeline_id'
     belongs_to :latest_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'latest_pipeline_id'

ee/app/models/vulnerabilities/finding_pipeline.rb

-# frozen_string_literal: true
-
-module Vulnerabilities
-  class FindingPipeline < Gitlab::Database::SecApplicationRecord
-    include EachBatch
-
-    self.table_name = "vulnerability_occurrence_pipelines"
-
-    alias_attribute :finding_id, :occurrence_id
-
-    belongs_to :finding, class_name: 'Vulnerabilities::Finding', inverse_of: :finding_pipelines, foreign_key: 'occurrence_id'
-    belongs_to :pipeline, class_name: '::Ci::Pipeline'
-
-    validates :finding, presence: true
-    validates :pipeline, presence: true
-    validates :pipeline_id, uniqueness: { scope: [:occurrence_id] }
-
-    scope :by_finding_id, ->(finding_ids) { where(occurrence_id: finding_ids) }
-  end
-end