Merge branch 'create-catalog-resource-component-usages-table' into 'master'

Create partitioned CI component usage table

See merge request !145881



Merged-by: Leaminn Ma <lma@gitlab.com>

app/models/ci/catalog/resource.rb

@@ -17,6 +17,8 @@ class Resource < ::ApplicationRecord
       belongs_to :project
       has_many :components, class_name: 'Ci::Catalog::Resources::Component', foreign_key: :catalog_resource_id,
         inverse_of: :catalog_resource
+      has_many :component_usages, class_name: 'Ci::Catalog::Resources::Components::Usage',
+        foreign_key: :catalog_resource_id, inverse_of: :catalog_resource
       has_many :versions, class_name: 'Ci::Catalog::Resources::Version', foreign_key: :catalog_resource_id,
         inverse_of: :catalog_resource
       has_many :sync_events, class_name: 'Ci::Catalog::Resources::SyncEvent', foreign_key: :catalog_resource_id,

app/models/ci/catalog/resources/component.rb

@@ -6,13 +6,16 @@ module Resources
       # This class represents a CI/CD Catalog resource component.
       # The data will be used as metadata of a component.
       class Component < ::ApplicationRecord
-        include BulkInsertSafe
-
         self.table_name = 'catalog_resource_components'
 
         belongs_to :project, inverse_of: :ci_components
         belongs_to :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :components
         belongs_to :version, class_name: 'Ci::Catalog::Resources::Version', inverse_of: :components
+        has_many :usages, class_name: 'Ci::Catalog::Resources::Components::Usage', inverse_of: :component
+
+        # BulkInsertSafe must be included after the `has_many` declaration, otherwise it raises
+        # an error about the save callback that is auto generated for this association.
+        include BulkInsertSafe
 
         enum resource_type: { template: 1 }
 

app/models/ci/catalog/resources/components/usage.rb

+# frozen_string_literal: true
+
+module Ci
+  module Catalog
+    module Resources
+      module Components
+        # This model is used to track when a project includes a component in a pipeline.
+        # The column `used_by_project_id` does not have an FK constraint because we want
+        # to preserve historical usage data.
+        class Usage < ::ApplicationRecord
+          include PartitionedTable
+
+          self.table_name = 'p_catalog_resource_component_usages'
+          self.primary_key = :id
+
+          # TO DO: Retention period to be shortened in https://gitlab.com/gitlab-org/gitlab/-/issues/443681
+          partitioned_by :used_date, strategy: :monthly, retain_for: 12.months
+
+          belongs_to :component, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :usages
+          belongs_to :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :component_usages
+          belongs_to :project, inverse_of: :ci_component_usages
+
+          validates :component, :catalog_resource, :project, :used_by_project_id, presence: true
+          validates :used_date, uniqueness: { scope: [:component_id, :used_by_project_id] }
+
+          before_validation :set_used_date, unless: :used_date?
+
+          private
+
+          def set_used_date
+            self.used_date = Date.today
+          end
+        end
+      end
+    end
+  end
+end

app/models/project.rb

@@ -189,6 +189,7 @@ class Project < ApplicationRecord
 
   has_one :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :project
   has_many :ci_components, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :project
+  has_many :ci_component_usages, class_name: 'Ci::Catalog::Resources::Components::Usage', inverse_of: :project
   has_many :catalog_resource_versions, class_name: 'Ci::Catalog::Resources::Version', inverse_of: :project
   has_many :catalog_resource_sync_events, class_name: 'Ci::Catalog::Resources::SyncEvent', inverse_of: :project
 

config/initializers/postgres_partitioning.rb

@@ -14,7 +14,8 @@
     BatchedGitRefUpdates::Deletion,
     Users::ProjectVisit,
     Users::GroupVisit,
-    Ci::Catalog::Resources::SyncEvent
+    Ci::Catalog::Resources::SyncEvent,
+    Ci::Catalog::Resources::Components::Usage
   ])
 
 if Gitlab.ee?

db/docs/p_catalog_resource_component_usages.yml

+---
+table_name: p_catalog_resource_component_usages
+classes:
+- Ci::Catalog::Resources::Components::Usage
+feature_categories:
+- pipeline_composition
+description: Tracks when a project includes a CI component in a pipeline.
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/145881
+milestone: '16.10'
+gitlab_schema: gitlab_main_cell
+sharding_key:
+  project_id: projects

db/migrate/20240301210341_create_catalog_resource_component_usages_table.rb

+# frozen_string_literal: true
+
+class CreateCatalogResourceComponentUsagesTable < Gitlab::Database::Migration[2.2]
+  milestone '16.10'
+
+  enable_lock_retries!
+
+  CATALOG_RESOURCE_INDEX_NAME = 'idx_p_catalog_resource_component_usages_on_catalog_resource_id'
+  UNIQUE_INDEX_NAME = 'idx_component_usages_on_component_used_by_project_and_used_date'
+
+  def up
+    options = {
+      primary_key: [:id, :used_date],
+      options: 'PARTITION BY RANGE (used_date)',
+      if_not_exists: true
+    }
+
+    create_table(:p_catalog_resource_component_usages, **options) do |t|
+      t.bigserial :id, null: false
+      t.bigint :component_id, null: false
+      t.bigint :catalog_resource_id, null: false
+      t.bigint :project_id, null: false, index: true
+      t.bigint :used_by_project_id, null: false
+      t.date :used_date, null: false
+
+      t.index :catalog_resource_id, name: CATALOG_RESOURCE_INDEX_NAME
+      t.index [:component_id, :used_by_project_id, :used_date], unique: true, name: UNIQUE_INDEX_NAME
+    end
+  end
+
+  def down
+    drop_table :p_catalog_resource_component_usages
+  end
+end

db/migrate/20240301210400_add_component_fk_to_catalog_resource_component_usages.rb

+# frozen_string_literal: true
+
+class AddComponentFkToCatalogResourceComponentUsages < Gitlab::Database::Migration[2.2]
+  include Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers
+
+  milestone '16.10'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_partitioned_foreign_key :p_catalog_resource_component_usages, :catalog_resource_components,
+      column: :component_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :p_catalog_resource_component_usages, column: :component_id
+    end
+  end
+end

db/migrate/20240301210420_add_catalog_resource_fk_to_catalog_resource_component_usages.rb

+# frozen_string_literal: true
+
+class AddCatalogResourceFkToCatalogResourceComponentUsages < Gitlab::Database::Migration[2.2]
+  include Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers
+
+  milestone '16.10'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_partitioned_foreign_key :p_catalog_resource_component_usages, :catalog_resources,
+      column: :catalog_resource_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :p_catalog_resource_component_usages, column: :catalog_resource_id
+    end
+  end
+end

db/migrate/20240301210440_add_project_fk_to_catalog_resource_component_usages.rb

+# frozen_string_literal: true
+
+class AddProjectFkToCatalogResourceComponentUsages < Gitlab::Database::Migration[2.2]
+  include Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers
+
+  milestone '16.10'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_partitioned_foreign_key :p_catalog_resource_component_usages, :projects,
+      column: :project_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :p_catalog_resource_component_usages, column: :project_id
+    end
+  end
+end

db/schema_migrations/20240301210341

+e7f5750d0c275f509cffa49a1365c9fff53362bf5b9f7239a1be5c67cb62273c
\ No newline at end of file

db/schema_migrations/20240301210400

+1b8a8f741ac2fcf7088e5f4ab53d725cbdda3b69665b8bd128d7292ec4f396fd
\ No newline at end of file

db/schema_migrations/20240301210420

+f7e58e44184c7591938373d2024d5d1a1192695f32272e09e24da12944f6c108
\ No newline at end of file

db/schema_migrations/20240301210440

+748b812798302fd55a6696ad37d79ae42634f499a3c0733b7b7ec6089424182f
\ No newline at end of file

db/structure.sql

@@ -12174,6 +12174,25 @@ CREATE SEQUENCE p_batched_git_ref_updates_deletions_id_seq
 ALTER SEQUENCE p_batched_git_ref_updates_deletions_id_seq OWNED BY p_batched_git_ref_updates_deletions.id;
+CREATE TABLE p_catalog_resource_component_usages (
+    id bigint NOT NULL,
+    component_id bigint NOT NULL,
+    catalog_resource_id bigint NOT NULL,
+    project_id bigint NOT NULL,
+    used_by_project_id bigint NOT NULL,
+    used_date date NOT NULL
+)
+PARTITION BY RANGE (used_date);
+
+CREATE SEQUENCE p_catalog_resource_component_usages_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE p_catalog_resource_component_usages_id_seq OWNED BY p_catalog_resource_component_usages.id;
+
 CREATE SEQUENCE p_catalog_resource_sync_events_id_seq
     START WITH 1
     INCREMENT BY 1
@@ -19140,6 +19159,8 @@ ALTER TABLE ONLY organizations ALTER COLUMN id SET DEFAULT nextval('organization
 ALTER TABLE ONLY p_batched_git_ref_updates_deletions ALTER COLUMN id SET DEFAULT nextval('p_batched_git_ref_updates_deletions_id_seq'::regclass);
+ALTER TABLE ONLY p_catalog_resource_component_usages ALTER COLUMN id SET DEFAULT nextval('p_catalog_resource_component_usages_id_seq'::regclass);
+
 ALTER TABLE ONLY p_catalog_resource_sync_events ALTER COLUMN id SET DEFAULT nextval('p_catalog_resource_sync_events_id_seq'::regclass);
 ALTER TABLE ONLY p_ci_builds_metadata ALTER COLUMN id SET DEFAULT nextval('ci_builds_metadata_id_seq'::regclass);
@@ -21447,6 +21468,9 @@ ALTER TABLE ONLY organizations
 ALTER TABLE ONLY p_batched_git_ref_updates_deletions
     ADD CONSTRAINT p_batched_git_ref_updates_deletions_pkey PRIMARY KEY (id, partition_id);
+ALTER TABLE ONLY p_catalog_resource_component_usages
+    ADD CONSTRAINT p_catalog_resource_component_usages_pkey PRIMARY KEY (id, used_date);
+
 ALTER TABLE ONLY p_catalog_resource_sync_events
     ADD CONSTRAINT p_catalog_resource_sync_events_pkey PRIMARY KEY (id, partition_id);
@@ -23554,6 +23578,8 @@ CREATE INDEX idx_ci_pipelines_artifacts_locked ON ci_pipelines USING btree (ci_r
 CREATE INDEX idx_compliance_security_policies_on_policy_configuration_id ON compliance_framework_security_policies USING btree (policy_configuration_id);
+CREATE UNIQUE INDEX idx_component_usages_on_component_used_by_project_and_used_date ON ONLY p_catalog_resource_component_usages USING btree (component_id, used_by_project_id, used_date);
+
 CREATE INDEX idx_container_exp_policies_on_project_id_next_run_at ON container_expiration_policies USING btree (project_id, next_run_at) WHERE (enabled = true);
 CREATE INDEX idx_container_exp_policies_on_project_id_next_run_at_enabled ON container_expiration_policies USING btree (project_id, next_run_at, enabled);
@@ -23662,6 +23688,8 @@ CREATE INDEX idx_on_protected_branch ON approval_group_rules_protected_branches
 CREATE INDEX idx_open_issues_on_project_and_confidential_and_author_and_id ON issues USING btree (project_id, confidential, author_id, id) WHERE (state_id = 1);
+CREATE INDEX idx_p_catalog_resource_component_usages_on_catalog_resource_id ON ONLY p_catalog_resource_component_usages USING btree (catalog_resource_id);
+
 CREATE INDEX idx_packages_debian_group_component_files_on_architecture_id ON packages_debian_group_component_files USING btree (architecture_id);
 CREATE INDEX idx_packages_debian_project_component_files_on_architecture_id ON packages_debian_project_component_files USING btree (architecture_id);
@@ -26024,6 +26052,8 @@ CREATE INDEX index_organizations_on_path_trigram ON organizations USING gin (pat
 CREATE UNIQUE INDEX index_organizations_on_unique_name_per_group ON customer_relations_organizations USING btree (group_id, lower(name), id);
+CREATE INDEX index_p_catalog_resource_component_usages_on_project_id ON ONLY p_catalog_resource_component_usages USING btree (project_id);
+
 CREATE INDEX index_p_catalog_resource_sync_events_on_id_where_pending ON ONLY p_catalog_resource_sync_events USING btree (id) WHERE (status = 1);
 CREATE INDEX index_p_ci_finished_build_ch_sync_events_finished_at ON ONLY p_ci_finished_build_ch_sync_events USING btree (partition, build_finished_at);
@@ -30604,6 +30634,9 @@ ALTER TABLE ONLY operations_user_lists
 ALTER TABLE ONLY resource_link_events
     ADD CONSTRAINT fk_rails_0cea73eba5 FOREIGN KEY (child_work_item_id) REFERENCES issues(id) ON DELETE CASCADE;
+ALTER TABLE p_catalog_resource_component_usages
+    ADD CONSTRAINT fk_rails_0e15a4677f FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY audit_events_google_cloud_logging_configurations
     ADD CONSTRAINT fk_rails_0eb52fc617 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
@@ -31585,6 +31618,9 @@ ALTER TABLE ONLY alert_management_alert_assignees
 ALTER TABLE ONLY scim_identities
     ADD CONSTRAINT fk_rails_9421a0bffb FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
+ALTER TABLE p_catalog_resource_component_usages
+    ADD CONSTRAINT fk_rails_9430673479 FOREIGN KEY (catalog_resource_id) REFERENCES catalog_resources(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY packages_debian_project_distributions
     ADD CONSTRAINT fk_rails_94b95e1f84 FOREIGN KEY (creator_id) REFERENCES users(id) ON DELETE SET NULL;
@@ -32131,6 +32167,9 @@ ALTER TABLE ONLY label_priorities
 ALTER TABLE ONLY packages_packages
     ADD CONSTRAINT fk_rails_e1ac527425 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
+ALTER TABLE p_catalog_resource_component_usages
+    ADD CONSTRAINT fk_rails_e1ba64b7ee FOREIGN KEY (component_id) REFERENCES catalog_resource_components(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY cluster_platforms_kubernetes
     ADD CONSTRAINT fk_rails_e1e2cf841a FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE;

ee/spec/models/factories_spec.rb

@@ -135,6 +135,7 @@
   # is being mutated.
   skip_factory_defaults = %i[
     ci_catalog_resource_component
+    ci_catalog_resource_component_usage
     ci_catalog_resource_version
     ci_job_token_project_scope_link
     ci_subscriptions_project

spec/db/schema_spec.rb

@@ -102,6 +102,7 @@
     p_ci_builds: %w[erased_by_id trigger_request_id partition_id auto_canceled_by_partition_id],
     p_batched_git_ref_updates_deletions: %w[project_id partition_id],
     p_catalog_resource_sync_events: %w[catalog_resource_id project_id partition_id],
+    p_catalog_resource_component_usages: %w[used_by_project_id], # No FK constraint because we want to preserve historical usage data
     p_ci_finished_build_ch_sync_events: %w[build_id],
     p_ci_job_artifacts: %w[partition_id project_id job_id],
     p_ci_pipeline_variables: %w[partition_id],

spec/factories/ci/catalog/resources/components/usages.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :ci_catalog_resource_component_usage, class: 'Ci::Catalog::Resources::Components::Usage' do
+    component factory: :ci_catalog_resource_component
+    catalog_resource { component.catalog_resource }
+    project { component.project }
+    used_by_project_id { 1 }
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -533,6 +533,7 @@ project:
 - catalog_resource_sync_events
 - catalog_resource_versions
 - ci_components
+- ci_component_usages
 - external_status_checks
 - base_tags
 - project_topics

spec/models/ci/catalog/resource_spec.rb

@@ -26,6 +26,12 @@
       have_many(:components).class_name('Ci::Catalog::Resources::Component').with_foreign_key(:catalog_resource_id))
   end
 
+  it do
+    is_expected.to(
+      have_many(:component_usages).class_name('Ci::Catalog::Resources::Components::Usage')
+        .with_foreign_key(:catalog_resource_id))
+  end
+
   it do
     is_expected.to(
       have_many(:versions).class_name('Ci::Catalog::Resources::Version').with_foreign_key(:catalog_resource_id))