Skip to content
Snippets Groups Projects
Commit af847017 authored by Aakriti Gupta's avatar Aakriti Gupta :red_circle:
Browse files

# This is a combination of 5 commits.

# This is the 1st commit message:

Add vulnerability export replication

- add a registry table and replicator

# The commit message #2 will be skipped:

# Add changelog

# The commit message #3 will be skipped:

# Move geo db migration

# The commit message #4 will be skipped:

# Fix docs for adding a registry table

# The commit message #5 will be skipped:

# Add verification related migrations
parent dac12de5
No related branches found
No related tags found
No related merge requests found
# frozen_string_literal: true
class CreateVulnerabilityExportStates < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
unless table_exists?(:vulnerability_export_states)
with_lock_retries do
create_table :vulnerability_export_states, id: false do |t|
t.references :vulnerability_export, primary_key: true, null: false, foreign_key: { on_delete: :cascade }
t.datetime_with_timezone :verification_retry_at
t.datetime_with_timezone :verified_at
t.binary :verification_checksum, using: 'verification_checksum::bytea'
t.text :verification_failure
t.integer :verification_retry_count, limit: 2
end
end
end
add_text_limit :vulnerability_export_states, :verification_failure, 255
end
def down
drop_table :vulnerability_export_states
end
end
# frozen_string_literal: true
class AddVerificationFailureIndexToVulnerabilityExports < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :vulnerability_export_states, :verification_failure, where: "(verification_failure IS NOT NULL)", name: "vulnerability_exports_verification_failure_partial"
add_concurrent_index :vulnerability_export_states, :verification_checksum, where: "(verification_checksum IS NOT NULL)", name: "vulnerability_exports_verification_checksum_partial"
end
def down
remove_concurrent_index :vulnerability_export_states, :verification_failure
remove_concurrent_index :vulnerability_export_states, :verification_checksum
end
end
......@@ -15972,6 +15972,45 @@ CREATE SEQUENCE public.vulnerabilities_id_seq
 
ALTER SEQUENCE public.vulnerabilities_id_seq OWNED BY public.vulnerabilities.id;
 
CREATE TABLE public.vulnerability_export_registry (
id bigint NOT NULL,
vulnerability_export_id integer NOT NULL,
state smallint DEFAULT 0 NOT NULL,
retry_count smallint DEFAULT 0,
last_sync_failure text,
retry_at timestamp with time zone,
last_synced_at timestamp with time zone,
created_at timestamp with time zone NOT NULL
);
CREATE SEQUENCE public.vulnerability_export_registry_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE public.vulnerability_export_registry_id_seq OWNED BY public.vulnerability_export_registry.id;
CREATE TABLE public.vulnerability_export_states (
vulnerability_export_id bigint NOT NULL,
verification_retry_at timestamp with time zone,
verified_at timestamp with time zone,
verification_checksum bytea,
verification_failure text,
verification_retry_count smallint,
CONSTRAINT check_0e7dabd138 CHECK ((char_length(verification_failure) <= 255))
);
CREATE SEQUENCE public.vulnerability_export_states_vulnerability_export_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE public.vulnerability_export_states_vulnerability_export_id_seq OWNED BY public.vulnerability_export_states.vulnerability_export_id;
CREATE TABLE public.vulnerability_exports (
id bigint NOT NULL,
created_at timestamp with time zone NOT NULL,
......@@ -17005,6 +17044,10 @@ ALTER TABLE ONLY public.users_statistics ALTER COLUMN id SET DEFAULT nextval('pu
 
ALTER TABLE ONLY public.vulnerabilities ALTER COLUMN id SET DEFAULT nextval('public.vulnerabilities_id_seq'::regclass);
 
ALTER TABLE ONLY public.vulnerability_export_registry ALTER COLUMN id SET DEFAULT nextval('public.vulnerability_export_registry_id_seq'::regclass);
ALTER TABLE ONLY public.vulnerability_export_states ALTER COLUMN vulnerability_export_id SET DEFAULT nextval('public.vulnerability_export_states_vulnerability_export_id_seq'::regclass);
ALTER TABLE ONLY public.vulnerability_exports ALTER COLUMN id SET DEFAULT nextval('public.vulnerability_exports_id_seq'::regclass);
 
ALTER TABLE ONLY public.vulnerability_feedback ALTER COLUMN id SET DEFAULT nextval('public.vulnerability_feedback_id_seq'::regclass);
......@@ -18258,6 +18301,12 @@ ALTER TABLE ONLY public.users_statistics
ALTER TABLE ONLY public.vulnerabilities
ADD CONSTRAINT vulnerabilities_pkey PRIMARY KEY (id);
 
ALTER TABLE ONLY public.vulnerability_export_registry
ADD CONSTRAINT vulnerability_export_registry_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.vulnerability_export_states
ADD CONSTRAINT vulnerability_export_states_pkey PRIMARY KEY (vulnerability_export_id);
ALTER TABLE ONLY public.vulnerability_exports
ADD CONSTRAINT vulnerability_exports_pkey PRIMARY KEY (id);
 
......@@ -18465,7 +18514,7 @@ CREATE INDEX backup_labels_group_id_title_idx ON public.backup_labels USING btre
 
CREATE INDEX backup_labels_project_id_idx ON public.backup_labels USING btree (project_id);
 
CREATE UNIQUE INDEX backup_labels_project_id_title_idx ON public.backup_labels USING btree (project_id, title) WHERE (group_id = NULL::integer);
CREATE INDEX backup_labels_project_id_title_idx ON public.backup_labels USING btree (project_id, title) WHERE (group_id = NULL::integer);
 
CREATE INDEX backup_labels_template_idx ON public.backup_labels USING btree (template) WHERE template;
 
......@@ -20473,6 +20522,14 @@ CREATE INDEX index_vulnerabilities_on_start_date_sourcing_milestone_id ON public
 
CREATE INDEX index_vulnerabilities_on_updated_by_id ON public.vulnerabilities USING btree (updated_by_id);
 
CREATE INDEX index_vulnerability_export_registry_on_retry_at ON public.vulnerability_export_registry USING btree (retry_at);
CREATE INDEX index_vulnerability_export_registry_on_state ON public.vulnerability_export_registry USING btree (state);
CREATE INDEX index_vulnerability_export_registry_on_vulnerability_export_id ON public.vulnerability_export_registry USING btree (vulnerability_export_id);
CREATE INDEX index_vulnerability_export_states_on_vulnerability_export_id ON public.vulnerability_export_states USING btree (vulnerability_export_id);
CREATE INDEX index_vulnerability_exports_on_author_id ON public.vulnerability_exports USING btree (author_id);
 
CREATE INDEX index_vulnerability_exports_on_group_id_not_null ON public.vulnerability_exports USING btree (group_id) WHERE (group_id IS NOT NULL);
......@@ -20621,6 +20678,10 @@ CREATE UNIQUE INDEX unique_merge_request_metrics_by_merge_request_id ON public.m
 
CREATE UNIQUE INDEX users_security_dashboard_projects_unique_index ON public.users_security_dashboard_projects USING btree (project_id, user_id);
 
CREATE INDEX vulnerability_exports_verification_checksum_partial ON public.vulnerability_export_states USING btree (verification_checksum) WHERE (verification_checksum IS NOT NULL);
CREATE INDEX vulnerability_exports_verification_failure_partial ON public.vulnerability_export_states USING btree (verification_failure) WHERE (verification_failure IS NOT NULL);
CREATE UNIQUE INDEX vulnerability_feedback_unique_idx ON public.vulnerability_feedback USING btree (project_id, category, feedback_type, project_fingerprint);
 
CREATE UNIQUE INDEX vulnerability_occurrence_pipelines_on_unique_keys ON public.vulnerability_occurrence_pipelines USING btree (occurrence_id, pipeline_id);
......@@ -21979,6 +22040,9 @@ ALTER TABLE ONLY public.design_management_versions
ALTER TABLE ONLY public.approval_merge_request_rules_approved_approvers
ADD CONSTRAINT fk_rails_6577725edb FOREIGN KEY (approval_merge_request_rule_id) REFERENCES public.approval_merge_request_rules(id) ON DELETE CASCADE;
 
ALTER TABLE ONLY public.vulnerability_export_states
ADD CONSTRAINT fk_rails_6642824de4 FOREIGN KEY (vulnerability_export_id) REFERENCES public.vulnerability_exports(id) ON DELETE CASCADE;
ALTER TABLE ONLY public.operations_feature_flags_clients
ADD CONSTRAINT fk_rails_6650ed902c FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE;
 
......@@ -22667,6 +22731,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20190220150130
20190222051615
20190225152525
20190225160300
20190225160301
20190228192410
20190301081611
......@@ -23767,5 +23832,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200708080631
20200710102846
20200710130234
20200713154007
20200713154244
\.
 
......@@ -225,12 +225,14 @@ For example, to add support for files referenced by a `Widget` model with a
```
1. Create the `widget_registry` table so Geo secondaries can track the sync and
verification state of each Widget's file:
verification state of each Widget's file. Add the migration in ee/db/geo/migrate/:
```ruby
# frozen_string_literal: true
class CreateWidgetRegistry < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
......@@ -361,7 +363,7 @@ Widgets should now be replicated by Geo!
```
1. Add a partial index on `verification_failure` and `verification_checksum` to ensure
re-verification can be performed efficiently:
re-verification can be performed efficiently. Add a migration in ee/db/geo/migrate/:
```ruby
# frozen_string_literal: true
......@@ -374,13 +376,13 @@ Widgets should now be replicated by Geo!
disable_ddl_transaction!
def up
add_concurrent_index :widgets, :verification_failure, where: "(verification_failure IS NOT NULL)", name: "widgets_verification_failure_partial"
add_concurrent_index :widgets, :verification_checksum, where: "(verification_checksum IS NOT NULL)", name: "widgets_verification_checksum_partial"
add_concurrent_index :widget_states, :verification_failure, where: "(verification_failure IS NOT NULL)", name: "widgets_verification_failure_partial"
add_concurrent_index :widget_states, :verification_checksum, where: "(verification_checksum IS NOT NULL)", name: "widgets_verification_checksum_partial"
end
def down
remove_concurrent_index :widgets, :verification_failure
remove_concurrent_index :widgets, :verification_checksum
remove_concurrent_index :widget_states, :verification_failure
remove_concurrent_index :widget_states, :verification_checksum
end
end
```
......
# frozen_string_literal: true
class CreateVulnerabilityExportRegistry < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
......
......@@ -10,10 +10,11 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2020_07_07_011052) do
ActiveRecord::Schema.define(version: 2020_07_10_194046) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
enable_extension "postgres_fdw"
create_table "container_repository_registry", id: :serial, force: :cascade do |t|
t.integer "container_repository_id", null: false
......@@ -168,4 +169,17 @@
t.index ["wiki_verification_checksum_sha"], name: "idx_project_registry_on_wiki_checksum_sha_partial", where: "(wiki_verification_checksum_sha IS NULL)"
end
create_table "vulnerability_export_registry", force: :cascade do |t|
t.integer "vulnerability_export_id", null: false
t.integer "state", limit: 2, default: 0, null: false
t.integer "retry_count", limit: 2, default: 0
t.text "last_sync_failure"
t.datetime_with_timezone "retry_at"
t.datetime_with_timezone "last_synced_at"
t.datetime_with_timezone "created_at", null: false
t.index ["retry_at"], name: "index_vulnerability_export_registry_on_retry_at"
t.index ["state"], name: "index_vulnerability_export_registry_on_state"
t.index ["vulnerability_export_id"], name: "index_vulnerability_export_registry_on_vulnerability_export_id"
end
end
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment