Geo: Add snippet repository verification [RUN ALL RSPEC] [RUN AS-IF-FOSS] (!56596) · Merge requests · GitLab.org / GitLab

You need to sign in or sign up before continuing.

Merged Michael Kozono requested to merge mk/verify-snippets into master Mar 13, 2021

What does this MR do?

This MR verifies snippet repositories which are replicated by Geo.

It looks big but the diff is mostly DB schema and documentation changes, so I prefer to keep it in one MR.

Resolves #324417 (closed)

Screenshots (strongly suggested)

Migration output

Up:

== 20210313045617 AddVerificationStateAndStartedAtToSnippetRepositories: migrating 
-- change_table(:snippet_repositories)
   -> 0.0079s
== 20210313045617 AddVerificationStateAndStartedAtToSnippetRepositories: migrated (0.0079s) 

== 20210313045845 AddVerificationIndexesToSnippetRepositories: migrating ======
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repositories, :verification_state, {:name=>"index_snippet_repositories_verification_state", :algorithm=>:concurrently})
   -> 0.0022s
-- execute("SET statement_timeout TO 0")
   -> 0.0007s
-- add_index(:snippet_repositories, :verification_state, {:name=>"index_snippet_repositories_verification_state", :algorithm=>:concurrently})
   -> 0.0071s
-- execute("RESET ALL")
   -> 0.0007s
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repositories, :verified_at, {:where=>"(verification_state = 0)", :order=>{:verified_at=>"ASC NULLS FIRST"}, :name=>"index_snippet_repositories_pending_verification", :algorithm=>:concurrently})
   -> 0.0016s
-- add_index(:snippet_repositories, :verified_at, {:where=>"(verification_state = 0)", :order=>{:verified_at=>"ASC NULLS FIRST"}, :name=>"index_snippet_repositories_pending_verification", :algorithm=>:concurrently})
   -> 0.0035s
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repositories, :verification_retry_at, {:where=>"(verification_state = 3)", :order=>{:verification_retry_at=>"ASC NULLS FIRST"}, :name=>"index_snippet_repositories_failed_verification", :algorithm=>:concurrently})
   -> 0.0019s
-- add_index(:snippet_repositories, :verification_retry_at, {:where=>"(verification_state = 3)", :order=>{:verification_retry_at=>"ASC NULLS FIRST"}, :name=>"index_snippet_repositories_failed_verification", :algorithm=>:concurrently})
   -> 0.0031s
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repositories, :verification_state, {:where=>"(verification_state = 0 OR verification_state = 3)", :name=>"index_snippet_repositories_needs_verification", :algorithm=>:concurrently})
   -> 0.0026s
-- add_index(:snippet_repositories, :verification_state, {:where=>"(verification_state = 0 OR verification_state = 3)", :name=>"index_snippet_repositories_needs_verification", :algorithm=>:concurrently})
   -> 0.0028s
== 20210313045845 AddVerificationIndexesToSnippetRepositories: migrated (0.0301s)

Geo tracking database up:

== 20210313050709 AddVerificationToSnippetRepositoryRegistry: migrating =======
-- add_column(:snippet_repository_registry, :verification_started_at, :datetime_with_timezone)
   -> 0.0022s
-- add_column(:snippet_repository_registry, :verified_at, :datetime_with_timezone)
   -> 0.0010s
-- add_column(:snippet_repository_registry, :verification_retry_at, :datetime_with_timezone)
   -> 0.0009s
-- add_column(:snippet_repository_registry, :verification_retry_count, :integer)
   -> 0.0009s
-- add_column(:snippet_repository_registry, :verification_state, :integer, {:limit=>2, :default=>0, :null=>false})
   -> 0.0016s
-- add_column(:snippet_repository_registry, :checksum_mismatch, :boolean)
   -> 0.0008s
-- add_column(:snippet_repository_registry, :verification_checksum, :binary)
   -> 0.0009s
-- add_column(:snippet_repository_registry, :verification_checksum_mismatched, :binary)
   -> 0.0009s
-- add_column(:snippet_repository_registry, :verification_failure, :string, {:limit=>255})
   -> 0.0010s
== 20210313050709 AddVerificationToSnippetRepositoryRegistry: migrated (0.0104s) 

== 20210313051642 AddVerificationIndexesToSnippetRepositoryRegistry: migrating 
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repository_registry, :verified_at, {:where=>"(state = 2 AND verification_state = 0)", :order=>{:verified_at=>"ASC NULLS FIRST"}, :name=>"snippet_repository_registry_pending_verification", :algorithm=>:concurrently})
   -> 0.0026s
-- add_index(:snippet_repository_registry, :verified_at, {:where=>"(state = 2 AND verification_state = 0)", :order=>{:verified_at=>"ASC NULLS FIRST"}, :name=>"snippet_repository_registry_pending_verification", :algorithm=>:concurrently})
   -> 0.0047s
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repository_registry, :verification_retry_at, {:where=>"(state = 2 AND verification_state = 3)", :order=>{:verification_retry_at=>"ASC NULLS FIRST"}, :name=>"snippet_repository_registry_failed_verification", :algorithm=>:concurrently})
   -> 0.0014s
-- add_index(:snippet_repository_registry, :verification_retry_at, {:where=>"(state = 2 AND verification_state = 3)", :order=>{:verification_retry_at=>"ASC NULLS FIRST"}, :name=>"snippet_repository_registry_failed_verification", :algorithm=>:concurrently})
   -> 0.0017s
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repository_registry, :verification_state, {:where=>"(state = 2 AND (verification_state IN (0, 3)))", :name=>"snippet_repository_registry_needs_verification", :algorithm=>:concurrently})
   -> 0.0015s
-- add_index(:snippet_repository_registry, :verification_state, {:where=>"(state = 2 AND (verification_state IN (0, 3)))", :name=>"snippet_repository_registry_needs_verification", :algorithm=>:concurrently})
   -> 0.0024s
== 20210313051642 AddVerificationIndexesToSnippetRepositoryRegistry: migrated (0.0163s)

Down:

== 20210313045845 AddVerificationIndexesToSnippetRepositories: reverting ======
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repositories)
   -> 0.0036s
-- execute("SET statement_timeout TO 0")
   -> 0.0005s
-- remove_index(:snippet_repositories, {:algorithm=>:concurrently, :name=>"index_snippet_repositories_verification_state"})
   -> 0.0113s
-- execute("RESET ALL")
   -> 0.0008s
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repositories)
   -> 0.0029s
-- remove_index(:snippet_repositories, {:algorithm=>:concurrently, :name=>"index_snippet_repositories_pending_verification"})
   -> 0.0020s
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repositories)
   -> 0.0018s
-- remove_index(:snippet_repositories, {:algorithm=>:concurrently, :name=>"index_snippet_repositories_failed_verification"})
   -> 0.0026s
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repositories)
   -> 0.0015s
-- remove_index(:snippet_repositories, {:algorithm=>:concurrently, :name=>"index_snippet_repositories_needs_verification"})
   -> 0.0023s
== 20210313045845 AddVerificationIndexesToSnippetRepositories: reverted (0.0323s) 

== 20210313045617 AddVerificationStateAndStartedAtToSnippetRepositories: reverting 
-- remove_column(:snippet_repositories, :verification_started_at, :datetime_with_timezone, {})
   -> 0.0038s
-- remove_column(:snippet_repositories, :verification_state, :integer, {:default=>0, :limit=>2, :null=>false})
   -> 0.0025s
== 20210313045617 AddVerificationStateAndStartedAtToSnippetRepositories: reverted (0.0088s)

Geo tracking database down:

== 20210313051642 AddVerificationIndexesToSnippetRepositoryRegistry: reverting 
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repository_registry)
   -> 0.0041s
-- remove_index(:snippet_repository_registry, {:algorithm=>:concurrently, :name=>"snippet_repository_registry_pending_verification"})
   -> 0.0091s
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repository_registry)
   -> 0.0026s
-- remove_index(:snippet_repository_registry, {:algorithm=>:concurrently, :name=>"snippet_repository_registry_failed_verification"})
   -> 0.0156s
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repository_registry)
   -> 0.0019s
-- remove_index(:snippet_repository_registry, {:algorithm=>:concurrently, :name=>"snippet_repository_registry_needs_verification"})
   -> 0.0011s
== 20210313051642 AddVerificationIndexesToSnippetRepositoryRegistry: reverted (0.0370s) 

== 20210313050709 AddVerificationToSnippetRepositoryRegistry: reverting =======
-- remove_column(:snippet_repository_registry, :verification_failure, :string, {:limit=>255})
   -> 0.0024s
-- remove_column(:snippet_repository_registry, :verification_checksum_mismatched, :binary)
   -> 0.0008s
-- remove_column(:snippet_repository_registry, :verification_checksum, :binary)
   -> 0.0012s
-- remove_column(:snippet_repository_registry, :checksum_mismatch, :boolean)
   -> 0.0010s
-- remove_column(:snippet_repository_registry, :verification_state, :integer, {:limit=>2, :default=>0, :null=>false})
   -> 0.0013s
-- remove_column(:snippet_repository_registry, :verification_retry_count, :integer)
   -> 0.0008s
-- remove_column(:snippet_repository_registry, :verification_retry_at, :datetime_with_timezone)
   -> 0.0008s
-- remove_column(:snippet_repository_registry, :verified_at, :datetime_with_timezone)
   -> 0.0008s
-- remove_column(:snippet_repository_registry, :verification_started_at, :datetime_with_timezone)
   -> 0.0008s
== 20210313050709 AddVerificationToSnippetRepositoryRegistry: reverted (0.0383s)

Does this MR meet the acceptance criteria?

Conformity

📋 Does this MR need a changelog?
- I have included a changelog entry for the database migrations (but note that the feature is behind a feature flag).
- I have not included a changelog entry because _____.
Documentation (if required)
Code review guidelines
Merge request performance guidelines
Style guides
Database guides
Separation of EE specific content

Availability and Testing

Review and add/update tests for this feature/bug. Consider all test levels. See the Test Planning Process.

Edited Mar 25, 2021 by Michael Kozono