Geo: Add snippet repository verification [RUN ALL RSPEC] [RUN AS-IF-FOSS] (!56596) · Merge requests · GitLab.org / GitLab

What does this MR do?

This MR verifies snippet repositories which are replicated by Geo.

It looks big but the diff is mostly DB schema and documentation changes, so I prefer to keep it in one MR.

Screenshots (strongly suggested)

Migration output

Up:

== 20210313045617 AddVerificationStateAndStartedAtToSnippetRepositories: migrating 
-- change_table(:snippet_repositories)
   -> 0.0079s
== 20210313045617 AddVerificationStateAndStartedAtToSnippetRepositories: migrated (0.0079s) 

== 20210313045845 AddVerificationIndexesToSnippetRepositories: migrating ======
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repositories, :verification_state, {:name=>"index_snippet_repositories_verification_state", :algorithm=>:concurrently})
   -> 0.0022s
-- execute("SET statement_timeout TO 0")
   -> 0.0007s
-- add_index(:snippet_repositories, :verification_state, {:name=>"index_snippet_repositories_verification_state", :algorithm=>:concurrently})
   -> 0.0071s
-- execute("RESET ALL")
   -> 0.0007s
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repositories, :verified_at, {:where=>"(verification_state = 0)", :order=>{:verified_at=>"ASC NULLS FIRST"}, :name=>"index_snippet_repositories_pending_verification", :algorithm=>:concurrently})
   -> 0.0016s
-- add_index(:snippet_repositories, :verified_at, {:where=>"(verification_state = 0)", :order=>{:verified_at=>"ASC NULLS FIRST"}, :name=>"index_snippet_repositories_pending_verification", :algorithm=>:concurrently})
   -> 0.0035s
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repositories, :verification_retry_at, {:where=>"(verification_state = 3)", :order=>{:verification_retry_at=>"ASC NULLS FIRST"}, :name=>"index_snippet_repositories_failed_verification", :algorithm=>:concurrently})
   -> 0.0019s
-- add_index(:snippet_repositories, :verification_retry_at, {:where=>"(verification_state = 3)", :order=>{:verification_retry_at=>"ASC NULLS FIRST"}, :name=>"index_snippet_repositories_failed_verification", :algorithm=>:concurrently})
   -> 0.0031s
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repositories, :verification_state, {:where=>"(verification_state = 0 OR verification_state = 3)", :name=>"index_snippet_repositories_needs_verification", :algorithm=>:concurrently})
   -> 0.0026s
-- add_index(:snippet_repositories, :verification_state, {:where=>"(verification_state = 0 OR verification_state = 3)", :name=>"index_snippet_repositories_needs_verification", :algorithm=>:concurrently})
   -> 0.0028s
== 20210313045845 AddVerificationIndexesToSnippetRepositories: migrated (0.0301s)

Geo tracking database up:

== 20210313050709 AddVerificationToSnippetRepositoryRegistry: migrating =======
-- add_column(:snippet_repository_registry, :verification_started_at, :datetime_with_timezone)
   -> 0.0022s
-- add_column(:snippet_repository_registry, :verified_at, :datetime_with_timezone)
   -> 0.0010s
-- add_column(:snippet_repository_registry, :verification_retry_at, :datetime_with_timezone)
   -> 0.0009s
-- add_column(:snippet_repository_registry, :verification_retry_count, :integer)
   -> 0.0009s
-- add_column(:snippet_repository_registry, :verification_state, :integer, {:limit=>2, :default=>0, :null=>false})
   -> 0.0016s
-- add_column(:snippet_repository_registry, :checksum_mismatch, :boolean)
   -> 0.0008s
-- add_column(:snippet_repository_registry, :verification_checksum, :binary)
   -> 0.0009s
-- add_column(:snippet_repository_registry, :verification_checksum_mismatched, :binary)
   -> 0.0009s
-- add_column(:snippet_repository_registry, :verification_failure, :string, {:limit=>255})
   -> 0.0010s
== 20210313050709 AddVerificationToSnippetRepositoryRegistry: migrated (0.0104s) 

== 20210313051642 AddVerificationIndexesToSnippetRepositoryRegistry: migrating 
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repository_registry, :verified_at, {:where=>"(state = 2 AND verification_state = 0)", :order=>{:verified_at=>"ASC NULLS FIRST"}, :name=>"snippet_repository_registry_pending_verification", :algorithm=>:concurrently})
   -> 0.0026s
-- add_index(:snippet_repository_registry, :verified_at, {:where=>"(state = 2 AND verification_state = 0)", :order=>{:verified_at=>"ASC NULLS FIRST"}, :name=>"snippet_repository_registry_pending_verification", :algorithm=>:concurrently})
   -> 0.0047s
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repository_registry, :verification_retry_at, {:where=>"(state = 2 AND verification_state = 3)", :order=>{:verification_retry_at=>"ASC NULLS FIRST"}, :name=>"snippet_repository_registry_failed_verification", :algorithm=>:concurrently})
   -> 0.0014s
-- add_index(:snippet_repository_registry, :verification_retry_at, {:where=>"(state = 2 AND verification_state = 3)", :order=>{:verification_retry_at=>"ASC NULLS FIRST"}, :name=>"snippet_repository_registry_failed_verification", :algorithm=>:concurrently})
   -> 0.0017s
-- transaction_open?()
   -> 0.0000s
-- index_exists?(:snippet_repository_registry, :verification_state, {:where=>"(state = 2 AND (verification_state IN (0, 3)))", :name=>"snippet_repository_registry_needs_verification", :algorithm=>:concurrently})
   -> 0.0015s
-- add_index(:snippet_repository_registry, :verification_state, {:where=>"(state = 2 AND (verification_state IN (0, 3)))", :name=>"snippet_repository_registry_needs_verification", :algorithm=>:concurrently})
   -> 0.0024s
== 20210313051642 AddVerificationIndexesToSnippetRepositoryRegistry: migrated (0.0163s)

Down:

== 20210313045845 AddVerificationIndexesToSnippetRepositories: reverting ======
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repositories)
   -> 0.0036s
-- execute("SET statement_timeout TO 0")
   -> 0.0005s
-- remove_index(:snippet_repositories, {:algorithm=>:concurrently, :name=>"index_snippet_repositories_verification_state"})
   -> 0.0113s
-- execute("RESET ALL")
   -> 0.0008s
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repositories)
   -> 0.0029s
-- remove_index(:snippet_repositories, {:algorithm=>:concurrently, :name=>"index_snippet_repositories_pending_verification"})
   -> 0.0020s
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repositories)
   -> 0.0018s
-- remove_index(:snippet_repositories, {:algorithm=>:concurrently, :name=>"index_snippet_repositories_failed_verification"})
   -> 0.0026s
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repositories)
   -> 0.0015s
-- remove_index(:snippet_repositories, {:algorithm=>:concurrently, :name=>"index_snippet_repositories_needs_verification"})
   -> 0.0023s
== 20210313045845 AddVerificationIndexesToSnippetRepositories: reverted (0.0323s) 

== 20210313045617 AddVerificationStateAndStartedAtToSnippetRepositories: reverting 
-- remove_column(:snippet_repositories, :verification_started_at, :datetime_with_timezone, {})
   -> 0.0038s
-- remove_column(:snippet_repositories, :verification_state, :integer, {:default=>0, :limit=>2, :null=>false})
   -> 0.0025s
== 20210313045617 AddVerificationStateAndStartedAtToSnippetRepositories: reverted (0.0088s)

Geo tracking database down:

== 20210313051642 AddVerificationIndexesToSnippetRepositoryRegistry: reverting 
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repository_registry)
   -> 0.0041s
-- remove_index(:snippet_repository_registry, {:algorithm=>:concurrently, :name=>"snippet_repository_registry_pending_verification"})
   -> 0.0091s
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repository_registry)
   -> 0.0026s
-- remove_index(:snippet_repository_registry, {:algorithm=>:concurrently, :name=>"snippet_repository_registry_failed_verification"})
   -> 0.0156s
-- transaction_open?()
   -> 0.0000s
-- indexes(:snippet_repository_registry)
   -> 0.0019s
-- remove_index(:snippet_repository_registry, {:algorithm=>:concurrently, :name=>"snippet_repository_registry_needs_verification"})
   -> 0.0011s
== 20210313051642 AddVerificationIndexesToSnippetRepositoryRegistry: reverted (0.0370s) 

== 20210313050709 AddVerificationToSnippetRepositoryRegistry: reverting =======
-- remove_column(:snippet_repository_registry, :verification_failure, :string, {:limit=>255})
   -> 0.0024s
-- remove_column(:snippet_repository_registry, :verification_checksum_mismatched, :binary)
   -> 0.0008s
-- remove_column(:snippet_repository_registry, :verification_checksum, :binary)
   -> 0.0012s
-- remove_column(:snippet_repository_registry, :checksum_mismatch, :boolean)
   -> 0.0010s
-- remove_column(:snippet_repository_registry, :verification_state, :integer, {:limit=>2, :default=>0, :null=>false})
   -> 0.0013s
-- remove_column(:snippet_repository_registry, :verification_retry_count, :integer)
   -> 0.0008s
-- remove_column(:snippet_repository_registry, :verification_retry_at, :datetime_with_timezone)
   -> 0.0008s
-- remove_column(:snippet_repository_registry, :verified_at, :datetime_with_timezone)
   -> 0.0008s
-- remove_column(:snippet_repository_registry, :verification_started_at, :datetime_with_timezone)
   -> 0.0008s
== 20210313050709 AddVerificationToSnippetRepositoryRegistry: reverted (0.0383s)

Does this MR meet the acceptance criteria?

Conformity

📋 Does this MR need a changelog?
- I have included a changelog entry for the database migrations (but note that the feature is behind a feature flag).
- I have not included a changelog entry because _____.
Documentation (if required)
Code review guidelines
Merge request performance guidelines
Style guides
Database guides
Separation of EE specific content

Availability and Testing

Review and add/update tests for this feature/bug. Consider all test levels. See the Test Planning Process.

Edited Mar 25, 2021 by Michael Kozono

Geo: Add snippet repository verification [RUN ALL RSPEC] [RUN AS-IF-FOSS]