Migrate pages metadata when asked to serve virtual domain

On-demand migrate `project_pages_metadata` when asked for
namespace/custom domain virtual domain.

Related to
#28781 (comment 217282591).

app/models/namespace.rb

@@ -319,6 +319,12 @@ def pages_virtual_domain
   private
 
   def all_projects_with_pages
+    if all_projects.project_pages_metadata_not_migrated.exists?
+      Gitlab::BackgroundMigration::MigratePagesMetadata.new.perform_on_relation(
+        all_projects.project_pages_metadata_not_migrated
+      )
+    end
+
     all_projects.with_pages_deployed
   end
 

app/models/pages_domain.rb

@@ -194,7 +194,13 @@ def pages_virtual_domain
   private
 
   def pages_deployed?
-    project.project_pages_metadatum&.deployed?
+    return true if project.project_pages_metadatum&.deployed?
+
+    Gitlab::BackgroundMigration::MigratePagesMetadata.new.perform_on_relation(
+      Project.where(id: project_id)
+    )
+
+    project.reset.project_pages_metadatum&.deployed?
   end
 
   def set_verification_code

app/models/project.rb

@@ -434,6 +434,10 @@ class Project < ApplicationRecord
     where('EXISTS (?)', ProjectPagesMetadatum.project_scoped.deployed.select(1))
   end
 
+  scope :project_pages_metadata_not_migrated, -> do
+    where('NOT EXISTS (?)', ProjectPagesMetadatum.project_scoped.select(1))
+  end
+
   enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 }
 
   chronic_duration_attr :build_timeout_human_readable, :build_timeout,

changelogs/unreleased/28781-migrate-pages-metadata-on-demand.yml

+---
+title: Add index on ci_builds for successful Pages deploys
+merge_request: 17204
+author:
+type: added

db/migrate/20190919040324_add_successfull_pages_deploy_partial_index_on_ci_builds.rb

+# frozen_string_literal: true
+
+class AddSuccessfullPagesDeployPartialIndexOnCiBuilds < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(
+      :ci_builds, :project_id,
+      name: 'index_ci_builds_on_project_id_for_successfull_pages_deploy',
+      where: "type='GenericCommitStatus' AND stage='deploy' AND name='pages:deploy' AND status = 'success'"
+    )
+  end
+
+  def down
+    remove_concurrent_index_by_name :ci_builds, 'index_ci_builds_on_project_id_for_successfull_pages_deploy'
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_18_104222) do
+ActiveRecord::Schema.define(version: 2019_09_19_040324) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -639,6 +639,7 @@
     t.index ["name"], name: "index_ci_builds_on_name_for_security_products_values", where: "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text]))"
     t.index ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id"
     t.index ["project_id", "status"], name: "index_ci_builds_project_id_and_status_for_live_jobs_partial2", where: "(((type)::text = 'Ci::Build'::text) AND ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text])))"
+    t.index ["project_id"], name: "index_ci_builds_on_project_id_for_successfull_pages_deploy", where: "(((type)::text = 'GenericCommitStatus'::text) AND ((stage)::text = 'deploy'::text) AND ((name)::text = 'pages:deploy'::text) AND ((status)::text = 'success'::text))"
     t.index ["protected"], name: "index_ci_builds_on_protected"
     t.index ["queued_at"], name: "index_ci_builds_on_queued_at"
     t.index ["runner_id"], name: "index_ci_builds_on_runner_id"

lib/gitlab/background_migration/migrate_pages_metadata.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class that will insert record into project_pages_metadata
+    # for each existing project
+    class MigratePagesMetadata
+      def perform(start_id, stop_id)
+        perform_on_relation(Project.where(id: start_id..stop_id))
+      end
+
+      def perform_on_relation(relation)
+        successful_pages_deploy = <<~SQL
+          SELECT TRUE
+          FROM "ci_builds"
+          WHERE "ci_builds"."type" IN ('GenericCommitStatus')
+            AND ("ci_builds"."status" IN ('success'))
+            AND "ci_builds"."stage" = 'deploy'
+            AND "ci_builds"."name" = 'pages:deploy'
+            AND (ci_builds.project_id = projects.id)
+          LIMIT 1
+        SQL
+
+        select_from = relation
+          .select("projects.id", "COALESCE((#{successful_pages_deploy}), FALSE)")
+          .to_sql
+
+        ActiveRecord::Base.connection_pool.with_connection do |connection|
+          connection.execute <<~SQL
+            INSERT INTO project_pages_metadata (project_id, deployed)
+          #{select_from}
+            ON CONFLICT (project_id) DO NOTHING
+          SQL
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MigratePagesMetadata, :migration, schema: 20190919040324 do
+  let(:projects) { table(:projects) }
+
+  subject(:migrate_pages_metadata) { described_class.new }
+
+  describe '#perform_on_relation' do
+    let(:namespaces) { table(:namespaces) }
+    let(:builds) { table(:ci_builds) }
+    let(:pages_metadata) { table(:project_pages_metadata) }
+
+    it 'marks specified projects with successful pages deployment' do
+      namespace = namespaces.create!(name: 'gitlab', path: 'gitlab-org')
+      not_migrated_with_pages = projects.create!(namespace_id: namespace.id, name: 'Not Migrated With Pages')
+      builds.create!(project_id: not_migrated_with_pages.id, type: 'GenericCommitStatus', status: 'success', stage: 'deploy', name: 'pages:deploy')
+
+      migrated = projects.create!(namespace_id: namespace.id, name: 'Migrated')
+      pages_metadata.create!(project_id: migrated.id, deployed: true)
+
+      not_migrated_no_pages = projects.create!(namespace_id: namespace.id, name: 'Not Migrated No Pages')
+      other_project = projects.create!(namespace_id: namespace.id, name: 'Other')
+
+      projects_relation = projects.where(id: [not_migrated_with_pages, not_migrated_no_pages, migrated])
+
+      migrate_pages_metadata.perform_on_relation(projects_relation)
+
+      expect(pages_metadata.find_by_project_id(not_migrated_with_pages.id).deployed).to eq(true)
+      expect(pages_metadata.find_by_project_id(not_migrated_no_pages.id).deployed).to eq(false)
+      expect(pages_metadata.find_by_project_id(migrated.id).deployed).to eq(true)
+      expect(pages_metadata.find_by_project_id(other_project.id)).to be_nil
+    end
+  end
+
+  describe '#perform' do
+    it 'creates relation and delegates to #perform_on_relation' do
+      expect(migrate_pages_metadata).to receive(:perform_on_relation).with(projects.where(id: 3..5))
+
+      migrate_pages_metadata.perform(3, 5)
+    end
+  end
+end

spec/models/namespace_spec.rb

@@ -928,12 +928,34 @@ def project_rugged(project)
     let(:project) { create(:project, namespace: namespace) }
 
     context 'when there are pages deployed for the project' do
-      before do
-        project.mark_pages_as_deployed
+      context 'but pages metadata is not migrated' do
+        before do
+          generic_commit_status = create(:generic_commit_status, :success, stage: 'deploy', name: 'pages:deploy')
+          generic_commit_status.update!(project: project)
+          project.project_pages_metadatum.destroy!
+        end
+
+        it 'migrates pages metadata anbd returns the virual domain' do
+          virtual_domain = namespace.pages_virtual_domain
+
+          expect(project.reload.project_pages_metadatum.deployed).to eq(true)
+
+          expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain)
+          expect(virtual_domain.lookup_paths).not_to be_empty
+        end
       end
 
-      it 'returns the virual domain' do
-        expect(namespace.pages_virtual_domain).to be_an_instance_of(Pages::VirtualDomain)
+      context 'and pages metadata is migrated' do
+        before do
+          project.mark_pages_as_deployed
+        end
+
+        it 'returns the virual domain' do
+          virtual_domain = namespace.pages_virtual_domain
+
+          expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain)
+          expect(virtual_domain.lookup_paths).not_to be_empty
+        end
       end
     end
   end

spec/models/pages_domain_spec.rb

@@ -569,8 +569,9 @@
 
     context 'when there are pages deployed for the project' do
       before do
-        project.mark_pages_as_deployed
-        project.reload
+        generic_commit_status = create(:generic_commit_status, :success, stage: 'deploy', name: 'pages:deploy')
+        generic_commit_status.update!(project: project)
+        project.project_pages_metadatum.destroy!
       end
 
       it 'returns the virual domain' do
@@ -578,6 +579,12 @@
 
         expect(pages_domain.pages_virtual_domain).to be_an_instance_of(Pages::VirtualDomain)
       end
+
+      it 'migrates project pages metadata' do
+        expect { pages_domain.pages_virtual_domain }.to change {
+          project.reload.project_pages_metadatum&.deployed
+        }.from(nil).to(true)
+      end
     end
   end
 end