Commit 0d00d02e authored by Shinya Maeda's avatar Shinya Maeda 💍

Directly refer application code from migration code

parent b626fcc0
......@@ -55,6 +55,11 @@ module Ci
where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)',
'', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive)
end
scope :without_archived_trace, ->() do
where('NOT EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace)
end
scope :with_artifacts_stored_locally, -> { with_artifacts_archive.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) }
scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) }
......
......@@ -2,7 +2,7 @@ class ArchiveLegacyTraces < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
BATCH_SIZE = 10_000
BATCH_SIZE = 5000
BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces'
disable_ddl_transaction!
......@@ -14,25 +14,14 @@ class ArchiveLegacyTraces < ActiveRecord::Migration
scope :finished, -> { where(status: [:success, :failed, :canceled]) }
scope :without_new_traces, ->() do
where('NOT EXISTS (?)',
::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
scope :without_archived_trace, -> do
where('NOT EXISTS (SELECT 1 FROM ci_job_artifacts WHERE ci_builds.id = ci_job_artifacts.job_id AND ci_job_artifacts.file_type = 3)')
end
end
class JobArtifact < ActiveRecord::Base
self.table_name = 'ci_job_artifacts'
enum file_type: {
archive: 1,
metadata: 2,
trace: 3
}
end
def up
queue_background_migration_jobs_by_range_at_intervals(
::ArchiveLegacyTraces::Build.finished.without_new_traces,
::ArchiveLegacyTraces::Build.finished.without_archived_trace,
BACKGROUND_MIGRATION_CLASS,
5.minutes,
batch_size: BATCH_SIZE)
......
......@@ -5,73 +5,18 @@
module Gitlab
module BackgroundMigration
class ArchiveLegacyTraces
class Build < ActiveRecord::Base
include ::HasStatus
self.table_name = 'ci_builds'
self.inheritance_column = :_type_disabled # Disable STI
belongs_to :project, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Project'
has_one :job_artifacts_trace, -> () { where(file_type: ArchiveLegacyTraces::JobArtifact.file_types[:trace]) }, class_name: 'ArchiveLegacyTraces::JobArtifact', foreign_key: :job_id
has_many :trace_chunks, foreign_key: :build_id, class_name: 'ArchiveLegacyTraces::BuildTraceChunk'
scope :finished, -> { where(status: [:success, :failed, :canceled]) }
scope :without_new_traces, ->() do
finished.where('NOT EXISTS (?)',
BackgroundMigration::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
end
def trace
::Gitlab::Ci::Trace.new(self)
end
def trace=(data)
raise NotImplementedError
end
def old_trace
read_attribute(:trace)
end
def erase_old_trace!
update_column(:trace, nil)
end
end
class JobArtifact < ActiveRecord::Base
self.table_name = 'ci_job_artifacts'
belongs_to :build
belongs_to :project
mount_uploader :file, JobArtifactUploader
enum file_type: {
archive: 1,
metadata: 2,
trace: 3
}
end
class BuildTraceChunk < ActiveRecord::Base
self.table_name = 'ci_build_trace_chunks'
belongs_to :build
end
class Project < ActiveRecord::Base
self.table_name = 'projects'
has_many :builds, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Build'
end
def perform(start_id, stop_id)
BackgroundMigration::ArchiveLegacyTraces::Build
.finished
.without_new_traces
.where(id: (start_id..stop_id)).find_each do |build|
build.trace.archive!
# This background migrations directly refer ::Ci::Build model which is defined in application code.
# In general, migration code should be isolated as much as possible in order to be idempotent.
# However, `archive!` logic is too complicated to be replicated. So we chose a way to refer ::Ci::Build directly
# and we don't change the `archive!` logic until 11.1
::Ci::Build.finished.without_archived_trace
.where(id: start_id..stop_id).find_each do |build|
begin
build.trace.archive!
rescue => e
Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}"
end
end
end
end
......
......@@ -8,9 +8,7 @@ namespace :gitlab do
logger = Logger.new(STDOUT)
logger.info('Archiving legacy traces')
Ci::Build.finished
.where('NOT EXISTS (?)',
Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
Ci::Build.finished.without_archived_trace
.order(id: :asc)
.find_in_batches(batch_size: 1000) do |jobs|
job_ids = jobs.map { |job| [job.id] }
......
......@@ -14,7 +14,7 @@ describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 2
@legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path,
build.created_at.utc.strftime("%Y_%m"),
build.project_id.to_s)
FileUtils.mkdir_p(@legacy_trace_dir)
@legacy_trace_path = File.join(@legacy_trace_dir, "#{build.id}.log")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment