Commit 4f7e3503 authored by Shinya Maeda's avatar Shinya Maeda

Add background migration for legacy artifacts

parent 29cfa7bf
......@@ -25,6 +25,11 @@ module Ci
trace: 3
}
enum path_type: {
era_2: nil,
era_1: 1
}
def update_file_store
# The file.object_store is set during `uploader.store!`
# which happens after object is inserted/updated
......@@ -61,7 +66,7 @@ module Ci
end
def update_project_statistics_after_destroy
update_project_statistics(-self.size)
update_project_statistics(-self.size) if self.size
end
def update_project_statistics(difference)
......
......@@ -31,11 +31,15 @@ class JobArtifactUploader < GitlabUploader
creation_date = model.created_at.utc.strftime('%Y_%m_%d')
File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
creation_date, model.job_id.to_s, model.id.to_s)
if model.era_2
File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
creation_date, model.job_id.to_s, model.id.to_s)
elsif model.era_1
File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.id.to_s)
end
end
def disk_hash
@disk_hash ||= Digest::SHA2.hexdigest(model.project_id.to_s)
@disk_hash ||= Digest::SHA2.hexdigest(cproject_id.to_s)
end
end
class AddLegacyPathToCiJobArtifacts < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :ci_job_artifacts, :path_type, :integer
end
end
class MigrateLegacyArtifactsToJobArtifacts < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
MIGRATION = 'MigrateLegacyArtifacts'.freeze
BATCH_SIZE = 500
disable_ddl_transaction!
class Build < ActiveRecord::Base
include EachBatch
self.table_name = 'ci_builds'
self.inheritance_column = :_type_disabled # disable STI
scope :legacy_artifacts, -> { where('artifacts_file IS NOT NULL AND artifacts_file <> ?', '') }
end
def up
disable_statement_timeout
MigrateLegacyArtifactsToJobArtifacts::Build.legacy_artifacts.tap do |relation|
queue_background_migration_jobs_by_range_at_intervals(relation,
MIGRATION,
5.minutes,
batch_size: BATCH_SIZE)
end
end
def down
# There's nothing to revert for this migration.
end
end
......@@ -382,6 +382,7 @@ ActiveRecord::Schema.define(version: 20180603190921) do
t.datetime_with_timezone "expire_at"
t.string "file"
t.binary "file_sha256"
t.integer "path_type"
end
add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree
......
# frozen_string_literal: true
# rubocop:disable Metrics/AbcSize
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class MigrateLegacyArtifacts
class Build < ActiveRecord::Base
self.table_name = 'ci_builds'
self.inheritance_column = :_type_disabled
scope :legacy_artifacts, -> { where('artifacts_file IS NOT NULL AND artifacts_file <> ?', '') }
end
class JobArtifact < ActiveRecord::Base
self.table_name = 'ci_job_artifacts'
enum file_type: {
archive: 1,
metadata: 2,
trace: 3
}
enum path_type: {
era_2: nil,
era_1: 1
}
end
def perform(start_id, stop_id)
rows = []
Gitlab::BackgroundMigration::MigrateLegacyArtifacts::Build.legacy_artifacts
.where(id: (start_id..stop_id))
.each do |build|
base_param = {
project_id: build.project_id,
job_id: build.id,
expire_at: build.artifacts_expire_at,
path_type: Gitlab::BackgroundMigration::MigrateLegacyArtifacts::JobArtifact.path_types['era_1'],
created_at: build.created_at,
updated_at: build.created_at
}
rows << base_param.merge({
size: build.artifacts_size,
file: build.artifacts_file,
file_store: build.artifacts_file_store,
file_type: Gitlab::BackgroundMigration::MigrateLegacyArtifacts::JobArtifact.file_types['archive'],
file_sha256: nil # `file_sha256` of legacy artifacts had not been persisted
})
if build.legacy_artifacts_metadata.exists?
rows << base_param.merge({
size: nil, # `size`` of legacy metadatas had not been persisted
file: build.artifacts_metadata,
file_store: build.artifacts_metadata_store,
file_type: Gitlab::BackgroundMigration::MigrateLegacyArtifacts::JobArtifact.file_types['metadata'],
file_sha256: nil # `file_sha256` of legacy artifacts had not been persisted
})
end
end
Gitlab::Database.bulk_insert(
Gitlab::BackgroundMigration::MigrateLegacyArtifacts::JobArtifact.table_name,
rows)
end
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment