...
 
Commits (4)
......@@ -322,6 +322,7 @@ group :development do
gem 'letter_opener_web', '~> 1.3.4'
gem 'rblineprof', '~> 0.3.6', platform: :mri, require: false
gem 'active_record-pg_generate_series', '~> 0.1.3'
# Better errors handler
gem 'better_errors', '~> 2.5.0'
......
......@@ -29,6 +29,8 @@ GEM
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
active_record-pg_generate_series (0.1.3)
activerecord
activejob (5.2.3)
activesupport (= 5.2.3)
globalid (>= 0.3.6)
......@@ -1051,6 +1053,7 @@ DEPENDENCIES
RedCloth (~> 4.3.2)
ace-rails-ap (~> 4.1.0)
acme-client (~> 2.0.2)
active_record-pg_generate_series (~> 0.1.3)
activerecord-explain-analyze (~> 0.1)
acts-as-taggable-on (~> 6.0)
addressable (~> 2.5.2)
......
......@@ -31,6 +31,7 @@ class Project < ApplicationRecord
include FeatureGate
include OptionallySearch
include FromUnion
include EachBatch
extend Gitlab::Cache::RequestCache
extend Gitlab::ConfigHelper
......
class Gitlab::Seeder::Users
include ActionView::Helpers::NumberHelper
RANDOM_USERS_COUNT = 20
MASS_USERS_COUNT = 1_000_000
attr_reader :opts
def initialize(opts = {})
@opts = opts
end
def seed!
Sidekiq::Testing.inline! do
create_random_users!
create_mass_users!
end
end
private
def create_random_users!
RANDOM_USERS_COUNT.times do |i|
begin
User.create!(
username: FFaker::Internet.user_name,
name: FFaker::Name.name,
email: FFaker::Internet.email,
confirmed_at: DateTime.now,
password: '12345678'
)
print '.'
rescue ActiveRecord::RecordInvalid
print 'F'
end
end
end
def create_mass_users!
encrypted_password = Devise::Encryptor.digest(User, '12345678')
Gitlab::Seeder.with_mass_insert(MASS_USERS_COUNT, User) do
User.insert_using_generate_series(1, MASS_USERS_COUNT) do |sql|
sql.username = raw("'seed_user' || seq")
sql.name = raw("'Seed user ' || seq")
sql.email = raw("'seed_user' || seq || '@example.com'")
sql.confirmed_at = raw("('2019-09-10'::date + seq)")
sql.projects_limit = 10_000_000 # no limit
sql.encrypted_password = encrypted_password
end
end
# We can't use a sub-query here given we want to insert it just for the new
# namespaces.
Gitlab::Seeder.with_mass_insert(MASS_USERS_COUNT, Namespace, :batch) do
existing_namespaces = Namespace.pluck(:id)
User.where.not(id: existing_namespaces).find_in_batches(batch_size: 1_000) do |users|
rows = users.map do |user|
{
name: user.username,
path: user.username,
owner_id: user.id
}
end
Gitlab::Database.bulk_insert('namespaces', rows)
print '.'
end
end
end
end
Gitlab::Seeder.quiet do
users = Gitlab::Seeder::Users.new
users.seed!
end
This diff is collapsed.
......@@ -3,47 +3,45 @@
require 'digest/md5'
class Gitlab::Seeder::GroupLabels
def initialize(group, label_per_group: 10)
@group = group
@label_per_group = label_per_group
end
MASS_LABELS_COUNT = 200 # per group
def seed!
@label_per_group.times do
label_title = FFaker::Product.brand
Labels::CreateService
.new(title: label_title, color: "##{Digest::MD5.hexdigest(label_title)[0..5]}")
.execute(group: @group)
print '.'
end
end
end
class Gitlab::Seeder::ProjectLabels
def initialize(project, label_per_project: 5)
@project = project
@label_per_project = label_per_project
end
include ActionView::Helpers::NumberHelper
PROJECT_LIMIT = 500_000
MASS_LABELS_COUNT = 50 # per project
def seed!
@label_per_project.times do
label_title = FFaker::Vehicle.model
Labels::CreateService
.new(title: label_title, color: "##{Digest::MD5.hexdigest(label_title)[0..5]}")
.execute(project: @project)
print '.'
relation = Project.select(:id).limit(PROJECT_LIMIT)
total_labels = relation.count * MASS_LABELS_COUNT
Gitlab::Seeder.with_mass_insert(total_labels, Label) do
relation.find_in_batches(batch_size: 500) do |projects|
rows = projects.flat_map do |project|
MASS_LABELS_COUNT.times.map do
label_title = FFaker::Product.brand
{
title: label_title,
color: "##{Digest::MD5.hexdigest(label_title)[0..5]}",
project_id: project.id,
type: 'ProjectLabel'
}
end
end
Gitlab::Database.bulk_insert('labels', rows)
print '.'
end
end
end
end
Gitlab::Seeder.quiet do
puts "\nGenerating group labels"
Group.all.find_each do |group|
Gitlab::Seeder::GroupLabels.new(group).seed!
end
puts "\nGenerating project labels"
Project.all.find_each do |project|
Gitlab::Seeder::ProjectLabels.new(project).seed!
end
project_labels = Gitlab::Seeder::ProjectLabels.new
project_labels.seed!
end
require './spec/support/sidekiq'
Gitlab::Seeder.quiet do
20.times do |i|
begin
User.create!(
username: FFaker::Internet.user_name,
name: FFaker::Name.name,
email: FFaker::Internet.email,
confirmed_at: DateTime.now,
password: '12345678'
)
print '.'
rescue ActiveRecord::RecordInvalid
print 'F'
end
end
5.times do |i|
begin
User.create!(
username: "user#{i}",
name: "User #{i}",
email: "user#{i}@example.com",
confirmed_at: DateTime.now,
password: '12345678'
)
print '.'
rescue ActiveRecord::RecordInvalid
print 'F'
end
end
end
......@@ -14,7 +14,40 @@ end
module Gitlab
class Seeder
extend ActionView::Helpers::NumberHelper
ESTIMATED_INSERT_PER_MINUTE = 2_000_000
def self.with_mass_insert(size, model, strategy = :series)
humanized_size = number_with_delimiter(size)
humanized_model_name = model.model_name.human.pluralize(size)
estimative = humanized_insert_time_message(size, strategy)
puts "\nCreating #{humanized_size} #{humanized_model_name} (#{strategy} strategy)."
puts estimative
yield
puts "\n#{number_with_delimiter(size)} #{humanized_model_name} created!"
end
def self.humanized_insert_time_message(size, strategy)
estimated_minutes = (size.to_f / ESTIMATED_INSERT_PER_MINUTE).round
estimated_minutes = estimated_minutes * 3 if strategy == :batch
humanized_minutes = 'minute'.pluralize(estimated_minutes)
if estimated_minutes.zero?
"Estimated time: less than a minute ⏰"
else
"Estimated time: #{estimated_minutes} #{humanized_minutes} ⏰"
end
end
def self.quiet
# Disable database insertion logs so speed isn't limited by ability to print to console
old_logger = ActiveRecord::Base.logger
ActiveRecord::Base.logger = nil
mute_notifications
mute_mailer
......@@ -23,6 +56,7 @@ module Gitlab
yield
SeedFu.quiet = false
ActiveRecord::Base.logger = old_logger
puts "\nOK".color(:green)
end
......