Commit b9208980 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'stop-using-sidekiq-cron-for-push-mirrors' into 'master'

implements new logic for push mirrors to update on push

See merge request !1616
parents 0cbc32ba c54e08cf
Pipeline #7996658 failed with stages
in 143 minutes and 6 seconds
......@@ -49,6 +49,6 @@ class Projects::MirrorsController < Projects::ApplicationController
def mirror_params
params.require(:project).permit(:mirror, :import_url, :mirror_user_id,
:mirror_trigger_builds, :sync_time,
remote_mirrors_attributes: [:url, :id, :enabled, :sync_time])
remote_mirrors_attributes: [:url, :id, :enabled])
end
end
......@@ -182,7 +182,7 @@ class ApplicationSetting < ActiveRecord::Base
before_save :ensure_runners_registration_token
before_save :ensure_health_check_access_token
after_update :update_mirror_cron_jobs, if: :minimum_mirror_sync_time_changed?
after_update :update_mirror_cron_job, if: :minimum_mirror_sync_time_changed?
after_commit do
Rails.cache.write(CACHE_KEY, self)
......@@ -291,13 +291,11 @@ class ApplicationSetting < ActiveRecord::Base
end
end
def update_mirror_cron_jobs
def update_mirror_cron_job
Project.mirror.where('sync_time < ?', minimum_mirror_sync_time)
.update_all(sync_time: minimum_mirror_sync_time)
RemoteMirror.where('sync_time < ?', minimum_mirror_sync_time)
.update_all(sync_time: minimum_mirror_sync_time)
Gitlab::Mirror.configure_cron_jobs!
Gitlab::Mirror.configure_cron_job!
end
def elasticsearch_url
......
......@@ -639,6 +639,14 @@ class Project < ActiveRecord::Base
remote_mirrors.each(&:sync)
end
def mark_stuck_remote_mirrors_as_failed!
remote_mirrors.stuck.update_all(
update_status: :failed,
last_error: 'The remote mirror took to long to complete.',
last_update_at: Time.now
)
end
def fetch_mirror
return unless mirror?
......
class RemoteMirror < ActiveRecord::Base
include AfterCommitQueue
include IgnorableColumn
ignore_column :sync_time
BACKOFF_DELAY = 5.minutes
attr_encrypted :credentials,
key: Gitlab::Application.secrets.db_key_base,
......@@ -12,9 +17,6 @@ class RemoteMirror < ActiveRecord::Base
belongs_to :project, inverse_of: :remote_mirrors
validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true }
validates :sync_time,
presence: true,
inclusion: { in: Gitlab::Mirror::SYNC_TIME_OPTIONS.values }
validate :url_availability, if: -> (mirror) { mirror.url_changed? || mirror.enabled? }
......@@ -28,7 +30,7 @@ class RemoteMirror < ActiveRecord::Base
state_machine :update_status, initial: :none do
event :update_start do
transition [:none, :finished] => :started
transition [:none, :finished, :failed] => :started
end
event :update_finish do
......@@ -39,24 +41,22 @@ class RemoteMirror < ActiveRecord::Base
transition started: :failed
end
event :update_retry do
transition failed: :started
end
state :started
state :finished
state :failed
after_transition any => :started, do: :schedule_update_job
after_transition any => :started do |remote_mirror, _|
remote_mirror.update(last_update_started_at: Time.now)
end
after_transition started: :finished do |remote_mirror, transaction|
after_transition started: :finished do |remote_mirror, _|
timestamp = Time.now
remote_mirror.update_attributes!(
last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil
)
end
after_transition started: :failed do |remote_mirror, transaction|
after_transition started: :failed do |remote_mirror, _|
remote_mirror.update(last_update_at: Time.now)
end
end
......@@ -74,10 +74,13 @@ class RemoteMirror < ActiveRecord::Base
end
def sync
return unless project
return if !enabled || update_in_progress?
return unless project && enabled
update_failed? ? update_retry : update_start
RepositoryUpdateRemoteMirrorWorker.perform_in(BACKOFF_DELAY, self.id, Time.now) if project&.repository_exists?
end
def updated_since?(timestamp)
last_update_started_at && last_update_started_at > timestamp && !update_failed?
end
def mark_for_delete_if_blank_url
......@@ -130,16 +133,6 @@ class RemoteMirror < ActiveRecord::Base
)
end
def schedule_update_job
run_after_commit(:add_update_job)
end
def add_update_job
if project && project.repository_exists?
RepositoryUpdateRemoteMirrorWorker.perform_async(self.id)
end
end
def refresh_remote
return unless project
......
......@@ -59,6 +59,7 @@ class GitPushService < BaseService
execute_related_hooks
perform_housekeeping
update_remote_mirrors
update_caches
end
......@@ -96,6 +97,13 @@ class GitPushService < BaseService
protected
def update_remote_mirrors
return if @project.remote_mirrors.empty?
@project.mark_stuck_remote_mirrors_as_failed!
@project.update_remote_mirrors
end
def execute_related_hooks
# Update merge requests that may be affected by this push. A new branch
# could cause the last commit of a merge request to change.
......
......@@ -52,8 +52,8 @@
%h4.prepend-top-0
Push to a remote repository
%p.light
Set up the remote repository that you want to update with the content
of the current repository every hour.
Set up the remote repository that you want to update with the content of the current repository
every time someone pushes to it.
= link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pushing-to-a-remote-repository'), target: '_blank'
.col-lg-9
= render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror
......@@ -73,13 +73,10 @@
.prepend-left-20
= rm_form.label :enabled, "Remote mirror repository", class: "label-light append-bottom-0"
%p.light.append-bottom-0
Automatically update the remote mirror's branches, tags, and commits from this repository every hour.
Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it.
.form-group.has-feedback
= rm_form.label :url, "Git repository URL", class: "label-light"
= rm_form.text_field :url, class: "form-control", placeholder: 'https://username:password@gitlab.company.com/group/project.git'
= render "projects/mirrors/instructions"
.form-group
= rm_form.label :sync_time, "Synchronization time", class: "label-light append-bottom-0"
= rm_form.select :sync_time, options_for_select(mirror_sync_time_options, @remote_mirror.sync_time), {}, class: 'form-control remote-mirror-sync-time'
= f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'
%hr
class RepositoryUpdateRemoteMirrorWorker
UpdateRemoteMirrorError = Class.new(StandardError)
UpdateAlreadyInProgressError = Class.new(StandardError)
UpdateError = Class.new(StandardError)
include Sidekiq::Worker
include Gitlab::ShellAdapter
sidekiq_options queue: :project_mirror, retry: false
def perform(remote_mirror_id)
begin
remote_mirror = RemoteMirror.find(remote_mirror_id)
project = remote_mirror.project
current_user = project.creator
result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror)
if result[:status] == :error
remote_mirror.mark_as_failed(result[:message])
else
remote_mirror.update_finish
end
rescue => ex
remote_mirror.mark_as_failed("We're sorry, a temporary error occurred, please try again.")
raise UpdateRemoteMirrorError, "#{ex.class}: #{Gitlab::UrlSanitizer.sanitize(ex.message)}"
end
sidekiq_options queue: :project_mirror, retry: 3, dead: false
sidekiq_retry_in { |count| 30 * count }
sidekiq_retries_exhausted do |msg, _|
Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
end
def perform(remote_mirror_id, scheduled_time)
remote_mirror = RemoteMirror.find(remote_mirror_id)
return if remote_mirror.updated_since?(scheduled_time)
raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress?
remote_mirror.update_start
project = remote_mirror.project
current_user = project.creator
result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror)
raise UpdateError, result[:message] if result[:status] == :error
remote_mirror.update_finish
rescue UpdateAlreadyInProgressError
raise
rescue UpdateError => ex
remote_mirror.mark_as_failed(Gitlab::UrlSanitizer.sanitize(ex.message))
raise
rescue => ex
raise UpdateError, "#{ex.class}: #{ex.message}"
end
end
class UpdateAllRemoteMirrorsWorker
include Sidekiq::Worker
include CronjobQueue
def perform
fail_stuck_mirrors!
remote_mirrors_to_sync.find_each(batch_size: 50).each(&:sync)
end
def fail_stuck_mirrors!
RemoteMirror.stuck.find_each(batch_size: 50) do |remote_mirror|
remote_mirror.mark_as_failed('The mirror update took too long to complete.')
end
end
private
def remote_mirrors_to_sync
RemoteMirror.where("last_successful_update_at + #{Gitlab::Database.minute_interval('sync_time')} <= ? OR sync_time IN (?)", DateTime.now, Gitlab::Mirror.sync_times)
end
end
---
title: Stop using sidekiq cron for push mirrors
merge_request: 1616
author:
......@@ -40,7 +40,7 @@ Sidekiq.configure_server do |config|
end
Sidekiq::Cron::Job.load_from_hash! cron_jobs
Gitlab::Mirror.configure_cron_jobs!
Gitlab::Mirror.configure_cron_job!
Gitlab::Geo.configure_cron_jobs!
......
class AddLastUpdateStartedAtColumnToRemoteMirrors < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :remote_mirrors, :last_update_started_at, :datetime
end
end
class RemoveSyncTimeColumnFromRemoteMirrors < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
remove_concurrent_index :remote_mirrors, [:sync_time] if index_exists? :remote_mirrors, [:sync_time]
remove_column :remote_mirrors, :sync_time, :integer
end
def down
add_column :remote_mirrors, :sync_time, :integer
add_concurrent_index :remote_mirrors, [:sync_time]
end
end
......@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20170426175636) do
ActiveRecord::Schema.define(version: 20170427180205) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......@@ -1241,12 +1241,11 @@ ActiveRecord::Schema.define(version: 20170426175636) do
t.string "encrypted_credentials_salt"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.integer "sync_time", default: 60, null: false
t.datetime "last_update_started_at"
end
add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree
add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
add_index "remote_mirrors", ["sync_time"], name: "index_remote_mirrors_on_sync_time", using: :btree
create_table "routes", force: :cascade do |t|
t.integer "source_id", null: false
......@@ -1624,4 +1623,4 @@ ActiveRecord::Schema.define(version: 20170426175636) do
add_foreign_key "timelogs", "merge_requests", name: "fk_timelogs_merge_requests_merge_request_id", on_delete: :cascade
add_foreign_key "trending_projects", "projects", on_delete: :cascade
add_foreign_key "u2f_registrations", "users"
end
\ No newline at end of file
end
......@@ -71,9 +71,10 @@ repository to push to. Hit **Save changes** for the changes to take effect.
Similarly to the pull mirroring, since the upstream repository functions as a
mirror to the repository in GitLab, you are advised not to push commits directly
to the mirrored repository. Instead, any commits should be pushed to GitLab,
and will end up in the mirrored repository automatically within the configured time,
or when a [forced update](#forcing-an-update) is initiated.
to the mirrored repository. Instead, all changes will end up in the mirrored repository
whenever commits are pushed to GitLab, or when a [forced update](#forcing-an-update) is initiated.
Pushes into GitLab are automatically pushed to the remote mirror 5 minutes after they come in.
In case of a diverged branch, you will see an error indicated at the
**Mirror repository** settings.
......
......@@ -41,22 +41,12 @@ module Gitlab
sync_times
end
def configure_cron_jobs!
def configure_cron_job!
minimum_mirror_sync_time = current_application_settings.minimum_mirror_sync_time rescue FIFTEEN
sync_time = SYNC_TIME_TO_CRON[minimum_mirror_sync_time]
update_all_mirrors_worker_job = Sidekiq::Cron::Job.find("update_all_mirrors_worker")
update_all_remote_mirrors_worker_job = Sidekiq::Cron::Job.find("update_all_remote_mirrors_worker")
if update_all_mirrors_worker_job && update_all_remote_mirrors_worker_job
update_all_mirrors_worker_job.destroy
update_all_remote_mirrors_worker_job.destroy
end
Sidekiq::Cron::Job.find("update_all_mirrors_worker")&.destroy
Sidekiq::Cron::Job.create(
name: 'update_all_remote_mirrors_worker',
cron: sync_time,
class: 'UpdateAllRemoteMirrorsWorker'
)
Sidekiq::Cron::Job.create(
name: 'update_all_mirrors_worker',
cron: sync_time,
......
......@@ -41,24 +41,6 @@ describe Projects::MirrorsController do
end.to change { RemoteMirror.count }.to(1)
end
context 'sync_time update' do
it 'allows sync_time update with valid time' do
sync_times.each do |sync_time|
expect do
do_put(@project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => 'http://foo.com', 'sync_time' => sync_time } })
end.to change { RemoteMirror.where(sync_time: sync_time).count }.by(1)
end
end
it 'fails to update sync_time with invalid time' do
expect(@project.remote_mirrors.count).to eq(0)
expect do
do_put(@project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => 'http://foo.com', 'sync_time' => 1000 } })
end.not_to change { @project.remote_mirrors.count }
end
end
context 'when remote mirror has the same URL' do
it 'does not allow to create the remote mirror' do
expect do
......
......@@ -65,13 +65,12 @@ FactoryGirl.define do
trait :remote_mirror do
transient do
sync_time Gitlab::Mirror::HOURLY
url "http://foo.com"
enabled true
end
after(:create) do |project, evaluator|
project.remote_mirrors.create!(url: evaluator.url, enabled: evaluator.enabled, sync_time: evaluator.sync_time)
project.remote_mirrors.create!(url: evaluator.url, enabled: evaluator.enabled)
end
end
......
......@@ -30,7 +30,6 @@ feature 'Project mirror', feature: true do
it 'shows the correct selector options' do
expect(page).to have_selector('.project-mirror-sync-time > option', count: index + 1)
expect(page).to have_selector('.remote-mirror-sync-time > option', count: index + 1)
end
end
end
......
This diff is collapsed.
......@@ -89,7 +89,6 @@ describe ApplicationSetting, models: true do
Sidekiq::Logging.logger = nil
sync_times.each do |sync_time|
create(:project, :mirror, sync_time: sync_time)
create(:project, :remote_mirror, sync_time: sync_time)
end
end
......@@ -98,8 +97,8 @@ describe ApplicationSetting, models: true do
subject { setting.update_attributes(minimum_mirror_sync_time: sync_time) }
it "updates minimum mirror sync time to #{sync_time}" do
expect_any_instance_of(ApplicationSetting).to receive(:update_mirror_cron_jobs).and_call_original
expect(Gitlab::Mirror).to receive(:configure_cron_jobs!)
expect_any_instance_of(ApplicationSetting).to receive(:update_mirror_cron_job).and_call_original
expect(Gitlab::Mirror).to receive(:configure_cron_job!)
subject
end
......@@ -107,10 +106,6 @@ describe ApplicationSetting, models: true do
it 'updates every mirror to the current minimum_mirror_sync_time' do
expect { subject }.to change { Project.mirror.where('sync_time < ?', sync_time).count }.from(index + 1).to(0)
end
it 'updates every remote mirror to the current minimum_mirror_sync_time' do
expect { subject }.to change { RemoteMirror.where('sync_time < ?', sync_time).count }.from(index + 1).to(0)
end
end
end
......@@ -119,8 +114,8 @@ describe ApplicationSetting, models: true do
let(:sync_time) { Gitlab::Mirror::FIFTEEN }
it 'does not update minimum_mirror_sync_time' do
expect_any_instance_of(ApplicationSetting).not_to receive(:update_mirror_cron_jobs)
expect(Gitlab::Mirror).not_to receive(:configure_cron_jobs!)
expect_any_instance_of(ApplicationSetting).not_to receive(:update_mirror_cron_job)
expect(Gitlab::Mirror).not_to receive(:configure_cron_job!)
expect(setting.minimum_mirror_sync_time).to eq(Gitlab::Mirror::FIFTEEN)
setting.update_attributes(minimum_mirror_sync_time: sync_time)
......@@ -129,10 +124,6 @@ describe ApplicationSetting, models: true do
it 'updates every mirror to the current minimum_mirror_sync_time' do
expect { setting.update_attributes(minimum_mirror_sync_time: sync_time) }.not_to change { Project.mirror.where('sync_time < ?', sync_time).count }
end
it 'updates every remote mirror to the current minimum_mirror_sync_time' do
expect { setting.update_attributes(minimum_mirror_sync_time: sync_time) }.not_to change { RemoteMirror.where('sync_time < ?', sync_time).count }
end
end
end
......
......@@ -195,6 +195,21 @@ describe Project, models: true do
end
end
context '#mark_stuck_remote_mirrors_as_failed!' do
it 'fails stuck remote mirrors' do
project = create(:project, :remote_mirror)
project.remote_mirrors.first.update_attributes(
update_status: :started,
last_update_at: 2.days.ago
)
expect do
project.mark_stuck_remote_mirrors_as_failed!
end.to change { project.remote_mirrors.stuck.count }.from(1).to(0)
end
end
context 'mirror' do
subject { build(:project, mirror: true) }
......
......@@ -85,6 +85,66 @@ describe RemoteMirror do
end
end
context '#sync' do
let(:remote_mirror) { create(:project, :remote_mirror).remote_mirrors.first }
before do
Timecop.freeze(Time.now)
end
context 'with remote mirroring enabled' do
it 'schedules a RepositoryUpdateRemoteMirrorWorker to run within a certain backoff delay' do
expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::BACKOFF_DELAY, remote_mirror.id, Time.now)
remote_mirror.sync
end
end
context 'with remote mirroring disabled' do
it 'returns nil' do
remote_mirror.update_attributes(enabled: false)
expect(remote_mirror.sync).to be_nil
end
end
context 'without project' do
it 'returns nil' do
allow_any_instance_of(RemoteMirror).to receive(:project).and_return(nil)
expect(remote_mirror.sync).to be_nil
end
end
end
context '#updated_since?' do
let(:remote_mirror) { create(:project, :remote_mirror).remote_mirrors.first }
let(:timestamp) { Time.now - 5.minutes }
before do
Timecop.freeze(Time.now)
remote_mirror.update_attributes(last_update_started_at: Time.now)
end
context 'when remote mirror does not have status failed' do
it 'returns true when last update started after the timestamp' do
expect(remote_mirror.updated_since?(timestamp)).to be true
end
it 'returns false when last update started before the timestamp' do
expect(remote_mirror.updated_since?(Time.now + 5.minutes)).to be false
end
end
context 'when remote mirror has status failed' do
it 'returns false when last update started after the timestamp' do
remote_mirror.update_attributes(update_status: 'failed')
expect(remote_mirror.updated_since?(timestamp)).to be false
end
end
end
context 'no project' do
it 'includes mirror with a project in pending_delete' do
mirror = create_mirror(url: 'http://cantbeblank',
......
This diff is collapsed.
require 'rails_helper'
describe RepositoryUpdateRemoteMirrorWorker do
subject { described_class.new }
let(:remote_mirror) { create(:project, :remote_mirror).remote_mirrors.first }
let(:scheduled_time) { Time.now - 5.minutes }
before do
Timecop.freeze(Time.now)
end
describe '#perform' do
context 'with status none' do
before do
remote_mirror.update_attributes(update_status: 'none')
end
it 'sets status as finished when update remote mirror service executes successfully' do
expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success)
expect { subject.perform(remote_mirror.id, Time.now) }.to change { remote_mirror.reload.update_status }.to('finished')
end
it 'sets status as failed when update remote mirror service executes with errors' do
error_message = 'fail!'
expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :error, message: error_message)
expect do
subject.perform(remote_mirror.id, Time.now)
end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateError, error_message)
expect(remote_mirror.reload.update_status).to eq('failed')
end
it 'does nothing if last_update_started_at is higher than the time the job was scheduled in' do
remote_mirror.update_attributes(last_update_started_at: Time.now)
expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(true)
expect_any_instance_of(Projects::UpdateRemoteMirrorService).not_to receive(:execute).with(remote_mirror)
expect(subject.perform(remote_mirror.id, scheduled_time)).to be_nil
end
end
context 'with another worker already running' do
before do
remote_mirror.update_attributes(update_status: 'started')
end
it 'raises RemoteMirrorUpdateAlreadyInProgressError' do
expect do
subject.perform(remote_mirror.id, Time.now)
end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateAlreadyInProgressError)
end
end
context 'with status failed' do
before do
remote_mirror.update_attributes(update_status: 'failed')
end
it 'sets status as finished if last_update_started_at is higher than the time the job was scheduled in' do
remote_mirror.update_attributes(last_update_started_at: Time.now)
expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(false)
expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success)
expect { subject.perform(remote_mirror.id, scheduled_time) }.to change { remote_mirror.reload.update_status }.to('finished')
end
end
end
end
require 'rails_helper'
describe UpdateAllRemoteMirrorsWorker do
subject(:worker) { described_class.new }
describe "#perform" do
let!(:fifteen_mirror) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::FIFTEEN) }
let!(:hourly_mirror) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::HOURLY) }
let!(:three_mirror) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::THREE) }
let!(:six_mirror) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::SIX) }
let!(:twelve_mirror) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::TWELVE) }
let!(:daily_mirror) { create(:project, :remote_mirror, sync_time: Gitlab::Mirror::DAILY) }
let!(:outdated_mirror) { create(:project, :remote_mirror) }
it 'fails stuck mirrors' do
expect(worker).to receive(:fail_stuck_mirrors!)
worker.perform
end
describe 'sync time' do
def expect_worker_to_enqueue_mirrors(mirrors)
mirrors.each do |mirror|
expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(mirror.id)
end
worker.perform
end
before do
time = DateTime.now.change(time_params)
Timecop.freeze(time)
outdated_mirror.remote_mirrors.first.update_attributes(last_successful_update_at: time - (Gitlab::Mirror::DAILY + 5).minutes)
end
describe 'fifteen' do
let!(:time_params) { { hour: 1, min: 15 } }
let(:mirrors) { [fifteen_mirror, outdated_mirror] }
it { expect_worker_to_enqueue_mirrors(mirrors) }
end
describe "hourly" do
let!(:time_params) { { hour: 1 } }
let(:mirrors) { [fifteen_mirror, hourly_mirror, outdated_mirror] }
it { expect_worker_to_enqueue_mirrors(mirrors) }
end