Getting 502 errors after updated from 10.2.2 to 10.3.0
after run sudo gitlab-ctl reconfigure get below errors
* bash[migrate gitlab-rails database] action run
[execute] rake aborted!
NameError: uninitialized constant UseKeyWorker::DedicatedSidekiqQueue
/opt/gitlab/embedded/service/gitlab-rails/app/workers/use_key_worker.rb:3:in `<class:UseKeyWorker>'
/opt/gitlab/embedded/service/gitlab-rails/app/workers/use_key_worker.rb:1:in `<top (required)>'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:43:in `block in find_workers'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:40:in `map!'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:40:in `find_workers'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:23:in `workers'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:31:in `worker_queues'
/opt/gitlab/embedded/service/gitlab-rails/config/initializers/sidekiq.rb:68:in `<top (required)>'
/opt/gitlab/embedded/service/gitlab-rails/config/environment.rb:5:in `<top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Tasks: TOP => gitlab:db:configure => environment
(See full trace by running task with --trace)
================================================================================
Error executing action `run` on resource 'bash[migrate gitlab-rails database]'
================================================================================
Mixlib::ShellOut::ShellCommandFailed
------------------------------------
Expected process to exit with [0], but received '1'
---- Begin output of "bash" "/tmp/chef-script20171226-29335-1a8pvb5" ----
STDOUT: rake aborted!
NameError: uninitialized constant UseKeyWorker::DedicatedSidekiqQueue
/opt/gitlab/embedded/service/gitlab-rails/app/workers/use_key_worker.rb:3:in `<class:UseKeyWorker>'
/opt/gitlab/embedded/service/gitlab-rails/app/workers/use_key_worker.rb:1:in `<top (required)>'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:43:in `block in find_workers'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:40:in `map!'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:40:in `find_workers'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:23:in `workers'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:31:in `worker_queues'
/opt/gitlab/embedded/service/gitlab-rails/config/initializers/sidekiq.rb:68:in `<top (required)>'
/opt/gitlab/embedded/service/gitlab-rails/config/environment.rb:5:in `<top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Tasks: TOP => gitlab:db:configure => environment
(See full trace by running task with --trace)
STDERR:
---- End output of "bash" "/tmp/chef-script20171226-29335-1a8pvb5" ----
Ran "bash" "/tmp/chef-script20171226-29335-1a8pvb5" returned 1
Resource Declaration:
---------------------
# In /opt/gitlab/embedded/cookbooks/cache/cookbooks/gitlab/recipes/database_migrations.rb
51: bash "migrate gitlab-rails database" do
52: code <<-EOH
53: set -e
54: log_file="#{node['gitlab']['gitlab-rails']['log_directory']}/gitlab-rails-db-migrate-$(date +%Y-%m-%d-%H-%M-%S).log"
55: umask 077
56: /opt/gitlab/bin/gitlab-rake gitlab:db:configure 2>& 1 | tee ${log_file}
57: STATUS=${PIPESTATUS[0]}
58: echo $STATUS > #{db_migrate_status_file}
59: exit $STATUS
60: EOH
61: environment env_variables unless env_variables.empty?
62: notifies :run, "execute[clear the gitlab-rails cache]", :immediately
63: dependent_services.each do |svc|
64: notifies :restart, svc, :immediately
65: end
66: not_if "(test -f #{db_migrate_status_file}) && (cat #{db_migrate_status_file} | grep -Fx 0)"
67: only_if { node['gitlab']['gitlab-rails']['auto_migrate'] }
68: end
Compiled Resource:
------------------
# Declared in /opt/gitlab/embedded/cookbooks/cache/cookbooks/gitlab/recipes/database_migrations.rb:51:in `from_file'
bash("migrate gitlab-rails database") do
action [:run]
retries 0
retry_delay 2
default_guard_interpreter :default
command "migrate gitlab-rails database"
backup 5
returns 0
code " set -e\n log_file=\"/var/log/gitlab/gitlab-rails/gitlab-rails-db-migrate-$(date +%Y-%m-%d-%H-%M-%S).log\"\n umask 077\n /opt/gitlab/bin/gitlab-rake gitlab:db:configure 2>& 1 | tee ${log_file}\n STATUS=${PIPESTATUS[0]}\n echo $STATUS > /var/opt/gitlab/gitlab-rails/upgrade-status/db-migrate-873248b1f0d3a7a5535771a3a1635803-fd9dd4c\n exit $STATUS\n"
interpreter "bash"
declared_type :bash
cookbook_name "gitlab"
recipe_name "database_migrations"
not_if "(test -f /var/opt/gitlab/gitlab-rails/upgrade-status/db-migrate-873248b1f0d3a7a5535771a3a1635803-fd9dd4c) && (cat /var/opt/gitlab/gitlab-rails/upgrade-status/db-migrate-873248b1f0d3a7a5535771a3a1635803-fd9dd4c | grep -Fx 0)"
only_if { #code block }
end
Platform:
---------
x86_64-linux
Running handlers:
Running handlers complete
Chef Client failed. 2 resources updated in 25 secondsrun sudo gitlab-ctl status
run: gitaly: (pid 2490) 2824s; run: log: (pid 2488) 2824s
run: gitlab-monitor: (pid 2509) 2824s; run: log: (pid 2508) 2824s
run: gitlab-workhorse: (pid 2494) 2824s; run: log: (pid 2493) 2824s
run: logrotate: (pid 2497) 2824s; run: log: (pid 2489) 2824s
run: nginx: (pid 2504) 2824s; run: log: (pid 2503) 2824s
down: node-exporter: 0s, normally up, want up; run: log: (pid 2495) 2824s
down: postgres-exporter: 0s, normally up, want up; run: log: (pid 2506) 2824s
run: postgresql: (pid 2487) 2824s; run: log: (pid 2486) 2824s
run: prometheus: (pid 2502) 2824s; run: log: (pid 2501) 2824s
run: redis: (pid 2499) 2824s; run: log: (pid 2498) 2824s
run: redis-exporter: (pid 2492) 2824s; run: log: (pid 2491) 2824s
run: sidekiq: (pid 5034) 2s; run: log: (pid 2484) 2824s
run: unicorn: (pid 4951) 9s; run: log: (pid 2500) 2824sand cpu usage 100%
[root@izwz98qusdyye38qt6ym2cz ~]# ps auxw --sort=%cpu
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
git 9762 6.3 0.4 125756 38160 ? Sl 15:20 0:00 /opt/gitlab/embedded/bin/unicorn -D -E production -c /var/opt/gitlab/gitlab-rails/etc/unicorn.rb /opt/gitlab/embedded/service/gitlab-rails/config.ru
git 9771 93.2 3.2 497300 259668 ? Rl 15:20 0:06 /opt/gitlab/embedded/bin/unicorn -D -E production -c /var/opt/gitlab/gitlab-rails/etc/unicorn.rb /opt/gitlab/embedded/service/gitlab-rails/config.ru
git 9797 103 1.9 335380 156188 ? Rsl 15:21 0:04 /opt/gitlab/embedded/bin/sidekiq -C /opt/gitlab/embedded/service/gitlab-rails/config/sidekiq_queues.yml -e production -r /opt/gitlab/embedded/service/gitlab-rails -t 4 -c 25
get same error when i run sudo gitlab-rake gitlab:backup:create
rake aborted!
NameError: uninitialized constant UseKeyWorker::DedicatedSidekiqQueue
/opt/gitlab/embedded/service/gitlab-rails/app/workers/use_key_worker.rb:3:in `<class:UseKeyWorker>'
/opt/gitlab/embedded/service/gitlab-rails/app/workers/use_key_worker.rb:1:in `<top (required)>'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:43:in `block in find_workers'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:40:in `map!'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:40:in `find_workers'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:23:in `workers'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_config.rb:31:in `worker_queues'
/opt/gitlab/embedded/service/gitlab-rails/config/initializers/sidekiq.rb:68:in `<top (required)>'
/opt/gitlab/embedded/service/gitlab-rails/config/environment.rb:5:in `<top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Tasks: TOP => gitlab:backup:create => environment
(See full trace by running task with --trace)At this point I don't know how to solve this problem