``` ==> /var/log/gitlab/nginx/gitlab_access.log <== 10.0.2.117 - - [23/Nov/2021:16:11:29 +0000] "GET /admin/runners HTTP/1.1" 500 2926 "https://gitlab.chaowanpai.com/admin" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36" - ==> /var/log/gitlab/gitlab-rails/production.log <== Started GET "/admin/runners" for 111.199.81.98 at 2021-11-23 16:11:29 +0000 Processing by Admin::RunnersController#index as HTML Rendered layout layouts/admin.html.haml (Duration: 0.8ms | Allocations: 351) Completed 500 Internal Server Error in 11ms (ActiveRecord: 0.7ms | Elasticsearch: 0.0ms | Allocations: 4809) ==> /var/log/gitlab/gitlab-rails/production_json.log <== {"method":"GET","path":"/admin/runners","format":"html","controller":"Admin::RunnersController","action":"index","status":500,"time":"2021-11-23T16:11:29.998Z","params":[],"correlation_id":"01FN6S21SQ8JFAGM50YYC98AH7","meta.user":"jia.liu","meta.caller_id":"Admin::RunnersController#index","meta.remote_ip":"111.199.81.98","meta.feature_category":"runner","meta.client_id":"user/1","remote_ip":"111.199.81.98","user_id":1,"username":"jia.liu","ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36","request_urgency":"default","target_duration_s":1,"redis_calls":3,"redis_duration_s":0.000259,"redis_read_bytes":181,"redis_write_bytes":845,"redis_shared_state_calls":3,"redis_shared_state_duration_s":0.000259,"redis_shared_state_read_bytes":181,"redis_shared_state_write_bytes":845,"db_count":4,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_replica_cached_count":0,"db_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_count":4,"db_primary_cached_count":0,"db_primary_wal_count":0,"db_primary_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.001,"cpu_s":0.021439,"mem_objects":11821,"mem_bytes":1678808,"mem_mallocs":2859,"mem_total_bytes":2151648,"pid":703,"exception.class":"ActionView::Template::Error","exception.message":"","exception.backtrace":["lib/gitlab/crypto_helper.rb:28:in `aes256_gcm_decrypt'","app/models/concerns/token_authenticatable_strategies/encryption_helper.rb:18:in `decrypt_token'","app/models/concerns/token_authenticatable_strategies/encrypted.rb:45:in `get_token'","app/models/concerns/token_authenticatable_strategies/base.rb:38:in `ensure_token!'","app/models/concerns/token_authenticatable.rb:48:in `block in add_authentication_token_field'","app/models/application_setting_implementation.rb:394:in `runners_registration_token'","lib/gitlab/current_settings.rb:32:in `method_missing'","app/helpers/ci/runners_helper.rb:68:in `admin_runners_data_attributes'","app/views/admin/runners/index.html.haml:4","app/controllers/application_controller.rb:136:in `render'","app/controllers/application_controller.rb:504:in `set_current_admin'","lib/gitlab/session.rb:11:in `with_session'","app/controllers/application_controller.rb:495:in `set_session_storage'","lib/gitlab/i18n.rb:105:in `with_locale'","lib/gitlab/i18n.rb:111:in `with_user_locale'","app/controllers/application_controller.rb:489:in `set_locale'","app/controllers/application_controller.rb:483:in `set_current_context'","lib/gitlab/middleware/speedscope.rb:13:in `call'","lib/gitlab/request_profiler/middleware.rb:17:in `call'","lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'","lib/gitlab/jira/middleware.rb:19:in `call'","lib/gitlab/middleware/go.rb:20:in `call'","lib/gitlab/etag_caching/middleware.rb:21:in `call'","lib/gitlab/middleware/multipart.rb:173:in `call'","lib/gitlab/middleware/read_only/controller.rb:50:in `call'","lib/gitlab/middleware/read_only.rb:18:in `call'","lib/gitlab/middleware/same_site_cookies.rb:27:in `call'","lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'","lib/gitlab/middleware/basic_health_check.rb:25:in `call'","lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'","lib/gitlab/middleware/request_context.rb:21:in `call'","config/initializers/fix_local_cache_middleware.rb:11:in `call'","lib/gitlab/middleware/compressed_json.rb:26:in `call'","lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'","lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'","lib/gitlab/metrics/requests_rack_middleware.rb:75:in `call'","lib/gitlab/middleware/release_env.rb:13:in `call'"],"db_duration_s":0.00073,"view_duration_s":0.0,"duration_s":0.01124} ==> /var/log/gitlab/gitlab-rails/production.log <== ActionView::Template::Error (): 1: - breadcrumb_title _('Runners') 2: - page_title _('Runners') 3: 4: #js-admin-runners{ data: admin_runners_data_attributes } lib/gitlab/crypto_helper.rb:28:in `aes256_gcm_decrypt' app/models/concerns/token_authenticatable_strategies/encryption_helper.rb:18:in `decrypt_token' app/models/concerns/token_authenticatable_strategies/encrypted.rb:45:in `get_token' app/models/concerns/token_authenticatable_strategies/base.rb:38:in `ensure_token!' app/models/concerns/token_authenticatable.rb:48:in `block in add_authentication_token_field' app/models/application_setting_implementation.rb:394:in `runners_registration_token' lib/gitlab/current_settings.rb:32:in `method_missing' app/helpers/ci/runners_helper.rb:68:in `admin_runners_data_attributes' app/views/admin/runners/index.html.haml:4 app/controllers/application_controller.rb:136:in `render' app/controllers/application_controller.rb:504:in `set_current_admin' lib/gitlab/session.rb:11:in `with_session' app/controllers/application_controller.rb:495:in `set_session_storage' lib/gitlab/i18n.rb:105:in `with_locale' lib/gitlab/i18n.rb:111:in `with_user_locale' app/controllers/application_controller.rb:489:in `set_locale' app/controllers/application_controller.rb:483:in `set_current_context' lib/gitlab/middleware/speedscope.rb:13:in `call' lib/gitlab/request_profiler/middleware.rb:17:in `call' lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call' lib/gitlab/jira/middleware.rb:19:in `call' lib/gitlab/middleware/go.rb:20:in `call' lib/gitlab/etag_caching/middleware.rb:21:in `call' lib/gitlab/middleware/multipart.rb:173:in `call' lib/gitlab/middleware/read_only/controller.rb:50:in `call' lib/gitlab/middleware/read_only.rb:18:in `call' lib/gitlab/middleware/same_site_cookies.rb:27:in `call' lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call' lib/gitlab/middleware/basic_health_check.rb:25:in `call' lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call' lib/gitlab/middleware/request_context.rb:21:in `call' config/initializers/fix_local_cache_middleware.rb:11:in `call' lib/gitlab/middleware/compressed_json.rb:26:in `call' lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call' lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call' lib/gitlab/metrics/requests_rack_middleware.rb:75:in `call' lib/gitlab/middleware/release_env.rb:13:in `call' ``` ## Restore Steps ``` # Stop the processes that are connected to the database docker exec -it <name of container> gitlab-ctl stop puma docker exec -it <name of container> gitlab-ctl stop sidekiq # Verify that the processes are all down before continuing docker exec -it <name of container> gitlab-ctl status # Run the restore. NOTE: "_gitlab_backup.tar" is omitted from the name docker exec -it <name of container> gitlab-backup restore BACKUP=11493107454_2018_04_25_10.6.4-ce # Restart the GitLab container docker restart <name of container> # Check GitLab docker exec -it <name of container> gitlab-rake gitlab:check SANITIZE=true ```