Upgrade to 14.5.0-ee failed: database migrations problem
Upgrade failed after ~30 minutes of database migrations (consumed ~9GB of RAM)
* database_objects[postgresql] action create
* postgresql_user[gitlab] action create
* execute[create gitlab postgresql user] action run (skipped due to not_if)
(up to date)
* postgresql_user[gitlab_replicator] action create
* execute[create gitlab_replicator postgresql user] action run (skipped due to not_if)
* execute[set options for gitlab_replicator postgresql user] action run (skipped due to not_if)
(up to date)
* postgresql_database[gitlabhq_production] action create
* execute[create database gitlabhq_production] action run (skipped due to not_if)
(up to date)
* postgresql_extension[pg_trgm] action enable
* postgresql_query[enable pg_trgm extension] action run (skipped due to only_if)
(up to date)
* postgresql_extension[btree_gist] action enable
* postgresql_query[enable btree_gist extension] action run (skipped due to only_if)
(up to date)
(up to date)
* ruby_block[warn pending postgresql restart] action run (skipped due to only_if)
* execute[reload postgresql] action nothing (skipped due to action :nothing)
* execute[start postgresql] action nothing (skipped due to action :nothing)
Recipe: praefect::disable
* service[praefect] action nothing (skipped due to action :nothing)
* runit_service[praefect] action disable
* ruby_block[disable praefect] action run (skipped due to only_if)
(up to date)
* consul_service[praefect] action delete
* file[/var/opt/gitlab/consul/config.d/praefect-service.json] action delete (up to date)
(up to date)
Recipe: gitlab-kas::disable
* service[gitlab-kas] action nothing (skipped due to action :nothing)
* runit_service[gitlab-kas] action disable
* ruby_block[disable gitlab-kas] action run (skipped due to only_if)
(up to date)
Recipe: gitlab::database_migrations
* ruby_block[check remote PG version] action nothing (skipped due to action :nothing)
* rails_migration[gitlab-rails] action run
* bash[migrate gitlab-rails database] action run
[execute] rake aborted!
StandardError: An error has occurred, this and all later migrations canceled:
PG::UnableToSend: no connection to the server
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
ActiveRecord::StatementInvalid: PG::UnableToSend: no connection to the server
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
PG::UnableToSend: no connection to the server
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
ActiveRecord::StatementInvalid: PG::ConnectionBad: PQconsumeInput() server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:106:in `block in read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:105:in `read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:54:in `select_all'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `block in each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:25:in `block in each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `loop'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:119:in `get_data_to_update'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:87:in `perform'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:29:in `block in up'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `each'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
PG::ConnectionBad: PQconsumeInput() server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:106:in `block in read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:105:in `read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:54:in `select_all'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `block in each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:25:in `block in each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `loop'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:119:in `get_data_to_update'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:87:in `perform'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:29:in `block in up'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `each'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Tasks: TOP => db:migrate
(See full trace by running task with --trace)
== 20211012134316 CleanUpMigrateMergeRequestDiffCommitUsers: migrating ========
-- 8 MigrateMergeRequestDiffCommitUsers are still pending
================================================================================
Error executing action `run` on resource 'bash[migrate gitlab-rails database]'
================================================================================
Mixlib::ShellOut::ShellCommandFailed
------------------------------------
Expected process to exit with [0], but received '1'
---- Begin output of "bash" "/tmp/chef-script20211123-701709-y033qf" ----
STDOUT: rake aborted!
StandardError: An error has occurred, this and all later migrations canceled:
PG::UnableToSend: no connection to the server
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
ActiveRecord::StatementInvalid: PG::UnableToSend: no connection to the server
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
PG::UnableToSend: no connection to the server
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
ActiveRecord::StatementInvalid: PG::ConnectionBad: PQconsumeInput() server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:106:in `block in read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:105:in `read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:54:in `select_all'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `block in each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:25:in `block in each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `loop'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:119:in `get_data_to_update'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:87:in `perform'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:29:in `block in up'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `each'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
PG::ConnectionBad: PQconsumeInput() server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:106:in `block in read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:105:in `read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:54:in `select_all'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `block in each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:25:in `block in each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `loop'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:119:in `get_data_to_update'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:87:in `perform'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:29:in `block in up'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `each'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Tasks: TOP => db:migrate
(See full trace by running task with --trace)
== 20211012134316 CleanUpMigrateMergeRequestDiffCommitUsers: migrating ========
-- 8 MigrateMergeRequestDiffCommitUsers are still pending
STDERR:
---- End output of "bash" "/tmp/chef-script20211123-701709-y033qf" ----
Ran "bash" "/tmp/chef-script20211123-701709-y033qf" returned 1
Resource Declaration:
---------------------
# In /opt/gitlab/embedded/cookbooks/cache/cookbooks/gitlab/resources/rails_migration.rb
16: bash "migrate #{new_resource.name} database" do
17: code <<-EOH
18: set -e
19: log_file="#{node['gitlab']['gitlab-rails']['log_directory']}/#{new_resource.logfile_prefix}-$(date +%Y-%m-%d-%H-%M-%S).log"
20: umask 077
21: /opt/gitlab/bin/gitlab-rake #{new_resource.rake_task} 2>& 1 | tee ${log_file}
22: STATUS=${PIPESTATUS[0]}
23: chown #{account_helper.gitlab_user}:#{account_helper.gitlab_group} ${log_file}
24: echo $STATUS > #{new_resource.helper.db_migrate_status_file}
25: exit $STATUS
26: EOH
27:
28: environment new_resource.environment if new_resource.property_is_set?(:environment)
29: new_resource.dependent_services.each do |svc|
30: notifies :restart, svc, :immediately
31: end
32:
33: not_if { new_resource.helper.migrated? }
34: end
35: end
Compiled Resource:
------------------
# Declared in /opt/gitlab/embedded/cookbooks/cache/cookbooks/gitlab/resources/rails_migration.rb:16:in `block in class_from_file'
bash("migrate gitlab-rails database") do
action [:run]
default_guard_interpreter :default
command nil
backup 5
interpreter "bash"
declared_type :bash
cookbook_name "gitlab"
code " set -e\n log_file=\"/var/log/gitlab/gitlab-rails/gitlab-rails-db-migrate-$(date +%Y-%m-%d-%H-%M-%S).log\"\n umask 077\n /opt/gitlab/bin/gitlab-rake gitlab:db:configure 2>& 1 | tee ${log_file}\n STATUS=${PIPESTATUS[0]}\n chown git:git ${log_file}\n echo $STATUS > /var/opt/gitlab/gitlab-rails/upgrade-status/db-migrate-873248b1f0d3a7a5535771a3a1635803-490a8efda84\n exit $STATUS\n"
domain nil
user nil
environment {"GITLAB_ROOT_PASSWORD"=>"4Hw7dNe9i1xaKtO2Mx21MIkBTUB4hRezdz2GGpW07Io="}
not_if { #code block }
end
System Info:
------------
chef_version=15.17.4
platform=ubuntu
platform_version=20.04
ruby=ruby 2.7.4p191 (2021-07-07 revision a21a3b7d23) [x86_64-linux]
program_name=/opt/gitlab/embedded/bin/chef-client
executable=/opt/gitlab/embedded/bin/chef-client
================================================================================
Error executing action `run` on resource 'rails_migration[gitlab-rails]'
================================================================================
Mixlib::ShellOut::ShellCommandFailed
------------------------------------
bash[migrate gitlab-rails database] (/opt/gitlab/embedded/cookbooks/cache/cookbooks/gitlab/resources/rails_migration.rb line 16) had an error: Mixlib::ShellOut::ShellCommandFailed: Expected process to exit with [0], but received '1'
---- Begin output of "bash" "/tmp/chef-script20211123-701709-y033qf" ----
STDOUT: rake aborted!
StandardError: An error has occurred, this and all later migrations canceled:
PG::UnableToSend: no connection to the server
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
ActiveRecord::StatementInvalid: PG::UnableToSend: no connection to the server
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
PG::UnableToSend: no connection to the server
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
ActiveRecord::StatementInvalid: PG::ConnectionBad: PQconsumeInput() server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:106:in `block in read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:105:in `read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:54:in `select_all'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `block in each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:25:in `block in each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `loop'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:119:in `get_data_to_update'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:87:in `perform'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:29:in `block in up'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `each'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
PG::ConnectionBad: PQconsumeInput() server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:106:in `block in read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:105:in `read_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:54:in `select_all'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `block in each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:25:in `block in each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `loop'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/pagination/keyset/iterator.rb:19:in `each_batch'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:65:in `each_row_to_migrate'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:119:in `get_data_to_update'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb:87:in `perform'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:29:in `block in up'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `each'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20211012134316_clean_up_migrate_merge_request_diff_commit_users.rb:26:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/load_balancing/connection_proxy.rb:77:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:252:in `block in transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database.rb:251:in `transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migrations/lock_retry_mixin.rb:31:in `ddl_transaction'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:61:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Tasks: TOP => db:migrate
(See full trace by running task with --trace)
== 20211012134316 CleanUpMigrateMergeRequestDiffCommitUsers: migrating ========
-- 8 MigrateMergeRequestDiffCommitUsers are still pending
STDERR:
---- End output of "bash" "/tmp/chef-script20211123-701709-y033qf" ----
Ran "bash" "/tmp/chef-script20211123-701709-y033qf" returned 1
Resource Declaration:
---------------------
# In /opt/gitlab/embedded/cookbooks/cache/cookbooks/gitlab/recipes/database_migrations.rb
51: rails_migration "gitlab-rails" do
52: rake_task 'gitlab:db:configure'
53: logfile_prefix 'gitlab-rails-db-migrate'
54: helper migration_helper
55:
56: environment env_variables
57: dependent_services dependent_services
58: notifies :run, "execute[clear the gitlab-rails cache]", :immediately
59: notifies :run, "ruby_block[check remote PG version]", :immediately
60:
61: only_if { migration_helper.attributes_node['auto_migrate'] }
62: end
Compiled Resource:
------------------
# Declared in /opt/gitlab/embedded/cookbooks/cache/cookbooks/gitlab/recipes/database_migrations.rb:51:in `from_file'
rails_migration("gitlab-rails") do
action [:run]
default_guard_interpreter :default
declared_type :rails_migration
cookbook_name "gitlab"
recipe_name "database_migrations"
rake_task "gitlab:db:configure"
logfile_prefix "gitlab-rails-db-migrate"Edited by N0name