Transient failure due to `PG::LockNotAvailable` errors in migration jobs
Job [#440832044](https://gitlab.com/gitlab-org/gitlab/-/jobs/440832044) failed for 4cf01c0fa25d97e2b59f66c548d3b86a5fb88bda: ``` 355 1) Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData with jira service migrates data 356 Failure/Error: migration_context.down(migration_schema_version) 357 StandardError: 358 An error has occurred, this and all later migrations canceled: 359 PG::LockNotAvailable: ERROR: canceling statement due to lock timeout 360 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:92:in `exec' 361 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:92:in `block (2 levels) in execute' 362 # ./vendor/ruby/2.6.0/gems/activesupport-6.0.2/lib/active_support/dependencies/interlock.rb:48:in `block in permit_concurrent_loads' 363 # ./vendor/ruby/2.6.0/gems/activesupport-6.0.2/lib/active_support/concurrency/share_lock.rb:187:in `yield_shares' 364 # ./vendor/ruby/2.6.0/gems/activesupport-6.0.2/lib/active_support/dependencies/interlock.rb:47:in `permit_concurrent_loads' 365 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:91:in `block in execute' 366 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract_adapter.rb:718:in `block (2 levels) in log' 367 # /usr/local/lib/ruby/2.6.0/monitor.rb:235:in `mon_synchronize' 368 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract_adapter.rb:717:in `block in log' 369 # ./vendor/ruby/2.6.0/gems/activesupport-6.0.2/lib/active_support/notifications/instrumenter.rb:24:in `instrument' 370 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract_adapter.rb:708:in `log' 371 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:90:in `execute' 372 # ./vendor/ruby/2.6.0/gems/marginalia-1.8.0/lib/marginalia.rb:71:in `execute_with_marginalia' 373 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/schema_statements.rb:408:in `change_column_default' 374 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:890:in `block in method_missing' 375 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:858:in `block in say_with_time' 376 # /usr/local/lib/ruby/2.6.0/benchmark.rb:293:in `measure' 377 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:858:in `say_with_time' 378 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:879:in `method_missing' 379 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration/command_recorder.rb:116:in `block in replay' 380 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration/command_recorder.rb:115:in `each' 381 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration/command_recorder.rb:115:in `replay' 382 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:702:in `revert' 383 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:826:in `exec_migration' 384 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:812:in `block (2 levels) in migrate' 385 # /usr/local/lib/ruby/2.6.0/benchmark.rb:293:in `measure' 386 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:811:in `block in migrate' 387 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract/connection_pool.rb:473:in `with_connection' 388 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:810:in `migrate' 389 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1001:in `migrate' 390 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1311:in `block in execute_migration_in_transaction' 391 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1362:in `block in ddl_transaction' 392 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract/database_statements.rb:281:in `block in transaction' 393 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract/transaction.rb:280:in `block in within_new_transaction' 394 # /usr/local/lib/ruby/2.6.0/monitor.rb:235:in `mon_synchronize' 395 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract/transaction.rb:278:in `within_new_transaction' 396 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract/database_statements.rb:281:in `transaction' 397 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/transactions.rb:212:in `transaction' 398 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1362:in `ddl_transaction' 399 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1310:in `execute_migration_in_transaction' 400 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1282:in `block in migrate_without_lock' 401 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1281:in `each' 402 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1281:in `migrate_without_lock' 403 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1229:in `block in migrate' 404 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1382:in `with_advisory_lock' 405 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1229:in `migrate' 406 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1071:in `down' 407 # ./spec/support/helpers/migrations_helpers.rb:105:in `block in schema_migrate_down!' 408 # ./spec/support/helpers/migrations_helpers.rb:125:in `disable_migrations_output' 409 # ./spec/support/helpers/migrations_helpers.rb:104:in `schema_migrate_down!' 410 # ./ee/spec/support/helpers/ee/migrations_helpers.rb:33:in `block in schema_migrate_down!' 411 # ./ee/spec/support/helpers/ee/migrations_helpers.rb:50:in `with_db_config' 412 # ./ee/spec/support/helpers/ee/migrations_helpers.rb:33:in `schema_migrate_down!' 413 # ./spec/support/migration.rb:16:in `block (2 levels) in <top (required)>' 414 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:348:in `instance_exec' 415 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:348:in `run' 416 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:507:in `block in run_owned_hooks_for' 417 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:506:in `each' 418 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:506:in `run_owned_hooks_for' 419 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:458:in `run' 420 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/example_group.rb:542:in `block in run_before_context_hooks' 421 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/memoized_helpers.rb:186:in `block in isolate_for_context_hook' 422 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/memoized_helpers.rb:182:in `instance_exec' 423 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/memoized_helpers.rb:182:in `isolate_for_context_hook' 424 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/example_group.rb:541:in `run_before_context_hooks' 425 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/example_group.rb:594:in `run' 426 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:116:in `block (3 levels) in run_specs' 427 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:116:in `map' 428 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:116:in `block (2 levels) in run_specs' 429 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/configuration.rb:2008:in `with_suite_hooks' 430 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:111:in `block in run_specs' 431 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/reporter.rb:74:in `report' 432 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:110:in `run_specs' 433 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:87:in `run' 434 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:71:in `run' 435 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:45:in `invoke' 436 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/exe/rspec:4:in `<top (required)>' 437 # ./vendor/ruby/2.6.0/bin/rspec:23:in `load' 438 # ./vendor/ruby/2.6.0/bin/rspec:23:in `<top (required)>' 439 # /usr/local/lib/ruby/2.6.0/bundler/cli/exec.rb:74:in `load' 440 # /usr/local/lib/ruby/2.6.0/bundler/cli/exec.rb:74:in `kernel_load' 441 # /usr/local/lib/ruby/2.6.0/bundler/cli/exec.rb:28:in `run' 442 # /usr/local/lib/ruby/2.6.0/bundler/cli.rb:463:in `exec' 443 # /usr/local/lib/ruby/2.6.0/bundler/vendor/thor/lib/thor/command.rb:27:in `run' 444 # /usr/local/lib/ruby/2.6.0/bundler/vendor/thor/lib/thor/invocation.rb:126:in `invoke_command' 445 # /usr/local/lib/ruby/2.6.0/bundler/vendor/thor/lib/thor.rb:387:in `dispatch' 446 # /usr/local/lib/ruby/2.6.0/bundler/cli.rb:27:in `dispatch' 447 # /usr/local/lib/ruby/2.6.0/bundler/vendor/thor/lib/thor/base.rb:466:in `start' 448 # /usr/local/lib/ruby/2.6.0/bundler/cli.rb:18:in `start' 449 # /usr/local/lib/ruby/gems/2.6.0/gems/bundler-1.17.2/exe/bundle:30:in `block in <top (required)>' 450 # /usr/local/lib/ruby/2.6.0/bundler/friendly_errors.rb:124:in `with_friendly_errors' 451 # /usr/local/lib/ruby/gems/2.6.0/gems/bundler-1.17.2/exe/bundle:22:in `<top (required)>' 452 # /usr/local/bin/bundle:23:in `load' 453 # /usr/local/bin/bundle:23:in `<main>' 454 # ------------------ 455 # --- Caused by: --- 456 # PG::LockNotAvailable: 457 # ERROR: canceling statement due to lock timeout 458 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:92:in `exec' ```
issue