Transient failure due to `PG::LockNotAvailable` errors in migration jobs
Job #440832044 failed for 4cf01c0f:
355 1) Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData with jira service migrates data
356 Failure/Error: migration_context.down(migration_schema_version)
357 StandardError:
358 An error has occurred, this and all later migrations canceled:
359 PG::LockNotAvailable: ERROR: canceling statement due to lock timeout
360 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:92:in `exec'
361 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:92:in `block (2 levels) in execute'
362 # ./vendor/ruby/2.6.0/gems/activesupport-6.0.2/lib/active_support/dependencies/interlock.rb:48:in `block in permit_concurrent_loads'
363 # ./vendor/ruby/2.6.0/gems/activesupport-6.0.2/lib/active_support/concurrency/share_lock.rb:187:in `yield_shares'
364 # ./vendor/ruby/2.6.0/gems/activesupport-6.0.2/lib/active_support/dependencies/interlock.rb:47:in `permit_concurrent_loads'
365 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:91:in `block in execute'
366 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract_adapter.rb:718:in `block (2 levels) in log'
367 # /usr/local/lib/ruby/2.6.0/monitor.rb:235:in `mon_synchronize'
368 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract_adapter.rb:717:in `block in log'
369 # ./vendor/ruby/2.6.0/gems/activesupport-6.0.2/lib/active_support/notifications/instrumenter.rb:24:in `instrument'
370 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract_adapter.rb:708:in `log'
371 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:90:in `execute'
372 # ./vendor/ruby/2.6.0/gems/marginalia-1.8.0/lib/marginalia.rb:71:in `execute_with_marginalia'
373 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/schema_statements.rb:408:in `change_column_default'
374 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:890:in `block in method_missing'
375 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:858:in `block in say_with_time'
376 # /usr/local/lib/ruby/2.6.0/benchmark.rb:293:in `measure'
377 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:858:in `say_with_time'
378 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:879:in `method_missing'
379 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration/command_recorder.rb:116:in `block in replay'
380 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration/command_recorder.rb:115:in `each'
381 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration/command_recorder.rb:115:in `replay'
382 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:702:in `revert'
383 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:826:in `exec_migration'
384 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:812:in `block (2 levels) in migrate'
385 # /usr/local/lib/ruby/2.6.0/benchmark.rb:293:in `measure'
386 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:811:in `block in migrate'
387 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract/connection_pool.rb:473:in `with_connection'
388 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:810:in `migrate'
389 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1001:in `migrate'
390 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1311:in `block in execute_migration_in_transaction'
391 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1362:in `block in ddl_transaction'
392 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract/database_statements.rb:281:in `block in transaction'
393 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract/transaction.rb:280:in `block in within_new_transaction'
394 # /usr/local/lib/ruby/2.6.0/monitor.rb:235:in `mon_synchronize'
395 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract/transaction.rb:278:in `within_new_transaction'
396 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/abstract/database_statements.rb:281:in `transaction'
397 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/transactions.rb:212:in `transaction'
398 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1362:in `ddl_transaction'
399 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1310:in `execute_migration_in_transaction'
400 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1282:in `block in migrate_without_lock'
401 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1281:in `each'
402 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1281:in `migrate_without_lock'
403 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1229:in `block in migrate'
404 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1382:in `with_advisory_lock'
405 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1229:in `migrate'
406 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/migration.rb:1071:in `down'
407 # ./spec/support/helpers/migrations_helpers.rb:105:in `block in schema_migrate_down!'
408 # ./spec/support/helpers/migrations_helpers.rb:125:in `disable_migrations_output'
409 # ./spec/support/helpers/migrations_helpers.rb:104:in `schema_migrate_down!'
410 # ./ee/spec/support/helpers/ee/migrations_helpers.rb:33:in `block in schema_migrate_down!'
411 # ./ee/spec/support/helpers/ee/migrations_helpers.rb:50:in `with_db_config'
412 # ./ee/spec/support/helpers/ee/migrations_helpers.rb:33:in `schema_migrate_down!'
413 # ./spec/support/migration.rb:16:in `block (2 levels) in <top (required)>'
414 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:348:in `instance_exec'
415 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:348:in `run'
416 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:507:in `block in run_owned_hooks_for'
417 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:506:in `each'
418 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:506:in `run_owned_hooks_for'
419 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/hooks.rb:458:in `run'
420 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/example_group.rb:542:in `block in run_before_context_hooks'
421 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/memoized_helpers.rb:186:in `block in isolate_for_context_hook'
422 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/memoized_helpers.rb:182:in `instance_exec'
423 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/memoized_helpers.rb:182:in `isolate_for_context_hook'
424 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/example_group.rb:541:in `run_before_context_hooks'
425 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/example_group.rb:594:in `run'
426 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:116:in `block (3 levels) in run_specs'
427 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:116:in `map'
428 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:116:in `block (2 levels) in run_specs'
429 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/configuration.rb:2008:in `with_suite_hooks'
430 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:111:in `block in run_specs'
431 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/reporter.rb:74:in `report'
432 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:110:in `run_specs'
433 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:87:in `run'
434 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:71:in `run'
435 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/lib/rspec/core/runner.rb:45:in `invoke'
436 # ./vendor/ruby/2.6.0/gems/rspec-core-3.8.2/exe/rspec:4:in `<top (required)>'
437 # ./vendor/ruby/2.6.0/bin/rspec:23:in `load'
438 # ./vendor/ruby/2.6.0/bin/rspec:23:in `<top (required)>'
439 # /usr/local/lib/ruby/2.6.0/bundler/cli/exec.rb:74:in `load'
440 # /usr/local/lib/ruby/2.6.0/bundler/cli/exec.rb:74:in `kernel_load'
441 # /usr/local/lib/ruby/2.6.0/bundler/cli/exec.rb:28:in `run'
442 # /usr/local/lib/ruby/2.6.0/bundler/cli.rb:463:in `exec'
443 # /usr/local/lib/ruby/2.6.0/bundler/vendor/thor/lib/thor/command.rb:27:in `run'
444 # /usr/local/lib/ruby/2.6.0/bundler/vendor/thor/lib/thor/invocation.rb:126:in `invoke_command'
445 # /usr/local/lib/ruby/2.6.0/bundler/vendor/thor/lib/thor.rb:387:in `dispatch'
446 # /usr/local/lib/ruby/2.6.0/bundler/cli.rb:27:in `dispatch'
447 # /usr/local/lib/ruby/2.6.0/bundler/vendor/thor/lib/thor/base.rb:466:in `start'
448 # /usr/local/lib/ruby/2.6.0/bundler/cli.rb:18:in `start'
449 # /usr/local/lib/ruby/gems/2.6.0/gems/bundler-1.17.2/exe/bundle:30:in `block in <top (required)>'
450 # /usr/local/lib/ruby/2.6.0/bundler/friendly_errors.rb:124:in `with_friendly_errors'
451 # /usr/local/lib/ruby/gems/2.6.0/gems/bundler-1.17.2/exe/bundle:22:in `<top (required)>'
452 # /usr/local/bin/bundle:23:in `load'
453 # /usr/local/bin/bundle:23:in `<main>'
454 # ------------------
455 # --- Caused by: ---
456 # PG::LockNotAvailable:
457 # ERROR: canceling statement due to lock timeout
458 # ./vendor/ruby/2.6.0/gems/activerecord-6.0.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:92:in `exec'