CI pipelines not triggered on push, but work flawlessly if manually triggered
Hi all, I'm having the issue, that for about 3 weeks(~15.02.24), gitlab pipelines are not triggered automatically anymore on push events or merge events. Manually triggered pipelines still work as expected and run. the only hint I have is some warning that happens on this event from sidekiq:
{"severity":"WARN","time":"2024-03-12T13:15:07.236Z","retry":3,"queue":"default","version":0,"args":["project-146","key-266","1d0b46ea4ff345f34d71dc63ce3844d98133862e b6168f6a10d1ecad4fffe97c6d4dbe51292238eb refs/heads/master\n","{}"],"class":"PostReceive","jid":"d6ca5b9a6a3e9ae96e992fb3","created_at":"2024-03-12T13:14:08.031Z","correlation_id":"d478849759fb725acf4ce6927013fed0","meta.caller_id":"POST /api/:version/internal/post_receive","meta.remote_ip":"127.0.0.1","meta.feature_category":"source_code_management","meta.user":"fgoth","meta.user_id":45,"meta.project":"fgoth/zeta","meta.root_namespace":"fgoth","meta.client_id":"user/45","meta.root_caller_id":"POST /api/:version/internal/post_receive","worker_data_consistency":"always","size_limiter":"validated","enqueued_at":"2024-03-12T13:15:06.455Z","retry_count":1,"retried_at":"2024-03-12T13:14:29.238Z","job_size_bytes":132,"pid":14680,"message":"PostReceive JID-d6ca5b9a6a3e9ae96e992fb3: fail: 0.777912 sec","job_status":"fail","scheduling_latency_s":0.002138,"gitaly_calls":12,"gitaly_duration_s":0.054834,"redis_calls":80,"redis_allowed_cross_slot_calls":4,"redis_duration_s":0.021326,"redis_read_bytes":742,"redis_write_bytes":5886,"redis_cache_calls":13,"redis_cache_duration_s":0.003133,"redis_cache_read_bytes":146,"redis_cache_write_bytes":907,"redis_queues_calls":4,"redis_queues_duration_s":0.002961,"redis_queues_read_bytes":4,"redis_queues_write_bytes":1556,"redis_queues_metadata_calls":2,"redis_queues_metadata_duration_s":0.001971,"redis_queues_metadata_read_bytes":4,"redis_queues_metadata_write_bytes":416,"redis_repository_cache_calls":49,"redis_repository_cache_allowed_cross_slot_calls":4,"redis_repository_cache_duration_s":0.01216,"redis_repository_cache_read_bytes":540,"redis_repository_cache_write_bytes":2608,"redis_shared_state_calls":12,"redis_shared_state_duration_s":0.001101,"redis_shared_state_read_bytes":48,"redis_shared_state_write_bytes":399,"db_count":118,"db_write_count":82,"db_cached_count":0,"db_replica_count":0,"db_primary_count":118,"db_main_count":36,"db_ci_count":82,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.158,"db_main_duration_s":0.082,"db_ci_duration_s":0.076,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.575651,"mem_objects":361349,"mem_bytes":12116654,"mem_mallocs":69320,"mem_total_bytes":26570614,"worker_id":"sidekiq_0","rate_limiting_gates":["pipelines_create"],"duration_s":0.777912,"completed_at":"2024-03-12T13:15:07.235Z","load_balancing_strategy":"primary","exception.class":"ActiveRecord::NotNullViolation","exception.message":"PG::NotNullViolation: ERROR: null value in column "partition_id" of relation "ci_pipelines_config" violates not-null constraint\nDETAIL: Failing row contains (22369, ---\ninclude:\n- local: ".gitlab-ci.yml"\n, null).\n","exception.backtrace":["lib/gitlab/database/load_balancing/connection_proxy.rb:127:in
public_send'","lib/gitlab/database/load_balancing/connection_proxy.rb:127:inblock in write_using_load_balancer'","lib/gitlab/database/load_balancing/load_balancer.rb:141:inblock in read_write'","lib/gitlab/database/load_balancing/load_balancer.rb:228:inretry_with_backoff'","lib/gitlab/database/load_balancing/load_balancer.rb:130:inread_write'","lib/gitlab/database/load_balancing/connection_proxy.rb:126:inwrite_using_load_balancer'","lib/gitlab/database/load_balancing/connection_proxy.rb:61:inblock (2 levels) in \u003cclass:ConnectionProxy\u003e'","lib/gitlab/database/load_balancing/connection_proxy.rb:127:inpublic_send'","lib/gitlab/database/load_balancing/connection_proxy.rb:127:inblock in write_using_load_balancer'","lib/gitlab/database/load_balancing/load_balancer.rb:141:inblock in read_write'","lib/gitlab/database/load_balancing/load_balancer.rb:228:inretry_with_backoff'","lib/gitlab/database/load_balancing/load_balancer.rb:130:inread_write'","lib/gitlab/database/load_balancing/connection_proxy.rb:126:inwrite_using_load_balancer'","lib/gitlab/database/load_balancing/connection_proxy.rb:78:intransaction'","lib/gitlab/database/load_balancing/connection_proxy.rb:127:inpublic_send'","lib/gitlab/database/load_balancing/connection_proxy.rb:127:inblock in write_using_load_balancer'","lib/gitlab/database/load_balancing/load_balancer.rb:141:inblock in read_write'","lib/gitlab/database/load_balancing/load_balancer.rb:228:inretry_with_backoff'","lib/gitlab/database/load_balancing/load_balancer.rb:130:inread_write'","lib/gitlab/database/load_balancing/connection_proxy.rb:126:inwrite_using_load_balancer'","lib/gitlab/database/load_balancing/connection_proxy.rb:78:intransaction'","lib/gitlab/ci/pipeline/chain/create.rb:16:inblock (4 levels) in perform!'","app/models/concerns/cross_database_modification.rb:92:inblock in transaction'","lib/gitlab/database/load_balancing/connection_proxy.rb:127:inpublic_send'","lib/gitlab/database/load_balancing/connection_proxy.rb:127:inblock in write_using_load_balancer'","lib/gitlab/database/load_balancing/load_balancer.rb:141:inblock in read_write'","lib/gitlab/database/load_balancing/load_balancer.rb:228:inretry_with_backoff'","lib/gitlab/database/load_balancing/load_balancer.rb:130:inread_write'","lib/gitlab/database/load_balancing/connection_proxy.rb:126:inwrite_using_load_balancer'","lib/gitlab/database/load_balancing/connection_proxy.rb:78:intransaction'","lib/gitlab/database.rb:359:inblock in transaction'","lib/gitlab/database.rb:358:intransaction'","app/models/concerns/cross_database_modification.rb:83:intransaction'","lib/gitlab/ci/pipeline/chain/create.rb:15:inblock (3 levels) in perform!'","app/models/concerns/ci/bulk_insertable_tags.rb:13:inwith_bulk_insert_tags'","lib/gitlab/ci/pipeline/chain/create.rb:14:inblock (2 levels) in perform!'","app/models/concerns/bulk_insertable_associations.rb:54:inwith_bulk_insert'","lib/gitlab/ci/pipeline/chain/create.rb:13:inblock in perform!'","lib/gitlab/ci/pipeline/logger.rb:27:ininstrument'","lib/gitlab/ci/pipeline/logger.rb:43:ininstrument_once_with_sql'","lib/gitlab/ci/pipeline/chain/create.rb:12:inperform!'","lib/gitlab/ci/pipeline/chain/sequence.rb:20:inblock in build!'","lib/gitlab/ci/pipeline/chain/sequence.rb:16:ineach'","lib/gitlab/ci/pipeline/chain/sequence.rb:16:inbuild!'","app/services/ci/create_pipeline_service.rb:93:inexecute'","app/services/git/base_hooks_service.rb:59:increate_pipelines'","app/services/git/base_hooks_service.rb:13:inexecute'","app/services/git/branch_hooks_service.rb:13:inexecute'","app/services/git/branch_push_service.rb:78:inexecute_related_hooks'","app/services/git/branch_push_service.rb:29:inexecute'","app/services/git/process_ref_changes_service.rb:59:inblock in process_changes'","app/services/git/process_ref_changes_service.rb:49:ineach'","app/services/git/process_ref_changes_service.rb:49:inprocess_changes'","app/services/git/process_ref_changes_service.rb:22:inblock in process_changes_by_action'","app/services/git/process_ref_changes_service.rb:21:ineach'","app/services/git/process_ref_changes_service.rb:21:inprocess_changes_by_action'","app/services/git/process_ref_changes_service.rb:10:inexecute'","app/workers/post_receive.rb:134:inprocess_ref_changes'","app/workers/post_receive.rb:66:inprocess_project_changes'","app/workers/post_receive.rb:36:inperform'","lib/gitlab/sidekiq_middleware/skip_jobs.rb:49:incall'","lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:29:incall'","lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb:14:inperform'","lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:inperform'","lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:incall'","lib/click_house/migration_support/sidekiq_middleware.rb:7:incall'","lib/gitlab/sidekiq_middleware/pause_control/strategies/base.rb:31:inperform'","lib/gitlab/sidekiq_middleware/pause_control/strategy_handler.rb:22:inperform'","lib/gitlab/sidekiq_middleware/pause_control/server.rb:8:incall'","lib/gitlab/sidekiq_middleware/worker_context.rb:9:inwrap_in_optional_context'","lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:inblock in call'","lib/gitlab/application_context.rb:130:inblock in use'","lib/gitlab/application_context.rb:130:inuse'","lib/gitlab/application_context.rb:64:inwith_context'","lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:incall'","lib/gitlab/sidekiq_status/server_middleware.rb:7:incall'","lib/gitlab/sidekiq_versioning/middleware.rb:9:incall'","lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:inblock in call'","lib/gitlab/database/query_analyzer.rb:37:inwithin'","lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:incall'","lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:incall'","lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:incall'","lib/gitlab/sidekiq_middleware/batch_loader.rb:7:incall'","lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:incall'","lib/gitlab/sidekiq_middleware/request_store_middleware.rb:8:inblock in call'","lib/gitlab/sidekiq_middleware/request_store_middleware.rb:7:incall'","lib/gitlab/sidekiq_middleware/server_metrics.rb:105:inblock in call'","lib/gitlab/sidekiq_middleware/server_metrics.rb:133:inblock in instrument'","lib/gitlab/metrics/background_transaction.rb:33:inrun'","lib/gitlab/sidekiq_middleware/server_metrics.rb:133:ininstrument'","lib/gitlab/sidekiq_middleware/server_metrics.rb:104:incall'","lib/gitlab/sidekiq_middleware/monitor.rb:10:inblock in call'","lib/gitlab/sidekiq_daemon/monitor.rb:46:inwithin_job'","lib/gitlab/sidekiq_middleware/monitor.rb:9:incall'","lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:incall'","lib/gitlab/sidekiq_logging/structured_logger.rb:21:incall'"],"exception.cause_class":"PG::NotNullViolation","exception.sql":"/application:sidekiq,correlation_id:d478849759fb725acf4ce6927013fed0,jid:d6ca5b9a6a3e9ae96e992fb3,endpoint_id:PostReceive,db_config_name:ci/ INSERT INTO "ci_pipelines_config" ("pipeline_id", "content") VALUES ($1, $2) RETURNING "pipeline_id"","db_duration_s":0.155297,"urgency":"high","target_duration_s":10,"target_scheduling_latency_s":10}
The gist of it seems to be this:
"ActiveRecord::NotNullViolation","exception.message":"PG::NotNullViolation: ERROR: null value in column \"partition_id\" of relation \"ci_pipelines_config\" violates not-null constraint\nDETAIL: Failing row contains (22369, ---\ninclude:\n- local: \".gitlab-ci.yml\"\n, null).\n","exception.backtrace":
Any help appreciated, since I'm running out of ideas....