Skip to content

Expired artifacts are not deleted although they should have been

Summary

Some examples where artifacts are not deleted:

  • Artifacts of deleted/merged branches are not removed after any amount of time.
  • Artifacts of active branches keeps artifacts of previous pipelines although setting Keep artifacts from most recent successful jobs is enabled.

We updated our Gitlab instance from 16.3 to 16.5.

What is the current bug behavior?

An example of a job artifact of a test-project on our instance. Those artifacts previously did get cleaned automatically since the release/2.0.0 branch does not exist anymore and this job is not the latest successful job of this branch:

image

What is the expected correct behavior?

Expired artifacts should have been deleted

Relevant logs and/or screenshots

The cronjob which cleans up those job artifacts ExpireBuildArtifactsWorker which did destroy artifacts before the update but it did stop doing that after the update (We updated our instance on the 24th)
This job didn't fully stop cleaning up artifacts as shown in the logs below but it decreased significantly which i assume means that artifacts build before the update are still being destroyed but artifacts build after the update are not destroyed anymore.

sidekiq log 
{"severity":"INFO","time":"2023-10-23T07:49:04.401Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"6d48d52c03e4e19dafb0839b","created_at":"2023-10-23T07:49:02.736Z","meta.caller_id":"Cronjob","correlation_id":"2ed67509b5cceeb431e05e59920798be","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-23T07:49:02.737Z","job_size_bytes":2,"pid":25326,"message":"ExpireBuildArtifactsWorker JID-6d48d52c03e4e19dafb0839b: done: 1.626035 sec","job_status":"done","scheduling_latency_s":0.037416,"redis_calls":38,"redis_duration_s":0.20192000000000002,"redis_read_bytes":1529,"redis_write_bytes":43583,"redis_feature_flag_calls":8,"redis_feature_flag_duration_s":0.178289,"redis_feature_flag_read_bytes":1194,"redis_feature_flag_write_bytes":778,"redis_queues_calls":4,"redis_queues_duration_s":0.011768,"redis_queues_read_bytes":80,"redis_queues_write_bytes":519,"redis_shared_state_calls":26,"redis_shared_state_duration_s":0.011863,"redis_shared_state_read_bytes":255,"redis_shared_state_write_bytes":42286,"db_count":507,"db_write_count":2,"db_cached_count":498,"db_replica_count":0,"db_primary_count":507,"db_main_count":5,"db_ci_count":502,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":498,"db_main_cached_count":0,"db_ci_cached_count":498,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.437,"db_main_duration_s":0.078,"db_ci_duration_s":0.359,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.740338,"mem_objects":320515,"mem_bytes":14147528,"mem_mallocs":224502,"mem_total_bytes":26968128,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":24,"duration_s":1.626035,"completed_at":"2023-10-23T07:49:04.401Z","load_balancing_strategy":"primary","db_duration_s":0.532017,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-10-23T10:42:05.677Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"cb5c1d08dede58a4a781b155","created_at":"2023-10-23T10:42:03.701Z","meta.caller_id":"Cronjob","correlation_id":"54cc9cb58f12586e1067f4b9bd056ced","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-23T10:42:03.704Z","job_size_bytes":2,"pid":25326,"message":"ExpireBuildArtifactsWorker JID-cb5c1d08dede58a4a781b155: done: 1.96912 sec","job_status":"done","scheduling_latency_s":0.00401,"redis_calls":59,"redis_duration_s":0.016281,"redis_read_bytes":1196,"redis_write_bytes":85386,"redis_feature_flag_calls":5,"redis_feature_flag_duration_s":0.003302,"redis_feature_flag_read_bytes":598,"redis_feature_flag_write_bytes":601,"redis_queues_calls":4,"redis_queues_duration_s":0.000934,"redis_queues_read_bytes":80,"redis_queues_write_bytes":519,"redis_shared_state_calls":50,"redis_shared_state_duration_s":0.012045,"redis_shared_state_read_bytes":518,"redis_shared_state_write_bytes":84266,"db_count":507,"db_write_count":2,"db_cached_count":498,"db_replica_count":0,"db_primary_count":507,"db_main_count":5,"db_ci_count":502,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":498,"db_main_cached_count":0,"db_ci_cached_count":498,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.269,"db_main_duration_s":0.037,"db_ci_duration_s":0.232,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":1.130484,"mem_objects":332322,"mem_bytes":14508232,"mem_mallocs":227186,"mem_total_bytes":27801112,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":48,"duration_s":1.96912,"completed_at":"2023-10-23T10:42:05.677Z","load_balancing_strategy":"primary","db_duration_s":0.235632,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-10-23T22:07:03.351Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"0e5580732171475ea895dba4","created_at":"2023-10-23T22:07:01.473Z","meta.caller_id":"Cronjob","correlation_id":"15e17e7d0cdab0f81fba4f73c23c4066","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-23T22:07:01.475Z","job_size_bytes":2,"pid":25326,"message":"ExpireBuildArtifactsWorker JID-0e5580732171475ea895dba4: done: 1.823249 sec","job_status":"done","scheduling_latency_s":0.053576,"redis_calls":13,"redis_duration_s":0.105391,"redis_read_bytes":685,"redis_write_bytes":4955,"redis_feature_flag_calls":5,"redis_feature_flag_duration_s":0.006517,"redis_feature_flag_read_bytes":598,"redis_feature_flag_write_bytes":589,"redis_queues_calls":4,"redis_queues_duration_s":0.007056,"redis_queues_read_bytes":80,"redis_queues_write_bytes":519,"redis_shared_state_calls":4,"redis_shared_state_duration_s":0.091818,"redis_shared_state_read_bytes":7,"redis_shared_state_write_bytes":3847,"db_count":507,"db_write_count":2,"db_cached_count":498,"db_replica_count":0,"db_primary_count":507,"db_main_count":5,"db_ci_count":502,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":498,"db_main_cached_count":0,"db_ci_cached_count":498,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.529,"db_main_duration_s":0.14,"db_ci_duration_s":0.389,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.949291,"mem_objects":308549,"mem_bytes":13495480,"mem_mallocs":222357,"mem_total_bytes":25837440,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":2,"duration_s":1.823249,"completed_at":"2023-10-23T22:07:03.351Z","load_balancing_strategy":"primary","db_duration_s":0.626856,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-10-24T06:28:06.421Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"c17c9b3e886ce88929340876","created_at":"2023-10-24T06:28:03.102Z","meta.caller_id":"Cronjob","correlation_id":"52b7131db90c67dba0941ea051e4267d","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-24T06:28:03.103Z","job_size_bytes":2,"pid":25326,"message":"ExpireBuildArtifactsWorker JID-c17c9b3e886ce88929340876: done: 3.257925 sec","job_status":"done","scheduling_latency_s":0.059814,"redis_calls":13,"redis_duration_s":0.127056,"redis_read_bytes":244,"redis_write_bytes":8915,"redis_feature_flag_calls":3,"redis_feature_flag_duration_s":0.017377,"redis_feature_flag_read_bytes":201,"redis_feature_flag_write_bytes":421,"redis_queues_calls":4,"redis_queues_duration_s":0.071714,"redis_queues_read_bytes":5,"redis_queues_write_bytes":1044,"redis_shared_state_calls":6,"redis_shared_state_duration_s":0.037965,"redis_shared_state_read_bytes":38,"redis_shared_state_write_bytes":7450,"db_count":507,"db_write_count":2,"db_cached_count":498,"db_replica_count":0,"db_primary_count":507,"db_main_count":5,"db_ci_count":502,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":498,"db_main_cached_count":0,"db_ci_cached_count":498,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":1.147,"db_main_duration_s":0.627,"db_ci_duration_s":0.52,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.861228,"mem_objects":310878,"mem_bytes":13869232,"mem_mallocs":223348,"mem_total_bytes":26304352,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":4,"duration_s":3.257925,"completed_at":"2023-10-24T06:28:06.421Z","load_balancing_strategy":"primary","db_duration_s":1.170086,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-10-24T06:42:05.298Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"1e7851228869eba836b849a2","created_at":"2023-10-24T06:42:03.691Z","meta.caller_id":"Cronjob","correlation_id":"a8df1e816ff39f4c9937a6ec865eff51","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-24T06:42:03.692Z","job_size_bytes":2,"pid":25326,"message":"ExpireBuildArtifactsWorker JID-1e7851228869eba836b849a2: done: 1.603134 sec","job_status":"done","scheduling_latency_s":0.002743,"redis_calls":12,"redis_duration_s":0.057516000000000005,"redis_read_bytes":445,"redis_write_bytes":8589,"redis_feature_flag_calls":2,"redis_feature_flag_duration_s":0.000717,"redis_feature_flag_read_bytes":398,"redis_feature_flag_write_bytes":146,"redis_queues_calls":4,"redis_queues_duration_s":0.029201,"redis_queues_read_bytes":5,"redis_queues_write_bytes":1048,"redis_shared_state_calls":6,"redis_shared_state_duration_s":0.027598,"redis_shared_state_read_bytes":42,"redis_shared_state_write_bytes":7395,"db_count":506,"db_write_count":2,"db_cached_count":498,"db_replica_count":0,"db_primary_count":506,"db_main_count":4,"db_ci_count":502,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":498,"db_main_cached_count":0,"db_ci_cached_count":498,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.271,"db_main_duration_s":0.091,"db_ci_duration_s":0.18,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":1.118315,"mem_objects":310403,"mem_bytes":13812056,"mem_mallocs":223279,"mem_total_bytes":26228176,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":4,"duration_s":1.603134,"completed_at":"2023-10-24T06:42:05.298Z","load_balancing_strategy":"primary","db_duration_s":0.252607,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-10-26T05:21:05.816Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"e4b4257c4f9f618529ce104e","created_at":"2023-10-26T05:21:03.833Z","meta.caller_id":"Cronjob","correlation_id":"2a194a0b2f99481934b79e756a7622a1","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-26T05:21:03.850Z","job_size_bytes":2,"pid":2252,"message":"ExpireBuildArtifactsWorker JID-e4b4257c4f9f618529ce104e: done: 1.934073 sec","job_status":"done","scheduling_latency_s":0.031851,"redis_calls":5,"redis_duration_s":0.125525,"redis_read_bytes":203,"redis_write_bytes":569,"redis_cluster_shared_state_calls":2,"redis_cluster_shared_state_duration_s":0.012198,"redis_cluster_shared_state_read_bytes":2,"redis_cluster_shared_state_write_bytes":304,"redis_feature_flag_calls":1,"redis_feature_flag_duration_s":0.103983,"redis_feature_flag_read_bytes":199,"redis_feature_flag_write_bytes":79,"redis_queues_calls":1,"redis_queues_duration_s":0.000319,"redis_queues_read_bytes":1,"redis_queues_write_bytes":63,"redis_queues_metadata_calls":1,"redis_queues_metadata_duration_s":0.009025,"redis_queues_metadata_read_bytes":1,"redis_queues_metadata_write_bytes":123,"db_count":500,"db_write_count":0,"db_cached_count":499,"db_replica_count":0,"db_primary_count":500,"db_main_count":0,"db_ci_count":500,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":499,"db_main_cached_count":0,"db_ci_cached_count":499,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.147,"db_main_duration_s":0.0,"db_ci_duration_s":0.147,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":1.113361,"mem_objects":303103,"mem_bytes":13081408,"mem_mallocs":221030,"mem_total_bytes":25205528,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":0,"duration_s":1.934073,"completed_at":"2023-10-26T05:21:05.816Z","load_balancing_strategy":"primary","db_duration_s":0.069131,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-10-27T08:00:10.422Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"3cf5204b7297cd4829a806e0","created_at":"2023-10-27T08:00:08.187Z","meta.caller_id":"Cronjob","correlation_id":"3c3dd3ba20cd9dbb4ace108cffc3f4b6","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-27T08:00:08.188Z","job_size_bytes":2,"pid":9904,"message":"ExpireBuildArtifactsWorker JID-3cf5204b7297cd4829a806e0: done: 2.159146 sec","job_status":"done","scheduling_latency_s":0.074037,"redis_calls":5,"redis_duration_s":0.156935,"redis_read_bytes":203,"redis_write_bytes":569,"redis_cluster_shared_state_calls":2,"redis_cluster_shared_state_duration_s":0.067538,"redis_cluster_shared_state_read_bytes":2,"redis_cluster_shared_state_write_bytes":304,"redis_feature_flag_calls":1,"redis_feature_flag_duration_s":0.046411,"redis_feature_flag_read_bytes":199,"redis_feature_flag_write_bytes":79,"redis_queues_calls":1,"redis_queues_duration_s":0.000343,"redis_queues_read_bytes":1,"redis_queues_write_bytes":63,"redis_queues_metadata_calls":1,"redis_queues_metadata_duration_s":0.042643,"redis_queues_metadata_read_bytes":1,"redis_queues_metadata_write_bytes":123,"db_count":500,"db_write_count":0,"db_cached_count":499,"db_replica_count":0,"db_primary_count":500,"db_main_count":0,"db_ci_count":500,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":499,"db_main_cached_count":0,"db_ci_cached_count":499,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.126,"db_main_duration_s":0.0,"db_ci_duration_s":0.126,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":1.170198,"mem_objects":303102,"mem_bytes":13123152,"mem_mallocs":221038,"mem_total_bytes":25247232,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":0,"duration_s":2.159146,"completed_at":"2023-10-27T08:00:10.421Z","load_balancing_strategy":"primary","db_duration_s":0.18444,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-10-28T07:28:07.218Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"d4ee0a682ba8874f56b7ee60","created_at":"2023-10-28T07:28:05.390Z","meta.caller_id":"Cronjob","correlation_id":"72ed8597c34ec7054c3dc034738d725d","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-28T07:28:05.392Z","job_size_bytes":2,"pid":9904,"message":"ExpireBuildArtifactsWorker JID-d4ee0a682ba8874f56b7ee60: done: 1.734317 sec","job_status":"done","scheduling_latency_s":0.091927,"redis_calls":6,"redis_duration_s":0.076503,"redis_read_bytes":6,"redis_write_bytes":856,"redis_cluster_shared_state_calls":2,"redis_cluster_shared_state_duration_s":0.009836,"redis_cluster_shared_state_read_bytes":2,"redis_cluster_shared_state_write_bytes":304,"redis_feature_flag_calls":2,"redis_feature_flag_duration_s":0.046028,"redis_feature_flag_read_bytes":2,"redis_feature_flag_write_bytes":366,"redis_queues_calls":1,"redis_queues_duration_s":0.000342,"redis_queues_read_bytes":1,"redis_queues_write_bytes":63,"redis_queues_metadata_calls":1,"redis_queues_metadata_duration_s":0.020297,"redis_queues_metadata_read_bytes":1,"redis_queues_metadata_write_bytes":123,"db_count":501,"db_write_count":0,"db_cached_count":499,"db_replica_count":0,"db_primary_count":501,"db_main_count":1,"db_ci_count":500,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":499,"db_main_cached_count":0,"db_ci_cached_count":499,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.197,"db_main_duration_s":0.066,"db_ci_duration_s":0.131,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":1.145421,"mem_objects":303504,"mem_bytes":13131672,"mem_mallocs":221098,"mem_total_bytes":25271832,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":0,"duration_s":1.734317,"completed_at":"2023-10-28T07:28:07.217Z","load_balancing_strategy":"primary","db_duration_s":0.167429,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-10-28T07:56:10.872Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"6b1d653580ea0b15468f0680","created_at":"2023-10-28T07:56:08.784Z","meta.caller_id":"Cronjob","correlation_id":"0a8952e0a2fd82e91c0f31863c6e9db6","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-28T07:56:08.785Z","job_size_bytes":2,"pid":9904,"message":"ExpireBuildArtifactsWorker JID-6b1d653580ea0b15468f0680: done: 2.034691 sec","job_status":"done","scheduling_latency_s":0.052432,"redis_calls":12,"redis_duration_s":0.355488,"redis_read_bytes":226,"redis_write_bytes":7287,"redis_cluster_shared_state_calls":2,"redis_cluster_shared_state_duration_s":0.121086,"redis_cluster_shared_state_read_bytes":2,"redis_cluster_shared_state_write_bytes":505,"redis_feature_flag_calls":3,"redis_feature_flag_duration_s":0.107183,"redis_feature_flag_read_bytes":201,"redis_feature_flag_write_bytes":420,"redis_queues_calls":2,"redis_queues_duration_s":0.002648,"redis_queues_read_bytes":2,"redis_queues_write_bytes":716,"redis_queues_metadata_calls":2,"redis_queues_metadata_duration_s":0.123686,"redis_queues_metadata_read_bytes":3,"redis_queues_metadata_write_bytes":333,"redis_shared_state_calls":3,"redis_shared_state_duration_s":0.000885,"redis_shared_state_read_bytes":18,"redis_shared_state_write_bytes":5313,"db_count":507,"db_write_count":2,"db_cached_count":498,"db_replica_count":0,"db_primary_count":507,"db_main_count":5,"db_ci_count":502,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":498,"db_main_cached_count":0,"db_ci_cached_count":498,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.338,"db_main_duration_s":0.209,"db_ci_duration_s":0.129,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":1.037926,"mem_objects":311494,"mem_bytes":13936928,"mem_mallocs":223579,"mem_total_bytes":26396688,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":3,"duration_s":2.034691,"completed_at":"2023-10-28T07:56:10.872Z","load_balancing_strategy":"primary","db_duration_s":0.287597,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-10-29T09:00:06.249Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"320368f3ccd5f60809fe5ef1","created_at":"2023-10-29T09:00:03.656Z","meta.caller_id":"Cronjob","correlation_id":"53e84b501ea0ff30e3f0ca624308f3bf","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-29T09:00:03.657Z","job_size_bytes":2,"pid":9904,"message":"ExpireBuildArtifactsWorker JID-320368f3ccd5f60809fe5ef1: done: 2.454915 sec","job_status":"done","scheduling_latency_s":0.137493,"redis_calls":5,"redis_duration_s":0.002195,"redis_read_bytes":201,"redis_write_bytes":569,"redis_cluster_shared_state_calls":2,"redis_cluster_shared_state_duration_s":0.000815,"redis_cluster_shared_state_read_bytes":2,"redis_cluster_shared_state_write_bytes":304,"redis_feature_flag_calls":1,"redis_feature_flag_duration_s":0.000599,"redis_feature_flag_read_bytes":197,"redis_feature_flag_write_bytes":79,"redis_queues_calls":1,"redis_queues_duration_s":0.000388,"redis_queues_read_bytes":1,"redis_queues_write_bytes":63,"redis_queues_metadata_calls":1,"redis_queues_metadata_duration_s":0.000393,"redis_queues_metadata_read_bytes":1,"redis_queues_metadata_write_bytes":123,"db_count":500,"db_write_count":0,"db_cached_count":499,"db_replica_count":0,"db_primary_count":500,"db_main_count":0,"db_ci_count":500,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":499,"db_main_cached_count":0,"db_ci_cached_count":499,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.077,"db_main_duration_s":0.0,"db_ci_duration_s":0.077,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":1.169679,"mem_objects":303104,"mem_bytes":13020752,"mem_mallocs":221033,"mem_total_bytes":25144912,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":0,"duration_s":2.454915,"completed_at":"2023-10-29T09:00:06.249Z","load_balancing_strategy":"primary","db_duration_s":0.094478,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-10-30T08:07:07.867Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"affe4332e6f2a906655f2dd0","created_at":"2023-10-30T08:07:06.316Z","meta.caller_id":"Cronjob","correlation_id":"13767ffc860ec1a58e213c4dae9ca43a","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-30T08:07:06.317Z","job_size_bytes":2,"pid":9904,"message":"ExpireBuildArtifactsWorker JID-affe4332e6f2a906655f2dd0: done: 1.462078 sec","job_status":"done","scheduling_latency_s":0.08831,"redis_calls":5,"redis_duration_s":0.04549,"redis_read_bytes":202,"redis_write_bytes":751,"redis_cluster_shared_state_calls":2,"redis_cluster_shared_state_duration_s":0.025259,"redis_cluster_shared_state_read_bytes":2,"redis_cluster_shared_state_write_bytes":407,"redis_feature_flag_calls":1,"redis_feature_flag_duration_s":0.00127,"redis_feature_flag_read_bytes":198,"redis_feature_flag_write_bytes":158,"redis_queues_calls":1,"redis_queues_duration_s":0.018683,"redis_queues_read_bytes":1,"redis_queues_write_bytes":63,"redis_queues_metadata_calls":1,"redis_queues_metadata_duration_s":0.000278,"redis_queues_metadata_read_bytes":1,"redis_queues_metadata_write_bytes":123,"db_count":500,"db_write_count":0,"db_cached_count":499,"db_replica_count":0,"db_primary_count":500,"db_main_count":0,"db_ci_count":500,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":499,"db_main_cached_count":0,"db_ci_cached_count":499,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.072,"db_main_duration_s":0.0,"db_ci_duration_s":0.072,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":1.091443,"mem_objects":303189,"mem_bytes":13122336,"mem_mallocs":221063,"mem_total_bytes":25249896,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":0,"duration_s":1.462078,"completed_at":"2023-10-30T08:07:07.867Z","load_balancing_strategy":"primary","db_duration_s":0.059999,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-10-31T07:00:07.323Z","retry":0,"queue":"default","version":0,"queue_namespace":"cronjob","args":[],"class":"ExpireBuildArtifactsWorker","jid":"3f7c5f3f8d53098815e459a1","created_at":"2023-10-31T07:00:05.582Z","meta.caller_id":"Cronjob","correlation_id":"51c72b8e92037fe14176c024b0878463","meta.root_caller_id":"Cronjob","meta.feature_category":"build_artifacts","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:default:7be9ea93da0bda6381b334cf2932692b41842a57ea0aeac11899369225b9bb7e","size_limiter":"validated","enqueued_at":"2023-10-31T07:00:05.583Z","job_size_bytes":2,"pid":9904,"message":"ExpireBuildArtifactsWorker JID-3f7c5f3f8d53098815e459a1: done: 1.660609 sec","job_status":"done","scheduling_latency_s":0.080022,"redis_calls":5,"redis_duration_s":0.127726,"redis_read_bytes":202,"redis_write_bytes":648,"redis_cluster_shared_state_calls":2,"redis_cluster_shared_state_duration_s":0.026962,"redis_cluster_shared_state_read_bytes":2,"redis_cluster_shared_state_write_bytes":304,"redis_feature_flag_calls":1,"redis_feature_flag_duration_s":0.072599,"redis_feature_flag_read_bytes":198,"redis_feature_flag_write_bytes":158,"redis_queues_calls":1,"redis_queues_duration_s":0.000331,"redis_queues_read_bytes":1,"redis_queues_write_bytes":63,"redis_queues_metadata_calls":1,"redis_queues_metadata_duration_s":0.027834,"redis_queues_metadata_read_bytes":1,"redis_queues_metadata_write_bytes":123,"db_count":500,"db_write_count":0,"db_cached_count":499,"db_replica_count":0,"db_primary_count":500,"db_main_count":0,"db_ci_count":500,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":499,"db_main_cached_count":0,"db_ci_cached_count":499,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.073,"db_main_duration_s":0.0,"db_ci_duration_s":0.073,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.824907,"mem_objects":303140,"mem_bytes":13093072,"mem_mallocs":221050,"mem_total_bytes":25218672,"worker_id":"sidekiq_0","rate_limiting_gates":[],"extra.expire_build_artifacts_worker.destroyed_job_artifacts_count":0,"duration_s":1.660609,"completed_at":"2023-10-31T07:00:07.323Z","load_balancing_strategy":"primary","db_duration_s":0.165866,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}

If there are some additional infos which i can provide please let me know so i can help resolve this issue asap.😃

Thanks

Edited by Manuel Mair