Skip to content

Remove previous error details from Sidekiq logging

Marius Bobin requested to merge mb-clean-sidekiq-logging into master

What does this MR do and why?

In https://gitlab.com/gitlab-com/ops-sub-department/section-ops-request-for-help/-/issues/157#note_1476874451 we were seeing confusing logging messages which made us assume that the problem was not fixed. It turns out that we were still logging the error attributes for job retries that completed successfully.

  "enqueued_at": "2023-07-19T11:47:03.099Z",
  "error_message": "Unknown primary key for table ci_builds in model Ci::Build.",
  "error_class": "ActiveRecord::UnknownPrimaryKey",
  "failed_at": "2023-07-19T11:46:33.760Z",
  "retry_count": 0,
  "job_size_bytes": 9,
  "pid": 3848,
  "message": "Ci::ArchiveTraceWorker JID-9265e91ed98b1fbbbdac3a75: done: 0.026887 sec",
  "job_status": "done",
  • This job is actually successful, but the error_ attributes are confusing.
  • failed_at doesn't make sense because it's in the past, before the job was enqueued.

Before

Expand 

# Start job message

{
  "severity": "INFO",
  "time": "2023-07-24T07:36:55.625Z",
  "retry": 5,
  "queue": "default",
  "backtrace": true,
  "version": 0,
  "queue_namespace": "pipeline_background",
  "args": [
    "23"
  ],
  "class": "Ci::ArchiveTraceWorker",
  "jid": "88e182c9ade9c945876c8635",
  "created_at": "2023-07-24T07:36:55.579Z",
  "meta.feature_category": "continuous_integration",
  "correlation_id": "9786fc69dd7e68f9e9865bcadf2ddd3c",
  "worker_data_consistency": "sticky",
  "wal_locations": {},
  "wal_location_source": "primary",
  "idempotency_key": "resque:gitlab:duplicate:default:a10484e28216d88784f8efe9eeb2e7a81940e334692e080dc3130ac44eb08c2f",
  "size_limiter": "validated",
  "enqueued_at": "2023-07-24T07:36:55.622Z",
  "job_size_bytes": 4,
  "pid": 10638,
  "message": "Ci::ArchiveTraceWorker JID-88e182c9ade9c945876c8635: start",
  "job_status": "start",
  "scheduling_latency_s": 0.003117
}

# Job failed message

{
  "severity": "WARN",
  "time": "2023-07-24T07:36:55.654Z",
  "retry": 5,
  "queue": "default",
  "backtrace": true,
  "version": 0,
  "queue_namespace": "pipeline_background",
  "args": [
    "23"
  ],
  "class": "Ci::ArchiveTraceWorker",
  "jid": "88e182c9ade9c945876c8635",
  "created_at": "2023-07-24T07:36:55.579Z",
  "meta.feature_category": "continuous_integration",
  "correlation_id": "9786fc69dd7e68f9e9865bcadf2ddd3c",
  "worker_data_consistency": "sticky",
  "wal_locations": {},
  "wal_location_source": "primary",
  "idempotency_key": "resque:gitlab:duplicate:default:a10484e28216d88784f8efe9eeb2e7a81940e334692e080dc3130ac44eb08c2f",
  "size_limiter": "validated",
  "enqueued_at": "2023-07-24T07:36:55.622Z",
  "job_size_bytes": 4,
  "pid": 10638,
  "message": "Ci::ArchiveTraceWorker JID-88e182c9ade9c945876c8635: fail: 0.02891 sec",
  "job_status": "fail",
  "scheduling_latency_s": 0.003117,
  "duration_s": 0.02891,
  "completed_at": "2023-07-24T07:36:55.653Z",
  "load_balancing_strategy": "primary_no_wal",
  "exception.class": "ZeroDivisionError",
  "exception.message": "divided by 0",
  "exception.backtrace": [
    "app/workers/ci/archive_trace_worker.rb:24:in `/'",
    "app/workers/ci/archive_trace_worker.rb:24:in `perform'",
    "lib/gitlab/sidekiq_middleware/skip_jobs.rb:49:in `call'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
    "lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
    "lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
    "lib/gitlab/application_context.rb:124:in `block in use'",
    "lib/gitlab/application_context.rb:124:in `use'",
    "lib/gitlab/application_context.rb:62:in `with_context'",
    "lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
    "lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
    "lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
    "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
    "lib/gitlab/database/query_analyzer.rb:37:in `within'",
    "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
    "lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
    "lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
    "lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
    "lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
    "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
    "lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
    "lib/gitlab/with_request_store.rb:10:in `with_request_store'",
    "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:84:in `block in call'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:111:in `block in instrument'",
    "lib/gitlab/metrics/background_transaction.rb:33:in `run'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:111:in `instrument'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:83:in `call'",
    "lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
    "lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'",
    "lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
    "lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
    "lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"
  ]
}

# Retry job is starting

{
  "severity": "INFO",
  "time": "2023-07-24T07:37:10.766Z",
  "retry": 5,
  "queue": "default",
  "backtrace": true,
  "version": 0,
  "queue_namespace": "pipeline_background",
  "args": [
    "23"
  ],
  "class": "Ci::ArchiveTraceWorker",
  "jid": "88e182c9ade9c945876c8635",
  "created_at": "2023-07-24T07:36:55.579Z",
  "meta.feature_category": "continuous_integration",
  "correlation_id": "0880b46f6ecd0a16bc2ce694590b54b7",
  "worker_data_consistency": "sticky",
  "wal_locations": {},
  "wal_location_source": "primary",
  "idempotency_key": "resque:gitlab:duplicate:default:a10484e28216d88784f8efe9eeb2e7a81940e334692e080dc3130ac44eb08c2f",
  "size_limiter": "validated",
  "enqueued_at": "2023-07-24T07:37:10.765Z",
  "error_message": "divided by 0",
  "error_class": "ZeroDivisionError",
  "failed_at": "2023-07-24T07:36:55.652Z",
  "retry_count": 0,
  "error_backtrace": "eJzdWtuSmzgQ/ZWpecls1RoZsLGT/9jah60tVkAHaw0SkYQnk69fXbAtE7Lx\nRTgeqmaqjACdPq3W6ZbEX8/oDwFcoBpz0gr0J+Nb0eAcUElkhbNZATuoWFMD\nlbMtkV0zwk2DXtWz+tWcIMzzDdlBKrl6NbU3Ap59ihafCH36B314/v0uQA3w\nz4zX38MFJdSIt9kbioMwWCB1KZAgBWzJl1kSLIMVqki2b0ENZzkIwWzf88h0\nDl8hbyWk/7LML0C4mhuArGL59ukleqq0L8RvT6qxe/BmwJoURQWvmAPKN5hQ\ni7tycNUvQndsC9cPlgbsfna4qYMrtqTRzhMae/HRQOe4qkbi9nE0bgWWOMMC\nUMVwkWa4wjQntDxwVt3vgDvUTRS9Y8IDg1m0TUVybKeDslVNSAklAYFaKkmV\n2smivGJsS/5/do5i0smlCbmfaMQ4jjGxoOHXkxp/K71pzqiEr1IbYOFfOW5S\nQlPWSMIorvZP3McOx91hzx3Djr/CCJWRzAArei79sEtBB8BWDDh/cKA7aPWv\nEGfzII6DuUG0LchBiZfWy0RuvHr2J6QGufiDSaJxWF0UL6v3Pz+FxLLda04v\n/0yAniIlVOToVHvKbVqp9UsL/C3FSjzfvtnYXI0kZYdC5nvIeHWYkoTem+lU\nRhIXtUqFNSvAFZrFpDgSqmq/Vr9jZb1iZWmJTmtWZljmm1RX/FOMVJULOU4L\nRkEPYFqDxFocpkeUgxIdIVWqZBx6KTKcj6SzprA5QXYqDqA4q1ROO31gTNz5\nsdryj3mhz6clEvvKCyQnudnnWC9Giqmz0MMw7NPfq7UXIzooJY75tuSspYXe\nmaMC5zoXmDoiNgbwdrQi4sesPZM9b7zjBwnoq1aaLrXOiGPhEoXnhPI99yV9\ncRxYDSZXc/W9mh+R5PKRY/VCrHXUzfm7TYf5g3jPj5TVamEtu/OIseqgPWyB\nQcG5kIvEWW4On7B4pzmx0oN8U7U7qYkE7s7xh85H6l5J1PKf4FkYhOrP4B1b\nD0R7q8xHGr5hLGUpf5vtIZfBulMWe8ORE325F//+kVHyDihyvAM60+2RATTX\niCg+JTdjJY4ZqAJM2+ZBNyZ/VRI695A46fHwdTJ8ihK7hc9L4h5FF0SplMw3\nNyMqeVcLTxU8tli3CqUA8e0R0COzdMksRyHDMamEs693Tcmol0s7EG3TMC5n\nq2AeJAbDtqfdDdSdqTKa6lO2pttliw7nbn7RODibXdEpN/9ov5BbeCnIOcHg\nMe32Ytpdhb0sRonp089fEstFHy75VpvoRG3i88jcUC/pzWnzpYYqJnLZciic\n/epufTvCmEUnOhSNL6prW8GXFcu8i2r/nNsrg+NoxDbdNRwaPLQ1eRsHG9Lj\nTJgwWXemj5CgVyd9p4x6dk1n++B+3YX95qxulH1UHr9+edXeLljpt++wt10k\n8GdI5YYDLj48//0fPhVxIw==\n",
  "job_size_bytes": 4,
  "pid": 10638,
  "message": "Ci::ArchiveTraceWorker JID-88e182c9ade9c945876c8635: start",
  "job_status": "start",
  "scheduling_latency_s": 0.001119
}

# Retry job completes successfully

{
  "severity": "INFO",
  "time": "2023-07-24T07:37:10.783Z",
  "retry": 5,
  "queue": "default",
  "backtrace": true,
  "version": 0,
  "queue_namespace": "pipeline_background",
  "args": [
    "23"
  ],
  "class": "Ci::ArchiveTraceWorker",
  "jid": "88e182c9ade9c945876c8635",
  "created_at": "2023-07-24T07:36:55.579Z",
  "meta.feature_category": "continuous_integration",
  "correlation_id": "0880b46f6ecd0a16bc2ce694590b54b7",
  "worker_data_consistency": "sticky",
  "wal_locations": {},
  "wal_location_source": "primary",
  "idempotency_key": "resque:gitlab:duplicate:default:a10484e28216d88784f8efe9eeb2e7a81940e334692e080dc3130ac44eb08c2f",
  "size_limiter": "validated",
  "enqueued_at": "2023-07-24T07:37:10.765Z",
  "error_message": "divided by 0",
  "error_class": "ZeroDivisionError",
  "failed_at": "2023-07-24T07:36:55.652Z",
  "retry_count": 0,
  "error_backtrace": "eJzdWtuSmzgQ/ZWpecls1RoZsLGT/9jah60tVkAHaw0SkYQnk69fXbAtE7Lx\nRTgeqmaqjACdPq3W6ZbEX8/oDwFcoBpz0gr0J+Nb0eAcUElkhbNZATuoWFMD\nlbMtkV0zwk2DXtWz+tWcIMzzDdlBKrl6NbU3Ap59ihafCH36B314/v0uQA3w\nz4zX38MFJdSIt9kbioMwWCB1KZAgBWzJl1kSLIMVqki2b0ENZzkIwWzf88h0\nDl8hbyWk/7LML0C4mhuArGL59ukleqq0L8RvT6qxe/BmwJoURQWvmAPKN5hQ\ni7tycNUvQndsC9cPlgbsfna4qYMrtqTRzhMae/HRQOe4qkbi9nE0bgWWOMMC\nUMVwkWa4wjQntDxwVt3vgDvUTRS9Y8IDg1m0TUVybKeDslVNSAklAYFaKkmV\n2smivGJsS/5/do5i0smlCbmfaMQ4jjGxoOHXkxp/K71pzqiEr1IbYOFfOW5S\nQlPWSMIorvZP3McOx91hzx3Djr/CCJWRzAArei79sEtBB8BWDDh/cKA7aPWv\nEGfzII6DuUG0LchBiZfWy0RuvHr2J6QGufiDSaJxWF0UL6v3Pz+FxLLda04v\n/0yAniIlVOToVHvKbVqp9UsL/C3FSjzfvtnYXI0kZYdC5nvIeHWYkoTem+lU\nRhIXtUqFNSvAFZrFpDgSqmq/Vr9jZb1iZWmJTmtWZljmm1RX/FOMVJULOU4L\nRkEPYFqDxFocpkeUgxIdIVWqZBx6KTKcj6SzprA5QXYqDqA4q1ROO31gTNz5\nsdryj3mhz6clEvvKCyQnudnnWC9Giqmz0MMw7NPfq7UXIzooJY75tuSspYXe\nmaMC5zoXmDoiNgbwdrQi4sesPZM9b7zjBwnoq1aaLrXOiGPhEoXnhPI99yV9\ncRxYDSZXc/W9mh+R5PKRY/VCrHXUzfm7TYf5g3jPj5TVamEtu/OIseqgPWyB\nQcG5kIvEWW4On7B4pzmx0oN8U7U7qYkE7s7xh85H6l5J1PKf4FkYhOrP4B1b\nD0R7q8xHGr5hLGUpf5vtIZfBulMWe8ORE325F//+kVHyDihyvAM60+2RATTX\niCg+JTdjJY4ZqAJM2+ZBNyZ/VRI695A46fHwdTJ8ihK7hc9L4h5FF0SplMw3\nNyMqeVcLTxU8tli3CqUA8e0R0COzdMksRyHDMamEs693Tcmol0s7EG3TMC5n\nq2AeJAbDtqfdDdSdqTKa6lO2pttliw7nbn7RODibXdEpN/9ov5BbeCnIOcHg\nMe32Ytpdhb0sRonp089fEstFHy75VpvoRG3i88jcUC/pzWnzpYYqJnLZciic\n/epufTvCmEUnOhSNL6prW8GXFcu8i2r/nNsrg+NoxDbdNRwaPLQ1eRsHG9Lj\nTJgwWXemj5CgVyd9p4x6dk1n++B+3YX95qxulH1UHr9+edXeLljpt++wt10k\n8GdI5YYDLj48//0fPhVxIw==\n",
  "job_size_bytes": 4,
  "pid": 10638,
  "message": "Ci::ArchiveTraceWorker JID-88e182c9ade9c945876c8635: done: 0.017338 sec",
  "job_status": "done",
  "scheduling_latency_s": 0.001119,
  "duration_s": 0.017338,
  "completed_at": "2023-07-24T07:37:10.783Z",
  "load_balancing_strategy": "primary_no_wal"
}

After

Expand 

# Start job message

{
  "severity": "INFO",
  "time": "2023-07-24T07:33:14.162Z",
  "retry": 5,
  "queue": "default",
  "backtrace": true,
  "version": 0,
  "queue_namespace": "pipeline_background",
  "args": [
    "26"
  ],
  "class": "Ci::ArchiveTraceWorker",
  "jid": "3bc0fd45e394b662be503dbc",
  "created_at": "2023-07-24T07:33:14.102Z",
  "meta.feature_category": "continuous_integration",
  "correlation_id": "92aab1f03e899509f66052a7a1bbb711",
  "worker_data_consistency": "sticky",
  "wal_locations": {},
  "wal_location_source": "primary",
  "idempotency_key": "resque:gitlab:duplicate:default:f282b0e626a9aa2246005efbbb0879e2b82e3aeb193e71133bc8c21dec04bd4c",
  "size_limiter": "validated",
  "enqueued_at": "2023-07-24T07:33:14.159Z",
  "job_size_bytes": 4,
  "pid": 8015,
  "message": "Ci::ArchiveTraceWorker JID-3bc0fd45e394b662be503dbc: start",
  "job_status": "start",
  "scheduling_latency_s": 0.003256
}

# Job failed message

{
  "severity": "WARN",
  "time": "2023-07-24T07:33:14.192Z",
  "retry": 5,
  "queue": "default",
  "backtrace": true,
  "version": 0,
  "queue_namespace": "pipeline_background",
  "args": [
    "26"
  ],
  "class": "Ci::ArchiveTraceWorker",
  "jid": "3bc0fd45e394b662be503dbc",
  "created_at": "2023-07-24T07:33:14.102Z",
  "meta.feature_category": "continuous_integration",
  "correlation_id": "92aab1f03e899509f66052a7a1bbb711",
  "worker_data_consistency": "sticky",
  "wal_locations": {},
  "wal_location_source": "primary",
  "idempotency_key": "resque:gitlab:duplicate:default:f282b0e626a9aa2246005efbbb0879e2b82e3aeb193e71133bc8c21dec04bd4c",
  "size_limiter": "validated",
  "enqueued_at": "2023-07-24T07:33:14.159Z",
  "job_size_bytes": 4,
  "pid": 8015,
  "message": "Ci::ArchiveTraceWorker JID-3bc0fd45e394b662be503dbc: fail: 0.029699 sec",
  "job_status": "fail",
  "scheduling_latency_s": 0.003256,
  "duration_s": 0.029699,
  "completed_at": "2023-07-24T07:33:14.191Z",
  "load_balancing_strategy": "primary_no_wal",
  "exception.class": "ZeroDivisionError",
  "exception.message": "divided by 0",
  "exception.backtrace": [
    "app/workers/ci/archive_trace_worker.rb:24:in `/'",
    "app/workers/ci/archive_trace_worker.rb:24:in `perform'",
    "lib/gitlab/sidekiq_middleware/skip_jobs.rb:49:in `call'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
    "lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
    "lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
    "lib/gitlab/application_context.rb:124:in `block in use'",
    "lib/gitlab/application_context.rb:124:in `use'",
    "lib/gitlab/application_context.rb:62:in `with_context'",
    "lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
    "lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
    "lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
    "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
    "lib/gitlab/database/query_analyzer.rb:37:in `within'",
    "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
    "lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
    "lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
    "lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
    "lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
    "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
    "lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
    "lib/gitlab/with_request_store.rb:10:in `with_request_store'",
    "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:84:in `block in call'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:111:in `block in instrument'",
    "lib/gitlab/metrics/background_transaction.rb:33:in `run'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:111:in `instrument'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:83:in `call'",
    "lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
    "lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'",
    "lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
    "lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
    "lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"
  ]
}

# Retry job is starting

{
  "severity": "INFO",
  "time": "2023-07-24T07:33:32.880Z",
  "retry": 5,
  "queue": "default",
  "backtrace": true,
  "version": 0,
  "queue_namespace": "pipeline_background",
  "args": [
    "26"
  ],
  "class": "Ci::ArchiveTraceWorker",
  "jid": "3bc0fd45e394b662be503dbc",
  "created_at": "2023-07-24T07:33:14.102Z",
  "meta.feature_category": "continuous_integration",
  "correlation_id": "a5d4f5a3a7c1de1965b2fc2bc8695708",
  "worker_data_consistency": "sticky",
  "wal_locations": {},
  "wal_location_source": "primary",
  "idempotency_key": "resque:gitlab:duplicate:default:f282b0e626a9aa2246005efbbb0879e2b82e3aeb193e71133bc8c21dec04bd4c",
  "size_limiter": "validated",
  "enqueued_at": "2023-07-24T07:33:32.879Z",
  "retry_count": 0,
  "job_size_bytes": 4,
  "pid": 8015,
  "message": "Ci::ArchiveTraceWorker JID-3bc0fd45e394b662be503dbc: start",
  "job_status": "start",
  "scheduling_latency_s": 0.000621
}

# Retry job completes successfully

{
  "severity": "INFO",
  "time": "2023-07-24T07:33:32.890Z",
  "retry": 5,
  "queue": "default",
  "backtrace": true,
  "version": 0,
  "queue_namespace": "pipeline_background",
  "args": [
    "26"
  ],
  "class": "Ci::ArchiveTraceWorker",
  "jid": "3bc0fd45e394b662be503dbc",
  "created_at": "2023-07-24T07:33:14.102Z",
  "meta.feature_category": "continuous_integration",
  "correlation_id": "a5d4f5a3a7c1de1965b2fc2bc8695708",
  "worker_data_consistency": "sticky",
  "wal_locations": {},
  "wal_location_source": "primary",
  "idempotency_key": "resque:gitlab:duplicate:default:f282b0e626a9aa2246005efbbb0879e2b82e3aeb193e71133bc8c21dec04bd4c",
  "size_limiter": "validated",
  "enqueued_at": "2023-07-24T07:33:32.879Z",
  "retry_count": 0,
  "job_size_bytes": 4,
  "pid": 8015,
  "message": "Ci::ArchiveTraceWorker JID-3bc0fd45e394b662be503dbc: done: 0.010244 sec",
  "job_status": "done",
  "scheduling_latency_s": 0.000621,
  "duration_s": 0.010244,
  "completed_at": "2023-07-24T07:33:32.890Z",
  "load_balancing_strategy": "primary_no_wal"
}

How to set up and validate locally

Sample worker

This worker fails the first attempt at executing the job and the retry completes successfully.

# app/workers/ci/archive_trace_worker.rb
# frozen_string_literal: true

module Ci
  class ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker
    include ApplicationWorker

    data_consistency :sticky

    sidekiq_options retry: 3
    include PipelineBackgroundQueue

    def perform(job_id)
      cache_key = "cache:#{self.class.name}:#{job_id}"
      data = nil

      with_redis do |redis|
        data = redis.get(cache_key)
      end

      with_redis do |redis|
        redis.set(cache_key, 1, ex: 1.minute)
      end

      1/0 unless data
    end

    def with_redis(&block)
      Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord
    end
  end
end

You might also want to disable the cron jobs and instrumentation to reduce the noise:

diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb
index 2a3c4db5ffab..4d035d31d442 100644
--- a/lib/gitlab/instrumentation_helper.rb
+++ b/lib/gitlab/instrumentation_helper.rb
@@ -7,10 +7,12 @@ module InstrumentationHelper
     DURATION_PRECISION = 6 # microseconds
 
     def init_instrumentation_data
+      return
       Gitlab::RequestContext.start_thread_context
     end
 
     def add_instrumentation_data(payload)
+      return
       instrument_gitaly(payload)
       instrument_rugged(payload)
       instrument_redis(payload)

MR acceptance checklist

This checklist encourages us to confirm any changes have been analyzed to reduce risks in quality, performance, reliability, security, and maintainability.

Edited by Marius Bobin

Merge request reports