CustomersDot: Some log events missing in Kibana
Details
- Point of contact for this request: @mkaeppler
- If a call is needed, what is the proposed date and time of the call: not needed currently
- Additional call details (format, type of call): additional details
SRE Support Needed
We are investigating what appears to be missing log events in Kibana for ServiceCustomersDot.
So far, this was only visible for customers-staging
since these errors had not occurred in production yet.
Some evidence:
- The CDot application emits custom error log events from https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/faf30b8086312c7b5e732ec4165c7a9f68edac54/app/services/gitlab/cloud_connector/access_service.rb#L44
- These should reach both Kibana and Sentry with the given error message and context
- However, they are only present in Sentry, not Kibana:
- Sentry: https://new-sentry.gitlab.net/organizations/gitlab/issues/561897/events/af70e916d17a45ce83d708c6fe0d799c/
- Kibana: https://nonprod-log.gitlab.net/app/r/s/GJKOF (7 day search; due to log retention this will become invalid soon unfortunately)
- We verified that this log event exists in GCP, so the application definitely emitted it. I also verified with a local instance that this log event is written to
development_json.log
- We do see a corresponding access log event matching the correlation ID: https://nonprod-log.gitlab.net/app/discover#/doc/776ba910-538a-11ed-9af2-6131f0ee4ce6/pubsub-rails-inf-stgsub-000024?id=bF39-IwBajpF_RebQ2TY
- But this is just the request log logged by lograge, not the even emitted via the code linked above
So where is our event?
Additional data
Event JSON from GCP logs:
{
"insertId": "1lg4znvf2irvo1",
"jsonPayload": {
"fqdn": "customers.staging.gitlab.com",
"status": "error",
"params": null,
"class": "Gitlab::CloudConnector::AccessService",
"error": "service failed: service_token",
"path": null,
"progname": null,
"stage": "main",
"type": "customersdot",
"hostname": "10.185.2.3",
"backtrace": [
"/home/customersdot/CustomersDot/releases/20240111141517Z/lib/gitlab/logger_helpers.rb:34:in `public_send'",
"/home/customersdot/CustomersDot/releases/20240111141517Z/lib/gitlab/logger_helpers.rb:34:in `log'",
"/home/customersdot/CustomersDot/releases/20240111141517Z/lib/gitlab/logger_helpers.rb:15:in `log_error'",
"/home/customersdot/CustomersDot/releases/20240111141517Z/app/services/gitlab/cloud_connector/access_service.rb:44:in `handle_service_response'",
"/home/customersdot/CustomersDot/releases/20240111141517Z/app/services/gitlab/cloud_connector/access_service.rb:34:in `service_token'",
"/home/customersdot/CustomersDot/releases/20240111141517Z/app/services/gitlab/cloud_connector/access_service.rb:18:in `execute'",
...
"/home/customersdot/CustomersDot/shared/vendor/bundle/ruby/3.2.0/gems/puma-6.3.1/lib/puma/request.rb:100:in `block in handle_request'",
"/home/customersdot/CustomersDot/shared/vendor/bundle/ruby/3.2.0/gems/puma-6.3.1/lib/puma/thread_pool.rb:344:in `with_force_shutdown'",
"/home/customersdot/CustomersDot/shared/vendor/bundle/ruby/3.2.0/gems/puma-6.3.1/lib/puma/request.rb:99:in `handle_request'",
"/home/customersdot/CustomersDot/shared/vendor/bundle/ruby/3.2.0/gems/puma-6.3.1/lib/puma/server.rb:443:in `process_client'",
"/home/customersdot/CustomersDot/shared/vendor/bundle/ruby/3.2.0/gems/puma-6.3.1/lib/puma/server.rb:245:in `block in run'",
"/home/customersdot/CustomersDot/shared/vendor/bundle/ruby/3.2.0/gems/puma-6.3.1/lib/puma/thread_pool.rb:151:in `block in spawn_thread'"
],
"shard": "default",
"errors": "",
"tier": "sv",
"environment": "gstg",
"http_status": null,
"message": "There are no additional services for your license",
"correlation_id": "547eff97e6ba0f148cfa916cd82f6cdf"
},
"resource": {
"type": "gce_instance",
"labels": {
"project_id": "gitlab-subscriptions-staging",
"instance_id": "2850967734799309520",
"zone": "us-east1-d"
}
},
"timestamp": "2024-01-11T14:45:43.068Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "customers-02-inf-stgsub",
"tag": "rails.stg"
},
"logName": "projects/gitlab-subscriptions-staging/logs/rails.stg",
"receiveTimestamp": "2024-01-11T14:46:23.929354869Z"
}
Kibana access log event
{
"_index": "pubsub-rails-inf-stgsub-000024",
"_id": "bF39-IwBajpF_RebQ2TY",
"_version": 1,
"_score": 0,
"_source": {
"@timestamp": "2024-01-11T14:46:35.624Z",
"publish_time": "2024-01-11T14:46:35.589Z",
"json": {
"db": 31.58,
"tag": "rails.stg",
"format": "*/*",
"fqdn": "customers.staging.gitlab.com",
"stage": "main",
"ua": "Ruby",
"view": 0.81,
"shard": "default",
"action": "create",
"duration_threshold": 10.5,
"feature_category": "sm_provisioning",
"params": [
{
"value": "16.8.0-pre",
"key": "gitlab_version"
},
{
"value": "2024-01-11T14:45:41Z",
"key": "timestamp"
},
{
"key": "license_key",
"value": "[FILTERED]"
},
{
"key": "max_historical_user_count",
"value": "0"
},
{
"key": "billable_users_count",
"value": ""
},
{
"key": "hostname",
"value": "34-89-208-10.nip.io"
},
{
"key": "instance_id",
"value": "6f2c9b21-1aec-4203-9a82-8979e64ea97b"
}
],
"tier": "sv",
"controller": "Api::V1::SeatLinksController",
"remote_ip": "34.89.208.10",
"environment": "gstg",
"correlation_id": "547eff97e6ba0f148cfa916cd82f6cdf",
"type": "customersdot",
"zuora_requests": 8,
"hostname": "10.185.2.3",
"path": "/api/v1/seat_links",
"method": "POST",
"duration": 1778.06,
"zuora_requests_duration_s": 1.676208537,
"time": "2024-01-11T14:45:43.838Z",
"status": 200
},
"host": {
"name": "pubsubbeat-pubsub-rails-inf-stgsub-86f58788d5-48pcx"
},
"type": "pubsubbeat-pubsub-rails-inf-stgsub-86f58788d5-48pcx"
},
"fields": {
"json.type.keyword": [
"customersdot"
],
"host.name.keyword": [
"pubsubbeat-pubsub-rails-inf-stgsub-86f58788d5-48pcx"
],
"json.params.key": [
"gitlab_version",
"timestamp",
"license_key",
"max_historical_user_count",
"billable_users_count",
"hostname",
"instance_id"
],
"json.zuora_requests_duration_s": [
1.6762085
],
"json.duration": [
1778.06
],
"type": [
"pubsubbeat-pubsub-rails-inf-stgsub-86f58788d5-48pcx"
],
"type.keyword": [
"pubsubbeat-pubsub-rails-inf-stgsub-86f58788d5-48pcx"
],
"publish_time": [
"2024-01-11T14:46:35.589Z"
],
"json.zuora_requests": [
8
],
"host.name": [
"pubsubbeat-pubsub-rails-inf-stgsub-86f58788d5-48pcx"
],
"json.params.value": [
"16.8.0-pre",
"2024-01-11T14:45:41Z",
"[FILTERED]",
"0",
"",
"34-89-208-10.nip.io",
"6f2c9b21-1aec-4203-9a82-8979e64ea97b"
],
"json.tier.keyword": [
"sv"
],
"json.tag": [
"rails.stg"
],
"json.db": [
31.58
],
"json.path": [
"/api/v1/seat_links"
],
"json.view": [
0
],
"json.ua": [
"Ruby"
],
"json.environment": [
"gstg"
],
"json.remote_ip": [
"34.89.208.10"
],
"json.stage": [
"main"
],
"json.duration_threshold": [
10
],
"json.fqdn.keyword": [
"customers.staging.gitlab.com"
],
"json.params.value.keyword": [
"16.8.0-pre",
"2024-01-11T14:45:41Z",
"[FILTERED]",
"0",
"",
"34-89-208-10.nip.io",
"6f2c9b21-1aec-4203-9a82-8979e64ea97b"
],
"json.type": [
"customersdot"
],
"json.hostname.keyword": [
"10.185.2.3"
],
"json.hostname": [
"10.185.2.3"
],
"json.feature_category.keyword": [
"sm_provisioning"
],
"json.environment.keyword": [
"gstg"
],
"json.tag.keyword": [
"rails.stg"
],
"json.params.key.keyword": [
"gitlab_version",
"timestamp",
"license_key",
"max_historical_user_count",
"billable_users_count",
"hostname",
"instance_id"
],
"json.path.keyword": [
"/api/v1/seat_links"
],
"json.controller": [
"Api::V1::SeatLinksController"
],
"json.action": [
"create"
],
"json.time": [
"2024-01-11T14:45:43.838Z"
],
"json.stage.keyword": [
"main"
],
"json.shard": [
"default"
],
"json.fqdn": [
"customers.staging.gitlab.com"
],
"json.correlation_id.keyword": [
"547eff97e6ba0f148cfa916cd82f6cdf"
],
"json.correlation_id": [
"547eff97e6ba0f148cfa916cd82f6cdf"
],
"json.method": [
"POST"
],
"json.format": [
"*/*"
],
"json.tier": [
"sv"
],
"json.ua.keyword": [
"Ruby"
],
"json.status": [
200
],
"json.remote_ip.keyword": [
"34.89.208.10"
],
"json.feature_category": [
"sm_provisioning"
],
"json.shard.keyword": [
"default"
],
"@timestamp": [
"2024-01-11T14:46:35.624Z"
],
"json.action.keyword": [
"create"
],
"json.format.keyword": [
"*/*"
],
"json.controller.keyword": [
"Api::V1::SeatLinksController"
],
"json.method.keyword": [
"POST"
]
}
}