Skip to content

Format tracing duration in human-reable units

Daniele Rossetti requested to merge drosse/tracing-formatted-duration into master

What does this MR do and why?

Format tracing duration in human-reable units

Closes Tracing timeline: displayed durations should us... (gitlab-org/opstrace/opstrace#2541 - closed)

MR acceptance checklist

Please evaluate this MR against the MR acceptance checklist. It helps you analyze changes to reduce risks in quality, performance, reliability, security, and maintainability.

Screenshots or screen recordings

Before After
image Screenshot_2024-01-05_at_13.09.44
image Screenshot_2024-01-05_at_14.13.23

How to set up and validate locally

  • Enable observability_tracing feature flag

Apply patch to load mocks ( pbpaste | git apply --allow-empty )

diff --git a/app/assets/javascripts/observability/client.js b/app/assets/javascripts/observability/client.js
index b7e4ae8e8ea3..8dc408f440bf 100644
--- a/app/assets/javascripts/observability/client.js
+++ b/app/assets/javascripts/observability/client.js
@@ -1,22 +1,376 @@
+/* eslint-disable @gitlab/require-i18n-strings */
 import { isValidDate } from '~/lib/utils/datetime_utility';
 import * as Sentry from '~/sentry/sentry_browser_wrapper';
 import axios from '~/lib/utils/axios_utils';
 import { logError } from '~/lib/logger';
 import { DEFAULT_SORTING_OPTION, SORTING_OPTIONS } from './constants';
 
+const MOCK_METRICS = {
+  metrics: [],
+};
+
+const MOCK_TRACES = {
+  project_id: 1,
+  traces: [
+    {
+      timestamp: '2024-01-05T00:15:20.801Z',
+      timestamp_nano: 1704413720801000000,
+      trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 207700097663,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T00:15:17.942Z',
+      timestamp_nano: 1704413717942000000,
+      trace_id: '85a6e202-cd8a-dd0e-86a5-fb9ddd36a12c',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 196792486360,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T00:15:50.582Z',
+      timestamp_nano: 1704413750582000000,
+      trace_id: '968b0884-0c09-2108-e429-d4fd41b3b353',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 182400051401,
+      total_spans: 55,
+      matched_span_count: 55,
+      error_span_count: 0,
+      totalSpans: 55,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T08:38:11.464Z',
+      timestamp_nano: 1704443891464000000,
+      trace_id: 'f8471ace-5f81-8a90-9dc4-8a741d9df9bf',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 166341922617,
+      total_spans: 43,
+      matched_span_count: 43,
+      error_span_count: 0,
+      totalSpans: 43,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T00:15:15.09Z',
+      timestamp_nano: 1704413715090000000,
+      trace_id: '4817b6bd-a040-aaf3-6127-6512df80cc3d',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 163976687396,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T08:38:18.232Z',
+      timestamp_nano: 1704443898232000000,
+      trace_id: 'a23e6c1f-da9b-bded-88a9-33e0dd69e31e',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 159964955101,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T08:38:35.799Z',
+      timestamp_nano: 1704443915799000000,
+      trace_id: 'ba9e1234-077a-e17a-71a3-61987770ce03',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 158210647800,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T09:40:41.898Z',
+      timestamp_nano: 1704447641898000000,
+      trace_id: 'a9ea4a57-e7cd-20ec-95c8-e5b792b1a9c1',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 154867473296,
+      total_spans: 55,
+      matched_span_count: 55,
+      error_span_count: 0,
+      totalSpans: 55,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T08:51:04.623Z',
+      timestamp_nano: 1704444664623000000,
+      trace_id: 'f3046cda-61e5-7aa7-e4a6-aaec9ff42235',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 153347349088,
+      total_spans: 49,
+      matched_span_count: 49,
+      error_span_count: 0,
+      totalSpans: 49,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T00:15:04.171Z',
+      timestamp_nano: 1704413704171000000,
+      trace_id: '86b51939-070b-61fa-365a-176f30eca803',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 152202913183,
+      total_spans: 61,
+      matched_span_count: 61,
+      error_span_count: 0,
+      totalSpans: 61,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T04:34:24.214Z',
+      timestamp_nano: 1704429264214000000,
+      trace_id: 'ee245d86-6aaa-868a-2f15-315774486863',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 151873698263,
+      total_spans: 49,
+      matched_span_count: 49,
+      error_span_count: 0,
+      totalSpans: 49,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T08:38:48.836Z',
+      timestamp_nano: 1704443928836000000,
+      trace_id: '4d6e6cec-4a9b-1d2a-8513-e705762e8d73',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 151749660306,
+      total_spans: 43,
+      matched_span_count: 43,
+      error_span_count: 0,
+      totalSpans: 43,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T04:34:27.134Z',
+      timestamp_nano: 1704429267134000000,
+      trace_id: 'ccbb7811-2631-2226-cdc2-624ef56baa3d',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 150665386879,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T06:24:33.547Z',
+      timestamp_nano: 1704435873547000000,
+      trace_id: 'cf2660ad-065c-f02a-2bc2-f3c52bad600c',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 148598521297,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T00:15:10.015Z',
+      timestamp_nano: 1704413710015000000,
+      trace_id: '28113368-a062-28aa-3ad8-c56587d4283a',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 148532908597,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T13:16:44.075Z',
+      timestamp_nano: 1704460604075000000,
+      trace_id: '0a57c2b1-566c-50ce-740d-d4500bae0653',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 147986781541,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T09:41:07.345Z',
+      timestamp_nano: 1704447667345000000,
+      trace_id: '537a032e-ac51-8aa1-aa53-f8aaf54189e6',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 147962288501,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T09:41:09.2Z',
+      timestamp_nano: 1704447669200000000,
+      trace_id: 'f98255e2-e9ae-05d1-0c99-d039dbc9fa7d',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 147254839723,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T05:13:17.618Z',
+      timestamp_nano: 1704431597618000000,
+      trace_id: '8c66808f-1371-2bba-24fe-85773672f403',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 147249826168,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T05:13:12.647Z',
+      timestamp_nano: 1704431592647000000,
+      trace_id: '42255ecd-1451-a20b-064e-bea0ea25b3f0',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 146824417300,
+      total_spans: 43,
+      matched_span_count: 43,
+      error_span_count: 0,
+      totalSpans: 43,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T08:39:08.469Z',
+      timestamp_nano: 1704443948469000000,
+      trace_id: '83cb89f2-072d-97c2-7d47-cff34ebebc69',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 146591252147,
+      total_spans: 47,
+      matched_span_count: 47,
+      error_span_count: 0,
+      totalSpans: 47,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-04T22:55:29.76Z',
+      timestamp_nano: 1704408929760000000,
+      trace_id: '172ce5ae-5a5c-44bb-de21-93269366062e',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 145850782448,
+      total_spans: 55,
+      matched_span_count: 55,
+      error_span_count: 0,
+      totalSpans: 55,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T13:16:51.379Z',
+      timestamp_nano: 1704460611379000000,
+      trace_id: 'c1ac0783-0d59-cec8-9943-46ab1c57e110',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 145454855940,
+      total_spans: 59,
+      matched_span_count: 59,
+      error_span_count: 0,
+      totalSpans: 59,
+      statusCode: '',
+    },
+    {
+      timestamp: '2024-01-05T00:14:47.377Z',
+      timestamp_nano: 1704413687377000000,
+      trace_id: 'e4f6366e-0680-3948-6f75-8d0255022299',
+      service_name: 'frontend',
+      operation: 'HTTP POST',
+      status_code: 'STATUS_CODE_UNSET',
+      duration_nano: 144159234334,
+      total_spans: 37,
+      matched_span_count: 37,
+      error_span_count: 0,
+      totalSpans: 37,
+      statusCode: '',
+    },
+  ],
+  total_traces: 500,
+  next_page_token:
+    'eyJsYXN0X3NlZW5fZHVyYXRpb24iOjEwNTE1MjExMTMzNSwic2Vlbl90cmFjZV9pZHMiOlsiODgzY2UzY2UtMDViYS03ZTU5LWQ1NzktYThiMTUyNjk4ZDA1Il19',
+};
+
 function reportErrorAndThrow(e) {
   logError(e);
   Sentry.captureException(e);
   throw e;
 }
+
+function mockReturnDataWithDelay(data) {
+  return new Promise((resolve) => {
+    setTimeout(() => resolve(data), 500);
+  });
+}
+
 // Provisioning API spec: https://gitlab.com/gitlab-org/opstrace/opstrace/-/blob/main/provisioning-api/pkg/provisioningapi/routes.go#L59
 async function enableObservability(provisioningUrl) {
   try {
-    // Note: axios.put(url, undefined, {withCredentials: true}) does not send cookies properly, so need to use the API below for the correct behaviour
-    return await axios(provisioningUrl, {
-      method: 'put',
-      withCredentials: true,
-    });
+    console.log('[DEBUG] Enabling Observability');
+    return mockReturnDataWithDelay();
   } catch (e) {
     return reportErrorAndThrow(e);
   }
@@ -25,11 +379,12 @@ async function enableObservability(provisioningUrl) {
 // Provisioning API spec: https://gitlab.com/gitlab-org/opstrace/opstrace/-/blob/main/provisioning-api/pkg/provisioningapi/routes.go#L37
 async function isObservabilityEnabled(provisioningUrl) {
   try {
-    const { data } = await axios.get(provisioningUrl, { withCredentials: true });
+    console.log('[DEBUG] Checking Observability Enabled');
+    const data = { status: 'ready' };
     if (data && data.status) {
       // we currently ignore the 'status' payload and just check if the request was successful
       // We might improve this as part of https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/2315
-      return true;
+      return mockReturnDataWithDelay(true);
     }
   } catch (e) {
     if (e.response.status === 404) {
@@ -41,19 +396,1801 @@ async function isObservabilityEnabled(provisioningUrl) {
 }
 
 async function fetchTrace(tracingUrl, traceId) {
-  try {
-    if (!traceId) {
-      throw new Error('traceId is required.');
-    }
-
-    const { data } = await axios.get(`${tracingUrl}/${traceId}`, {
-      withCredentials: true,
-    });
-
-    return data;
-  } catch (e) {
-    return reportErrorAndThrow(e);
-  }
+  console.log(`[DEBUG] Fetch trace ${traceId} from ${tracingUrl}`);
+  return mockReturnDataWithDelay({
+    timestamp: '2024-01-05T00:15:20.801Z',
+    timestamp_nano: 1704413720801000000,
+    trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+    service_name: 'frontend',
+    operation: 'HTTP POST',
+    status_code: 'STATUS_CODE_UNSET',
+    duration_nano: 207700097663,
+    total_spans: 37,
+    spans: [
+      {
+        timestamp: '2024-01-05T00:15:20.801Z',
+        timestamp_nano: 1704413720801000000,
+        span_id: '49EDAF33EBF9C836',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'frontend',
+        operation: 'HTTP POST',
+        duration_nano: 207700097663,
+        parent_span_id: '',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'frontend',
+        scope_version: '',
+        span_attributes: {
+          'app.synthetic_request': 'true',
+          'http.flavor': '1.1',
+          'http.method': 'POST',
+          'http.status_code': '200',
+          'http.target': '/api/checkout',
+          'http.url': 'otel-demo-frontendproxy:8080/api/checkout',
+          'http.user_agent': 'python-requests/2.31.0',
+        },
+        resource_attributes: {
+          'cloud.account.id': 'opstrace-dev-bee41fca',
+          'cloud.availability_zone': 'europe-west1-b',
+          'cloud.provider': 'gcp',
+          'container.name': '',
+          'host.arch': 'amd64',
+          'host.id': '6325675042675770930',
+          'host.name': 'rossetd-0201.c.opstrace-dev-bee41fca.internal',
+          'k8s.cluster.name': '',
+          'k8s.deployment.name': 'otel-demo-frontend',
+          'k8s.namespace.name': '',
+          'k8s.node.name': 'opstrace-worker',
+          'k8s.pod.ip': '192.168.95.23',
+          'k8s.pod.name': 'otel-demo-frontend-868cdc975b-kvtb4',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': 'd2707722-9050-494f-aa30-4c377550bee3',
+          'os.type': 'linux',
+          'os.version': '6.2.0-1019-gcp',
+          'process.command': '/app/server.js',
+          'process.command_args':
+            '["/usr/local/bin/node","--require","./Instrumentation.js","/app/server.js"]',
+          'process.executable.name': 'node',
+          'process.executable.path': '/usr/local/bin/node',
+          'process.owner': 'nextjs',
+          'process.pid': '23',
+          'process.runtime.description': 'Node.js',
+          'process.runtime.name': 'nodejs',
+          'process.runtime.version': '18.19.0',
+          'service.name': 'frontend',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'nodejs',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.18.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.801Z',
+        timestamp_nano: 1704413720801000000,
+        span_id: '1F68638BD5C4BAB7',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'frontend',
+        operation: 'grpc.oteldemo.CheckoutService/PlaceOrder',
+        duration_nano: 207696402643,
+        parent_span_id: '49EDAF33EBF9C836',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: '@opentelemetry/instrumentation-grpc',
+        scope_version: '0.45.1',
+        span_attributes: {
+          'net.peer.name': 'otel-demo-checkoutservice',
+          'net.peer.port': '8080',
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'PlaceOrder',
+          'rpc.service': 'oteldemo.CheckoutService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'cloud.account.id': 'opstrace-dev-bee41fca',
+          'cloud.availability_zone': 'europe-west1-b',
+          'cloud.provider': 'gcp',
+          'container.name': '',
+          'host.arch': 'amd64',
+          'host.id': '6325675042675770930',
+          'host.name': 'rossetd-0201.c.opstrace-dev-bee41fca.internal',
+          'k8s.cluster.name': '',
+          'k8s.deployment.name': 'otel-demo-frontend',
+          'k8s.namespace.name': '',
+          'k8s.node.name': 'opstrace-worker',
+          'k8s.pod.ip': '192.168.95.23',
+          'k8s.pod.name': 'otel-demo-frontend-868cdc975b-kvtb4',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': 'd2707722-9050-494f-aa30-4c377550bee3',
+          'os.type': 'linux',
+          'os.version': '6.2.0-1019-gcp',
+          'process.command': '/app/server.js',
+          'process.command_args':
+            '["/usr/local/bin/node","--require","./Instrumentation.js","/app/server.js"]',
+          'process.executable.name': 'node',
+          'process.executable.path': '/usr/local/bin/node',
+          'process.owner': 'nextjs',
+          'process.pid': '23',
+          'process.runtime.description': 'Node.js',
+          'process.runtime.name': 'nodejs',
+          'process.runtime.version': '18.19.0',
+          'service.name': 'frontend',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'nodejs',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.18.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.80284427Z',
+        timestamp_nano: 1704413720802844270,
+        span_id: 'C11AA1CA8C022D46',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'oteldemo.CheckoutService/PlaceOrder',
+        duration_nano: 207694202123,
+        parent_span_id: '1F68638BD5C4BAB7',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'app.order.amount': '154',
+          'app.order.id': '834ae5b9-ab5f-11ee-8874-beb360c28a91',
+          'app.order.items.count': '1',
+          'app.shipping.amount': '44',
+          'app.shipping.tracking.id': '050cb161-26bc-46b1-8463-f2c9e3935a45',
+          'app.user.currency': 'USD',
+          'app.user.id': '83454efc-ab5f-11ee-898a-0290e1d7bf90',
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'PlaceOrder',
+          'rpc.service': 'oteldemo.CheckoutService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.803068854Z',
+        timestamp_nano: 1704413720803068854,
+        span_id: 'CE8F88E5A300B03A',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'prepareOrderItemsAndShippingQuoteFromCart',
+        duration_nano: 28305203,
+        parent_span_id: 'C11AA1CA8C022D46',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_INTERNAL',
+        scope_name: 'checkoutservice',
+        scope_version: '',
+        span_attributes: {
+          'app.cart.items.count': '5',
+          'app.order.items.count': '1',
+          'app.shipping.amount': '44',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.803354245Z',
+        timestamp_nano: 1704413720803354245,
+        span_id: '0C4F86E8DCC7604C',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'oteldemo.CartService/GetCart',
+        duration_nano: 4325569,
+        parent_span_id: 'CE8F88E5A300B03A',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'GetCart',
+          'rpc.service': 'oteldemo.CartService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.8060353Z',
+        timestamp_nano: 1704413720806035300,
+        span_id: '822B9633DE49EB20',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'cartservice',
+        operation: 'oteldemo.CartService/GetCart',
+        duration_nano: 1342000,
+        parent_span_id: '0C4F86E8DCC7604C',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'Microsoft.AspNetCore',
+        scope_version: '',
+        span_attributes: {
+          'app.cart.items.count': '5',
+          'app.user.id': '83454efc-ab5f-11ee-898a-0290e1d7bf90',
+          'http.flavor': '2',
+          'http.method': 'POST',
+          'http.route': '/oteldemo.CartService/GetCart',
+          'http.scheme': 'http',
+          'http.status_code': '200',
+          'http.target': '/oteldemo.CartService/GetCart',
+          'http.url': 'http://otel-demo-cartservice:8080/oteldemo.CartService/GetCart',
+          'http.user_agent': 'grpc-go/1.59.0',
+          'net.host.name': 'otel-demo-cartservice',
+          'net.host.port': '8080',
+          'net.peer.ip': '::ffff:192.168.62.219',
+          'net.peer.port': '54066',
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'GetCart',
+          'rpc.service': 'oteldemo.CartService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'container.id': 'e1beb90eee0e5bb19aadcc48e179a9cd145de723fed1ecc22599eedef573c571',
+          'k8s.deployment.name': 'otel-demo-cartservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.217',
+          'k8s.pod.name': 'otel-demo-cartservice-6dcc867f5f-vtfpj',
+          'k8s.pod.start_time': '2024-01-02T12:09:31Z',
+          'k8s.pod.uid': '70418413-82d1-46c1-a978-c7f71302a489',
+          'service.name': 'cartservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'dotnet',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.6.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.8064211Z',
+        timestamp_nano: 1704413720806421100,
+        span_id: 'E50F244D645865D1',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'cartservice',
+        operation: 'HGET',
+        duration_nano: 864600,
+        parent_span_id: '822B9633DE49EB20',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'OpenTelemetry.Instrumentation.StackExchangeRedis',
+        scope_version: '1.0.0.12',
+        span_attributes: {
+          'db.redis.database_index': '0',
+          'db.redis.flags': 'None',
+          'db.statement': 'HGET 83454efc-ab5f-11ee-898a-0290e1d7bf90',
+          'db.system': 'redis',
+          'net.peer.name': 'otel-demo-redis',
+          'net.peer.port': '6379',
+          'peer.service': 'otel-demo-redis:6379',
+        },
+        resource_attributes: {
+          'container.id': 'e1beb90eee0e5bb19aadcc48e179a9cd145de723fed1ecc22599eedef573c571',
+          'k8s.deployment.name': 'otel-demo-cartservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.217',
+          'k8s.pod.name': 'otel-demo-cartservice-6dcc867f5f-vtfpj',
+          'k8s.pod.start_time': '2024-01-02T12:09:31Z',
+          'k8s.pod.uid': '70418413-82d1-46c1-a978-c7f71302a489',
+          'service.name': 'cartservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'dotnet',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.6.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.808140926Z',
+        timestamp_nano: 1704413720808140926,
+        span_id: 'E4C4943514DC5D52',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'oteldemo.ProductCatalogService/GetProduct',
+        duration_nano: 3198278,
+        parent_span_id: 'CE8F88E5A300B03A',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'GetProduct',
+          'rpc.service': 'oteldemo.ProductCatalogService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.810666775Z',
+        timestamp_nano: 1704413720810666775,
+        span_id: '2BEC237F76B69509',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'productcatalogservice',
+        operation: 'oteldemo.ProductCatalogService/GetProduct',
+        duration_nano: 214006,
+        parent_span_id: 'E4C4943514DC5D52',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'app.product.id': 'L9ECAV7KIM',
+          'app.product.name': 'Lens Cleaning Kit',
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'GetProduct',
+          'rpc.service': 'oteldemo.ProductCatalogService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-productcatalogservice-5cf4c48bdb-8gnwn',
+          'k8s.deployment.name': 'otel-demo-productcatalogservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker',
+          'k8s.pod.ip': '192.168.95.20',
+          'k8s.pod.name': 'otel-demo-productcatalogservice-5cf4c48bdb-8gnwn',
+          'k8s.pod.start_time': '2024-01-02T12:09:32Z',
+          'k8s.pod.uid': 'd96c0f64-ed60-437f-a51f-e72c40b52c04',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-productcatalogservice-5cf4c48bdb-8gnwn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./productcatalogservice"]',
+          'process.executable.name': 'productcatalogservice',
+          'process.executable.path': '/usr/src/app/productcatalogservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'productcatalogservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.811589444Z',
+        timestamp_nano: 1704413720811589444,
+        span_id: '2264D728D9223DE9',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'oteldemo.CurrencyService/Convert',
+        duration_nano: 6412096,
+        parent_span_id: 'CE8F88E5A300B03A',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'Convert',
+          'rpc.service': 'oteldemo.CurrencyService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.814444261Z',
+        timestamp_nano: 1704413720814444261,
+        span_id: 'D946B353ABE35B01',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'currencyservice',
+        operation: 'CurrencyService/Convert',
+        duration_nano: 293163,
+        parent_span_id: '2264D728D9223DE9',
+        status_code: 'STATUS_CODE_OK',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'currencyservice',
+        scope_version: '',
+        span_attributes: {
+          'app.currency.conversion.from': 'USD',
+          'app.currency.conversion.to': 'USD',
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'Convert',
+          'rpc.service': 'CurrencyService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'k8s.deployment.name': 'otel-demo-currencyservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker3',
+          'k8s.pod.ip': '192.168.37.216',
+          'k8s.pod.name': 'otel-demo-currencyservice-8c6556cc-9kz4v',
+          'k8s.pod.start_time': '2024-01-02T12:09:29Z',
+          'k8s.pod.uid': '8c7eaff6-28c1-43b8-9729-fc000ac0fe8c',
+          'service.name': 'currencyservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'cpp',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.13.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.81893761Z',
+        timestamp_nano: 1704413720818937610,
+        span_id: 'E59B54C0229C37C0',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'oteldemo.ShippingService/GetQuote',
+        duration_nano: 6421560,
+        parent_span_id: 'CE8F88E5A300B03A',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'GetQuote',
+          'rpc.service': 'oteldemo.ShippingService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.821418874Z',
+        timestamp_nano: 1704413720821418874,
+        span_id: '7CDD4D475A6FEB4B',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'shippingservice',
+        operation: 'oteldemo.ShippingService/GetQuote',
+        duration_nano: 3561228,
+        parent_span_id: 'E59B54C0229C37C0',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'shippingservice',
+        scope_version: '',
+        span_attributes: {
+          'app.shipping.cost.total': '44.50',
+          'app.shipping.items.count': '5',
+          'app.shipping.zip_code': '98052',
+          'rpc.grpc.status_code': '0',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'k8s.deployment.name': 'otel-demo-shippingservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker3',
+          'k8s.pod.ip': '192.168.37.219',
+          'k8s.pod.name': 'otel-demo-shippingservice-545578f46d-vkhjg',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '3047135f-b88f-4d77-8311-35bc4a730eb8',
+          'os.type': 'linux',
+          'process.command_args': '["/app/shippingservice"]',
+          'process.pid': '1',
+          'service.name': 'shippingservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'rust',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '0.21.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.821490449Z',
+        timestamp_nano: 1704413720821490449,
+        span_id: 'C1D7B84A8AA334FB',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'shippingservice',
+        operation: 'POST',
+        duration_nano: 3216233,
+        parent_span_id: '7CDD4D475A6FEB4B',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'opentelemetry-otlp',
+        scope_version: '0.14.0',
+        span_attributes: {
+          busy_ns: '470871',
+          'code.filepath':
+            '/usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/reqwest-tracing-0.4.6/src/reqwest_otel_span_builder.rs',
+          'code.lineno': '138',
+          'code.namespace': 'reqwest_tracing::reqwest_otel_span_builder',
+          'http.host': 'otel-demo-quoteservice',
+          'http.method': 'POST',
+          'http.scheme': 'http',
+          'http.status_code': '200',
+          'http.url': 'http://otel-demo-quoteservice:8080/getquote',
+          'http.user_agent': '',
+          idle_ns: '2745364',
+          'net.host.port': '8080',
+          'thread.id': '6',
+          'thread.name': 'tokio-runtime-worker',
+        },
+        resource_attributes: {
+          'k8s.deployment.name': 'otel-demo-shippingservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker3',
+          'k8s.pod.ip': '192.168.37.219',
+          'k8s.pod.name': 'otel-demo-shippingservice-545578f46d-vkhjg',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '3047135f-b88f-4d77-8311-35bc4a730eb8',
+          'os.type': 'linux',
+          'process.command_args': '["/app/shippingservice"]',
+          'process.pid': '1',
+          'service.name': 'shippingservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'rust',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '0.21.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.823700104Z',
+        timestamp_nano: 1704413720823700104,
+        span_id: 'DFD22125C4C78612',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'quoteservice',
+        operation: 'POST /getquote',
+        duration_nano: 493890,
+        parent_span_id: '7CDD4D475A6FEB4B',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'io.opentelemetry.contrib.php.slim',
+        scope_version: '',
+        span_attributes: {
+          'code.filepath': '/var/www/vendor/slim/slim/Slim/App.php',
+          'code.function': 'handle',
+          'code.lineno': '197',
+          'code.namespace': 'Slim\\App',
+          'http.request.body.size': '19',
+          'http.request.method': 'POST',
+          'http.response.body.size': '',
+          'http.response.status_code': '200',
+          'network.protocol.version': '1.1',
+          'url.full': 'http://otel-demo-quoteservice:8080/getquote',
+          'url.scheme': 'http',
+        },
+        resource_attributes: {
+          'container.id': '8a306cf680ff52c50b5375f431ef5dba2dfc8555470045f195ed6cbe74ab1ac0',
+          'host.arch': 'x86_64',
+          'host.name': 'otel-demo-quoteservice-5f7f59687b-96f7f',
+          'k8s.deployment.name': 'otel-demo-quoteservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker',
+          'k8s.pod.ip': '192.168.95.21',
+          'k8s.pod.name': 'otel-demo-quoteservice-5f7f59687b-96f7f',
+          'k8s.pod.start_time': '2024-01-02T12:09:32Z',
+          'k8s.pod.uid': '2606f261-2d77-493c-a1e3-65331c7a539e',
+          'os.description': '6.2.0-1019-gcp',
+          'os.name': 'Linux',
+          'os.type': 'linux',
+          'os.version': '#21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023',
+          'process.command': 'public/index.php',
+          'process.command_args': '["public/index.php"]',
+          'process.executable.path': '/usr/local/bin/php',
+          'process.owner': 'www-data',
+          'process.pid': '13',
+          'process.runtime.name': 'cli',
+          'process.runtime.version': '8.2.13',
+          'service.name': 'quoteservice',
+          'service.namespace': 'opentelemetry-demo',
+          'service.version': '1.0.0+no-version-set',
+          'telemetry.distro.name': 'opentelemetry-php-instrumentation',
+          'telemetry.distro.version': '1.0.0',
+          'telemetry.sdk.language': 'php',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.0.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.823876053Z',
+        timestamp_nano: 1704413720823876053,
+        span_id: '5F3CE501411A9819',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'quoteservice',
+        operation: '{closure}',
+        duration_nano: 285838,
+        parent_span_id: 'DFD22125C4C78612',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_INTERNAL',
+        scope_name: 'io.opentelemetry.contrib.php.slim',
+        scope_version: '',
+        span_attributes: {
+          'code.filepath': '/var/www/vendor/php-di/slim-bridge/src/ControllerInvoker.php',
+          'code.function': '__invoke',
+          'code.lineno': '31',
+          'code.namespace': 'DI\\Bridge\\Slim\\ControllerInvoker',
+        },
+        resource_attributes: {
+          'container.id': '8a306cf680ff52c50b5375f431ef5dba2dfc8555470045f195ed6cbe74ab1ac0',
+          'host.arch': 'x86_64',
+          'host.name': 'otel-demo-quoteservice-5f7f59687b-96f7f',
+          'k8s.deployment.name': 'otel-demo-quoteservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker',
+          'k8s.pod.ip': '192.168.95.21',
+          'k8s.pod.name': 'otel-demo-quoteservice-5f7f59687b-96f7f',
+          'k8s.pod.start_time': '2024-01-02T12:09:32Z',
+          'k8s.pod.uid': '2606f261-2d77-493c-a1e3-65331c7a539e',
+          'os.description': '6.2.0-1019-gcp',
+          'os.name': 'Linux',
+          'os.type': 'linux',
+          'os.version': '#21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023',
+          'process.command': 'public/index.php',
+          'process.command_args': '["public/index.php"]',
+          'process.executable.path': '/usr/local/bin/php',
+          'process.owner': 'www-data',
+          'process.pid': '13',
+          'process.runtime.name': 'cli',
+          'process.runtime.version': '8.2.13',
+          'service.name': 'quoteservice',
+          'service.namespace': 'opentelemetry-demo',
+          'service.version': '1.0.0+no-version-set',
+          'telemetry.distro.name': 'opentelemetry-php-instrumentation',
+          'telemetry.distro.version': '1.0.0',
+          'telemetry.sdk.language': 'php',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.0.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.823970653Z',
+        timestamp_nano: 1704413720823970653,
+        span_id: 'FE569EBF17E22B5D',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'quoteservice',
+        operation: 'calculate-quote',
+        duration_nano: 18778,
+        parent_span_id: '5F3CE501411A9819',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_INTERNAL',
+        scope_name: 'manual-instrumentation',
+        scope_version: '',
+        span_attributes: {
+          'app.quote.cost.total': '44.5',
+          'app.quote.items.count': '5',
+        },
+        resource_attributes: {
+          'container.id': '8a306cf680ff52c50b5375f431ef5dba2dfc8555470045f195ed6cbe74ab1ac0',
+          'host.arch': 'x86_64',
+          'host.name': 'otel-demo-quoteservice-5f7f59687b-96f7f',
+          'k8s.deployment.name': 'otel-demo-quoteservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker',
+          'k8s.pod.ip': '192.168.95.21',
+          'k8s.pod.name': 'otel-demo-quoteservice-5f7f59687b-96f7f',
+          'k8s.pod.start_time': '2024-01-02T12:09:32Z',
+          'k8s.pod.uid': '2606f261-2d77-493c-a1e3-65331c7a539e',
+          'os.description': '6.2.0-1019-gcp',
+          'os.name': 'Linux',
+          'os.type': 'linux',
+          'os.version': '#21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023',
+          'process.command': 'public/index.php',
+          'process.command_args': '["public/index.php"]',
+          'process.executable.path': '/usr/local/bin/php',
+          'process.owner': 'www-data',
+          'process.pid': '13',
+          'process.runtime.name': 'cli',
+          'process.runtime.version': '8.2.13',
+          'service.name': 'quoteservice',
+          'service.namespace': 'opentelemetry-demo',
+          'service.version': '1.0.0+no-version-set',
+          'telemetry.distro.name': 'opentelemetry-php-instrumentation',
+          'telemetry.distro.version': '1.0.0',
+          'telemetry.sdk.language': 'php',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.0.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.825836851Z',
+        timestamp_nano: 1704413720825836851,
+        span_id: 'A48D01451A13178B',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'oteldemo.CurrencyService/Convert',
+        duration_nano: 5262192,
+        parent_span_id: 'CE8F88E5A300B03A',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'Convert',
+          'rpc.service': 'oteldemo.CurrencyService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.828578105Z',
+        timestamp_nano: 1704413720828578105,
+        span_id: '80278E2918A99C22',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'currencyservice',
+        operation: 'CurrencyService/Convert',
+        duration_nano: 489278,
+        parent_span_id: 'A48D01451A13178B',
+        status_code: 'STATUS_CODE_OK',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'currencyservice',
+        scope_version: '',
+        span_attributes: {
+          'app.currency.conversion.from': 'USD',
+          'app.currency.conversion.to': 'USD',
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'Convert',
+          'rpc.service': 'CurrencyService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'k8s.deployment.name': 'otel-demo-currencyservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker3',
+          'k8s.pod.ip': '192.168.37.216',
+          'k8s.pod.name': 'otel-demo-currencyservice-8c6556cc-9kz4v',
+          'k8s.pod.start_time': '2024-01-02T12:09:29Z',
+          'k8s.pod.uid': '8c7eaff6-28c1-43b8-9729-fc000ac0fe8c',
+          'service.name': 'currencyservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'cpp',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.13.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.831737833Z',
+        timestamp_nano: 1704413720831737833,
+        span_id: 'F708DDC07E31E000',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'oteldemo.PaymentService/Charge',
+        duration_nano: 8172104,
+        parent_span_id: 'C11AA1CA8C022D46',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'Charge',
+          'rpc.service': 'oteldemo.PaymentService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.836Z',
+        timestamp_nano: 1704413720836000000,
+        span_id: 'CA4C8E8B6FC00007',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'paymentservice',
+        operation: 'charge',
+        duration_nano: 79836,
+        parent_span_id: '298954BEA621C8D0',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_INTERNAL',
+        scope_name: 'paymentservice',
+        scope_version: '',
+        span_attributes: {
+          'app.payment.card_type': 'visa',
+          'app.payment.card_valid': 'true',
+          'app.payment.charged': 'false',
+        },
+        resource_attributes: {
+          'cloud.account.id': 'opstrace-dev-bee41fca',
+          'cloud.availability_zone': 'europe-west1-b',
+          'cloud.provider': 'gcp',
+          'container.name': '',
+          'host.arch': 'amd64',
+          'host.id': '6325675042675770930',
+          'host.name': 'rossetd-0201.c.opstrace-dev-bee41fca.internal',
+          'k8s.cluster.name': '',
+          'k8s.deployment.name': 'otel-demo-paymentservice',
+          'k8s.namespace.name': '',
+          'k8s.node.name': 'opstrace-worker3',
+          'k8s.pod.ip': '192.168.37.220',
+          'k8s.pod.name': 'otel-demo-paymentservice-556654fdf8-5ljm4',
+          'k8s.pod.start_time': '2024-01-02T12:09:36Z',
+          'k8s.pod.uid': '436bbd33-13ab-41c6-839e-1d6cbb7238ca',
+          'os.type': 'linux',
+          'os.version': '6.2.0-1019-gcp',
+          'process.command': '/usr/src/app/index.js',
+          'process.command_args':
+            '["/usr/local/bin/node","--require","./opentelemetry.js","/usr/src/app/index.js"]',
+          'process.executable.name': 'node',
+          'process.executable.path': '/usr/local/bin/node',
+          'process.owner': 'node',
+          'process.pid': '27',
+          'process.runtime.description': 'Node.js',
+          'process.runtime.name': 'nodejs',
+          'process.runtime.version': '21.4.0',
+          'service.name': 'paymentservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'nodejs',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.18.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.836Z',
+        timestamp_nano: 1704413720836000000,
+        span_id: '298954BEA621C8D0',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'paymentservice',
+        operation: 'grpc.oteldemo.PaymentService/Charge',
+        duration_nano: 755070,
+        parent_span_id: 'F708DDC07E31E000',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: '@opentelemetry/instrumentation-grpc',
+        scope_version: '0.45.1',
+        span_attributes: {
+          'app.payment.amount': '154.249999995',
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'Charge',
+          'rpc.service': 'oteldemo.PaymentService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'cloud.account.id': 'opstrace-dev-bee41fca',
+          'cloud.availability_zone': 'europe-west1-b',
+          'cloud.provider': 'gcp',
+          'container.name': '',
+          'host.arch': 'amd64',
+          'host.id': '6325675042675770930',
+          'host.name': 'rossetd-0201.c.opstrace-dev-bee41fca.internal',
+          'k8s.cluster.name': '',
+          'k8s.deployment.name': 'otel-demo-paymentservice',
+          'k8s.namespace.name': '',
+          'k8s.node.name': 'opstrace-worker3',
+          'k8s.pod.ip': '192.168.37.220',
+          'k8s.pod.name': 'otel-demo-paymentservice-556654fdf8-5ljm4',
+          'k8s.pod.start_time': '2024-01-02T12:09:36Z',
+          'k8s.pod.uid': '436bbd33-13ab-41c6-839e-1d6cbb7238ca',
+          'os.type': 'linux',
+          'os.version': '6.2.0-1019-gcp',
+          'process.command': '/usr/src/app/index.js',
+          'process.command_args':
+            '["/usr/local/bin/node","--require","./opentelemetry.js","/usr/src/app/index.js"]',
+          'process.executable.name': 'node',
+          'process.executable.path': '/usr/local/bin/node',
+          'process.owner': 'node',
+          'process.pid': '27',
+          'process.runtime.description': 'Node.js',
+          'process.runtime.name': 'nodejs',
+          'process.runtime.version': '21.4.0',
+          'service.name': 'paymentservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'nodejs',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.18.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.840862845Z',
+        timestamp_nano: 1704413720840862845,
+        span_id: 'ABAFA0955C975AD9',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'oteldemo.ShippingService/ShipOrder',
+        duration_nano: 6135070,
+        parent_span_id: 'C11AA1CA8C022D46',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'ShipOrder',
+          'rpc.service': 'oteldemo.ShippingService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.846424838Z',
+        timestamp_nano: 1704413720846424838,
+        span_id: 'BF7685E87BFDEC95',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'shippingservice',
+        operation: 'oteldemo.ShippingService/ShipOrder',
+        duration_nano: 169012,
+        parent_span_id: 'ABAFA0955C975AD9',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'shippingservice',
+        scope_version: '',
+        span_attributes: {
+          'app.shipping.tracking.id': '050cb161-26bc-46b1-8463-f2c9e3935a45',
+          'rpc.grpc.status_code': '0',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'k8s.deployment.name': 'otel-demo-shippingservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker3',
+          'k8s.pod.ip': '192.168.37.219',
+          'k8s.pod.name': 'otel-demo-shippingservice-545578f46d-vkhjg',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '3047135f-b88f-4d77-8311-35bc4a730eb8',
+          'os.type': 'linux',
+          'process.command_args': '["/app/shippingservice"]',
+          'process.pid': '1',
+          'service.name': 'shippingservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'rust',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '0.21.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.847696575Z',
+        timestamp_nano: 1704413720847696575,
+        span_id: 'D088842B1B1D7B5B',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'oteldemo.CartService/EmptyCart',
+        duration_nano: 7242190,
+        parent_span_id: 'C11AA1CA8C022D46',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'EmptyCart',
+          'rpc.service': 'oteldemo.CartService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.8519939Z',
+        timestamp_nano: 1704413720851993900,
+        span_id: 'D0F5932EA325C271',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'cartservice',
+        operation: 'oteldemo.CartService/EmptyCart',
+        duration_nano: 2231300,
+        parent_span_id: 'D088842B1B1D7B5B',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'Microsoft.AspNetCore',
+        scope_version: '',
+        span_attributes: {
+          'app.user.id': '83454efc-ab5f-11ee-898a-0290e1d7bf90',
+          'http.flavor': '2',
+          'http.method': 'POST',
+          'http.route': '/oteldemo.CartService/EmptyCart',
+          'http.scheme': 'http',
+          'http.status_code': '200',
+          'http.target': '/oteldemo.CartService/EmptyCart',
+          'http.url': 'http://otel-demo-cartservice:8080/oteldemo.CartService/EmptyCart',
+          'http.user_agent': 'grpc-go/1.59.0',
+          'net.host.name': 'otel-demo-cartservice',
+          'net.host.port': '8080',
+          'net.peer.ip': '::ffff:192.168.62.219',
+          'net.peer.port': '54078',
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'EmptyCart',
+          'rpc.service': 'oteldemo.CartService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'container.id': 'e1beb90eee0e5bb19aadcc48e179a9cd145de723fed1ecc22599eedef573c571',
+          'k8s.deployment.name': 'otel-demo-cartservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.217',
+          'k8s.pod.name': 'otel-demo-cartservice-6dcc867f5f-vtfpj',
+          'k8s.pod.start_time': '2024-01-02T12:09:31Z',
+          'k8s.pod.uid': '70418413-82d1-46c1-a978-c7f71302a489',
+          'service.name': 'cartservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'dotnet',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.6.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.8524231Z',
+        timestamp_nano: 1704413720852423100,
+        span_id: '4B802666480132D2',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'cartservice',
+        operation: 'HMSET',
+        duration_nano: 855400,
+        parent_span_id: 'D0F5932EA325C271',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'OpenTelemetry.Instrumentation.StackExchangeRedis',
+        scope_version: '1.0.0.12',
+        span_attributes: {
+          'db.redis.database_index': '0',
+          'db.redis.flags': 'DemandMaster',
+          'db.statement': 'HMSET 83454efc-ab5f-11ee-898a-0290e1d7bf90',
+          'db.system': 'redis',
+          'net.peer.name': 'otel-demo-redis',
+          'net.peer.port': '6379',
+          'peer.service': 'otel-demo-redis:6379',
+        },
+        resource_attributes: {
+          'container.id': 'e1beb90eee0e5bb19aadcc48e179a9cd145de723fed1ecc22599eedef573c571',
+          'k8s.deployment.name': 'otel-demo-cartservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.217',
+          'k8s.pod.name': 'otel-demo-cartservice-6dcc867f5f-vtfpj',
+          'k8s.pod.start_time': '2024-01-02T12:09:31Z',
+          'k8s.pod.uid': '70418413-82d1-46c1-a978-c7f71302a489',
+          'service.name': 'cartservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'dotnet',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.6.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.853307Z',
+        timestamp_nano: 1704413720853307000,
+        span_id: '930CB541D64F6C52',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'cartservice',
+        operation: 'EXPIRE',
+        duration_nano: 822000,
+        parent_span_id: 'D0F5932EA325C271',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'OpenTelemetry.Instrumentation.StackExchangeRedis',
+        scope_version: '1.0.0.12',
+        span_attributes: {
+          'db.redis.database_index': '0',
+          'db.redis.flags': 'DemandMaster',
+          'db.statement': 'EXPIRE 83454efc-ab5f-11ee-898a-0290e1d7bf90',
+          'db.system': 'redis',
+          'net.peer.name': 'otel-demo-redis',
+          'net.peer.port': '6379',
+          'peer.service': 'otel-demo-redis:6379',
+        },
+        resource_attributes: {
+          'container.id': 'e1beb90eee0e5bb19aadcc48e179a9cd145de723fed1ecc22599eedef573c571',
+          'k8s.deployment.name': 'otel-demo-cartservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.217',
+          'k8s.pod.name': 'otel-demo-cartservice-6dcc867f5f-vtfpj',
+          'k8s.pod.start_time': '2024-01-02T12:09:31Z',
+          'k8s.pod.uid': '70418413-82d1-46c1-a978-c7f71302a489',
+          'service.name': 'cartservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'dotnet',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.6.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.855293783Z',
+        timestamp_nano: 1704413720855293783,
+        span_id: '0ABF82C9A845D30C',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'HTTP POST',
+        duration_nano: 16435941,
+        parent_span_id: 'C11AA1CA8C022D46',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'http.flavor': '1.1',
+          'http.method': 'POST',
+          'http.request_content_length': '472',
+          'http.status_code': '200',
+          'http.url': 'http://otel-demo-emailservice:8080/send_order_confirmation',
+          'net.peer.name': 'otel-demo-emailservice',
+          'net.peer.port': '8080',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.864435971Z',
+        timestamp_nano: 1704413720864435971,
+        span_id: 'FF57608A99C229AC',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'emailservice',
+        operation: 'POST /send_order_confirmation',
+        duration_nano: 6499598,
+        parent_span_id: '0ABF82C9A845D30C',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'OpenTelemetry::Instrumentation::Rack',
+        scope_version: '0.23.4',
+        span_attributes: {
+          'app.order.id': '834ae5b9-ab5f-11ee-8874-beb360c28a91',
+          'http.host': 'otel-demo-emailservice:8080',
+          'http.method': 'POST',
+          'http.route': '/send_order_confirmation',
+          'http.scheme': 'http',
+          'http.status_code': '200',
+          'http.target': '/send_order_confirmation',
+          'http.user_agent': 'Go-http-client/1.1',
+        },
+        resource_attributes: {
+          'k8s.deployment.name': 'otel-demo-emailservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.220',
+          'k8s.pod.name': 'otel-demo-emailservice-6465bcc66f-zs4vn',
+          'k8s.pod.start_time': '2024-01-02T12:09:36Z',
+          'k8s.pod.uid': '3b57a0e5-2b62-44f6-b110-f6ad4cbb6063',
+          'process.command': 'email_server.rb',
+          'process.pid': '1',
+          'process.runtime.description':
+            'ruby 3.2.2 (2023-03-30 revision e51014f9c0) [x86_64-linux]',
+          'process.runtime.name': 'ruby',
+          'process.runtime.version': '3.2.2',
+          'service.name': 'emailservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'ruby',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.3.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.867642458Z',
+        timestamp_nano: 1704413720867642458,
+        span_id: '141D7DE7D1F18A5D',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'emailservice',
+        operation: 'send_email',
+        duration_nano: 3099825,
+        parent_span_id: 'FF57608A99C229AC',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_INTERNAL',
+        scope_name: 'emailservice',
+        scope_version: '',
+        span_attributes: {
+          'app.email.recipient': 'bill@example.com',
+        },
+        resource_attributes: {
+          'k8s.deployment.name': 'otel-demo-emailservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.220',
+          'k8s.pod.name': 'otel-demo-emailservice-6465bcc66f-zs4vn',
+          'k8s.pod.start_time': '2024-01-02T12:09:36Z',
+          'k8s.pod.uid': '3b57a0e5-2b62-44f6-b110-f6ad4cbb6063',
+          'process.command': 'email_server.rb',
+          'process.pid': '1',
+          'process.runtime.description':
+            'ruby 3.2.2 (2023-03-30 revision e51014f9c0) [x86_64-linux]',
+          'process.runtime.name': 'ruby',
+          'process.runtime.version': '3.2.2',
+          'service.name': 'emailservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'ruby',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.3.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.867697239Z',
+        timestamp_nano: 1704413720867697239,
+        span_id: '137B2CBEFA2C2BEA',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'emailservice',
+        operation: 'sinatra.render_template',
+        duration_nano: 479405,
+        parent_span_id: '141D7DE7D1F18A5D',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_INTERNAL',
+        scope_name: 'OpenTelemetry::Instrumentation::Sinatra',
+        scope_version: '0.23.2',
+        span_attributes: {
+          'sinatra.template_name': 'confirmation',
+        },
+        resource_attributes: {
+          'k8s.deployment.name': 'otel-demo-emailservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.220',
+          'k8s.pod.name': 'otel-demo-emailservice-6465bcc66f-zs4vn',
+          'k8s.pod.start_time': '2024-01-02T12:09:36Z',
+          'k8s.pod.uid': '3b57a0e5-2b62-44f6-b110-f6ad4cbb6063',
+          'process.command': 'email_server.rb',
+          'process.pid': '1',
+          'process.runtime.description':
+            'ruby 3.2.2 (2023-03-30 revision e51014f9c0) [x86_64-linux]',
+          'process.runtime.name': 'ruby',
+          'process.runtime.version': '3.2.2',
+          'service.name': 'emailservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'ruby',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.3.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.867924176Z',
+        timestamp_nano: 1704413720867924176,
+        span_id: 'A1BF11653AF04736',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'emailservice',
+        operation: 'sinatra.render_template',
+        duration_nano: 223020,
+        parent_span_id: '137B2CBEFA2C2BEA',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_INTERNAL',
+        scope_name: 'OpenTelemetry::Instrumentation::Sinatra',
+        scope_version: '0.23.2',
+        span_attributes: {
+          'sinatra.template_name': 'layout',
+        },
+        resource_attributes: {
+          'k8s.deployment.name': 'otel-demo-emailservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.220',
+          'k8s.pod.name': 'otel-demo-emailservice-6465bcc66f-zs4vn',
+          'k8s.pod.start_time': '2024-01-02T12:09:36Z',
+          'k8s.pod.uid': '3b57a0e5-2b62-44f6-b110-f6ad4cbb6063',
+          'process.command': 'email_server.rb',
+          'process.pid': '1',
+          'process.runtime.description':
+            'ruby 3.2.2 (2023-03-30 revision e51014f9c0) [x86_64-linux]',
+          'process.runtime.name': 'ruby',
+          'process.runtime.version': '3.2.2',
+          'service.name': 'emailservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'ruby',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.3.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.8720101Z',
+        timestamp_nano: 1704413720872010100,
+        span_id: '24F5A9DAA3A2EB5F',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'checkoutservice',
+        operation: 'orders publish',
+        duration_nano: 207624972759,
+        parent_span_id: 'C11AA1CA8C022D46',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_PRODUCER',
+        scope_name: 'checkoutservice',
+        scope_version: '',
+        span_attributes: {
+          'messaging.destination.kind': 'topic',
+          'messaging.destination.name': 'orders',
+          'messaging.kafka.destination.partition': '0',
+          'messaging.operation': 'publish',
+          'messaging.system': 'kafka',
+          'net.transport': 'ip_tcp',
+          'peer.service': 'kafka',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.deployment.name': 'otel-demo-checkoutservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker2',
+          'k8s.pod.ip': '192.168.62.219',
+          'k8s.pod.name': 'otel-demo-checkoutservice-78d7b99f47-lk7kn',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': '07e168bb-0953-455d-be4d-ca85bdf76412',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-checkoutservice-78d7b99f47-lk7kn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./checkoutservice"]',
+          'process.executable.name': 'checkoutservice',
+          'process.executable.path': '/usr/src/app/checkoutservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'checkoutservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:15:20.874902347Z',
+        timestamp_nano: 1704413720874902347,
+        span_id: 'FDE699876DD9BECD',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'frauddetectionservice',
+        operation: 'orders process',
+        duration_nano: 361429,
+        parent_span_id: '24F5A9DAA3A2EB5F',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CONSUMER',
+        scope_name: 'io.opentelemetry.kafka-clients-0.11',
+        scope_version: '1.31.0-alpha',
+        span_attributes: {
+          'messaging.client_id': 'consumer-frauddetectionservice-1',
+          'messaging.destination.name': 'orders',
+          'messaging.kafka.consumer.group': 'frauddetectionservice',
+          'messaging.kafka.destination.partition': '0',
+          'messaging.kafka.message.offset': '1559',
+          'messaging.message.payload_size_bytes': '180',
+          'messaging.operation': 'process',
+          'messaging.system': 'kafka',
+          'thread.id': '1',
+          'thread.name': 'main',
+        },
+        resource_attributes: {
+          'container.id': 'bd506193f6781dcb807237fcd4775a9e58b30593ded040a49c25538458396262',
+          'host.arch': 'amd64',
+          'host.name': 'otel-demo-frauddetectionservice-5675fccd44-7qbhg',
+          'k8s.deployment.name': 'otel-demo-frauddetectionservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker3',
+          'k8s.pod.ip': '192.168.37.213',
+          'k8s.pod.name': 'otel-demo-frauddetectionservice-5675fccd44-7qbhg',
+          'k8s.pod.start_time': '2024-01-02T12:09:27Z',
+          'k8s.pod.uid': '36ac6939-c5f6-41f0-86a1-755733e19f95',
+          'os.description': 'Linux 6.2.0-1019-gcp',
+          'os.type': 'linux',
+          'process.command_args':
+            '["/usr/lib/jvm/java-17-openjdk-amd64/bin/java","-jar","frauddetectionservice-1.0-all.jar"]',
+          'process.executable.path': '/usr/lib/jvm/java-17-openjdk-amd64/bin/java',
+          'process.pid': '1',
+          'process.runtime.description': 'Debian OpenJDK 64-Bit Server VM 17.0.9+9-Debian-1deb11u1',
+          'process.runtime.name': 'OpenJDK Runtime Environment',
+          'process.runtime.version': '17.0.9+9-Debian-1deb11u1',
+          'service.name': 'frauddetectionservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.auto.version': '1.31.0',
+          'telemetry.sdk.language': 'java',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.31.0',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:18:48.498Z',
+        timestamp_nano: 1704413928498000000,
+        span_id: '3A2FD3EBC1C35519',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'frontend',
+        operation: 'grpc.oteldemo.ProductCatalogService/GetProduct',
+        duration_nano: 2592220,
+        parent_span_id: '49EDAF33EBF9C836',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_CLIENT',
+        scope_name: '@opentelemetry/instrumentation-grpc',
+        scope_version: '0.45.1',
+        span_attributes: {
+          'net.peer.name': 'otel-demo-productcatalogservice',
+          'net.peer.port': '8080',
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'GetProduct',
+          'rpc.service': 'oteldemo.ProductCatalogService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'cloud.account.id': 'opstrace-dev-bee41fca',
+          'cloud.availability_zone': 'europe-west1-b',
+          'cloud.provider': 'gcp',
+          'container.name': '',
+          'host.arch': 'amd64',
+          'host.id': '6325675042675770930',
+          'host.name': 'rossetd-0201.c.opstrace-dev-bee41fca.internal',
+          'k8s.cluster.name': '',
+          'k8s.deployment.name': 'otel-demo-frontend',
+          'k8s.namespace.name': '',
+          'k8s.node.name': 'opstrace-worker',
+          'k8s.pod.ip': '192.168.95.23',
+          'k8s.pod.name': 'otel-demo-frontend-868cdc975b-kvtb4',
+          'k8s.pod.start_time': '2024-01-02T12:09:34Z',
+          'k8s.pod.uid': 'd2707722-9050-494f-aa30-4c377550bee3',
+          'os.type': 'linux',
+          'os.version': '6.2.0-1019-gcp',
+          'process.command': '/app/server.js',
+          'process.command_args':
+            '["/usr/local/bin/node","--require","./Instrumentation.js","/app/server.js"]',
+          'process.executable.name': 'node',
+          'process.executable.path': '/usr/local/bin/node',
+          'process.owner': 'nextjs',
+          'process.pid': '23',
+          'process.runtime.description': 'Node.js',
+          'process.runtime.name': 'nodejs',
+          'process.runtime.version': '18.19.0',
+          'service.name': 'frontend',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'nodejs',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.18.1',
+        },
+      },
+      {
+        timestamp: '2024-01-05T00:18:48.499650576Z',
+        timestamp_nano: 1704413928499650576,
+        span_id: '418C6E0E89EE5DD6',
+        trace_id: 'acff541d-9491-4ba7-4bba-96e79109f09c',
+        service_name: 'productcatalogservice',
+        operation: 'oteldemo.ProductCatalogService/GetProduct',
+        duration_nano: 266314,
+        parent_span_id: '3A2FD3EBC1C35519',
+        status_code: 'STATUS_CODE_UNSET',
+        status_message: '',
+        trace_state: '',
+        span_kind: 'SPAN_KIND_SERVER',
+        scope_name: 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc',
+        scope_version: '0.46.0',
+        span_attributes: {
+          'app.product.id': 'L9ECAV7KIM',
+          'app.product.name': 'Lens Cleaning Kit',
+          'rpc.grpc.status_code': '0',
+          'rpc.method': 'GetProduct',
+          'rpc.service': 'oteldemo.ProductCatalogService',
+          'rpc.system': 'grpc',
+        },
+        resource_attributes: {
+          'host.name': 'otel-demo-productcatalogservice-5cf4c48bdb-8gnwn',
+          'k8s.deployment.name': 'otel-demo-productcatalogservice',
+          'k8s.namespace.name': 'otel-demo-app',
+          'k8s.node.name': 'opstrace-worker',
+          'k8s.pod.ip': '192.168.95.20',
+          'k8s.pod.name': 'otel-demo-productcatalogservice-5cf4c48bdb-8gnwn',
+          'k8s.pod.start_time': '2024-01-02T12:09:32Z',
+          'k8s.pod.uid': 'd96c0f64-ed60-437f-a51f-e72c40b52c04',
+          'os.description':
+            'Alpine Linux 3.19.0 (Linux otel-demo-productcatalogservice-5cf4c48bdb-8gnwn 6.2.0-1019-gcp #21~22.04.1-Ubuntu SMP Thu Nov 16 18:18:34 UTC 2023 x86_64)',
+          'os.type': 'linux',
+          'process.command_args': '["./productcatalogservice"]',
+          'process.executable.name': 'productcatalogservice',
+          'process.executable.path': '/usr/src/app/productcatalogservice',
+          'process.owner': 'root',
+          'process.pid': '1',
+          'process.runtime.description': 'go version go1.21.4 linux/amd64',
+          'process.runtime.name': 'go',
+          'process.runtime.version': 'go1.21.4',
+          'service.name': 'productcatalogservice',
+          'service.namespace': 'opentelemetry-demo',
+          'telemetry.sdk.language': 'go',
+          'telemetry.sdk.name': 'opentelemetry',
+          'telemetry.sdk.version': '1.20.0',
+        },
+      },
+    ],
+  });
 }
 
 /**
@@ -219,15 +2356,15 @@ async function fetchTraces(tracingUrl, { filters = {}, pageToken, pageSize, sort
     : DEFAULT_SORTING_OPTION;
   params.append('sort', sortOrder);
 
+  console.log(`[DEBUG] Fetching traces with params: ${params.toString()}`);
+
   try {
-    const { data } = await axios.get(tracingUrl, {
-      withCredentials: true,
-      params,
-    });
+    const data = MOCK_TRACES;
+
     if (!Array.isArray(data.traces)) {
       throw new Error('traces are missing/invalid in the response'); // eslint-disable-line @gitlab/require-i18n-strings
     }
-    return data;
+    return mockReturnDataWithDelay(data);
   } catch (e) {
     return reportErrorAndThrow(e);
   }
@@ -235,15 +2372,17 @@ async function fetchTraces(tracingUrl, { filters = {}, pageToken, pageSize, sort
 
 async function fetchServices(servicesUrl) {
   try {
-    const { data } = await axios.get(servicesUrl, {
-      withCredentials: true,
-    });
+    console.log(`[DEBUG] Fetching services from ${servicesUrl}`);
+    const uniqueServices = new Set(
+      MOCK_TRACES.traces.map((t) => t.spans.map((s) => s.service_name)).flat(),
+    );
+    const data = { services: Array.from(uniqueServices).map((s) => ({ name: s })) };
 
     if (!Array.isArray(data.services)) {
       throw new Error('failed to fetch services. invalid response'); // eslint-disable-line @gitlab/require-i18n-strings
     }
 
-    return data.services;
+    return mockReturnDataWithDelay(data.services);
   } catch (e) {
     return reportErrorAndThrow(e);
   }
@@ -258,15 +2397,21 @@ async function fetchOperations(operationsUrl, serviceName) {
       throw new Error('fetchOperations() - operationsUrl must contain $SERVICE_NAME$');
     }
     const url = operationsUrl.replace('$SERVICE_NAME$', serviceName);
-    const { data } = await axios.get(url, {
-      withCredentials: true,
-    });
+
+    console.log('[DEBUG] fetching operations suggestions from', url); // eslint-disable-line @gitlab/require-i18n-strings
+    const uniqOps = new Set(
+      MOCK_TRACES.traces
+        .map((t) => t.spans.filter((s) => s.service_name === serviceName))
+        .flat()
+        .map((s) => s.operation),
+    );
+    const data = { operations: Array.from(uniqOps).map((s) => ({ name: s })) };
 
     if (!Array.isArray(data.operations)) {
       throw new Error('failed to fetch operations. invalid response'); // eslint-disable-line @gitlab/require-i18n-strings
     }
 
-    return data.operations;
+    return mockReturnDataWithDelay(data.operations);
   } catch (e) {
     return reportErrorAndThrow(e);
   }
@@ -289,14 +2434,12 @@ async function fetchMetrics(metricsUrl, { filters = {}, limit } = {}) {
         }
       }
     }
-    const { data } = await axios.get(metricsUrl, {
-      withCredentials: true,
-      params,
-    });
+    console.log(`[DEBUG] Fetching metrics with params: ${params.toString()}`);
+    const data = MOCK_METRICS;
     if (!Array.isArray(data.metrics)) {
       throw new Error('metrics are missing/invalid in the response'); // eslint-disable-line @gitlab/require-i18n-strings
     }
-    return data;
+    return mockReturnDataWithDelay(data);
   } catch (e) {
     return reportErrorAndThrow(e);
   }
diff --git a/app/assets/javascripts/observability/components/observability_container.vue b/app/assets/javascripts/observability/components/observability_container.vue
index b89c2624f81c..f6cbf7ee771f 100644
--- a/app/assets/javascripts/observability/components/observability_container.vue
+++ b/app/assets/javascripts/observability/components/observability_container.vue
@@ -27,12 +27,12 @@ export default {
 
     // TODO: Improve local GDK dev experience with tracing https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/2308
     // Uncomment the lines below to to test this locally
-    // setTimeout(() => {
-    //   this.messageHandler({
-    //     data: { type: 'AUTH_COMPLETION', status: 'success' },
-    //     origin: new URL(this.oauthUrl).origin,
-    //   });
-    // }, 2000);
+    setTimeout(() => {
+      this.messageHandler({
+        data: { type: 'AUTH_COMPLETION', status: 'success' },
+        origin: new URL(this.apiConfig.oauthUrl).origin,
+      });
+    }, 2000);
   },
   destroyed() {
     window.removeEventListener('message', this.messageHandler);
Edited by Daniele Rossetti

Merge request reports