Incubation:APM Datadog agent testing
Evaluate suitability of using datadog as a single agent for o11y telemetry.
-
Review protocol and wire format, document and make inspection reproducible for new releases -
Document DD_DD_URL
endpoint (akaapp
endpoint) that takes metrics and event data. -
Document DD_PROCESS_AGENT_URL
(akaprocess
endpoint). -
Document DD_ORCHESTRATOR_EXPLORER_ORCHESTRATOR_DD_URL
(akaorchestrator
endpoint). -
Document DD_LOGS_CONFIG_LOGS_DD_URL
(akaagent-http-intake.logs
endpoint). -
Document DD_APM_CONFIG_APM_DD_URL
(akatrace.agent
endpoint). -
Document DD_APPSEC_CONFIG_APPSEC_DD_URL
(akaappsecevts-http-intake.logs
endpoint).
-
-
Review authentication options and compatibility with existing GitLab AuthN - We can hopefully use existing JWT auth endpoints used for existing container registries.
Endpoint Findings
app
endpoint
Main First call is always to /api/v1/validate
- https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
Example `/api/v1/validate` request:
{
"method": "GET",
"path": "/api/v1/validate",
"queryStringParameters": {
"api_key": [
"123456"
]
},
"headers": {
"Host": [
"mock-server-app:1080"
],
"User-Agent": [
"datadog-agent/7.30.1"
],
"Accept-Encoding": [
"gzip"
],
"content-length": [
"0"
]
},
"keepAlive": true,
"secure": true
}
This is followed by two requests to an undocumented /intake
endpoint.
The purpose of the first call to this endpoint seems to be auditing of the agent starting:
- agent metadata, such as build versions.
- host metadata, such as OS version, hardware details
- processes in the running environment
Example initial `/intake/` request:
{
"method": "POST",
"path": "/intake/",
"queryStringParameters": {
"api_key": [
"123456"
]
},
"headers": {
"Host": [
"mock-server-app:1080"
],
"User-Agent": [
"datadog-agent/7.30.1"
],
"Content-Type": [
"application/json"
],
"Dd-Agent-Version": [
"7.30.1"
],
"Dd-Api-Key": [
"123456"
],
"Accept-Encoding": [
"gzip"
],
"content-length": [
"3847"
]
},
"keepAlive": true,
"secure": true,
"body": {
"apiKey": "123456",
"agentVersion": "7.30.1",
"uuid": "4c4c4544-0047-3110-8053-b8c04f334633",
"internalHostname": "joe-xps159500",
"os": "linux",
"agent-flavor": "agent",
"python": "3.8.10 (default, Aug 19 2021, 16:19:59) [GCC 4.7.2]",
"systemStats": {
"cpuCores": 1,
"machine": "amd64",
"platform": "linux",
"pythonV": "3.8.10",
"processor": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz",
"macV": [
"",
"",
""
],
"nixV": [
"ubuntu",
"21.04",
""
],
"fbsdV": [
"",
"",
""
],
"winV": [
"",
"",
""
]
},
"meta": {
"socket-hostname": "2ab1d975fedc",
"timezones": [
"UTC"
],
"socket-fqdn": "2ab1d975fedc",
"ec2-hostname": "",
"hostname": "joe-xps159500",
"host_aliases": [],
"instance-id": ""
},
"host-tags": {
"system": []
},
"container-meta": {
"docker_swarm": "inactive",
"docker_version": "20.10.8"
},
"network": null,
"logs": {
"transport": "HTTP"
},
"install-method": {
"tool": "docker",
"tool_version": "docker",
"installer_version": "docker"
},
"proxy-info": {
"no-proxy-nonexact-match": false,
"proxy-behavior-changed": false
},
"resources": {
"processes": {
"snaps": [
[
1629885273,
[
[
"dd-agent",
0,
1.8226879668427374,
4758814720,
608763904,
"clickhouse-server",
1
],
[
"root",
0,
0.40396801073716065,
2486964224,
134922240,
"agent",
1
],
[
"root",
0,
0.3135841540543169,
3539959808,
104734720,
"dockerd",
1
],
[
"root",
0,
0.25362666757908986,
8011350016,
84709376,
"containerd-shim-runc-v2",
11
],
[
"root",
0,
0.25124749957414116,
1537617920,
83914752,
"security-agent",
1
],
[
"root",
0,
0.1985746821449941,
2312699904,
66322432,
"process-agent",
1
],
[
"dd-agent",
0,
0.1829138700711825,
397987840,
61091840,
"clickhouse-watchdog",
1
],
[
"root",
0,
0.18196956112076473,
103378944,
60776448,
"systemd-journald",
1
],
[
"root",
0,
0.15102811330382915,
16020987904,
50442240,
"docker-proxy",
14
],
[
"root",
0,
0.11899519150524196,
2279555072,
39743488,
"containerd",
1
],
[
"root",
0,
0.11199258876902705,
1978093568,
37404672,
"trace-agent",
1
],
[
"root",
0,
0.10301552186375681,
770064384,
34406400,
"coredns",
1
],
[
"root",
0,
0.09409977372150073,
2336276480,
31428608,
"snapd",
1
],
[
"root",
0,
0.06558654891992517,
416792576,
21905408,
"NetworkManager",
1
],
[
"root",
0,
0.043609904255657056,
403570688,
14565376,
"udisksd",
1
],
[
"root",
0,
0.039035524534802135,
326750208,
13037568,
"gdm-session-worker [pam/gdm-password]",
1
],
[
"root",
0,
0.03569978382683287,
169979904,
11923456,
"systemd",
1
],
[
"root",
0,
0.03377437466818884,
32661504,
11280384,
"cupsd",
1
],
[
"root",
0,
0.03216782307721835,
247275520,
10743808,
"ModemManager",
1
],
[
"root",
0,
0.03159142670488543,
33247232,
10551296,
"systemd-udevd",
1
]
]
]
]
},
"meta": {
"host": "joe-xps159500"
}
},
"gohai": "{\"cpu\":{\"cache_size\":\"16384 KB\",\"cpu_cores\":\"8\",\"cpu_logical_processors\":\"16\",\"family\":\"6\",\"mhz\":\"3500.001\",\"model\":\"165\",\"model_name\":\"Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz\",\"stepping\":\"2\",\"vendor_id\":\"GenuineIntel\"},\"filesystem\":[{\"kb_size\":\"947746588\",\"mounted_on\":\"/\",\"name\":\"overlay\"},{\"kb_size\":\"65536\",\"mounted_on\":\"/dev\",\"name\":\"tmpfs\"},{\"kb_size\":\"65536\",\"mounted_on\":\"/dev/shm\",\"name\":\"shm\"},{\"kb_size\":\"947746588\",\"mounted_on\":\"/etc/hosts\",\"name\":\"/dev/mapper/luks-b0c127c2-6224-4046-957c-a3efc822fe21\"},{\"kb_size\":\"16308220\",\"mounted_on\":\"/var/run/docker.sock\",\"name\":\"run\"},{\"kb_size\":\"16308220\",\"mounted_on\":\"/proc/asound\",\"name\":\"tmpfs\"},{\"kb_size\":\"16308220\",\"mounted_on\":\"/proc/acpi\",\"name\":\"tmpfs\"},{\"kb_size\":\"16308220\",\"mounted_on\":\"/proc/scsi\",\"name\":\"tmpfs\"},{\"kb_size\":\"16308220\",\"mounted_on\":\"/sys/firmware\",\"name\":\"tmpfs\"}],\"memory\":{\"swap_total\":\"35881832kB\",\"total\":\"32616444kB\"},\"network\":null,\"platform\":{\"GOOARCH\":\"amd64\",\"GOOS\":\"linux\",\"goV\":\"1.15.13\",\"hardware_platform\":\"x86_64\",\"hostname\":\"2ab1d975fedc\",\"kernel_name\":\"Linux\",\"kernel_release\":\"5.13.11-1-MANJARO\",\"kernel_version\":\"#1 SMP PREEMPT Sun Aug 15 13:15:18 UTC 2021\",\"machine\":\"x86_64\",\"os\":\"GNU/Linux\",\"processor\":\"x86_64\",\"pythonV\":\"3.8.10\"}}"
}
}
The second /intake
call appears to send a number of events about services in the same environment. In this case events are sent about docker containers.
Example second `/intake` request:
{
"method": "POST",
"path": "/intake/",
"queryStringParameters": {
"api_key": [
"123456"
]
},
"headers": {
"Host": [
"mock-server-app:1080"
],
"User-Agent": [
"datadog-agent/7.30.1"
],
"Content-Type": [
"application/json"
],
"Dd-Agent-Version": [
"7.30.1"
],
"Dd-Api-Key": [
"123456"
],
"Accept-Encoding": [
"gzip"
],
"content-length": [
"8104"
]
},
"keepAlive": true,
"secure": true,
"body": {
"apiKey": "",
"events": {
"docker": [
{
"msg_title": "gcr.io/datadoghq/agent:7.30.1 1 kill 1 die 1 stop 1 start 1 restart on joe-xps159500",
"msg_text": "%%% \ngcr.io/datadoghq/agent:7.30.1 1 kill 1 die 1 stop 1 start 1 restart on joe-xps159500\n```\nKILL\tdatadog-agent-testing_datadog-agent_1\nDIE\tdatadog-agent-testing_datadog-agent_1\nSTOP\tdatadog-agent-testing_datadog-agent_1\nSTART\tdatadog-agent-testing_datadog-agent_1\nRESTART\tdatadog-agent-testing_datadog-agent_1\n```\n %%%",
"timestamp": 1629885257,
"priority": "normal",
"host": "joe-xps159500",
"tags": [
"container_id:2ab1d975fedc1fb44d22c07e671139a3c227c2949702bb85eb53655d0a207da4",
"container_name:datadog-agent-testing_datadog-agent_1",
"docker_image:gcr.io/datadoghq/agent:7.30.1",
"image_name:gcr.io/datadoghq/agent",
"image_tag:7.30.1",
"short_image:agent"
],
"alert_type": "error",
"aggregation_key": "docker:gcr.io/datadoghq/agent:7.30.1",
"source_type_name": "docker",
"event_type": "docker"
},
{
"msg_title": "coredns/coredns 1 stop 1 start 1 restart 1 kill 1 die on joe-xps159500",
"msg_text": "%%% \ncoredns/coredns 1 stop 1 start 1 restart 1 kill 1 die on joe-xps159500\n```\nKILL\tdatadog-agent-testing_dns-proxy_1\nDIE\tdatadog-agent-testing_dns-proxy_1\nSTOP\tdatadog-agent-testing_dns-proxy_1\nSTART\tdatadog-agent-testing_dns-proxy_1\nRESTART\tdatadog-agent-testing_dns-proxy_1\n```\n %%%",
"timestamp": 1629885255,
"priority": "normal",
"host": "joe-xps159500",
"tags": [
"container_id:f0fdfe0667054a998102329058e35284dfbb1adb0bb62a149967e5a22fbc0bd0",
"container_name:datadog-agent-testing_dns-proxy_1",
"docker_image:coredns/coredns",
"image_name:coredns/coredns",
"short_image:coredns"
],
"alert_type": "error",
"aggregation_key": "docker:coredns/coredns",
"source_type_name": "docker",
"event_type": "docker"
},
{
"msg_title": "praqma/network-multitool 1 die 1 start 1 restart on joe-xps159500",
"msg_text": "%%% \npraqma/network-multitool 1 die 1 start 1 restart on joe-xps159500\n```\nSTART\tdatadog-agent-testing_dig_1\nRESTART\tdatadog-agent-testing_dig_1\nDIE\tdatadog-agent-testing_dig_1\n```\n %%%",
"timestamp": 1629885253,
"priority": "normal",
"host": "joe-xps159500",
"aggregation_key": "docker:praqma/network-multitool",
"source_type_name": "docker",
"event_type": "docker"
},
{
"msg_title": "yandex/clickhouse-server:21 1 die 1 stop 1 start 1 restart 1 kill on joe-xps159500",
"msg_text": "%%% \nyandex/clickhouse-server:21 1 die 1 stop 1 start 1 restart 1 kill on joe-xps159500\n```\nKILL\tdatadog-agent-testing_clickhouse_1\nDIE\tdatadog-agent-testing_clickhouse_1\nSTOP\tdatadog-agent-testing_clickhouse_1\nSTART\tdatadog-agent-testing_clickhouse_1\nRESTART\tdatadog-agent-testing_clickhouse_1\n```\n %%%",
"timestamp": 1629885256,
"priority": "normal",
"host": "joe-xps159500",
"tags": [
"container_id:cecd312fae0e86e122711feb04d1c7225cff2a01b3b977d32ef27fa873fed28b",
"container_name:datadog-agent-testing_clickhouse_1",
"docker_image:yandex/clickhouse-server:21",
"image_name:yandex/clickhouse-server",
"image_tag:21",
"short_image:clickhouse-server"
],
"alert_type": "error",
"aggregation_key": "docker:yandex/clickhouse-server:21",
"source_type_name": "docker",
"event_type": "docker"
},
{
"msg_title": "datadog-agent-testing_series-store 1 kill 2 die 1 stop 2 start 1 restart on joe-xps159500",
"msg_text": "%%% \ndatadog-agent-testing_series-store 1 kill 2 die 1 stop 2 start 1 restart on joe-xps159500\n```\nKILL\tdatadog-agent-testing_series-store_1\nDIE\tdatadog-agent-testing_series-store_1\nSTOP\tdatadog-agent-testing_series-store_1\nSTART\tdatadog-agent-testing_series-store_1\nRESTART\tdatadog-agent-testing_series-store_1\nDIE\tdatadog-agent-testing_series-store_1\nSTART\tdatadog-agent-testing_series-store_1\n```\n %%%",
"timestamp": 1629885261,
"priority": "normal",
"host": "joe-xps159500",
"tags": [
"container_id:5b21feb65f28ee03aeb781a0f1912d4e551773920c178bfc9bdbb28399e88597",
"container_name:datadog-agent-testing_series-store_1",
"docker_image:datadog-agent-testing_series-store",
"image_name:datadog-agent-testing_series-store",
"short_image:datadog-agent-testing_series-store"
],
"alert_type": "error",
"aggregation_key": "docker:datadog-agent-testing_series-store",
"source_type_name": "docker",
"event_type": "docker"
},
{
"msg_title": "grafana/grafana:8.1.2 1 die 1 stop 1 start 1 restart 1 kill on joe-xps159500",
"msg_text": "%%% \ngrafana/grafana:8.1.2 1 die 1 stop 1 start 1 restart 1 kill on joe-xps159500\n```\nKILL\tdatadog-agent-testing_grafana_1\nDIE\tdatadog-agent-testing_grafana_1\nSTOP\tdatadog-agent-testing_grafana_1\nSTART\tdatadog-agent-testing_grafana_1\nRESTART\tdatadog-agent-testing_grafana_1\n```\n %%%",
"timestamp": 1629885257,
"priority": "normal",
"host": "joe-xps159500",
"tags": [
"container_id:2e2d9252c531ede9c8a490d4e0572bc22cb45d00b3f943fc4f8f3087d86139c7",
"container_name:datadog-agent-testing_grafana_1",
"docker_image:grafana/grafana:8.1.2",
"image_name:grafana/grafana",
"image_tag:8.1.2",
"short_image:grafana"
],
"alert_type": "error",
"aggregation_key": "docker:grafana/grafana:8.1.2",
"source_type_name": "docker",
"event_type": "docker"
},
{
"msg_title": "mockserver/mockserver:mockserver-5.11.2 6 kill 6 die 6 stop 6 start 6 restart on joe-xps159500",
"msg_text": "%%% \nmockserver/mockserver:mockserver-5.11.2 6 kill 6 die 6 stop 6 start 6 restart on joe-xps159500\n```\nKILL\tdatadog-agent-testing_mock-server-orchestrator_1\nKILL\tdatadog-agent-testing_mock-server-appsec_1\nKILL\tdatadog-agent-testing_mock-server-app_1\nKILL\tdatadog-agent-testing_mock-server-apm_1\nKILL\tdatadog-agent-testing_mock-server-logs_1\nKILL\tdatadog-agent-testing_mock-server-process_1\nDIE\tdatadog-agent-testing_mock-server-appsec_1\nDIE\tdatadog-agent-testing_mock-server-orchestrator_1\nDIE\tdatadog-agent-testing_mock-server-logs_1\nDIE\tdatadog-agent-testing_mock-server-apm_1\nDIE\tdatadog-agent-testing_mock-server-process_1\nDIE\tdatadog-agent-testing_mock-server-app_1\nSTOP\tdatadog-agent-testing_mock-server-appsec_1\nSTOP\tdatadog-agent-testing_mock-server-orchestrator_1\nSTOP\tdatadog-agent-testing_mock-server-app_1\nSTOP\tdatadog-agent-testing_mock-server-process_1\nSTOP\tdatadog-agent-testing_mock-server-logs_1\nSTOP\tdatadog-agent-testing_mock-server-apm_1\nSTART\tdatadog-agent-testing_mock-server-orchestrator_1\nRESTART\tdatadog-agent-testing_mock-server-orchestrator_1\nSTART\tdatadog-agent-testing_mock-server-appsec_1\nRESTART\tdatadog-agent-testing_mock-server-appsec_1\nSTART\tdatadog-agent-testing_mock-server-process_1\nRESTART\tdatadog-agent-testing_mock-server-process_1\nSTART\tdatadog-agent-testing_mock-server-app_1\nRESTART\tdatadog-agent-testing_mock-server-app_1\nSTART\tdatadog-agent-testing_mock-server-logs_1\nRESTART\tdatadog-agent-testing_mock-server-logs_1\nSTART\tdatadog-agent-testing_mock-server-apm_1\nRESTART\tdatadog-agent-testing_mock-server-apm_1\n```\n %%%",
"timestamp": 1629885257,
"priority": "normal",
"host": "joe-xps159500",
"tags": [
"container_id:282e8c465d488d2246738195b24a5c22164b73263e1a777568e0e81af77eec18",
"container_id:327006ce003cc1918edba0623d5faf3f4c0a8cc7aa762ff814a9e9e237d2fd1b",
"container_id:7efa358cf05f86d37f0045eda408a67378c0422384382d47a55b846c47d7b81a",
"container_id:94e3c7f201c3582a375e48ce9e12e8077a6dafcdae12400b8fff993a48b76a14",
"container_id:9bbe7f691b025aab54b279137cf39d091842bccef94e0b02e0015322f4009fde",
"container_id:9e5eaf26671326bc3c79ca62940a58fd8c33cc8e55f218bd68da0857c608a937",
"container_name:datadog-agent-testing_mock-server-apm_1",
"container_name:datadog-agent-testing_mock-server-app_1",
"container_name:datadog-agent-testing_mock-server-appsec_1",
"container_name:datadog-agent-testing_mock-server-logs_1",
"container_name:datadog-agent-testing_mock-server-orchestrator_1",
"container_name:datadog-agent-testing_mock-server-process_1",
"docker_image:mockserver/mockserver:mockserver-5.11.2",
"image_name:mockserver/mockserver",
"image_tag:mockserver-5.11.2",
"short_image:mockserver"
],
"alert_type": "error",
"aggregation_key": "docker:mockserver/mockserver:mockserver-5.11.2",
"source_type_name": "docker",
"event_type": "docker"
}
],
"System": [
{
"msg_title": "",
"msg_text": "Version 7.30.1",
"timestamp": 1629885261,
"host": "joe-xps159500",
"source_type_name": "System",
"event_type": "Agent Startup"
}
]
},
"internalHostname": "joe-xps159500"
}
}
After these requests, the agent alternates between /api/v1/check_run
and /api/v1/series
.
The /api/v1/check_run
endpoint submits service checks - https://docs.datadoghq.com/api/latest/service-checks/#submit-a-service-check. This provides status checks for various services that are being monitored.
Example `/api/v1/check_run` request:
{
"method": "POST",
"path": "/api/v1/check_run",
"queryStringParameters": {
"api_key": [
"123456"
]
},
"headers": {
"Host": [
"mock-server-app:1080"
],
"User-Agent": [
"datadog-agent/7.30.1"
],
"Content-Type": [
"application/json"
],
"Dd-Agent-Version": [
"7.30.1"
],
"Dd-Api-Key": [
"123456"
],
"Accept-Encoding": [
"gzip"
],
"content-length": [
"1571"
]
},
"keepAlive": true,
"secure": true,
"body": [
{
"check": "ntp.in_sync",
"host_name": "joe-xps159500",
"timestamp": 1629885267,
"status": 3,
"message": "",
"tags": null
},
{
"check": "datadog.agent.check_status",
"host_name": "joe-xps159500",
"timestamp": 1629885267,
"status": 0,
"message": "",
"tags": [
"check:ntp"
]
},
{
"check": "coredns.prometheus.health",
"host_name": "joe-xps159500",
"timestamp": 1629885267,
"status": 2,
"message": "",
"tags": [
"dns-pod:172.28.0.10",
"docker_image:coredns/coredns",
"endpoint:http://172.28.0.10:9153/metrics",
"image_name:coredns/coredns",
"short_image:coredns"
]
},
{
"check": "datadog.agent.check_status",
"host_name": "joe-xps159500",
"timestamp": 1629885267,
"status": 2,
"message": "",
"tags": [
"check:coredns"
]
},
{
"check": "datadog.agent.check_status",
"host_name": "joe-xps159500",
"timestamp": 1629885270,
"status": 0,
"message": "",
"tags": [
"check:uptime"
]
},
{
"check": "datadog.agent.check_status",
"host_name": "joe-xps159500",
"timestamp": 1629885271,
"status": 0,
"message": "",
"tags": [
"check:memory"
]
},
{
"check": "datadog.agent.check_status",
"host_name": "joe-xps159500",
"timestamp": 1629885272,
"status": 0,
"message": "",
"tags": [
"check:io"
]
},
{
"check": "docker.service_up",
"host_name": "joe-xps159500",
"timestamp": 1629885273,
"status": 0,
"message": "",
"tags": null
},
{
"check": "datadog.agent.check_status",
"host_name": "joe-xps159500",
"timestamp": 1629885273,
"status": 0,
"message": "",
"tags": [
"check:docker"
]
},
{
"check": "datadog.agent.check_status",
"host_name": "joe-xps159500",
"timestamp": 1629885274,
"status": 0,
"message": "",
"tags": [
"check:cpu"
]
},
{
"check": "datadog.agent.up",
"host_name": "joe-xps159500",
"timestamp": 1629885276,
"status": 0,
"message": "",
"tags": null
}
]
}
The /api/v1/series
endpoint submits metrics series data - https://docs.datadoghq.com/api/latest/metrics/#submit-metrics.
Example `/api/v1/series` request:
{
"method": "POST",
"path": "/api/v1/series",
"queryStringParameters": {
"api_key": [
"123456"
]
},
"headers": {
"Host": [
"mock-server-app:1080"
],
"User-Agent": [
"datadog-agent/7.30.1"
],
"Content-Type": [
"application/json"
],
"Dd-Agent-Version": [
"7.30.1"
],
"Dd-Api-Key": [
"123456"
],
"Accept-Encoding": [
"gzip"
],
"content-length": [
"16912"
]
},
"keepAlive": true,
"secure": false,
"body": {
"series": [
{
"metric": "datadog.trace_agent.trace_writer.bytes_estimated",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.trace_writer.events",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.stats_writer.errors",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.sampler.kept",
"points": [
[
1629885260,
0
]
],
"tags": [
"sampler:error",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.service_checks",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.bytes_dropped",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.bytes_dropped_queue",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.stats_writer.stats_entries",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.trace_writer.spans",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.sampler.exception.misses",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.stats_writer.splits",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.sampler.seen",
"points": [
[
1629885260,
0
]
],
"tags": [
"sampler:priority",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.sampler.kept",
"points": [
[
1629885260,
0
]
],
"tags": [
"sampler:priority",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.stats_writer.stats_buckets",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.bytes_dropped_writer",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.packets_dropped_queue",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.sampler.kept",
"points": [
[
1629885260,
0
]
],
"tags": [
"sampler:no_priority",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.sampler.exception.hits",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.metrics",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.started",
"points": [
[
1629885260,
0.1
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.packets_sent",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.stats_writer.payloads",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.metrics",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.trace_writer.bytes_uncompressed",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.agent.started",
"points": [
[
1629885260,
1
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "count",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.packets_dropped",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.trace_writer.traces",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.bytes_sent",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.bytes_dropped_queue",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.packets_dropped_writer",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.packets_sent",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.bytes_dropped_writer",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.metric_dropped_on_receive",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.packets_dropped_writer",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.stats_writer.retries",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.stats_writer.client_payloads",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.packets_dropped",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.trace_writer.errors",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.sampler.seen",
"points": [
[
1629885260,
0
]
],
"tags": [
"sampler:no_priority",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.events",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.metric_dropped_on_receive",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.sampler.seen",
"points": [
[
1629885260,
0
]
],
"tags": [
"sampler:error",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.packets_dropped_queue",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0",
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.events",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.service_checks",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.trace_writer.bytes",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.trace_writer.retries",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.bytes_dropped",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.dogstatsd.client.bytes_sent",
"points": [
[
1629885260,
0
]
],
"tags": [
"client:go",
"client_transport:udp",
"client_version:4.8.0"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.stats_writer.bytes",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "datadog.trace_agent.trace_writer.payloads",
"points": [
[
1629885260,
0
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "rate",
"interval": 10
},
{
"metric": "docker.containers.stopped",
"points": [
[
1629885273,
0
]
],
"tags": [
"docker_image:mockserver/mockserver:mockserver-5.11.2",
"image_name:mockserver/mockserver",
"image_tag:mockserver-5.11.2",
"short_image:mockserver"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.running",
"points": [
[
1629885273,
1
]
],
"tags": [
"docker_image:gcr.io/datadoghq/agent:7.30.1",
"image_name:gcr.io/datadoghq/agent",
"image_tag:7.30.1",
"short_image:agent"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.stopped",
"points": [
[
1629885273,
0
]
],
"tags": [
"docker_image:grafana/grafana:8.1.2",
"image_name:grafana/grafana",
"image_tag:8.1.2",
"short_image:grafana"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.running.total",
"points": [
[
1629885273,
11
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.running",
"points": [
[
1629885273,
1
]
],
"tags": [
"docker_image:coredns/coredns",
"image_name:coredns/coredns",
"short_image:coredns"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.stopped",
"points": [
[
1629885273,
0
]
],
"tags": [
"docker_image:gcr.io/datadoghq/agent:7.30.1",
"image_name:gcr.io/datadoghq/agent",
"image_tag:7.30.1",
"short_image:agent"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.running",
"points": [
[
1629885273,
1
]
],
"tags": [
"docker_image:grafana/grafana:8.1.2",
"image_name:grafana/grafana",
"image_tag:8.1.2",
"short_image:grafana"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.running",
"points": [
[
1629885273,
1
]
],
"tags": [
"docker_image:datadog-agent-testing_series-store",
"image_name:datadog-agent-testing_series-store",
"short_image:datadog-agent-testing_series-store"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.running",
"points": [
[
1629885273,
6
]
],
"tags": [
"docker_image:mockserver/mockserver:mockserver-5.11.2",
"image_name:mockserver/mockserver",
"image_tag:mockserver-5.11.2",
"short_image:mockserver"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.stopped",
"points": [
[
1629885273,
7
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.running",
"points": [
[
1629885273,
1
]
],
"tags": [
"docker_image:yandex/clickhouse-server:21",
"image_name:yandex/clickhouse-server",
"image_tag:21",
"short_image:clickhouse-server"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.stopped",
"points": [
[
1629885273,
0
]
],
"tags": [
"docker_image:yandex/clickhouse-server:21",
"image_name:yandex/clickhouse-server",
"image_tag:21",
"short_image:clickhouse-server"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.stopped",
"points": [
[
1629885273,
0
]
],
"tags": [
"docker_image:datadog-agent-testing_series-store",
"image_name:datadog-agent-testing_series-store",
"short_image:datadog-agent-testing_series-store"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.stopped.total",
"points": [
[
1629885273,
7
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.running",
"points": [
[
1629885273,
0
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "docker.containers.stopped",
"points": [
[
1629885273,
0
]
],
"tags": [
"docker_image:coredns/coredns",
"image_name:coredns/coredns",
"short_image:coredns"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.cpu.num_cores",
"points": [
[
1629885274,
16
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.slab",
"points": [
[
1629885271,
429.98046875
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.page_tables",
"points": [
[
1629885271,
72.828125
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.swap.cached",
"points": [
[
1629885271,
0
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.total",
"points": [
[
1629885271,
31851.99609375
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.usable",
"points": [
[
1629885271,
23516.55859375
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.slab_reclaimable",
"points": [
[
1629885271,
231.1484375
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.swap.total",
"points": [
[
1629885271,
35040.8515625
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.swap.used",
"points": [
[
1629885271,
0
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.used",
"points": [
[
1629885271,
12151.0625
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.cached",
"points": [
[
1629885271,
5527.375
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.committed_as",
"points": [
[
1629885271,
25278.34375
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.shared",
"points": [
[
1629885271,
1598.04296875
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.commit_limit",
"points": [
[
1629885271,
50966.84765625
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.buffered",
"points": [
[
1629885271,
327.5625
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.swap.free",
"points": [
[
1629885271,
35040.8515625
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.swap.pct_free",
"points": [
[
1629885271,
1
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.free",
"points": [
[
1629885271,
19700.93359375
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.mem.pct_usable",
"points": [
[
1629885271,
0.738307217058978
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "system.uptime",
"points": [
[
1629885270,
4438
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "datadog.agent.python.version",
"points": [
[
1629885276,
1
]
],
"tags": [
"python_version:3",
"agent_version_major:7",
"agent_version_minor:30",
"agent_version_patch:1"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "datadog.agent.running",
"points": [
[
1629885276,
1
]
],
"tags": [
"version:7.30.1"
],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
},
{
"metric": "n_o_i_n_d_e_x.datadog.agent.payload.dropped",
"points": [
[
1629885276,
0
]
],
"tags": [],
"host": "joe-xps159500",
"type": "gauge",
"interval": 0,
"source_type_name": "System"
}
]
}
}