ValidationError not_a_dict on tags, headers when ingested via getsentry/relay
Description
Hello! We run a GlitchTip locally. To implement #134 (closed) between SDKs and GT, we tried to have sentry's relay in the middle, in static configuration mode.
However, error reports passing via relay result in this error on glitchtip:
{'tags': [ErrorDetail(string='Expected a dictionary of items but got type \"list\".', code='not_a_dict')]}event payload, partially redacted:
{
"sdk": {
"name": "sentry.python.django",
"version": "1.25.1",
"packages": [
{
"name": "pypi:sentry-sdk",
"version": "1.25.1"
}
],
"integrations": [
"aiohttp",
"argv",
"atexit",
"boto3",
"celery",
"dedupe",
"django",
"excepthook",
"logging",
"modules",
"redis",
"stdlib",
"threading",
"tornado"
]
},
"type": "error",
"extra": {
"sys.argv": [
"uwsgi"
]
},
"title": "ValidationError: {'tags': [ErrorDetail(string='Expected a dictionary of items but got type \"list\".', code='not_a_dict…",
"culprit": "/api/{id}/envelope/",
"message": "",
"modules": {
"pip": "23.0.1",
"rsa": "4.9",
"six": "1.16.0",
"amqp": "5.1.1",
"cffi": "1.15.1",
"idna": "3.4",
"pytz": "2023.3",
"vine": "5.0.0",
"xlrd": "2.0.1",
"xlwt": "1.3.0",
"yarl": "1.9.2",
"attrs": "23.1.0",
"boto3": "1.26.155",
"click": "8.1.3",
"fido2": "1.1.1",
"kombu": "5.3.1",
"odfpy": "1.4.1",
"pyjwt": "2.7.0",
"pyotp": "2.8.0",
"redis": "4.5.4",
"uwsgi": "2.0.21",
"wheel": "0.40.0",
"celery": "5.3.1",
"django": "4.2.2",
"flower": "2.0.0",
"grpcio": "1.54.2",
"itypes": "1.2.0",
"jinja2": "3.1.2",
"pyasn1": "0.5.0",
"pyyaml": "6.0",
"stripe": "4.2.0",
"tablib": "3.5.0",
"tzdata": "2023.3",
"aiohttp": "3.8.4",
"asgiref": "3.7.2",
"certifi": "2023.5.7",
"coreapi": "2.3.3",
"hiredis": "2.2.3",
"isodate": "0.6.1",
"psycopg": "3.1.9",
"tornado": "6.3.2",
"urllib3": "1.26.16",
"wcwidth": "0.2.6",
"billiard": "4.1.0",
"botocore": "1.29.155",
"drf-yasg": "1.21.5",
"humanize": "4.6.0",
"jmespath": "1.0.1",
"markuppy": "1.14",
"oauthlib": "3.2.2",
"openpyxl": "3.1.2",
"protobuf": "4.23.3",
"requests": "2.31.0",
"sqlparse": "0.4.4",
"symbolic": "10.2.1",
"aiosignal": "1.3.1",
"dj-stripe": "2.7.3",
"jsonfield": "3.1.0",
"milksnake": "0.1.5",
"multidict": "6.0.4",
"packaging": "23.1",
"psycopg-c": "3.1.9",
"pycparser": "2.21",
"ua-parser": "0.16.1",
"azure-core": "1.27.1",
"cachetools": "5.3.1",
"click-repl": "0.3.0",
"coreschema": "0.0.4",
"defusedxml": "0.7.1",
"django-csp": "3.7",
"et-xmlfile": "1.1.0",
"frozenlist": "1.3.3",
"inflection": "0.5.1",
"markupsafe": "2.1.3",
"proto-plus": "1.22.2",
"s3transfer": "0.6.1",
"sentry-sdk": "1.25.1",
"setuptools": "65.5.1",
"whitenoise": "6.5.0",
"anonymizeip": "1.0.0",
"google-auth": "2.20.0",
"ruamel.yaml": "0.17.32",
"uritemplate": "4.1.1",
"user-agents": "2.2.0",
"cryptography": "41.0.1",
"dj-rest-auth": "4.0.1",
"django-redis": "5.3.0",
"async-timeout": "4.0.2",
"click-plugins": "1.1.1",
"django-filter": "22.1",
"django-ipware": "5.0.0",
"google-crc32c": "1.5.0",
"grpcio-status": "1.54.2",
"uwsgi-chunked": "0.1.8",
"django-allauth": "0.54.0",
"django-anymail": "8.6",
"django-environ": "0.9.0",
"prompt-toolkit": "3.0.38",
"pyasn1-modules": "0.3.0",
"python3-openid": "3.2.0",
"django-rest-mfa": "1.2.2",
"django-storages": "1.13.2",
"google-api-core": "2.11.1",
"python-dateutil": "2.8.2",
"click-didyoumean": "0.3.0",
"diff-match-patch": "20230430",
"django-sql-utils": "0.6.1",
"ruamel.yaml.clib": "0.2.7",
"django-extensions": "3.2.3",
"django-prometheus": "2.3.1",
"google-cloud-core": "2.3.2",
"prometheus-client": "0.17.0",
"requests-oauthlib": "1.3.1",
"typing_extensions": "4.6.3",
"azure-storage-blob": "12.16.0",
"charset-normalizer": "3.1.0",
"drf-nested-routers": "0.93.4",
"grpc-google-iam-v1": "0.12.6",
"django-cors-headers": "4.1.0",
"djangorestframework": "3.14.0",
"django-import-export": "3.2.0",
"django-organizations": "2.1.0",
"google-cloud-logging": "3.5.0",
"google-cloud-storage": "2.9.0",
"google-cloud-audit-log": "0.2.5",
"google-resumable-media": "2.5.0",
"googleapis-common-protos": "1.59.1",
"google-cloud-appengine-logging": "1.3.0"
},
"request": {
"env": {
"SERVER_NAME": "glitchtip-web",
"SERVER_PORT": "8080"
},
"url": "http://glitchtip-web/api/35/envelope/",
"method": "POST",
"headers": [
[
"Accept",
"*/*"
],
[
"Accept-Encoding",
"gzip"
],
[
"Content-Encoding",
"gzip"
],
[
"Content-Length",
"684"
],
[
"Content-Type",
"application/x-sentry-envelope"
],
[
"Host",
"glitchtip-web"
],
[
"X-Forwarded-For",
""
],
[
"X-Sentry-Auth",
"Sentry sentry_key=[removed], sentry_version=7, sentry_client=sentry-cli/x86_64"
]
],
"query_string": null,
"inferred_content_type": "application/x-sentry-envelope"
},
"contexts": {
"trace": {
"op": "view.render",
"span_id": "90ccfe9c5e6f33ae",
"trace_id": "07b793317aa64477b02c773c03bb14fb",
"description": "envelope_store",
"parent_span_id": "a5c359e74073a9c9"
},
"runtime": {
"name": "CPython",
"build": "3.10.12 (main, Jun 14 2023, 18:47:16) [GCC 12.2.0]",
"version": "3.10.12"
},
"incoming event": {
"sdk": {
"name": "sentry-cli",
"version": "2.20.1"
},
"tags": [
[
"abc",
"def"
]
],
"extra": {
"environ": {[redacted]}
},
"contexts": {
"os": {
"name": "Linux",
"type": "os",
"version": "6.4.4-200.fc38.x86_64",
"kernel_version": "#1 SMP PREEMPT_DYNAMIC Wed Jul 19 16:32:49 UTC 2023"
},
"rust": {
"name": "rustc",
"type": "runtime",
"channel": "stable",
"version": "1.69.0"
},
"device": {
"arch": "x86_64",
"type": "device"
}
},
"event_id": "183bf03b83d74d95813b881d68ae945b",
"platform": "native",
"timestamp": 1690534494.249482,
"environment": "production",
"server_name": "ric"
}
},
"logentry": null,
"metadata": {
"type": "ValidationError",
"value": "{'tags': [ErrorDetail(string='Expected a dictionary of items but got type \"list\".', code='not_a_dict')]}",
"filename": "events/views.py",
"function": "process_event"
},
"platform": "python",
"exception": {
"values": [
{
"type": "ValidationError",
"value": "{'tags': [ErrorDetail(string='Expected a dictionary of items but got type \"list\".', code='not_a_dict')]}",
"mechanism": {
"type": "generic",
"handled": true
},
"stacktrace": {
"frames": [
{
"vars": {
"err": "ValidationError({'tags': [ErrorDetail(string='Expected a dictionary of items but got type \"list\".', code='not_a_dict')]})",
"data": {
"sdk": {
"name": "'sentry-cli'",
"version": "'2.20.1'"
},
"tags": [
[
"'abc'",
"'def'"
]
],
"extra": {
"environ": {[redacted]}
},
"contexts": {
"os": {
"name": "'Linux'",
"type": "'os'",
"version": "'6.4.4-200.fc38.x86_64'",
"kernel_version": "'#1 SMP PREEMPT_DYNAMIC Wed Jul 19 16:32:49 UTC 2023'"
},
"rust": {
"name": "'rustc'",
"type": "'runtime'",
"channel": "'stable'",
"version": "'1.69.0'"
},
"device": {
"arch": "'x86_64'",
"type": "'device'"
}
},
"event_id": "'183bf03b83d74d95813b881d68ae945b'",
"platform": "'native'",
"timestamp": "1690534494.249482",
"environment": "'production'",
"server_name": "'ric'"
},
"self": "<events.views.EnvelopeAPIView object at 0x7f308cc7a020>",
"project": "<Project: example DSN>",
"request": "<rest_framework.request.Request: POST '/api/35/envelope/'>",
"serializer": "StoreDefaultSerializer(context={'request': <rest_framework.request.Request: POST '/api/35/envelope/'>, 'project': <Project: example DSN>}, data={'event_id': '183bf03b83d74d95813b881d68ae945b', 'platform': 'native', 'timestamp': 1690534494.249482, 'server_name': 'ric', 'environment': 'production', 'contexts': {'device': {'arch': 'x86_64', 'type': 'device'}, 'os': {'name': 'Linux', 'version': '6.4.4-200.fc38.x86_64', 'kernel_version': '#1 SMP PREEMPT_DYNAMIC Wed Jul 19 16:32:49 UTC 2023', 'type': 'os'}, 'rust': {'name': 'rustc', 'version': '1.69.0', 'channel': 'stable', 'type': 'runtime'}}, 'tags': [['abc', 'def']], 'extra': {'environ': [redacted]
},
"in_app": true,
"lineno": 154,
"module": "events.views",
"abs_path": "/code/events/views.py",
"filename": "events/views.py",
"function": "process_event",
"pre_context": [
" set_context(\"incoming event\", data)",
" serializer = self.get_event_serializer_class(data)(",
" data=data, context={\"request\": self.request, \"project\": project}",
" )",
" try:"
],
"context_line": " serializer.is_valid(raise_exception=True)",
"post_context": [
" except exceptions.ValidationError as err:",
" set_level(\"warning\")",
" capture_exception(err)",
" logger.warning(\"Invalid event %s\", serializer.errors)",
" return Response()"
]
},
{
"vars": {
"self": "StoreDefaultSerializer(context={'request': <rest_framework.request.Request: POST '/api/35/envelope/'>, 'project': <Project: example DSN>}, data={'event_id': '183bf03b83d74d95813b881d68ae945b', 'platform': 'native', 'timestamp': 1690534494.249482, 'server_name': 'ric', 'environment': 'production', 'contexts': {'device': {'arch': 'x86_64', 'type': 'device'}, 'os': {'name': 'Linux', 'version': '6.4.4-200.fc38.x86_64', 'kernel_version': '#1 SMP PREEMPT_DYNAMIC Wed Jul 19 16:32:49 UTC 2023', 'type': 'os'}, 'rust': {'name': 'rustc', 'version': '1.69.0', 'channel': 'stable', 'type': 'runtime'}}, 'tags': [['abc', 'def']], 'extra': {'environ': [redacted]
"raise_exception": "True"
},
"in_app": false,
"lineno": 235,
"module": "rest_framework.serializers",
"abs_path": "/usr/local/lib/python3.10/site-packages/rest_framework/serializers.py",
"filename": "rest_framework/serializers.py",
"function": "is_valid",
"pre_context": [
" self._errors = exc.detail",
" else:",
" self._errors = {}",
"",
" if self._errors and raise_exception:"
],
"context_line": " raise ValidationError(self.errors)",
"post_context": [
"",
" return not bool(self._errors)",
"",
" @property",
" def data(self):"
]
}
]
}
}
]
},
"breadcrumbs": {
"values": [
{
"type": "default",
"level": "info",
"message": "connect",
"category": "query",
"timestamp": "2023-07-28T08:54:54.271783Z"
},
{
"data": {},
"type": "default",
"level": "info",
"message": "SELECT \"projects_project\".\"id\", \"projects_project\".\"organization_id\", \"projects_project\".\"first_event\", EXISTS(SELECT %s AS \"a\" FROM \"difs_debuginformationfile\" U0 WHERE U0.\"project_id\" = (\"projects_project\".\"id\") LIMIT 1) AS \"has_difs\", (select releases_release.id from releases_release inner join releases_releaseproject on releases_releaseproject.release_id = releases_release.id and releases_releaseproject.project_id=%s where version=%s limit 1) AS \"release_id\", (select environments_environment.id from environments_environment inner join environments_environmentproject on environments_environmentproject.environment_id = environments_environment.id and environments_environmentproject.project_id=%s where environments_environment.name=%s limit 1) AS \"environment_id\", \"organizations_ext_organization\".\"id\", \"organizations_ext_organization\".\"is_accepting_events\" FROM \"projects_project\" INNER JOIN \"projects_projectkey\" ON (\"projects_project\".\"id\" = \"projects_projectkey\".\"project_id\") INNER JOIN \"organizations_ex...",
"category": "query",
"timestamp": "2023-07-28T08:54:54.304980Z"
}
]
},
"environment": "production",
"event_id": "af1852212cf64ea1abcf1cb0cf9a1298",
"project": 4,
"level": "warning",
"tags": [
[
"release",
"glitchtip@3.2.1"
],
[
"environment",
"production"
],
[
"server_name",
"glitchtip-web-668d4d9599-wvb55"
]
],
"datetime": "2023-07-28T08:54:54.316851Z",
"release": "glitchtip@3.2.1"
}For real php sdk data, the error also happens with headers parallel-array data.
we can try updating to GT 3.3 next week, but i don' think there are update around this area.
Reproduction on app.GT
i have registered project with id 3699 and 3700. sentry-cli can send directly to 3699, result is https://app.glitchtip.com/hieuhg-test-relay/issues/2525280
podman run --rm -it --network host -e SENTRY_URL=https://app.glitchtip.com/ -e SENTRY_LOG_LEVEL=trace -e SENTRY_DSN=https://PUBKEY3699@app.glitchtip.com/3699 docker.io/getsentry/sentry-cli:2.20.1 send-event --no-environ -m 'sent %s' -a directly DEBUG 2023-07-28 10:11:55.388972677 +00:00 sentry-cli version: 2.20.1, platform: "linux", architecture: "x86_64"
INFO 2023-07-28 10:11:55.389046695 +00:00 sentry-cli was invoked with the following command line: "/bin/sentry-cli" "send-event" "--no-environ" "-m" "sent %s" "-a" "directly"
DEBUG 2023-07-28 10:11:55.389518107 +00:00 Event { event_id: da5a8c0c-e2fe-432d-a14e-bbab4b68c177, level: Error, fingerprint: ["{{ default }}"], culprit: None, transaction: None, message: None, logentry: Some(LogEntry { message: "sent %s", params: [String("directly")] }), logger: None, modules: {}, platform: "other", timestamp: SystemTime { tv_sec: 1690539115, tv_nsec: 389513545 }, server_name: None, release: None, dist: None, environment: None, user: None, request: None, contexts: {}, breadcrumbs: Values { values: [] }, exception: Values { values: [] }, stacktrace: None, template: None, threads: Values { values: [] }, tags: {}, extra: {}, debug_meta: DebugMeta { sdk_info: None, images: [] }, sdk: Some(ClientSdkInfo { name: "sentry-cli", version: "2.20.1", integrations: [], packages: [] }) }
Event dispatched: da5a8c0c-e2fe-432d-a14e-bbab4b68c177To run a relay in proxy mode
podman run --entrypoint bash --rm -it --network host docker.io/getsentry/relay:23.7.0
# inside, write a config
# cat .relay/config.yml
logging:
level: debug
relay:
port: 3000
host: 0.0.0.0
upstream: https://app.glitchtip.com/
mode: proxy
# run relay
relay runSend event
podman run --rm -it --network host -e SENTRY_LOG_LEVEL=trace -e SENTRY_DSN=http://PUBKEY3700@127.0.0.1:3000/3700 docker.io/getsentry/sentry-cli:2.20.1 send-event --no-environ -m 'sent %s' -a 'via relay proxy' DEBUG 2023-07-28 10:14:41.257278325 +00:00 sentry-cli version: 2.20.1, platform: "linux", architecture: "x86_64"
INFO 2023-07-28 10:14:41.257350508 +00:00 sentry-cli was invoked with the following command line: "/bin/sentry-cli" "send-event" "--no-environ" "-m" "sent %s" "-a" "via relay proxy"
DEBUG 2023-07-28 10:14:41.257736637 +00:00 Event { event_id: b1b0fa8a-75b5-4110-951f-6b84db721a8b, level: Error, fingerprint: ["{{ default }}"], culprit: None, transaction: None, message: None, logentry: Some(LogEntry { message: "sent %s", params: [String("via relay proxy")] }), logger: None, modules: {}, platform: "other", timestamp: SystemTime { tv_sec: 1690539281, tv_nsec: 257732530 }, server_name: None, release: None, dist: None, environment: None, user: None, request: None, contexts: {}, breadcrumbs: Values { values: [] }, exception: Values { values: [] }, stacktrace: None, template: None, threads: Values { values: [] }, tags: {}, extra: {}, debug_meta: DebugMeta { sdk_info: None, images: [] }, sdk: Some(ClientSdkInfo { name: "sentry-cli", version: "2.20.1", integrations: [], packages: [] }) }
Event dispatched: b1b0fa8a-75b5-4110-951f-6b84db721a8bIt doesn't show up in issue dashboard
relay log showing it connected to app.glitchtip.com
DEBUG hyper::proto::h1::role: setting h1 header read timeout timer
DEBUG hyper::proto::h1::io: parsed 4 headers
DEBUG hyper::proto::h1::conn: incoming body is content-length (611 bytes)
DEBUG hyper::proto::h1::conn: incoming body completed
DEBUG request: tower_http::trace::on_request: started processing request method=POST uri=/api/3700/envelope/ version=HTTP/1.1
DEBUG relay_server::actors::project: project PUBKEY3700 state requested 1 times
DEBUG relay_server::actors::project: project state PUBKEY3700 updated
DEBUG request: tower_http::trace::on_response: finished processing request latency=0 ms status=200 method=POST uri=/api/3700/envelope/ version=HTTP/1.1
DEBUG hyper::proto::h1::io: flushed 379 bytes
DEBUG hyper::proto::h1::conn: read eof
DEBUG reqwest::connect: starting new connection: https://app.glitchtip.com/
DEBUG trust_dns_proto::xfer::dns_handle: querying: app.glitchtip.com A
DEBUG trust_dns_resolver::name_server::name_server_pool: sending request: [Query { name: Name("app.glitchtip.com"), query_type: A, query_class: IN }]
DEBUG trust_dns_resolver::name_server::name_server: existing connection: NameServerConfig { socket_addr: 127.0.0.53:53, protocol: Udp, tls_dns_name: None, trust_nx_responses: false, bind_addr: None }
DEBUG trust_dns_proto::xfer: enqueueing message:QUERY:[Query { name: Name("app.glitchtip.com"), query_type: A, query_class: IN }]
DEBUG trust_dns_proto::udp::udp_client_stream: final message: ; header 30679:QUERY:RD:NoError:QUERY:0/0/0
; query
;; app.glitchtip.com. IN A
DEBUG trust_dns_proto::udp::udp_stream: created socket successfully
DEBUG trust_dns_proto::udp::udp_client_stream: received message id: 30679
DEBUG trust_dns_resolver::error: Response:; header 30679:RESPONSE:RD,RA:NoError:QUERY:2/13/10
; query
;; app.glitchtip.com. IN A
; answers 2
app.glitchtip.com. 300 IN A 172.67.209.149
app.glitchtip.com. 300 IN A 104.21.77.157
; nameservers 13
com. 159957 IN NS f.gtld-servers.net.
com. 159957 IN NS d.gtld-servers.net.
com. 159957 IN NS m.gtld-servers.net.
com. 159957 IN NS c.gtld-servers.net.
com. 159957 IN NS k.gtld-servers.net.
com. 159957 IN NS b.gtld-servers.net.
com. 159957 IN NS a.gtld-servers.net.
com. 159957 IN NS i.gtld-servers.net.
com. 159957 IN NS e.gtld-servers.net.
com. 159957 IN NS j.gtld-servers.net.
com. 159957 IN NS l.gtld-servers.net.
com. 159957 IN NS g.gtld-servers.net.
com. 159957 IN NS h.gtld-servers.net.
; additionals 10
a.gtld-servers.net. 64481 IN A 192.5.6.30
a.gtld-servers.net. 13055 IN AAAA 2001:503:a83e::2:30
b.gtld-servers.net. 1296 IN A 192.33.14.30
b.gtld-servers.net. 3331 IN AAAA 2001:503:231d::2:30
c.gtld-servers.net. 1934 IN A 192.26.92.30
c.gtld-servers.net. 2411 IN AAAA 2001:503:83eb::30
d.gtld-servers.net. 9701 IN A 192.31.80.30
d.gtld-servers.net. 66699 IN AAAA 2001:500:856e::30
e.gtld-servers.net. 958 IN A 192.12.94.30
e.gtld-servers.net. 12187 IN AAAA 2001:502:1ca1::30
DEBUG trust_dns_resolver::error: Response:; header 30679:RESPONSE:RD,RA:NoError:QUERY:2/13/10
; query
;; app.glitchtip.com. IN A
; answers 2
app.glitchtip.com. 300 IN A 172.67.209.149
app.glitchtip.com. 300 IN A 104.21.77.157
; nameservers 13
com. 159957 IN NS f.gtld-servers.net.
com. 159957 IN NS d.gtld-servers.net.
com. 159957 IN NS m.gtld-servers.net.
com. 159957 IN NS c.gtld-servers.net.
com. 159957 IN NS k.gtld-servers.net.
com. 159957 IN NS b.gtld-servers.net.
com. 159957 IN NS a.gtld-servers.net.
com. 159957 IN NS i.gtld-servers.net.
com. 159957 IN NS e.gtld-servers.net.
com. 159957 IN NS j.gtld-servers.net.
com. 159957 IN NS l.gtld-servers.net.
com. 159957 IN NS g.gtld-servers.net.
com. 159957 IN NS h.gtld-servers.net.
; additionals 10
a.gtld-servers.net. 64481 IN A 192.5.6.30
a.gtld-servers.net. 13055 IN AAAA 2001:503:a83e::2:30
b.gtld-servers.net. 1296 IN A 192.33.14.30
b.gtld-servers.net. 3331 IN AAAA 2001:503:231d::2:30
c.gtld-servers.net. 1934 IN A 192.26.92.30
c.gtld-servers.net. 2411 IN AAAA 2001:503:83eb::30
d.gtld-servers.net. 9701 IN A 192.31.80.30
d.gtld-servers.net. 66699 IN AAAA 2001:500:856e::30
e.gtld-servers.net. 958 IN A 192.12.94.30
e.gtld-servers.net. 12187 IN AAAA 2001:502:1ca1::30
DEBUG hyper::client::connect::http: connecting to 172.67.209.149:443
DEBUG hyper::client::connect::http: connected to 172.67.209.149:443
DEBUG hyper::proto::h1::io: flushed 909 bytes
DEBUG hyper::proto::h1::io: parsed 18 headers
DEBUG hyper::proto::h1::conn: incoming body is empty
DEBUG hyper::client::pool: pooling idle connection for ("https", app.glitchtip.com)
Proposed Solution(s)
i have no idea why result payload (with parallel-array tags) becomes invalid when ingested back.