Internal server error when generations endpoint uses Anthropic
Problem to solve
When /v2/code/generations
endpoint is sent following params:
{
"current_file": {
"file_name": "snippets/python/censor.py",
"content_above_cursor": "# Ask the user to provide a line of text.\n# Scan the text for the following mildly offensive words: \\\n# arse, bloody, damn, dummy.\n# If you find any, then replace its letters with asterisks \\\n# except for the first letter in each offensive word.\n# Print the resulting text",
"content_below_cursor": "\n"
},
"prompt_version": 2,
"prompt": "\n\nHuman: You are a code completion AI that writes high-quality code like a senior engineer.\nYou are looking at 'snippets/python/censor.py' file. You write code in between tags as in this example:\n\n\u003cnew_code\u003e\n// Code goes here\n\u003c/new_code\u003e\n\nThis is a task to write new Python code in a file 'snippets/python/censor.py', based on a given description.\n\nYou get the description of the code that needs to be created in \u003cinstruction\u003e XML tags.\n\nIt is your task to write valid and working Python code.\nOnly return in your response new code.\nDo not provide any explanation.\n\n\n\n\u003cinstruction\u003e\n Ask the user to provide a line of text.\nScan the text for the following mildly offensive words: \\\narse, bloody, damn, dummy.\nIf you find any, then replace its letters with asterisks \\\nexcept for the first letter in each offensive word.\nPrint the resulting text\n\u003c/instruction\u003e\n\n\nAssistant:\n",
"model_provider": "anthropic"
}
It raises the following exception:
Log
--- Logging error ---
Traceback (most recent call last):
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 98, in receive
return self.receive_nowait()
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 93, in receive_nowait
raise WouldBlock
anyio.WouldBlock
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 43, in call_next
message = await recv_stream.receive()
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 118, in receive
raise EndOfStream
anyio.EndOfStream
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/gitlab-development-kit/ai-assist/ai_gateway/api/middleware.py", line 107, in dispatch
response = await call_next(request)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 46, in call_next
raise app_exc
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 36, in coro
await self.app(scope, request.receive, send_stream.send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/authentication.py", line 48, in __call__
await self.app(scope, receive, send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 68, in __call__
response = await self.dispatch_func(request, call_next)
File "/opt/gitlab-development-kit/ai-assist/ai_gateway/api/middleware.py", line 274, in dispatch
return await call_next(request)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 46, in call_next
raise app_exc
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 36, in coro
await self.app(scope, request.receive, send_stream.send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/exceptions.py", line 93, in __call__
raise exc
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/exceptions.py", line 82, in __call__
await self.app(scope, receive, sender)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__
raise e
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
await self.app(scope, receive, send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/routing.py", line 670, in __call__
await route.handle(scope, receive, send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/routing.py", line 266, in handle
await self.app(scope, receive, send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/routing.py", line 65, in app
response = await func(request)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/fastapi/routing.py", line 231, in app
raw_response = await run_endpoint_function(
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/fastapi/routing.py", line 160, in run_endpoint_function
return await dependant.call(**values)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/dependency_injector/wiring.py", line 994, in _patched
return await _async_inject(
File "src/dependency_injector/_cwiring.pyx", line 66, in _async_inject
File "/opt/gitlab-development-kit/ai-assist/ai_gateway/api/v2/endpoints/code.py", line 181, in generations
suggestion = await code_generations.execute(
File "/opt/gitlab-development-kit/ai-assist/ai_gateway/code_suggestions/generations.py", line 106, in execute
watch_container.register_safety_attributes(res.safety_attributes)
File "/opt/gitlab-development-kit/ai-assist/ai_gateway/instrumentators/base.py", line 128, in register_safety_attributes
"safety_categories": safety_attributes.categories,
AttributeError: 'NoneType' object has no attribute 'categories'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/honza/.asdf/installs/python/3.9.17/lib/python3.9/logging/__init__.py", line 1083, in emit
msg = self.format(record)
File "/home/honza/.asdf/installs/python/3.9.17/lib/python3.9/logging/__init__.py", line 927, in format
return fmt.format(record)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/structlog/stdlib.py", line 1046, in format
ed = p(logger, meth_name, ed)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/structlog/dev.py", line 434, in __call__
sio.write("\n" + exc)
TypeError: can only concatenate str (not "dict") to str
Call stack:
File "<string>", line 1, in <module>
File "/opt/gitlab-development-kit/ai-assist/ai_gateway/app.py", line 82, in main
uvicorn.run(
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/uvicorn/main.py", line 569, in run
server.run()
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/uvicorn/server.py", line 60, in run
return asyncio.run(self.serve(sockets=sockets))
File "/home/honza/.asdf/installs/python/3.9.17/lib/python3.9/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/uvicorn/protocols/http/httptools_impl.py", line 419, in run_asgi
result = await app( # type: ignore[func-returns-value]
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
return await self.app(scope, receive, send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/uvicorn/middleware/message_logger.py", line 82, in __call__
await self.app(scope, inner_receive, inner_send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/fastapi/applications.py", line 269, in __call__
await super().__call__(scope, receive, send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/applications.py", line 124, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/errors.py", line 162, in __call__
await self.app(scope, receive, _send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/prometheus_fastapi_instrumentator/middleware.py", line 167, in __call__
await self.app(scope, receive, send_wrapper)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/asgi_correlation_id/middleware.py", line 90, in __call__
await self.app(scope, receive, handle_outgoing_request)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette_context/middleware/raw_middleware.py", line 92, in __call__
await self.app(scope, receive, send_wrapper)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/cors.py", line 84, in __call__
await self.app(scope, receive, send)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 68, in __call__
response = await self.dispatch_func(request, call_next)
File "/opt/gitlab-development-kit/ai-assist/ai_gateway/api/middleware.py", line 157, in dispatch
access_logger.info(
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/structlog/_log_levels.py", line 161, in meth
return self._proxy_to_logger(name, event, **kw)
File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/structlog/_base.py", line 205, in _proxy_to_logger
return getattr(self._logger, method_name)(*args, **kw)
Message: {'url': 'http://0.0.0.0:5052/v2/code/generations', 'path': '/v2/code/generations', 'status_code': 500, 'method': 'POST', 'correlation_id': '479ef40657ea419797aa0ac3cedeaab0', 'http_version': '1.1', 'client_ip': '127.0.0.1', 'client_port': 55104, 'duration_s': 2.9355666720075533, 'cpu_s': 0.01561118100000014, 'user_agent': 'gitlab-workhorse (11-10-0cfa69752d8-74ffd66ae-ee-230447-g7f43e3a1213e)-(20230915.103825)', 'gitlab_instance_id': '292c3c7c-c5d5-48ec-b4bf-f00b724ce560', 'gitlab_global_user_id': 'Df0Jhs9xlbetQR8YoZCKDZJflhxO0ZBI8uoRzmpnd1w=', 'gitlab_realm': 'saas', 'auth_duration_s': 0.0004465689416974783, 'model_engine': 'anthropic', 'model_name': 'claude-2', 'prompt_length': 873, 'prompt_length_stripped': 716, 'inference_duration_s': 2.9326032089302316, 'lang': 'python', 'editor_lang': None, 'model_output_length': 271, 'model_output_length_stripped': 212, 'model_output_score': 100000, 'exception': {'message': "'NoneType' object has no attribute 'categories'", 'backtrace': 'Traceback (most recent call last):\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 98, in receive\n return self.receive_nowait()\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 93, in receive_nowait\n raise WouldBlock\nanyio.WouldBlock\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 43, in call_next\n message = await recv_stream.receive()\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 118, in receive\n raise EndOfStream\nanyio.EndOfStream\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/opt/gitlab-development-kit/ai-assist/ai_gateway/api/middleware.py", line 107, in dispatch\n response = await call_next(request)\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 46, in call_next\n raise app_exc\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 36, in coro\n await self.app(scope, request.receive, send_stream.send)\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/authentication.py", line 48, in __call__\n await self.app(scope, receive, send)\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 68, in __call__\n response = await self.dispatch_func(request, call_next)\n File "/opt/gitlab-development-kit/ai-assist/ai_gateway/api/middleware.py", line 274, in dispatch\n return await call_next(request)\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 46, in call_next\n raise app_exc\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 36, in coro\n await self.app(scope, request.receive, send_stream.send)\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/exceptions.py", line 93, in __call__\n raise exc\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/exceptions.py", line 82, in __call__\n await self.app(scope, receive, sender)\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__\n raise e\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__\n await self.app(scope, receive, send)\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/routing.py", line 670, in __call__\n await route.handle(scope, receive, send)\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/routing.py", line 266, in handle\n await self.app(scope, receive, send)\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/starlette/routing.py", line 65, in app\n response = await func(request)\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/fastapi/routing.py", line 231, in app\n raw_response = await run_endpoint_function(\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/fastapi/routing.py", line 160, in run_endpoint_function\n return await dependant.call(**values)\n File "/home/honza/.cache/pypoetry/virtualenvs/ai-gateway-5E8YQAKd-py3.9/lib/python3.9/site-packages/dependency_injector/wiring.py", line 994, in _patched\n return await _async_inject(\n File "src/dependency_injector/_cwiring.pyx", line 66, in _async_inject\n File "/opt/gitlab-development-kit/ai-assist/ai_gateway/api/v2/endpoints/code.py", line 181, in generations\n suggestion = await code_generations.execute(\n File "/opt/gitlab-development-kit/ai-assist/ai_gateway/code_suggestions/generations.py", line 106, in execute\n watch_container.register_safety_attributes(res.safety_attributes)\n File "/opt/gitlab-development-kit/ai-assist/ai_gateway/instrumentators/base.py", line 128, in register_safety_attributes\n "safety_categories": safety_attributes.categories,\nAttributeError: \'NoneType\' object has no attribute \'categories\'\n'}, 'event': '127.0.0.1:55104 - "POST /v2/code/generations HTTP/1.1" 500', 'logger': 'api.access', 'level': 'info', 'type': 'mlops', 'stage': 'main', 'timestamp': '2023-10-04T10:41:49.881597Z'}
When model_provider
param is not specified, it works.
Proposal
Further details
Links / references
Edited by Tan Le