Response time degraded for Git Clone requests under load
Summary
Git Clone over HTTPs under load on smaller environments (up to 3k) is degraded between 16.11 and 17.0. TTFB (Time to First Byte) P90 increased from 500ms to 3 seconds.
Latest git_clone results against 1k with GitLab nightly package can be found in https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/1k
More details can be found in gitlab-org/quality/quality-engineering/team-tasks#2770 (comment 2024842403)
Reviewing logs for POST .git/git-upload-pack requests between 16.11 and 17.0 shows that CPU and User time significantly increased between versions:
-
grpc.time_ms:7 218.83in 16.11 and50 683.953in 17.0 -
command.cpu_time_ms:10 683in 16.11 and94 989in 17.0 -
command.user_time_ms:8 666in 16.11 and91 822in 17.0
The test data, RPS, environment was consistent between the runs, the only difference was that GitLab version was upgraded.
Detailed logs - `01J44R9QRWPM1S95WN0E817H83` - 16.11 when test passed
// 01J44R9QRWPM1S95WN0E817H83 - 16.11 when test passed
{
"component": "gitaly.UnaryServerInterceptor",
"correlation_id": "01J44R9QRWPM1S95WN0E817H83",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-workhorse",
"grpc.meta.deadline_type": "none",
"grpc.meta.method_operation": "accessor",
"grpc.meta.method_scope": "repository",
"grpc.meta.method_type": "unary",
"grpc.method": "PostUploadPackWithSidechannel",
"grpc.request.fullMethod": "/gitaly.SmartHTTPService/PostUploadPackWithSidechannel",
"grpc.request.glProjectPath": "gpt/large_projects/gitlabhq1",
"grpc.request.glRepository": "project-5001",
"grpc.request.repoPath": "@hashed/ad/b0/adb019dcde61d092941e0fec4e89b405130df238877e2611c330ae95a7266487.git",
"grpc.request.repoStorage": "default",
"grpc.service": "gitaly.SmartHTTPService",
"grpc.start_time": "2024-07-31T15:59:48.569",
"level": "info",
"msg": "request details",
"pid": 28666,
"remote_ip": "35.227.22.222",
"request_sha": "71c6e19431bce3c5490e4a41745a35688ab97760",
"response_bytes": 623923986,
"span.kind": "server",
"system": "grpc",
"time": "2024-07-31T15:59:55.787Z"
}
{
"command.count": 1,
"command.cpu_time_ms": 10683,
"command.inblock": 0,
"command.majflt": 0,
"command.maxrss": 915164,
"command.minflt": 47674,
"command.oublock": 0,
"command.real_time_ms": 7217,
"command.spawn_token_fork_ms": 0,
"command.spawn_token_wait_ms": 0,
"command.system_time_ms": 2017,
"command.user_time_ms": 8666,
"component": "gitaly.UnaryServerInterceptor",
"correlation_id": "01J44R9QRWPM1S95WN0E817H83",
"grpc.code": "OK",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-workhorse",
"grpc.meta.deadline_type": "none",
"grpc.meta.method_operation": "accessor",
"grpc.meta.method_scope": "repository",
"grpc.meta.method_type": "unary",
"grpc.method": "PostUploadPackWithSidechannel",
"grpc.request.fullMethod": "/gitaly.SmartHTTPService/PostUploadPackWithSidechannel",
"grpc.request.glProjectPath": "gpt/large_projects/gitlabhq1",
"grpc.request.glRepository": "project-5001",
"grpc.request.payload_bytes": 140,
"grpc.request.repoPath": "@hashed/ad/b0/adb019dcde61d092941e0fec4e89b405130df238877e2611c330ae95a7266487.git",
"grpc.request.repoStorage": "default",
"grpc.response.payload_bytes": 45,
"grpc.service": "gitaly.SmartHTTPService",
"grpc.start_time": "2024-07-31T15:59:48.569",
"grpc.time_ms": 7218.83,
"level": "info",
"msg": "finished unary call with code OK",
"pid": 28666,
"remote_ip": "35.227.22.222",
"span.kind": "server",
"system": "grpc",
"time": "2024-07-31T15:59:55.788Z"
}Detailed logs - `01J44Y3ZZC19YVCGN72C71KTG2 ` - 17.0 when test failed
{
"component": "gitaly.UnaryServerInterceptor",
"correlation_id": "01J44Y3ZZC19YVCGN72C71KTG2",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-workhorse",
"grpc.meta.deadline_type": "none",
"grpc.meta.method_operation": "accessor",
"grpc.meta.method_scope": "repository",
"grpc.meta.method_type": "unary",
"grpc.method": "PostUploadPackWithSidechannel",
"grpc.request.fullMethod": "/gitaly.SmartHTTPService/PostUploadPackWithSidechannel",
"grpc.request.glProjectPath": "gpt/large_projects/gitlabhq1",
"grpc.request.glRepository": "project-5001",
"grpc.request.repoPath": "@hashed/ad/b0/adb019dcde61d092941e0fec4e89b405130df238877e2611c330ae95a7266487.git",
"grpc.request.repoStorage": "default",
"grpc.service": "gitaly.SmartHTTPService",
"grpc.start_time": "2024-07-31T17:41:31.909",
"level": "info",
"msg": "request details",
"pid": 526777,
"remote_ip": "35.227.22.222",
"request_sha": "71c6e19431bce3c5490e4a41745a35688ab97760",
"response_bytes": 690962010,
"span.kind": "server",
"system": "grpc",
"time": "2024-07-31T17:42:22.592Z"
}
{
"command.count": 1,
"command.cpu_time_ms": 94989,
"command.inblock": 0,
"command.majflt": 0,
"command.maxrss": 1199196,
"command.minflt": 130708,
"command.oublock": 0,
"command.real_time_ms": 50682,
"command.spawn_token_fork_ms": 0,
"command.spawn_token_wait_ms": 0,
"command.system_time_ms": 3167,
"command.user_time_ms": 91822,
"component": "gitaly.UnaryServerInterceptor",
"correlation_id": "01J44Y3ZZC19YVCGN72C71KTG2",
"grpc.code": "OK",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-workhorse",
"grpc.meta.deadline_type": "none",
"grpc.meta.method_operation": "accessor",
"grpc.meta.method_scope": "repository",
"grpc.meta.method_type": "unary",
"grpc.method": "PostUploadPackWithSidechannel",
"grpc.request.fullMethod": "/gitaly.SmartHTTPService/PostUploadPackWithSidechannel",
"grpc.request.glProjectPath": "gpt/large_projects/gitlabhq1",
"grpc.request.glRepository": "project-5001",
"grpc.request.payload_bytes": 140,
"grpc.request.repoPath": "@hashed/ad/b0/adb019dcde61d092941e0fec4e89b405130df238877e2611c330ae95a7266487.git",
"grpc.request.repoStorage": "default",
"grpc.response.payload_bytes": 45,
"grpc.service": "gitaly.SmartHTTPService",
"grpc.start_time": "2024-07-31T17:41:31.909",
"grpc.time_ms": 50683.953,
"level": "info",
"msg": "finished unary call with code OK",
"pid": 526777,
"remote_ip": "35.227.22.222",
"span.kind": "server",
"system": "grpc",
"time": "2024-07-31T17:42:22.593Z"
}Steps to reproduce
- Build 1k environment on 16.11.7, seed with GPT Test data
- Run
git_cloneGPT test -bin/run-k6 -s -e 1k.json -o 60s_20rps.json -t git_clone - Upgrade to 17.0 or later
- Run
git_cloneGPT test - Compare TTFB results
What is the current bug behavior?
TTFB P90 is over 2 seconds.
Click to expand
* Environment: 1k
* Environment Version: 17.3.0-pre `ff0d3cbaef3`
* Option: 60s_20rps
* Date: 2024-08-01
* Run Time: 1h 33m 45.27s (Start: 04:45:29 UTC, End: 06:19:15 UTC)
* GPT Version: v2.15.0
❯ Overall Results Score: 96.91%
NAME | RPS | RPS RESULT | TTFB AVG | TTFB P90 | REQ STATUS | RESULT
---------------------------------------------------------|------|--------------------|------------|-----------------------|----------------|---------
git_clone | 1/s | 0.09/s (>0.03/s) | 959.62ms | 2410.97ms (<800ms) | 100.00% (>99%) | FAILED¹²
█ setup
█ Git - Git Clone HTTP
█ Git - Get Refs List
█ Git - Git Clone Refs
data_received..................: 1.8 GB 22 MB/s
data_sent......................: 11 MB 128 kB/s
group_duration.................: avg=18572.61ms min=90.76ms med=27252.18ms max=28295.54ms p(90)=28211.43ms p(95)=28253.49ms
http_req_blocked...............: avg=2.79ms min=0.00ms med=0.00ms max=19.23ms p(90)=7.92ms p(95)=13.57ms
http_req_connecting............: avg=0.17ms min=0.00ms med=0.00ms max=0.68ms p(90)=0.67ms p(95)=0.67ms
http_req_duration..............: avg=10029.46ms min=84.69ms med=125.58ms max=27294.87ms p(90)=26627.97ms p(95)=26961.42ms
{ expected_response:true }...: avg=10029.46ms min=84.69ms med=125.58ms max=27294.87ms p(90)=26627.97ms p(95)=26961.42ms
http_req_failed................: 0.00% ✓ 0 ✗ 8
http_req_receiving.............: avg=9069.73ms min=0.05ms med=15.33ms max=24784.51ms p(90)=24216.87ms p(95)=24500.69ms
http_req_sending...............: avg=0.11ms min=0.06ms med=0.11ms max=0.20ms p(90)=0.15ms p(95)=0.18ms
http_req_tls_handshaking.......: avg=0.53ms min=0.00ms med=0.00ms max=2.25ms p(90)=2.05ms p(95)=2.15ms
✗ http_req_waiting...............: avg=959.62ms min=72.56ms med=125.37ms max=2510.26ms p(90)=2410.97ms p(95)=2460.62ms
✓ http_reqs......................: 8 0.094442/s
iteration_duration.............: avg=21176.69ms min=1129.78ms med=27640.70ms max=28295.59ms p(90)=28188.42ms p(95)=28242.01ms
iterations.....................: 3 0.035416/s
✓ successful_requests............: 100.00% ✓ 6 ✗ 0
vus............................: 1 min=0 max=1
vus_max........................: 1 min=1 max=1What is the expected correct behavior?
TTFB is below 800ms
Click to expand
* Environment: 1k
* Environment Version: 16.11.7-ee `013d9311c68`
* Option: 60s_20rps
* Date: 2024-07-31
* Run Time: 1m 5s (Start: 15:59:12 UTC, End: 16:00:17 UTC)
* GPT Version: v2.15.0
NAME | RPS | RPS RESULT | TTFB AVG | TTFB P90 | REQ STATUS | RESULT
----------|-----|------------------|----------|-------------------|----------------|--------
git_clone | 1/s | 0.29/s (>0.03/s) | 274.76ms | 427.03ms (<800ms) | 100.00% (>99%) | Passed¹
█ Git - Git Clone HTTP
█ Git - Get Refs List
█ Git - Git Clone Refs
data_received..................: 5.1 GB 81 MB/s
data_sent......................: 31 MB 487 kB/s
group_duration.................: avg=5138.19ms min=79.82ms med=6634.00ms max=8974.15ms p(90)=8448.85ms p(95)=8820.38ms
http_req_blocked...............: avg=0.36ms min=0.00ms med=0.00ms max=3.59ms p(90)=0.88ms p(95)=3.03ms
http_req_connecting............: avg=0.11ms min=0.00ms med=0.00ms max=1.14ms p(90)=0.26ms p(95)=0.89ms
http_req_duration..............: avg=3034.06ms min=79.66ms med=378.91ms max=7972.95ms p(90)=7427.37ms p(95)=7540.24ms
{ expected_response:true }...: avg=3034.06ms min=79.66ms med=378.91ms max=7972.95ms p(90)=7427.37ms p(95)=7540.24ms
http_req_failed................: 0.00% ✓ 0 ✗ 18
http_req_receiving.............: avg=2759.21ms min=0.05ms med=15.95ms max=7412.63ms p(90)=7043.10ms p(95)=7134.86ms
http_req_sending...............: avg=0.09ms min=0.04ms med=0.10ms max=0.20ms p(90)=0.13ms p(95)=0.14ms
http_req_tls_handshaking.......: avg=0.18ms min=0.00ms med=0.00ms max=1.66ms p(90)=0.49ms p(95)=1.65ms
✓ http_req_waiting...............: avg=274.76ms min=64.49ms med=321.42ms max=560.21ms p(90)=427.03ms p(95)=489.65ms
✓ http_reqs......................: 18 0.286333/s
iteration_duration.............: avg=6984.75ms min=1203.83ms med=7681.88ms max=8974.20ms p(90)=8566.58ms p(95)=8770.39ms
iterations.....................: 8 0.127259/s
✓ successful_requests............: 100.00% ✓ 16 ✗ 0
vus............................: 1 min=0 max=1
vus_max........................: 1 min=1 max=1Relevant logs and/or screenshots
Logs when git clone test passed in 16.11:
Click to expand
33101:{"content_type":"application/x-git-upload-pack-result","correlation_id":"01J44R9QRWPM1S95WN0E817H83","duration_ms":7280,"host":"test-1k.distr-qe-test.k8s-ft.win","level":"info","method":"POST","msg":"access","proto":"HTTP/1.1","referrer":"","remote_addr":"35.227.22.222:0","remote_ip":"35.227.22.222","route":"^/.+\\.git/git-upload-pack\\z","status":200,"system":"http","time":"2024-07-31T15:59:55Z","ttfb_ms":65,"uri":"/gpt/large_projects/gitlabhq1.git/git-upload-pack","user_agent":"GPT/2.15.0","written_bytes":623923986}
33106:{"content_type":"application/x-git-upload-pack-result","correlation_id":"01J44R9ZVQ7VAYHAZYYA5WX1J0","duration_ms":7461,"host":"test-1k.distr-qe-test.k8s-ft.win","level":"info","method":"POST","msg":"access","proto":"HTTP/1.1","referrer":"","remote_addr":"35.227.22.222:0","remote_ip":"35.227.22.222","route":"^/.+\\.git/git-upload-pack\\z","status":200,"system":"http","time":"2024-07-31T16:00:04Z","ttfb_ms":59,"uri":"/gpt/large_projects/gitlabhq1.git/git-upload-pack","user_agent":"GPT/2.15.0","written_bytes":623929946}
33110:{"content_type":"application/x-git-upload-pack-result","correlation_id":"01J44RA8485DS2BB2V3RA0Q5W2","duration_ms":5747,"host":"test-1k.distr-qe-test.k8s-ft.win","level":"info","method":"POST","msg":"access","proto":"HTTP/1.1","referrer":"","remote_addr":"35.227.22.222:0","remote_ip":"35.227.22.222","route":"^/.+\\.git/git-upload-pack\\z","status":200,"system":"http","time":"2024-07-31T16:00:11Z","ttfb_ms":85,"uri":"/gpt/large_projects/gitlabhq1.git/git-upload-pack","user_agent":"GPT/2.15.0","written_bytes":650345477}
33114:{"content_type":"application/x-git-upload-pack-result","correlation_id":"01J44RAEQ9KNRM07AACMFRFVD9","duration_ms":5491,"host":"test-1k.distr-qe-test.k8s-ft.win","level":"info","method":"POST","msg":"access","proto":"HTTP/1.1","referrer":"","remote_addr":"35.227.22.222:0","remote_ip":"35.227.22.222","route":"^/.+\\.git/git-upload-pack\\z","status":200,"system":"http","time":"2024-07-31T16:00:17Z","ttfb_ms":64,"uri":"/gpt/large_projects/gitlabhq1.git/git-upload-pack","user_agent":"GPT/2.15.0","written_bytes":650300892}
72762:{"component":"gitaly.UnaryServerInterceptor","correlation_id":"01J44R9QRWPM1S95WN0E817H83","grpc.meta.auth_version":"v2","grpc.meta.client_name":"gitlab-workhorse","grpc.meta.deadline_type":"none","grpc.meta.method_operation":"accessor","grpc.meta.method_scope":"repository","grpc.meta.method_type":"unary","grpc.method":"PostUploadPackWithSidechannel","grpc.request.fullMethod":"/gitaly.SmartHTTPService/PostUploadPackWithSidechannel","grpc.request.glProjectPath":"gpt/large_projects/gitlabhq1","grpc.request.glRepository":"project-5001","grpc.request.repoPath":"@hashed/ad/b0/adb019dcde61d092941e0fec4e89b405130df238877e2611c330ae95a7266487.git","grpc.request.repoStorage":"default","grpc.service":"gitaly.SmartHTTPService","grpc.start_time":"2024-07-31T15:59:48.569","level":"info","msg":"request details","pid":28666,"remote_ip":"35.227.22.222","request_sha":"71c6e19431bce3c5490e4a41745a35688ab97760","response_bytes":623923986,"span.kind":"server","system":"grpc","time":"2024-07-31T15:59:55.787Z"}
72763:{"command.count":1,"command.cpu_time_ms":10683,"command.inblock":0,"command.majflt":0,"command.maxrss":915164,"command.minflt":47674,"command.oublock":0,"command.real_time_ms":7217,"command.spawn_token_fork_ms":0,"command.spawn_token_wait_ms":0,"command.system_time_ms":2017,"command.user_time_ms":8666,"component":"gitaly.UnaryServerInterceptor","correlation_id":"01J44R9QRWPM1S95WN0E817H83","grpc.code":"OK","grpc.meta.auth_version":"v2","grpc.meta.client_name":"gitlab-workhorse","grpc.meta.deadline_type":"none","grpc.meta.method_operation":"accessor","grpc.meta.method_scope":"repository","grpc.meta.method_type":"unary","grpc.method":"PostUploadPackWithSidechannel","grpc.request.fullMethod":"/gitaly.SmartHTTPService/PostUploadPackWithSidechannel","grpc.request.glProjectPath":"gpt/large_projects/gitlabhq1","grpc.request.glRepository":"project-5001","grpc.request.payload_bytes":140,"grpc.request.repoPath":"@hashed/ad/b0/adb019dcde61d092941e0fec4e89b405130df238877e2611c330ae95a7266487.git","grpc.request.repoStorage":"default","grpc.response.payload_bytes":45,"grpc.service":"gitaly.SmartHTTPService","grpc.start_time":"2024-07-31T15:59:48.569","grpc.time_ms":7218.83,"level":"info","msg":"finished unary call with code OK","pid":28666,"remote_ip":"35.227.22.222","span.kind":"server","system":"grpc","time":"2024-07-31T15:59:55.788Z"}Logs when git clone failed in 17.0 against the same environment:
Click to expand
35554:{"content_type":"application/x-git-upload-pack-result","correlation_id":"01J44Y3ZZC19YVCGN72C71KTG2","duration_ms":50837,"host":"test-1k.distr-qe-test.k8s-ft.win","level":"info","method":"POST","msg":"access","proto":"HTTP/1.1","referrer":"","remote_addr":"35.227.22.222:0","remote_ip":"35.227.22.222","route":"^/.+\\.git/git-upload-pack\\z","status":200,"system":"http","time":"2024-07-31T17:42:22Z","ttfb_ms":158,"uri":"/gpt/large_projects/gitlabhq1.git/git-upload-pack","user_agent":"GPT/2.15.0","written_bytes":690962010}
35569:{"correlation_id":"01J44Y5JKDZSBAVE6XCJ24SC8N","error":"handleUploadPack: do gitaly call: PostUploadPackWithSidechannel: rpc error: code = Canceled desc = context canceled","level":"error","method":"POST","msg":"","time":"2024-07-31T17:42:59Z","uri":"/gpt/large_projects/gitlabhq1.git/git-upload-pack"}
35570:{"content_type":"application/x-git-upload-pack-result","correlation_id":"01J44Y5JKDZSBAVE6XCJ24SC8N","duration_ms":36288,"host":"test-1k.distr-qe-test.k8s-ft.win","level":"info","method":"POST","msg":"access","proto":"HTTP/1.1","referrer":"","remote_addr":"35.227.22.222:0","remote_ip":"35.227.22.222","route":"^/.+\\.git/git-upload-pack\\z","status":200,"system":"http","time":"2024-07-31T17:42:59Z","ttfb_ms":77,"uri":"/gpt/large_projects/gitlabhq1.git/git-upload-pack","user_agent":"GPT/2.15.0","written_bytes":548206941}
35595:{"content_type":"application/x-git-upload-pack-result","correlation_id":"01J44Y6TFPQGZB3BT9QYWEH91Q","duration_ms":54800,"host":"test-1k.distr-qe-test.k8s-ft.win","level":"info","method":"POST","msg":"access","proto":"HTTP/1.1","referrer":"","remote_addr":"35.227.22.222:0","remote_ip":"35.227.22.222","route":"^/.+\\.git/git-upload-pack\\z","status":200,"system":"http","time":"2024-07-31T17:43:59Z","ttfb_ms":79,"uri":"/gpt/large_projects/gitlabhq1.git/git-upload-pack","user_agent":"GPT/2.15.0","written_bytes":719284047}
35610:{"correlation_id":"01J44Y8GZKD0QAQ576CWZQVWJ5","error":"handleUploadPack: do gitaly call: PostUploadPackWithSidechannel: rpc error: code = Canceled desc = context canceled","level":"error","method":"POST","msg":"","time":"2024-07-31T17:44:32Z","uri":"/gpt/large_projects/gitlabhq1.git/git-upload-pack"}
35611:{"content_type":"application/x-git-upload-pack-result","correlation_id":"01J44Y8GZKD0QAQ576CWZQVWJ5","duration_ms":32320,"host":"test-1k.distr-qe-test.k8s-ft.win","level":"info","method":"POST","msg":"access","proto":"HTTP/1.1","referrer":"","remote_addr":"35.227.22.222:0","remote_ip":"35.227.22.222","route":"^/.+\\.git/git-upload-pack\\z","status":200,"system":"http","time":"2024-07-31T17:44:32Z","ttfb_ms":129,"uri":"/gpt/large_projects/gitlabhq1.git/git-upload-pack","user_agent":"GPT/2.15.0","written_bytes":522493666}
/var/log/gitlab/gitaly/current
72992:{"component":"gitaly.UnaryServerInterceptor","correlation_id":"01J44Y3ZZC19YVCGN72C71KTG2","grpc.meta.auth_version":"v2","grpc.meta.client_name":"gitlab-workhorse","grpc.meta.deadline_type":"none","grpc.meta.method_operation":"accessor","grpc.meta.method_scope":"repository","grpc.meta.method_type":"unary","grpc.method":"PostUploadPackWithSidechannel","grpc.request.fullMethod":"/gitaly.SmartHTTPService/PostUploadPackWithSidechannel","grpc.request.glProjectPath":"gpt/large_projects/gitlabhq1","grpc.request.glRepository":"project-5001","grpc.request.repoPath":"@hashed/ad/b0/adb019dcde61d092941e0fec4e89b405130df238877e2611c330ae95a7266487.git","grpc.request.repoStorage":"default","grpc.service":"gitaly.SmartHTTPService","grpc.start_time":"2024-07-31T17:41:31.909","level":"info","msg":"request details","pid":526777,"remote_ip":"35.227.22.222","request_sha":"71c6e19431bce3c5490e4a41745a35688ab97760","response_bytes":690962010,"span.kind":"server","system":"grpc","time":"2024-07-31T17:42:22.592Z"}
72993:{"command.count":1,"command.cpu_time_ms":94989,"command.inblock":0,"command.majflt":0,"command.maxrss":1199196,"command.minflt":130708,"command.oublock":0,"command.real_time_ms":50682,"command.spawn_token_fork_ms":0,"command.spawn_token_wait_ms":0,"command.system_time_ms":3167,"command.user_time_ms":91822,"component":"gitaly.UnaryServerInterceptor","correlation_id":"01J44Y3ZZC19YVCGN72C71KTG2","grpc.code":"OK","grpc.meta.auth_version":"v2","grpc.meta.client_name":"gitlab-workhorse","grpc.meta.deadline_type":"none","grpc.meta.method_operation":"accessor","grpc.meta.method_scope":"repository","grpc.meta.method_type":"unary","grpc.method":"PostUploadPackWithSidechannel","grpc.request.fullMethod":"/gitaly.SmartHTTPService/PostUploadPackWithSidechannel","grpc.request.glProjectPath":"gpt/large_projects/gitlabhq1","grpc.request.glRepository":"project-5001","grpc.request.payload_bytes":140,"grpc.request.repoPath":"@hashed/ad/b0/adb019dcde61d092941e0fec4e89b405130df238877e2611c330ae95a7266487.git","grpc.request.repoStorage":"default","grpc.response.payload_bytes":45,"grpc.service":"gitaly.SmartHTTPService","grpc.start_time":"2024-07-31T17:41:31.909","grpc.time_ms":50683.953,"level":"info","msg":"finished unary call with code OK","pid":526777,"remote_ip":"35.227.22.222","span.kind":"server","system":"grpc","time":"2024-07-31T17:42:22.593Z"}
Edited by Nailia Iskhakova