Flaky limithandler tests
The limithandler tests in internal/middleware/limithandler
are quite flaky. The flake can be easily reproduced by running the tests in a loop for some iterations. See e.g. this job:
=== FAIL: internal/middleware/limithandler TestStreamLimitHandler/Multiple_requests,_single_response (unknown)
panic: test timed out after 20m0s
goroutine 2041 [running]:
testing.(*M).startAlarm.func1()
/usr/local/go/src/testing/testing.go:2036 +0x8e
created by time.goFunc
/usr/local/go/src/time/sleep.go:176 +0x32
goroutine 1 [chan receive, 19 minutes]:
testing.tRunner.func1()
/usr/local/go/src/testing/testing.go:1412 +0x4a5
testing.tRunner(0xc00023c820, 0xc000131bb8)
/usr/local/go/src/testing/testing.go:1452 +0x144
testing.runTests(0xc0001a9360?, {0x10c4dc0, 0xb, 0xb}, {0xb3b8c3?, 0x8?, 0x10d5320?})
/usr/local/go/src/testing/testing.go:1844 +0x456
testing.(*M).Run(0xc0001a9360)
/usr/local/go/src/testing/testing.go:1726 +0x5d9
gitlab.com/gitlab-org/gitaly/v15/internal/testhelper.Run.func1({0x0, 0x0, 0x0?}, 0xb3e7f6?)
/builds/gitlab-org/gitaly/internal/testhelper/configure.go:71 +0x255
gitlab.com/gitlab-org/gitaly/v15/internal/testhelper.Run(0xffffffffffffffff?, {0x0?, 0x442405?, 0xc000070718?})
/builds/gitlab-org/gitaly/internal/testhelper/configure.go:74 +0x31
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestMain(...)
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:32
main.main()
_testmain.go:71 +0x1db
goroutine 215 [select, 20 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0002e9fc0)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/balancer_conn_wrappers.go:115 +0x73
created by google.golang.org/grpc.newCCBalancerWrapper
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/balancer_conn_wrappers.go:76 +0x22a
goroutine 214 [IO wait, 20 minutes]:
internal/poll.runtime_pollWait(0x7f6fdbf21110, 0x72)
/usr/local/go/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc000381180?, 0x18?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000381180)
/usr/local/go/src/internal/poll/fd_unix.go:614 +0x234
net.(*netFD).accept(0xc000381180)
/usr/local/go/src/net/fd_unix.go:172 +0x35
net.(*UnixListener).accept(0x446bc0?)
/usr/local/go/src/net/unixsock_posix.go:166 +0x1c
net.(*UnixListener).Accept(0xc0001882d0)
/usr/local/go/src/net/unixsock.go:260 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0004603c0, {0xc3c098?, 0xc0001882d0})
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/server.go:823 +0x475
gitlab.com/gitlab-org/gitaly/v15/internal/testhelper.MustServe({0xc435f0, 0xc0003c3ba0}, {0xc378c0, 0xc0004603c0}, {0xc3c098, 0xc0001882d0})
/builds/gitlab-org/gitaly/internal/testhelper/testhelper.go:121 +0x62
created by gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.runServer
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:614 +0x195
goroutine 101 [chan receive, 20 minutes]:
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func6(0xc0003c3ba0)
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:305 +0x505
testing.tRunner(0xc0003c3ba0, 0xc00039b860)
/usr/local/go/src/testing/testing.go:1446 +0x10b
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:1493 +0x35f
goroutine 47 [chan receive, 20 minutes]:
testing.tRunner.func1()
/usr/local/go/src/testing/testing.go:1412 +0x4a5
testing.tRunner(0xc0002b6d00, 0xb847a8)
/usr/local/go/src/testing/testing.go:1452 +0x144
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:1493 +0x35f
goroutine 217 [chan send, 20 minutes]:
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func1(0x0?, {0xc3cb70, 0xc00029a100}, {0xc3e888, 0xc000358530}, 0xc3cb70?, 0xc0002e9c40?)
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:109 +0x237
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func6.1()
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:291 +0x43
created by gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func6
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:290 +0x34a
goroutine 226 [IO wait, 20 minutes]:
internal/poll.runtime_pollWait(0x7f6fdbf20c60, 0x72)
/usr/local/go/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc000319380?, 0xc000322000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000319380, {0xc000322000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000319380, {0xc000322000?, 0xaf83c0?, 0x437101?})
/usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000014688, {0xc000322000?, 0xc0002e6a80?, 0x800010601?})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00023a060, {0xc0003b6580, 0x9, 0x18?})
/usr/local/go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0xc33c60, 0xc00023a060}, {0xc0003b6580, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:332 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc0003b6580?, 0x9?, 0x2065d03?}, {0xc33c60?, 0xc00023a060?})
/builds/gitlab-org/gitaly/_build/cache/go-mod/golang.org/x/net@v0.0.0-20221002022538-bcab6841153b/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003b6540)
/builds/gitlab-org/gitaly/_build/cache/go-mod/golang.org/x/net@v0.0.0-20221002022538-bcab6841153b/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0001470e0, 0xc000177a00?)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_client.go:1561 +0x265
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_client.go:388 +0x1bdb
goroutine 227 [select, 20 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000037db0, 0x1)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00023a120)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_client.go:442 +0x6a
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_client.go:440 +0x222c
goroutine 224 [chan send, 20 minutes]:
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func1(0x7bca40?, {0xc3cb70, 0xc00029a100}, {0xc3e888, 0xc000358530}, 0xc40478?, 0xc000386468?)
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:109 +0x237
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func6.1()
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:291 +0x43
created by gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func6
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:290 +0x34a
goroutine 230 [select, 20 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0000c8680)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_server.go:1129 +0x233
created by google.golang.org/grpc/internal/transport.NewServerTransport
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_server.go:343 +0x19f8
goroutine 229 [select, 20 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000037ea0, 0x1)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00023a1e0)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_server.go:334 +0xce
created by google.golang.org/grpc/internal/transport.NewServerTransport
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_server.go:331 +0x19b3
goroutine 231 [IO wait, 20 minutes]:
internal/poll.runtime_pollWait(0x7f6fdbf21020, 0x72)
/usr/local/go/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc000319800?, 0xc000340000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000319800, {0xc000340000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000319800, {0xc000340000?, 0x46de6e?, 0x43e9a7?})
/usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000014690, {0xc000340000?, 0x0?, 0x7f6fd9ee5268?})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00023a180, {0xc0003b6660, 0x9, 0x18?})
/usr/local/go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0xc33c60, 0xc00023a180}, {0xc0003b6660, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:332 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc0003b6660?, 0x9?, 0xc000186168?}, {0xc33c60?, 0xc00023a180?})
/builds/gitlab-org/gitaly/_build/cache/go-mod/golang.org/x/net@v0.0.0-20221002022538-bcab6841153b/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003b6620)
/builds/gitlab-org/gitaly/_build/cache/go-mod/golang.org/x/net@v0.0.0-20221002022538-bcab6841153b/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0000c8680, 0xc00006c780?, 0xc00006c758?)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_server.go:610 +0x174
google.golang.org/grpc.(*Server).serveStreams(0xc0004603c0, {0xc406b8?, 0xc0000c8680})
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/server.go:949 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/server.go:891 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/server.go:890 +0x185
goroutine 247 [select, 20 minutes]:
google.golang.org/grpc.newClientStreamWithParams.func4()
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/stream.go:365 +0xe6f
goroutine 236 [select, 20 minutes]:
google.golang.org/grpc.newClientStreamWithParams.func4()
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/stream.go:365 +0xe6f
=== FAIL: internal/middleware/limithandler TestStreamLimitHandler/Single_request,_multiple_responses (unknown)
=== FAIL: internal/middleware/limithandler TestStreamLimitHandler (unknown)
panic: test timed out after 20m0s
goroutine 2041 [running]:
testing.(*M).startAlarm.func1()
/usr/local/go/src/testing/testing.go:2036 +0x8e
created by time.goFunc
/usr/local/go/src/time/sleep.go:176 +0x32
goroutine 1 [chan receive, 19 minutes]:
testing.tRunner.func1()
/usr/local/go/src/testing/testing.go:1412 +0x4a5
testing.tRunner(0xc00023c820, 0xc000131bb8)
/usr/local/go/src/testing/testing.go:1452 +0x144
testing.runTests(0xc0001a9360?, {0x10c4dc0, 0xb, 0xb}, {0xb3b8c3?, 0x8?, 0x10d5320?})
/usr/local/go/src/testing/testing.go:1844 +0x456
testing.(*M).Run(0xc0001a9360)
/usr/local/go/src/testing/testing.go:1726 +0x5d9
gitlab.com/gitlab-org/gitaly/v15/internal/testhelper.Run.func1({0x0, 0x0, 0x0?}, 0xb3e7f6?)
/builds/gitlab-org/gitaly/internal/testhelper/configure.go:71 +0x255
gitlab.com/gitlab-org/gitaly/v15/internal/testhelper.Run(0xffffffffffffffff?, {0x0?, 0x442405?, 0xc000070718?})
/builds/gitlab-org/gitaly/internal/testhelper/configure.go:74 +0x31
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestMain(...)
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:32
main.main()
_testmain.go:71 +0x1db
goroutine 215 [select, 20 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0002e9fc0)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/balancer_conn_wrappers.go:115 +0x73
created by google.golang.org/grpc.newCCBalancerWrapper
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/balancer_conn_wrappers.go:76 +0x22a
goroutine 214 [IO wait, 20 minutes]:
internal/poll.runtime_pollWait(0x7f6fdbf21110, 0x72)
/usr/local/go/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc000381180?, 0x18?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000381180)
/usr/local/go/src/internal/poll/fd_unix.go:614 +0x234
net.(*netFD).accept(0xc000381180)
/usr/local/go/src/net/fd_unix.go:172 +0x35
net.(*UnixListener).accept(0x446bc0?)
/usr/local/go/src/net/unixsock_posix.go:166 +0x1c
net.(*UnixListener).Accept(0xc0001882d0)
/usr/local/go/src/net/unixsock.go:260 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0004603c0, {0xc3c098?, 0xc0001882d0})
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/server.go:823 +0x475
gitlab.com/gitlab-org/gitaly/v15/internal/testhelper.MustServe({0xc435f0, 0xc0003c3ba0}, {0xc378c0, 0xc0004603c0}, {0xc3c098, 0xc0001882d0})
/builds/gitlab-org/gitaly/internal/testhelper/testhelper.go:121 +0x62
created by gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.runServer
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:614 +0x195
goroutine 101 [chan receive, 20 minutes]:
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func6(0xc0003c3ba0)
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:305 +0x505
testing.tRunner(0xc0003c3ba0, 0xc00039b860)
/usr/local/go/src/testing/testing.go:1446 +0x10b
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:1493 +0x35f
goroutine 47 [chan receive, 20 minutes]:
testing.tRunner.func1()
/usr/local/go/src/testing/testing.go:1412 +0x4a5
testing.tRunner(0xc0002b6d00, 0xb847a8)
/usr/local/go/src/testing/testing.go:1452 +0x144
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:1493 +0x35f
goroutine 217 [chan send, 20 minutes]:
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func1(0x0?, {0xc3cb70, 0xc00029a100}, {0xc3e888, 0xc000358530}, 0xc3cb70?, 0xc0002e9c40?)
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:109 +0x237
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func6.1()
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:291 +0x43
created by gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func6
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:290 +0x34a
goroutine 226 [IO wait, 20 minutes]:
internal/poll.runtime_pollWait(0x7f6fdbf20c60, 0x72)
/usr/local/go/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc000319380?, 0xc000322000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000319380, {0xc000322000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000319380, {0xc000322000?, 0xaf83c0?, 0x437101?})
/usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000014688, {0xc000322000?, 0xc0002e6a80?, 0x800010601?})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00023a060, {0xc0003b6580, 0x9, 0x18?})
/usr/local/go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0xc33c60, 0xc00023a060}, {0xc0003b6580, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:332 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc0003b6580?, 0x9?, 0x2065d03?}, {0xc33c60?, 0xc00023a060?})
/builds/gitlab-org/gitaly/_build/cache/go-mod/golang.org/x/net@v0.0.0-20221002022538-bcab6841153b/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003b6540)
/builds/gitlab-org/gitaly/_build/cache/go-mod/golang.org/x/net@v0.0.0-20221002022538-bcab6841153b/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0001470e0, 0xc000177a00?)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_client.go:1561 +0x265
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_client.go:388 +0x1bdb
goroutine 227 [select, 20 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000037db0, 0x1)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00023a120)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_client.go:442 +0x6a
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_client.go:440 +0x222c
goroutine 224 [chan send, 20 minutes]:
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func1(0x7bca40?, {0xc3cb70, 0xc00029a100}, {0xc3e888, 0xc000358530}, 0xc40478?, 0xc000386468?)
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:109 +0x237
gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func6.1()
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:291 +0x43
created by gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler_test.TestStreamLimitHandler.func6
/builds/gitlab-org/gitaly/internal/middleware/limithandler/middleware_test.go:290 +0x34a
goroutine 230 [select, 20 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0000c8680)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_server.go:1129 +0x233
created by google.golang.org/grpc/internal/transport.NewServerTransport
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_server.go:343 +0x19f8
goroutine 229 [select, 20 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000037ea0, 0x1)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00023a1e0)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_server.go:334 +0xce
created by google.golang.org/grpc/internal/transport.NewServerTransport
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_server.go:331 +0x19b3
goroutine 231 [IO wait, 20 minutes]:
internal/poll.runtime_pollWait(0x7f6fdbf21020, 0x72)
/usr/local/go/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc000319800?, 0xc000340000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000319800, {0xc000340000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000319800, {0xc000340000?, 0x46de6e?, 0x43e9a7?})
/usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000014690, {0xc000340000?, 0x0?, 0x7f6fd9ee5268?})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00023a180, {0xc0003b6660, 0x9, 0x18?})
/usr/local/go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0xc33c60, 0xc00023a180}, {0xc0003b6660, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:332 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc0003b6660?, 0x9?, 0xc000186168?}, {0xc33c60?, 0xc00023a180?})
/builds/gitlab-org/gitaly/_build/cache/go-mod/golang.org/x/net@v0.0.0-20221002022538-bcab6841153b/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003b6620)
/builds/gitlab-org/gitaly/_build/cache/go-mod/golang.org/x/net@v0.0.0-20221002022538-bcab6841153b/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0000c8680, 0xc00006c780?, 0xc00006c758?)
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/internal/transport/http2_server.go:610 +0x174
google.golang.org/grpc.(*Server).serveStreams(0xc0004603c0, {0xc406b8?, 0xc0000c8680})
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/server.go:949 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/server.go:891 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/server.go:890 +0x185
goroutine 247 [select, 20 minutes]:
google.golang.org/grpc.newClientStreamWithParams.func4()
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/stream.go:365 +0xe6f
goroutine 236 [select, 20 minutes]:
google.golang.org/grpc.newClientStreamWithParams.func4()
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
/builds/gitlab-org/gitaly/_build/cache/go-mod/google.golang.org/grpc@v1.51.0/stream.go:365 +0xe6f
=== FAIL: internal/middleware/limithandler TestStreamLimitHandler/Multiple_requests,_multiple_responses (unknown)
=== FAIL: internal/middleware/limithandler TestStreamLimitHandler/Multiple_requests_with_same_id,_multiple_responses (unknown)
=== FAIL: internal/middleware/limithandler TestStreamLimitHandler/With_a_max_concurrency_of_0 (unknown)