High rate of stack trace dumps in Workhorse logs
On GitLab.com, I noticed a high number of these stack traces showing up in the JSON logs:
goroutine 11212236 [running]:
runtime/debug.Stack(0x0, 0x209, 0xc001bf0ba0)
/usr/local/go/src/runtime/debug/stack.go:24 +0xa7
runtime/debug.PrintStack()
/usr/local/go/src/runtime/debug/stack.go:16 +0x22
gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/getsentry/raven-go.Recoverer.func1.1(0xc000f22300, 0xd31fc0, 0xc0028c0d20)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/getsentry/raven-go/http.go:84 +0x6e
panic(0xb0a420, 0xc000082ad0)
/usr/local/go/src/runtime/panic.go:513 +0x1b9
main.wrapRaven.func1.1(0xc000f22300)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/raven.go:29 +0x74
panic(0xb0a420, 0xc000082ad0)
/usr/local/go/src/runtime/panic.go:513 +0x1b9
net/http/httputil.(*ReverseProxy).ServeHTTP(0xc0000c6c30, 0xd2ad80, 0xc0005b5890, 0xc001bf10c8)
/usr/local/go/src/net/http/httputil/reverseproxy.go:284 +0x1058
gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy.(*Proxy).ServeHTTP(0xc00000d820, 0xd2ad80, 0xc0005b5890, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy/proxy.go:50 +0x1e1
gitlab.com/gitlab-org/gitlab-workhorse/internal/api.Block.func1(0xd2b080, 0xc0005b5830, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/internal/api/block.go:16 +0xc9
net/http.HandlerFunc.ServeHTTP(0xc00000d880, 0xd2b080, 0xc0005b5830, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/internal/sendfile.SendFile.func1(0xd2b000, 0xc0023c5a80, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/internal/sendfile/sendfile.go:64 +0x105
net/http.HandlerFunc.ServeHTTP(0xc00000d8a0, 0xd2b000, 0xc0023c5a80, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata.SendData.func1(0xd2b040, 0xc0005b5800, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata/senddata.go:51 +0x146
net/http.HandlerFunc.ServeHTTP(0xc0003b7260, 0xd2b040, 0xc0005b5800, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata/contentprocessor.SetContentHeaders.func1(0xd2b100, 0xc0005b54a0, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata/contentprocessor/contentprocessor.go:35 +0xf3
net/http.HandlerFunc.ServeHTTP(0xc00000d8c0, 0xd2b100, 0xc0005b54a0, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/internal/staticpages.(*Static).ErrorPagesUnless.func1(0x7f0460037aa0, 0xc0018ebc70, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/internal/staticpages/error_pages.go:93 +0xf8
net/http.HandlerFunc.ServeHTTP(0xc0001ea600, 0x7f0460037aa0, 0xc0018ebc70, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream.denyWebsocket.func1(0x7f0460037aa0, 0xc0018ebc70, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream/routes.go:238 +0xc7
net/http.HandlerFunc.ServeHTTP(0xc0001ea640, 0x7f0460037aa0, 0xc0018ebc70, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1(0x7f0460037aa0, 0xc0018ebc20, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:100 +0xda
net/http.HandlerFunc.ServeHTTP(0xc0001e7560, 0x7f0460037aa0, 0xc0018ebc20, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func1(0x7f0460037aa0, 0xc0018ebbd0, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:68 +0x11c
net/http.HandlerFunc.ServeHTTP(0xc0001e7800, 0x7f0460037aa0, 0xc0018ebbd0, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerInFlight.func1(0x7f0460037aa0, 0xc0018ebbd0, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:40 +0xa9
net/http.HandlerFunc.ServeHTTP(0xc0001e7830, 0x7f0460037aa0, 0xc0018ebbd0, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerRequestSize.func1(0x7f0460037aa0, 0xc0018ebb80, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:163 +0xe9
net/http.HandlerFunc.ServeHTTP(0xc0001e7ad0, 0x7f0460037aa0, 0xc0018ebb80, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1(0x7f0460037aa0, 0xc0018ebb30, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196 +0xe9
net/http.HandlerFunc.ServeHTTP(0xc0001e7d70, 0x7f0460037aa0, 0xc0018ebb30, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerTimeToWriteHeader.func1(0x7f04600379d0, 0xc0023c5a00, 0xc000f22500)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:137 +0x18c
net/http.HandlerFunc.ServeHTTP(0xc000216030, 0x7f04600379d0, 0xc0023c5a00, 0xc000f22500)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/vendor/gitlab.com/gitlab-org/labkit/tracing.Handler.func1(0x7f04600379d0, 0xc0023c5a00, 0xc000f22400)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor/gitlab.com/gitlab-org/labkit/tracing/inbound_http.go:45 +0x35a
net/http.HandlerFunc.ServeHTTP(0xc0001eaa40, 0x7f04600379d0, 0xc0023c5a00, 0xc000f22400)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream.(*upstream).ServeHTTP(0xc0000f8580, 0xd31fc0, 0xc0028c0d20, 0xc000f22400)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream/upstream.go:107 +0x3f7
gitlab.com/gitlab-org/gitlab-workhorse/vendor/gitlab.com/gitlab-org/labkit/correlation.InjectCorrelationID.func1(0xd31fc0, 0xc0028c0d20, 0xc000f22300)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor/gitlab.com/gitlab-org/labkit/correlation/inbound_http.go:33 +0x186
net/http.HandlerFunc.ServeHTTP(0xc0001eb320, 0xd31fc0, 0xc0028c0d20, 0xc000f22300)
/usr/local/go/src/net/http/server.go:1964 +0x44
main.wrapRaven.func1(0xd31fc0, 0xc0028c0d20, 0xc000f22300)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/raven.go:33 +0x8b
net/http.HandlerFunc.ServeHTTP(0xc0001eb340, 0xd31fc0, 0xc0028c0d20, 0xc000f22300)
/usr/local/go/src/net/http/server.go:1964 +0x44
gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/getsentry/raven-go.Recoverer.func1(0xd31fc0, 0xc0028c0d20, 0xc000f22300)
/var/cache/omnibus/src/gitlab-workhorse/_build/src/gitlab.com/gitlab-org/gitlab-workhorse/vendor/github.com/getsentry/raven-go/http.go:97 +0x9f
net/http.HandlerFunc.ServeHTTP(0xc0001eb360, 0xd31fc0, 0xc0028c0d20, 0xc000f22300)
/usr/local/go/src/net/http/server.go:1964 +0x44
net/http.Handler.ServeHTTP-fm(0xd31fc0, 0xc0028c0d20, 0xc000f22300)
/usr/local/go/src/net/http/h2_bundle.go:5592 +0x4d
net/http.HandlerFunc.ServeHTTP(0xc0001eb380, 0xd31fc0, 0xc0028c0d20, 0xc000f22300)
/usr/local/go/src/net/http/server.go:1964 +0x44
net/http.serverHandler.ServeHTTP(0xc000420d00, 0xd31fc0, 0xc0028c0d20, 0xc000f22300)
/usr/local/go/src/net/http/server.go:2741 +0xab
net/http.(*conn).serve(0xc002b530e0, 0xd32a40, 0xc0023c5900)
/usr/local/go/src/net/http/server.go:1847 +0x646
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2851 +0x2f5
2019/04/22 22:04:39 http: multiple response.WriteHeader calls