Repeated panic from within ql library in application
Created by: wrouesnel
Within the github.com/prometheus/alertmanager
I've been able to get a repeated crash that seems to originate from within QL after a running time of an hour or so with data being pushed into the DB. The panic is below:
OS:
Linux Kernel: 3.2.0-4-amd64
Userspace: Debian Squeeze
also
Linux Kernel: 3.19.0-31-generic
Userspace: Ubuntu vivid
Seen with golang 1.5.1
and 1.5.2
builds, and with totally static builds (CGO_DISABLED=1, -tags purego) and normal dynamic builds.
unexpected fault address 0x0
fatal error: fault
unexpected fault address 0x0
[signal 0xb code=0x80 addr=0x0 pc=0x41cbde]
goroutine 79731 [running]:
runtime.throw(0xbe7a98, 0x5)
/usr/local/go/src/runtime/panic.go:527 +0x90 fp=0xc821b8d3f8 sp=0xc821b8d3e0
runtime.sigpanic()
/usr/local/go/src/runtime/sigpanic_unix.go:27 +0x2ab fp=0xc821b8d448 sp=0xc821b8d3f8
runtime.mapaccess1_fast64(0xaafea0, 0xc8206b9560, 0x468, 0xc820381ea8)
/usr/local/go/src/runtime/hashmap_fast.go:120 +0x5e fp=0xc821b8d468 sp=0xc821b8d448
github.com/cznic/exp/lldb.(*bitFiler).ReadAt(0xc8205c0600, 0xc8207680bd, 0x266, 0x1ac5, 0x8d000, 0xbd, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/exp/lldb/xact.go:151 +0xec fp=0xc821b8d570 sp=0xc821b8d468
github.com/cznic/exp/lldb.(*RollbackFiler).ReadAt(0xc8200810e0, 0xc820768000, 0x323, 0x1b82, 0x8cf43, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/exp/lldb/xact.go:553 +0x1de fp=0xc821b8d5d8 sp=0xc821b8d570
github.com/cznic/exp/lldb.(*Allocator).read(0xc82007ec60, 0xc820768000, 0x323, 0x1b82, 0x8cf43, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/exp/lldb/falloc.go:964 +0x86 fp=0xc821b8d648 sp=0xc821b8d5d8
github.com/cznic/exp/lldb.(*Allocator).Get(0xc82007ec60, 0x0, 0x0, 0x0, 0x8cee, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/exp/lldb/falloc.go:773 +0x8d9 fp=0xc821b8d7d0 sp=0xc821b8d648
github.com/cznic/ql.(*file).Read(0xc8201a01c0, 0x0, 0x0, 0x0, 0x8cee, 0xc8205ab040, 0x7, 0x7, 0x0, 0x0, ...)
/home/will/src/go/src/github.com/cznic/ql/file.go:786 +0x9e fp=0xc821b8d960 sp=0xc821b8d7d0
github.com/cznic/ql.(*table).row0(0xc821638780, 0xc8202ae6e0, 0x8cee, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/storage.go:778 +0xc0 fp=0xc821b8da38 sp=0xc821b8d960
github.com/cznic/ql.(*tableDefaultPlan).do(0xc8202ae720, 0xc8202ae6e0, 0xc82077f980, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:2402 +0xa8 fp=0xc821b8db38 sp=0xc821b8da38
github.com/cznic/ql.(*filterDefaultPlan).do(0xc82077f280, 0xc8202ae6e0, 0xc82077f940, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1460 +0x1a5 fp=0xc821b8dba0 sp=0xc821b8db38
github.com/cznic/ql.(*selectFieldsDefaultPlan).do(0xc82077f2c0, 0xc8202ae6e0, 0xc8201235e0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1987 +0x1a5 fp=0xc821b8dc08 sp=0xc821b8dba0
github.com/cznic/ql.(*orderByDefaultPlan).do(0xc8205a5bd0, 0xc8202ae6e0, 0xc8202ae840, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1926 +0x3b5 fp=0xc821b8dd58 sp=0xc821b8dc08
github.com/cznic/ql.(*DB).do(0xc8201a0230, 0xc8202ae6e0, 0x7f3d7842d0b0, 0xc8205a5bd0, 0x0, 0xc8202bd100, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/ql.go:1402 +0x178 fp=0xc821b8ddc8 sp=0xc821b8dd58
github.com/cznic/ql.recordset.Do(0xc8202ae6e0, 0x7f3d7842d0b0, 0xc8205a5bd0, 0x0, 0x0, 0xc8202bd100, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/ql.go:75 +0x115 fp=0xc821b8de38 sp=0xc821b8ddc8
github.com/cznic/ql.(*recordset).Do(0xc8202ae7c0, 0xc8202bd100, 0xc8202bd100, 0x0, 0x0)
<autogenerated>:60 +0xb5 fp=0xc821b8de80 sp=0xc821b8de38
github.com/cznic/ql.newdriverRows.func1(0xc8202ae800)
/home/will/src/go/src/github.com/cznic/ql/driver.go:387 +0xae fp=0xc821b8df98 sp=0xc821b8de80
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1721 +0x1 fp=0xc821b8dfa0 sp=0xc821b8df98
created by github.com/cznic/ql.newdriverRows
/home/will/src/go/src/github.com/cznic/ql/driver.go:402 +0xeb
goroutine 1 [chan receive, 222 minutes]:
main.main()
/home/will/src/go/src/github.com/prometheus/alertmanager/main.go:196 +0x1005
goroutine 17 [syscall, 222 minutes, locked to thread]:
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1721 +0x1
goroutine 8 [syscall, 220 minutes]:
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
/usr/local/go/src/os/signal/signal_unix.go:28 +0x37
goroutine 10 [chan receive, 222 minutes]:
database/sql.(*DB).connectionOpener(0xc820093c20)
/usr/local/go/src/database/sql/sql.go:634 +0x45
created by database/sql.Open
/usr/local/go/src/database/sql/sql.go:481 +0x336
goroutine 15 [IO wait]:
net.runtime_pollWait(0x7f3d7a2c4808, 0x72, 0xc82000e250)
/usr/local/go/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc82020ca70, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc82020ca70, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).accept(0xc82020ca10, 0x0, 0x7f3d7a2c4900, 0xc8201d4ea0)
/usr/local/go/src/net/fd_unix.go:408 +0x27c
net.(*TCPListener).AcceptTCP(0xc820032b80, 0x462ba0, 0x0, 0x0)
/usr/local/go/src/net/tcpsock_posix.go:254 +0x4d
net/http.tcpKeepAliveListener.Accept(0xc820032b80, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2135 +0x41
net/http.(*Server).Serve(0xc820204f60, 0x7f3d7a2c48c8, 0xc820032b80, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:1887 +0xb3
net/http.(*Server).ListenAndServe(0xc820204f60, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:1877 +0x136
net/http.ListenAndServe(0xbe2d18, 0x5, 0x7f3d7a2c3830, 0xc820217d00, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:1967 +0x8f
created by main.main
/home/will/src/go/src/github.com/prometheus/alertmanager/main.go:181 +0xd48
goroutine 16 [select, 222 minutes, locked to thread]:
runtime.gopark(0xd1d110, 0xc82002b728, 0xbeaf70, 0x6, 0x43ef18, 0x2)
/usr/local/go/src/runtime/proc.go:185 +0x163
runtime.selectgoImpl(0xc82002b728, 0x0, 0x18)
/usr/local/go/src/runtime/select.go:392 +0xa64
runtime.selectgo(0xc82002b728)
/usr/local/go/src/runtime/select.go:212 +0x12
runtime.ensureSigM.func1()
/usr/local/go/src/runtime/signal1_unix.go:227 +0x353
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1721 +0x1
goroutine 18 [chan receive, 220 minutes]:
main.main.func4(0xc820204d80, 0xc820015d10)
/home/will/src/go/src/github.com/prometheus/alertmanager/main.go:191 +0x4d
created by main.main
/home/will/src/go/src/github.com/prometheus/alertmanager/main.go:194 +0xfe2
goroutine 79717 [runnable]:
time.(*Time).GobDecode(0xc8206921c0, 0xc821e80250, 0xf, 0xf, 0xc8206921c0, 0xf)
/usr/local/go/src/time/time.go:926
encoding/gob.(*Decoder).decodeGobDecoder(0xc820090780, 0xc8200fac80, 0xc8200fbd00, 0xbd91e0, 0xc8206921c0, 0x16)
/usr/local/go/src/encoding/gob/decode.go:725 +0x2e7
encoding/gob.(*Decoder).gobDecodeOpFor.func1(0xc8200fbf00, 0xc8200fbd00, 0xbd91e0, 0xc8206921c0, 0x16)
/usr/local/go/src/encoding/gob/decode.go:951 +0x102
encoding/gob.(*Decoder).decodeSingle(0xc820090780, 0xc82013d780, 0xc8200fac80, 0xbd70e0, 0xc8206921c0, 0xd9)
/usr/local/go/src/encoding/gob/decode.go:437 +0x204
encoding/gob.(*Decoder).decodeValue(0xc820090780, 0x45, 0xbd70e0, 0xc8206921c0, 0xd9)
/usr/local/go/src/encoding/gob/decode.go:1196 +0x4cd
encoding/gob.(*Decoder).DecodeValue(0xc820090780, 0xbd91e0, 0xc8206921c0, 0x16, 0x0, 0x0)
/usr/local/go/src/encoding/gob/decoder.go:210 +0x1a7
encoding/gob.(*Decoder).Decode(0xc820090780, 0xbd91e0, 0xc8206921c0, 0x0, 0x0)
/usr/local/go/src/encoding/gob/decoder.go:185 +0x289
github.com/cznic/ql.(*gobCoder).decode(0xc820081290, 0xc8206920c0, 0x14, 0x20, 0x54, 0x0, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/blob.go:145 +0x4f6
github.com/cznic/ql.(*file).loadChunks(0xc8201a01c0, 0xc820513fc0, 0x17, 0x20, 0x0, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/file.go:960 +0x5eb
github.com/cznic/ql.(*file).expandBytes(0xc8201a01c0, 0xc8206920a0, 0x2, 0x2, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/file.go:620 +0x174
github.com/cznic/ql.(*file).collate(0xc8201a01c0, 0xc8205137c0, 0x1a, 0x20, 0xc820513fa0, 0x1a, 0x20, 0xb)
/home/will/src/go/src/github.com/cznic/ql/file.go:634 +0xc7
github.com/cznic/ql.(*file).CreateTemp.func1(0xc8205137c0, 0x1a, 0x20, 0xc820513fa0, 0x1a, 0x20, 0xc820513fa0)
/home/will/src/go/src/github.com/cznic/ql/file.go:672 +0x65
github.com/cznic/exp/lldb.btreeDataPage.cmp(0xc821c18000, 0xd91, 0x2021e, 0x7f3d7a2c2800, 0xc821d7c000, 0xc82089e660, 0xc8205137c0, 0x1a, 0x20, 0x16, ...)
/home/will/src/go/src/github.com/cznic/exp/lldb/btree.go:1399 +0x124
github.com/cznic/exp/lldb.btreePage.find(0xc821c18000, 0xd91, 0x2021e, 0x7f3d7a2c2800, 0xc821d7c000, 0xc82089e660, 0xc8205137c0, 0x1a, 0x20, 0x16, ...)
/home/will/src/go/src/github.com/cznic/exp/lldb/btree.go:1606 +0x2ff
github.com/cznic/exp/lldb.btree.put2(0x1, 0xc821bf6000, 0x1010f, 0x2021e, 0x7f3d7a2c2800, 0xc821d7c000, 0xc82089e660, 0xc8205137c0, 0x1a, 0x20, ...)
/home/will/src/go/src/github.com/cznic/exp/lldb/btree.go:1909 +0x6db
github.com/cznic/exp/lldb.btree.put(0x1, 0xc821bf6000, 0x1010f, 0x2021e, 0x7f3d7a2c2800, 0xc821d7c000, 0xc82089e660, 0xc8205137c0, 0x1a, 0x20, ...)
/home/will/src/go/src/github.com/cznic/exp/lldb/btree.go:1862 +0x110
github.com/cznic/exp/lldb.(*BTree).Set(0xc8206b8d20, 0xc8205137c0, 0x1a, 0x20, 0xc821d9f380, 0x50c, 0x680, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/exp/lldb/btree.go:483 +0x20b
github.com/cznic/ql.(*fileTemp).Set(0xc820674b80, 0xc82089f000, 0x2, 0x2, 0xc82192a5a0, 0x6, 0x6, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/file.go:373 +0x33d
github.com/cznic/ql.(*orderByDefaultPlan).do.func2(0xa104e0, 0xc8211a9b70, 0xc82192a5a0, 0x6, 0x6, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1921 +0x709
github.com/cznic/ql.(*selectFieldsDefaultPlan).do.func1(0xa104e0, 0xc8211a9b70, 0xc820726000, 0x7, 0x9, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1986 +0x458
github.com/cznic/ql.(*filterDefaultPlan).do.func1(0xa104e0, 0xc8211a9b70, 0xc820726000, 0x7, 0x9, 0x9, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1459 +0x507
github.com/cznic/ql.(*tableDefaultPlan).do(0xc82089e4a0, 0xc82089e460, 0xc820674c00, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:2412 +0x32f
github.com/cznic/ql.(*filterDefaultPlan).do(0xc820674500, 0xc82089e460, 0xc820674bc0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1460 +0x1a5
github.com/cznic/ql.(*selectFieldsDefaultPlan).do(0xc820674540, 0xc82089e460, 0xc82020cbd0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1987 +0x1a5
github.com/cznic/ql.(*orderByDefaultPlan).do(0xc82048cd70, 0xc82089e460, 0xc82089e5a0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1926 +0x3b5
github.com/cznic/ql.(*DB).do(0xc8201a0230, 0xc82089e460, 0x7f3d7842d0b0, 0xc82048cd70, 0x0, 0xc8202eafc0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/ql.go:1402 +0x178
github.com/cznic/ql.recordset.Do(0xc82089e460, 0x7f3d7842d0b0, 0xc82048cd70, 0x0, 0x0, 0xc8202eafc0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/ql.go:75 +0x115
github.com/cznic/ql.(*recordset).Do(0xc82089e560, 0xc8202eaf00, 0xc8202eafc0, 0x0, 0x0)
<autogenerated>:60 +0xb5
github.com/cznic/ql.newdriverRows.func1(0xc82089e580)
/home/will/src/go/src/github.com/cznic/ql/driver.go:387 +0xae
created by github.com/cznic/ql.newdriverRows
/home/will/src/go/src/github.com/cznic/ql/driver.go:402 +0xeb
goroutine 77458 [select, 6 minutes]:
main.(*aggrGroup).run(0xc8200a8080, 0xc821f3e160)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:312 +0x1305
created by main.(*Dispatcher).processAlert
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f
goroutine 1554 [select]:
github.com/cznic/ql.(*driverRows).Next(0xc8202ae800, 0xc8207700c0, 0x6, 0x6, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/driver.go:428 +0x15dc
database/sql.(*Rows).Next(0xc82192bf80, 0xd289e0)
/usr/local/go/src/database/sql/sql.go:1639 +0x12b
github.com/prometheus/alertmanager/provider.(*SQLAlerts).getPending(0xc8201a4f00, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/provider/sql.go:302 +0x11b
github.com/prometheus/alertmanager/provider.(*SQLAlerts).GetPending(0xc8201a4f00, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/provider/sql.go:264 +0x88
main.(*Inhibitor).Mutes(0xc8204abbd0, 0xc8218e2f90, 0x100)
/home/will/src/go/src/github.com/prometheus/alertmanager/inhibit.go:51 +0x6c
github.com/prometheus/alertmanager/notify.(*InhibitNotifier).Notify(0xc82185bdd0, 0x7f3d783b1658, 0xc821872060, 0xc8200329b0, 0x1, 0x1, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/notify/notify.go:406 +0x13e
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc821d385a0, 0x7f3d783b1658, 0xc821872060, 0xc8200329b0, 0x1, 0x1, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/notify/notify.go:434 +0x1bf
main.(*Dispatcher).processAlert.func1(0x7f3d783b1658, 0xc821872060, 0xc8200329b0, 0x1, 0x1, 0xc8202ae2a0)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:236 +0x8a
main.(*aggrGroup).run.func1(0xc8200329b0, 0x1, 0x1, 0x2)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:337 +0x5b
main.(*aggrGroup).flush(0xc821c57180, 0xc821c7dec8)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:401 +0x466
main.(*aggrGroup).run(0xc821c57180, 0xc8202a5680)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:338 +0xf59
created by main.(*Dispatcher).processAlert
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f
goroutine 79732 [semacquire]:
sync.runtime_Semacquire(0xc8201a025c)
/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*RWMutex).Lock(0xc8201a0254)
/usr/local/go/src/sync/rwmutex.go:87 +0xa1
github.com/cznic/ql.(*DB).run1(0xc8201a0230, 0xc820601970, 0x7f3d7a2c1820, 0x11a5640, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/home/will/src/go/src/github.com/cznic/ql/ql.go:1239 +0x401
github.com/cznic/ql.(*DB).Execute(0xc8201a0230, 0xc820601970, 0xc82000f980, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/home/will/src/go/src/github.com/cznic/ql/ql.go:1202 +0x273
github.com/cznic/ql.(*driverConn).Begin(0xc8206b9170, 0x0, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/driver.go:249 +0xee
database/sql.(*DB).begin(0xc820093c20, 0x1, 0x0, 0x0, 0x0)
/usr/local/go/src/database/sql/sql.go:1055 +0xb8
database/sql.(*DB).Begin(0xc820093c20, 0x421f11, 0x0, 0x0)
/usr/local/go/src/database/sql/sql.go:1038 +0x5c
github.com/prometheus/alertmanager/provider.(*SQLAlerts).Put(0xc8201a4f00, 0xc8205c05e0, 0x4, 0x4, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/provider/sql.go:342 +0x64
main.(*API).legacyAddAlerts(0xc8201d0300, 0x7f3d78471358, 0xc8201ea0b0, 0xc8203020e0)
/home/will/src/go/src/github.com/prometheus/alertmanager/api.go:203 +0x782
main.(*API).(main.legacyAddAlerts)-fm(0x7f3d78471358, 0xc8201ea0b0, 0xc8203020e0)
/home/will/src/go/src/github.com/prometheus/alertmanager/api.go:64 +0x3e
github.com/prometheus/common/route.handle.func1(0x7f3d78471358, 0xc8201ea0b0, 0xc8203020e0, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/common/route/route.go:49 +0x417
github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc820288680, 0x7f3d78471358, 0xc8201ea0b0, 0xc8203020e0)
/home/will/src/go/src/github.com/julienschmidt/httprouter/router.go:299 +0x193
github.com/prometheus/common/route.(*Router).ServeHTTP(0xc820217d00, 0x7f3d78471358, 0xc8201ea0b0, 0xc8203020e0)
/home/will/src/go/src/github.com/prometheus/common/route/route.go:102 +0x42
net/http.serverHandler.ServeHTTP(0xc820204f60, 0x7f3d78471358, 0xc8201ea0b0, 0xc8203020e0)
/usr/local/go/src/net/http/server.go:1862 +0x19e
net/http.(*conn).serve(0xc8201ea000)
/usr/local/go/src/net/http/server.go:1361 +0xbee
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:1910 +0x3f6
goroutine 1311 [chan receive, 220 minutes]:
github.com/prometheus/alertmanager/provider.(*SQLAlerts).Subscribe.func1(0xc8201a4f00, 0x2, 0xc8207918c0, 0xc820492000, 0xa2, 0x100, 0xc820791920)
/home/will/src/go/src/github.com/prometheus/alertmanager/provider/sql.go:247 +0x165
created by github.com/prometheus/alertmanager/provider.(*SQLAlerts).Subscribe
/home/will/src/go/src/github.com/prometheus/alertmanager/provider/sql.go:248 +0x1cc
goroutine 1555 [select]:
github.com/cznic/ql.(*driverRows).Next(0xc82089e580, 0xc82067e480, 0x6, 0x6, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/driver.go:428 +0x15dc
database/sql.(*Rows).Next(0xc82067e420, 0xd289e0)
/usr/local/go/src/database/sql/sql.go:1639 +0x12b
github.com/prometheus/alertmanager/provider.(*SQLAlerts).getPending(0xc8201a4f00, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/provider/sql.go:302 +0x11b
github.com/prometheus/alertmanager/provider.(*SQLAlerts).GetPending(0xc8201a4f00, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/provider/sql.go:264 +0x88
main.(*Inhibitor).Mutes(0xc8204abbd0, 0xc8218e2f00, 0x100)
/home/will/src/go/src/github.com/prometheus/alertmanager/inhibit.go:51 +0x6c
github.com/prometheus/alertmanager/notify.(*InhibitNotifier).Notify(0xc82185bdd0, 0x7f3d783b1658, 0xc82185bfb0, 0xc820258000, 0x1, 0x1, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/notify/notify.go:406 +0x13e
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc821d385a0, 0x7f3d783b1658, 0xc82185bfb0, 0xc820258000, 0x1, 0x1, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/notify/notify.go:434 +0x1bf
main.(*Dispatcher).processAlert.func1(0x7f3d783b1658, 0xc82185bfb0, 0xc820258000, 0x1, 0x1, 0xc82089e080)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:236 +0x8a
main.(*aggrGroup).run.func1(0xc820258000, 0x1, 0x1, 0x2)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:337 +0x5b
main.(*aggrGroup).flush(0xc821c57200, 0xc821f6bec8)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:401 +0x466
main.(*aggrGroup).run(0xc821c57200, 0xc8202a5740)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:338 +0xf59
created by main.(*Dispatcher).processAlert
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f
goroutine 1515 [select]:
main.(*Dispatcher).run(0xc82060b320, 0x7f3d783b15e0, 0xc82064a720)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:150 +0x8ea
main.(*Dispatcher).Run(0xc82060b320)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:59 +0x1de
created by main.main.func3
/home/will/src/go/src/github.com/prometheus/alertmanager/main.go:167 +0x55a
goroutine 1312 [select, 9 minutes]:
main.(*aggrGroup).run(0xc821c57000, 0xc8202a5470)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:312 +0x1305
created by main.(*Dispatcher).processAlert
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f
goroutine 1540 [select]:
main.(*aggrGroup).run(0xc820090b00, 0xc8202d2710)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:312 +0x1305
created by main.(*Dispatcher).processAlert
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f
goroutine 1588 [select]:
github.com/cznic/ql.(*driverRows).Next(0xc8201d46a0, 0xc8202fea20, 0x6, 0x6, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/driver.go:428 +0x15dc
database/sql.(*Rows).Next(0xc8202fe9c0, 0xd289e0)
/usr/local/go/src/database/sql/sql.go:1639 +0x12b
github.com/prometheus/alertmanager/provider.(*SQLAlerts).getPending(0xc8201a4f00, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/provider/sql.go:302 +0x11b
github.com/prometheus/alertmanager/provider.(*SQLAlerts).GetPending(0xc8201a4f00, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/provider/sql.go:264 +0x88
main.(*Inhibitor).Mutes(0xc8204abbd0, 0xc8218e2e40, 0x100)
/home/will/src/go/src/github.com/prometheus/alertmanager/inhibit.go:51 +0x6c
github.com/prometheus/alertmanager/notify.(*InhibitNotifier).Notify(0xc82185bdd0, 0x7f3d783b1658, 0xc82192c9c0, 0xc82026c040, 0x1, 0x1, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/notify/notify.go:406 +0x13e
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc821d385a0, 0x7f3d783b1658, 0xc82192c9c0, 0xc82026c040, 0x1, 0x1, 0x0, 0x0)
/home/will/src/go/src/github.com/prometheus/alertmanager/notify/notify.go:434 +0x1bf
main.(*Dispatcher).processAlert.func1(0x7f3d783b1658, 0xc82192c9c0, 0xc82026c040, 0x1, 0x1, 0xc8201d4120)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:236 +0x8a
main.(*aggrGroup).run.func1(0xc82026c040, 0x1, 0x1, 0x2)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:337 +0x5b
main.(*aggrGroup).flush(0xc821c57b80, 0xc821c79ec8)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:401 +0x466
main.(*aggrGroup).run(0xc821c57b80, 0xc82196da40)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:338 +0xf59
created by main.(*Dispatcher).processAlert
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f
goroutine 79685 [runnable]:
github.com/cznic/exp/lldb.(*bitFiler).ReadAt(0xc8205c0600, 0xc821c800bd, 0x266, 0x1ac3, 0x8d000, 0xbd, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/exp/lldb/xact.go:151 +0xec
github.com/cznic/exp/lldb.(*RollbackFiler).ReadAt(0xc8200810e0, 0xc821c80000, 0x323, 0x1b80, 0x8cf43, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/exp/lldb/xact.go:553 +0x1de
github.com/cznic/exp/lldb.(*Allocator).read(0xc82007ec60, 0xc821c80000, 0x323, 0x1b80, 0x8cf43, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/exp/lldb/falloc.go:964 +0x86
github.com/cznic/exp/lldb.(*Allocator).Get(0xc82007ec60, 0x0, 0x0, 0x0, 0x8cee, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/exp/lldb/falloc.go:773 +0x8d9
github.com/cznic/ql.(*file).Read(0xc8201a01c0, 0x0, 0x0, 0x0, 0x8cee, 0xc8205ab040, 0x7, 0x7, 0x0, 0x0, ...)
/home/will/src/go/src/github.com/cznic/ql/file.go:786 +0x9e
github.com/cznic/ql.(*table).row0(0xc821638780, 0xc8201d45a0, 0x8cee, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/storage.go:778 +0xc0
github.com/cznic/ql.(*tableDefaultPlan).do(0xc8201d45c0, 0xc8201d45a0, 0xc8201c4d80, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:2402 +0xa8
github.com/cznic/ql.(*filterDefaultPlan).do(0xc8201c4640, 0xc8201d45a0, 0xc8201c4d40, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1460 +0x1a5
github.com/cznic/ql.(*selectFieldsDefaultPlan).do(0xc8201c4680, 0xc8201d45a0, 0xc82001abd0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1987 +0x1a5
github.com/cznic/ql.(*orderByDefaultPlan).do(0xc8211845a0, 0xc8201d45a0, 0xc8201d46c0, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/plan.go:1926 +0x3b5
github.com/cznic/ql.(*DB).do(0xc8201a0230, 0xc8201d45a0, 0x7f3d7842d0b0, 0xc8211845a0, 0x0, 0xc820648f40, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/ql.go:1402 +0x178
github.com/cznic/ql.recordset.Do(0xc8201d45a0, 0x7f3d7842d0b0, 0xc8211845a0, 0x0, 0x0, 0xc820648f40, 0x0, 0x0)
/home/will/src/go/src/github.com/cznic/ql/ql.go:75 +0x115
github.com/cznic/ql.(*recordset).Do(0xc8201d4660, 0xc820648f00, 0xc820648f40, 0x0, 0x0)
<autogenerated>:60 +0xb5
github.com/cznic/ql.newdriverRows.func1(0xc8201d46a0)
/home/will/src/go/src/github.com/cznic/ql/driver.go:387 +0xae
created by github.com/cznic/ql.newdriverRows
/home/will/src/go/src/github.com/cznic/ql/driver.go:402 +0xeb
goroutine 77784 [select, 5 minutes]:
main.(*aggrGroup).run(0xc821f54300, 0xc821a140d0)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:312 +0x1305
created by main.(*Dispatcher).processAlert
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f
goroutine 78071 [select, 4 minutes]:
main.(*aggrGroup).run(0xc821bdc080, 0xc8201ca0a0)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:312 +0x1305
created by main.(*Dispatcher).processAlert
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f
goroutine 76910 [select, 8 minutes]:
main.(*aggrGroup).run(0xc821db2080, 0xc82052c180)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:312 +0x1305
created by main.(*Dispatcher).processAlert
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f
goroutine 77926 [select, 5 minutes]:
main.(*aggrGroup).run(0xc8203e2100, 0xc8205ae0a0)
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:312 +0x1305
created by main.(*Dispatcher).processAlert
/home/will/src/go/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f