Go runtime panic with qemu-x86_64-static on aarch64 (bisected)
Host environment
- Operating system: Fedora 39 Asahi Remix
- OS/kernel version: Linux mbp 6.6.3-403.asahi.fc39.aarch64+16k #1 SMP PREEMPT_DYNAMIC Fri Dec 8 11:17:57 UTC 2023 aarch64 GNU/Linux
- Architecture: aarch64
- QEMU flavor: qemu-x86_64-static
- QEMU version: qemu-x86_64 version 8.1.1 (qemu-8.1.1-1.fc39)
- QEMU command line:
qemu-x86_64-static <affected go binary>
Emulated/Virtualized environment
- Operating system: Fedora 39 Asahi Remix
- OS/kernel version: Linux mbp 6.6.3-403.asahi.fc39.aarch64+16k #1 SMP PREEMPT_DYNAMIC Fri Dec 8 11:17:57 UTC 2023 aarch64 GNU/Linux
- Architecture: x86
Description of problem
I have run into some crashes with certain x86 Go binaries running on arm64 (Asahi Linux) using qemu-user-static. The issue is also reproducible on current master (9c74490b). I have bisected the issue to commit 2d708164:
first bad commit: [2d708164e0475064e0e2167bd73e8570e22df1e0] linux-user: Define TASK_UNMAPPED_BASE in $guest/target_mman.h
Steps to reproduce
- Build example Go program
GOARCH=amd64 go build -o crashing .
- Run it with
qemu-x86_64-static ./crashing
Go program to reproduce
package main
import "crypto/x509"
func main() {
x509.SystemCertPool()
}
Additional information
Go program stacktrace
runtime: lfstack.push invalid packing: node=0xffff3c3a9780 cnt=0x1 packed=0xffff3c3a97800001 -> node=0xffffffff3c3a9780
fatal error: lfstack.push
runtime stack:
runtime.throw({0x52cb61?, 0x2ce5?})
/usr/lib/golang/src/runtime/panic.go:1077 +0x5c fp=0xc000613f08 sp=0xc000613ed8 pc=0x433d5c
runtime.(*lfstack).push(0xa0000000002?, 0xffffffffffffefe8?)
/usr/lib/golang/src/runtime/lfstack.go:29 +0x125 fp=0xc000613f48 sp=0xc000613f08 pc=0x40ac25
runtime.(*spanSetBlockAlloc).free(...)
/usr/lib/golang/src/runtime/mspanset.go:322
runtime.(*spanSet).reset(0x64d220)
/usr/lib/golang/src/runtime/mspanset.go:264 +0x79 fp=0xc000613f78 sp=0xc000613f48 pc=0x42ef79
runtime.finishsweep_m()
/usr/lib/golang/src/runtime/mgcsweep.go:260 +0x95 fp=0xc000613fb8 sp=0xc000613f78 pc=0x423455
runtime.gcStart.func2()
/usr/lib/golang/src/runtime/mgc.go:687 +0xf fp=0xc000613fc8 sp=0xc000613fb8 pc=0x45bd8f
traceback: unexpected SPWRITE function runtime.systemstack
runtime.systemstack()
/usr/lib/golang/src/runtime/asm_amd64.s:509 +0x4a fp=0xc000613fd8 sp=0xc000613fc8 pc=0x46016a
goroutine 1 [running]:
runtime.systemstack_switch()
/usr/lib/golang/src/runtime/asm_amd64.s:474 +0x8 fp=0xc0001bb9f0 sp=0xc0001bb9e0 pc=0x460108
runtime.gcStart({0xc000600000?, 0x98370?, 0x307800?})
/usr/lib/golang/src/runtime/mgc.go:686 +0x2e5 fp=0xc0001bba88 sp=0xc0001bb9f0 pc=0x418e05
runtime.mallocgc(0x98370, 0x50bb80, 0x1)
/usr/lib/golang/src/runtime/malloc.go:1242 +0x76f fp=0xc0001bbaf0 sp=0xc0001bba88 pc=0x40caaf
runtime.makeslice(0xc0001840a8?, 0x26?, 0x0?)
/usr/lib/golang/src/runtime/slice.go:103 +0x49 fp=0xc0001bbb18 sp=0xc0001bbaf0 pc=0x449729
os.ReadFile({0xc00035a0f0?, 0x52dcd6?})
/usr/lib/golang/src/os/file.go:738 +0xe5 fp=0xc0001bbbf0 sp=0xc0001bbb18 pc=0x49ed25
crypto/x509.loadSystemRoots()
/usr/lib/golang/src/crypto/x509/root_unix.go:70 +0x3d4 fp=0xc0001bbcd8 sp=0xc0001bbbf0 pc=0x4fdef4
crypto/x509.initSystemRoots()
/usr/lib/golang/src/crypto/x509/root.go:30 +0x5c fp=0xc0001bbd10 sp=0xc0001bbcd8 pc=0x4fd9fc
sync.(*Once).doSlow(0x1?, 0xb30000c00018ada0?)
/usr/lib/golang/src/sync/once.go:74 +0xbf fp=0xc0001bbd70 sp=0xc0001bbd10 pc=0x467bff
sync.(*Once).Do(...)
/usr/lib/golang/src/sync/once.go:65
crypto/x509.systemRootsPool()
/usr/lib/golang/src/crypto/x509/root.go:21 +0x45 fp=0xc0001bbdc0 sp=0xc0001bbd70 pc=0x4fd8a5
crypto/x509.SystemCertPool()
/usr/lib/golang/src/crypto/x509/cert_pool.go:112 +0x25 fp=0xc0001bbf30 sp=0xc0001bbdc0 pc=0x4f6705
main.main()
/home/cyrill/dev/goruntime-crash/main.go:6 +0xf fp=0xc0001bbf40 sp=0xc0001bbf30 pc=0x4ff18f
runtime.main()
/usr/lib/golang/src/runtime/proc.go:267 +0x2bb fp=0xc0001bbfe0 sp=0xc0001bbf40 pc=0x43673b
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0001bbfe8 sp=0xc0001bbfe0 pc=0x461f61
goroutine 2 [force gc (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00004efa8 sp=0xc00004ef88 pc=0x436b8e
runtime.goparkunlock(...)
/usr/lib/golang/src/runtime/proc.go:404
runtime.forcegchelper()
/usr/lib/golang/src/runtime/proc.go:322 +0xb3 fp=0xc00004efe0 sp=0xc00004efa8 pc=0x436a13
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00004efe8 sp=0xc00004efe0 pc=0x461f61
created by runtime.init.6 in goroutine 1
/usr/lib/golang/src/runtime/proc.go:310 +0x1a
goroutine 3 [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00004f778 sp=0xc00004f758 pc=0x436b8e
runtime.goparkunlock(...)
/usr/lib/golang/src/runtime/proc.go:404
runtime.bgsweep(0x0?)
/usr/lib/golang/src/runtime/mgcsweep.go:321 +0xdf fp=0xc00004f7c8 sp=0xc00004f778 pc=0x4235bf
runtime.gcenable.func1()
/usr/lib/golang/src/runtime/mgc.go:200 +0x25 fp=0xc00004f7e0 sp=0xc00004f7c8 pc=0x418945
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00004f7e8 sp=0xc00004f7e0 pc=0x461f61
created by runtime.gcenable in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:200 +0x66
goroutine 4 [GC scavenge wait]:
runtime.gopark(0xc00006c000?, 0x570658?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00004ff70 sp=0xc00004ff50 pc=0x436b8e
runtime.goparkunlock(...)
/usr/lib/golang/src/runtime/proc.go:404
runtime.(*scavengerState).park(0x625680)
/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0xc00004ffa0 sp=0xc00004ff70 pc=0x420e49
runtime.bgscavenge(0x0?)
/usr/lib/golang/src/runtime/mgcscavenge.go:658 +0x59 fp=0xc00004ffc8 sp=0xc00004ffa0 pc=0x4213f9
runtime.gcenable.func2()
/usr/lib/golang/src/runtime/mgc.go:201 +0x25 fp=0xc00004ffe0 sp=0xc00004ffc8 pc=0x4188e5
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00004ffe8 sp=0xc00004ffe0 pc=0x461f61
created by runtime.gcenable in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:201 +0xa5
goroutine 17 [finalizer wait]:
runtime.gopark(0x400000?, 0x10004e670?, 0x0?, 0x0?, 0x654640?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00004e628 sp=0xc00004e608 pc=0x436b8e
runtime.runfinq()
/usr/lib/golang/src/runtime/mfinal.go:193 +0x107 fp=0xc00004e7e0 sp=0xc00004e628 pc=0x4179c7
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00004e7e8 sp=0xc00004e7e0 pc=0x461f61
created by runtime.createfing in goroutine 1
/usr/lib/golang/src/runtime/mfinal.go:163 +0x3d
goroutine 18 [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00004a750 sp=0xc00004a730 pc=0x436b8e
runtime.gcBgMarkWorker()
/usr/lib/golang/src/runtime/mgc.go:1293 +0xe5 fp=0xc00004a7e0 sp=0xc00004a750 pc=0x41a2c5
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00004a7e8 sp=0xc00004a7e0 pc=0x461f61
created by runtime.gcBgMarkStartWorkers in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:1217 +0x1c
goroutine 19 [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00004af50 sp=0xc00004af30 pc=0x436b8e
runtime.gcBgMarkWorker()
/usr/lib/golang/src/runtime/mgc.go:1293 +0xe5 fp=0xc00004afe0 sp=0xc00004af50 pc=0x41a2c5
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00004afe8 sp=0xc00004afe0 pc=0x461f61
created by runtime.gcBgMarkStartWorkers in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:1217 +0x1c
goroutine 33 [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc000090750 sp=0xc000090730 pc=0x436b8e
runtime.gcBgMarkWorker()
/usr/lib/golang/src/runtime/mgc.go:1293 +0xe5 fp=0xc0000907e0 sp=0xc000090750 pc=0x41a2c5
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000907e8 sp=0xc0000907e0 pc=0x461f61
created by runtime.gcBgMarkStartWorkers in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:1217 +0x1c
goroutine 20 [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00004b750 sp=0xc00004b730 pc=0x436b8e
runtime.gcBgMarkWorker()
/usr/lib/golang/src/runtime/mgc.go:1293 +0xe5 fp=0xc00004b7e0 sp=0xc00004b750 pc=0x41a2c5
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00004b7e8 sp=0xc00004b7e0 pc=0x461f61
created by runtime.gcBgMarkStartWorkers in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:1217 +0x1c
goroutine 49 [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00008c750 sp=0xc00008c730 pc=0x436b8e
runtime.gcBgMarkWorker()
/usr/lib/golang/src/runtime/mgc.go:1293 +0xe5 fp=0xc00008c7e0 sp=0xc00008c750 pc=0x41a2c5
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00008c7e8 sp=0xc00008c7e0 pc=0x461f61
created by runtime.gcBgMarkStartWorkers in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:1217 +0x1c
goroutine 21 [GC worker (idle)]:
runtime.gopark(0xa740c76b8ab?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00004bf50 sp=0xc00004bf30 pc=0x436b8e
runtime.gcBgMarkWorker()
/usr/lib/golang/src/runtime/mgc.go:1293 +0xe5 fp=0xc00004bfe0 sp=0xc00004bf50 pc=0x41a2c5
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00004bfe8 sp=0xc00004bfe0 pc=0x461f61
created by runtime.gcBgMarkStartWorkers in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:1217 +0x1c
goroutine 22 [GC worker (idle)]:
runtime.gopark(0xa740cc9dc5e?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00004c750 sp=0xc00004c730 pc=0x436b8e
runtime.gcBgMarkWorker()
/usr/lib/golang/src/runtime/mgc.go:1293 +0xe5 fp=0xc00004c7e0 sp=0xc00004c750 pc=0x41a2c5
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00004c7e8 sp=0xc00004c7e0 pc=0x461f61
created by runtime.gcBgMarkStartWorkers in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:1217 +0x1c
goroutine 23 [GC worker (idle)]:
runtime.gopark(0x654640?, 0x1?, 0xba?, 0x5f?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00004cf50 sp=0xc00004cf30 pc=0x436b8e
runtime.gcBgMarkWorker()
/usr/lib/golang/src/runtime/mgc.go:1293 +0xe5 fp=0xc00004cfe0 sp=0xc00004cf50 pc=0x41a2c5
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00004cfe8 sp=0xc00004cfe0 pc=0x461f61
created by runtime.gcBgMarkStartWorkers in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:1217 +0x1c
goroutine 24 [GC worker (idle)]:
runtime.gopark(0xa740c58ec16?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc00004d750 sp=0xc00004d730 pc=0x436b8e
runtime.gcBgMarkWorker()
/usr/lib/golang/src/runtime/mgc.go:1293 +0xe5 fp=0xc00004d7e0 sp=0xc00004d750 pc=0x41a2c5
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00004d7e8 sp=0xc00004d7e0 pc=0x461f61
created by runtime.gcBgMarkStartWorkers in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:1217 +0x1c
goroutine 34 [GC worker (idle)]:
runtime.gopark(0x654640?, 0x1?, 0x7a?, 0xa3?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:398 +0xce fp=0xc000090f50 sp=0xc000090f30 pc=0x436b8e
runtime.gcBgMarkWorker()
/usr/lib/golang/src/runtime/mgc.go:1293 +0xe5 fp=0xc000090fe0 sp=0xc000090f50 pc=0x41a2c5
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000090fe8 sp=0xc000090fe0 pc=0x461f61
created by runtime.gcBgMarkStartWorkers in goroutine 1
/usr/lib/golang/src/runtime/mgc.go:1217 +0x1c
exit status 2