Windows 11 Guest keeps crashing with abort in cpu_asidx_from_attrs
This has been happening across multiple distros, and kernels for me. The hardware remains the same, but I just can't get past this issue. I have researched this in depth and tried everything that gives any hope of solving it. Primarily, the number one suggestion is to disable nested KVM which I have done and verified.
The log shows this
If you're running a guest on an Intel machine without unrestricted mode
support, the failure can be most likely due to the guest entering an invalid
state for Intel VT. For example, the guest maybe running in big real mode
which is not supported on less recent Intel processors.
I am running on an Alder Lake, so it is fairly recent and should have support. I cannot find any documentation on how to run in unrestricted mode, unless it inherently is or isn't supported.
Host environment
- Operating system: Arch Linux
- OS/kernel version: 5.19.5-zen1-1.1-zen #1 ZEN SMP PREEMPT_DYNAMIC Tue, 30 Aug 2022 05:11:44 +0000 x86_64 GNU/Linux
- Architecture: x64
- QEMU flavor: qemu-system-x86_64
- QEMU version: 7.0.0
- QEMU command line:
/usr/bin/qemu-system-x86_64 \
-name guest=Windows,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-2-Windows/master-key.aes"}' \
-machine pc-q35-7.0,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram \
-accel kvm \
-cpu host,migratable=on,hv-time=on,hv-relaxed=on,hv-vapic=on,hv-spinlocks=0x1fff \
-m 12384 \
-object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":12985565184}' \
-overcommit mem-lock=off \
-smp 8,sockets=8,cores=1,threads=1 \
-uuid e723c7e4-d6df-45a7-a0aa-bb9e21570326 \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=33,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot strict=on \
-device '{"driver":"pcie-root-port","port":16,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x2"}' \
-device '{"driver":"pcie-root-port","port":17,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x2.0x1"}' \
-device '{"driver":"pcie-root-port","port":18,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x2.0x2"}' \
-device '{"driver":"pcie-root-port","port":19,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x2.0x3"}' \
-device '{"driver":"pcie-root-port","port":20,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x2.0x4"}' \
-device '{"driver":"pcie-root-port","port":21,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x2.0x5"}' \
-device '{"driver":"pcie-root-port","port":22,"chassis":7,"id":"pci.7","bus":"pcie.0","addr":"0x2.0x6"}' \
-device '{"driver":"pcie-root-port","port":23,"chassis":8,"id":"pci.8","bus":"pcie.0","addr":"0x2.0x7"}' \
-device '{"driver":"pcie-root-port","port":24,"chassis":9,"id":"pci.9","bus":"pcie.0","multifunction":true,"addr":"0x3"}' \
-device '{"driver":"pcie-root-port","port":25,"chassis":10,"id":"pci.10","bus":"pcie.0","addr":"0x3.0x1"}' \
-device '{"driver":"pcie-root-port","port":26,"chassis":11,"id":"pci.11","bus":"pcie.0","addr":"0x3.0x2"}' \
-device '{"driver":"pcie-root-port","port":27,"chassis":12,"id":"pci.12","bus":"pcie.0","addr":"0x3.0x3"}' \
-device '{"driver":"pcie-root-port","port":28,"chassis":13,"id":"pci.13","bus":"pcie.0","addr":"0x3.0x4"}' \
-device '{"driver":"pcie-root-port","port":29,"chassis":14,"id":"pci.14","bus":"pcie.0","addr":"0x3.0x5"}' \
-device '{"driver":"qemu-xhci","p2":15,"p3":15,"id":"usb","bus":"pci.2","addr":"0x0"}' \
-device '{"driver":"virtio-serial-pci","id":"virtio-serial0","bus":"pci.3","addr":"0x0"}' \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/Microsoft-Windows.img","node-name":"libvirt-1-storage","cache":{"direct":false,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":false,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \
-device '{"driver":"virtio-blk-pci","bus":"pci.4","addr":"0x0","drive":"libvirt-1-format","id":"virtio-disk0","bootindex":1,"write-cache":"on"}' \
-netdev tap,fd=34,id=hostnet0 \
-device '{"driver":"e1000e","netdev":"hostnet0","id":"net0","mac":"52:54:00:65:47:3f","bus":"pci.1","addr":"0x0"}' \
-chardev pty,id=charserial0 \
-device '{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \
-chardev spicevmc,id=charchannel0,name=vdagent \
-device '{"driver":"virtserialport","bus":"virtio-serial0.0","nr":1,"chardev":"charchannel0","id":"channel0","name":"com.redhat.spice.0"}' \
-chardev socket,id=chrtpm,path=/run/libvirt/qemu/swtpm/2-Windows-swtpm.sock \
-tpmdev emulator,id=tpm-tpm0,chardev=chrtpm \
-device '{"driver":"tpm-crb","tpmdev":"tpm-tpm0","id":"tpm0"}' \
-device '{"driver":"usb-tablet","id":"input0","bus":"usb.0","port":"1"}' \
-audiodev '{"id":"audio1","driver":"spice"}' \
-spice port=5900,addr=127.0.0.1,disable-ticketing=on,image-compression=off,seamless-migration=on \
-device '{"driver":"qxl-vga","id":"video0","max_outputs":1,"ram_size":67108864,"vram_size":67108864,"vram64_size_mb":0,"vgamem_mb":16,"bus":"pcie.0","addr":"0x1"}' \
-device '{"driver":"ich9-intel-hda","id":"sound0","bus":"pcie.0","addr":"0x1b"}' \
-device '{"driver":"hda-duplex","id":"sound0-codec0","bus":"sound0.0","cad":0,"audiodev":"audio1"}' \
-chardev spicevmc,id=charredir0,name=usbredir \
-device '{"driver":"usb-redir","chardev":"charredir0","id":"redir0","bus":"usb.0","port":"2"}' \
-chardev spicevmc,id=charredir1,name=usbredir \
-device '{"driver":"usb-redir","chardev":"charredir1","id":"redir1","bus":"usb.0","port":"3"}' \
-device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.5","addr":"0x0"}' \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
char device redirected to /dev/pts/1 (label charserial0)
qxl_send_events: spice-server bug: guest stopped, ignoring
2022-08-23T18:31:15.829881Z qemu-system-x86_64: terminating on signal 15 from pid 863 (/usr/bin/libvirtd)
2022-08-23 18:31:16.030+0000: shutting down, reason=destroyed
2022-08-23 18:31:32.023+0000: Starting external device: TPM Emulator
/usr/bin/swtpm socket --ctrl type=unixio,path=/run/libvirt/qemu/swtpm/3-Windows-swtpm.sock,mode=0600 --tpmstate dir=/var/lib/libvirt/swtpm/e723c7e4-d6df-45a7-a0aa-bb9e21570326/tpm2,mode=0600 --log file=/var/log/swtpm/libvirt/qemu/Windows-swtpm.log --terminate --tpm2
2022-08-23 18:31:32.042+0000: starting up libvirt version: 8.6.0, qemu version: 7.0.0, kernel: 5.19.3-zen1-1.1-zen, hostname: rayzerarch.localdomain
LC_ALL=C \
PATH=/usr/local/sbin:/usr/local/bin:/usr/bin \
HOME=/var/lib/libvirt/qemu/domain-3-Windows \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-3-Windows/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-3-Windows/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-3-Windows/.config \
/usr/bin/qemu-system-x86_64 \
-name guest=Windows,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-3-Windows/master-key.aes"}' \
-machine pc-q35-7.0,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram \
-accel kvm \
-cpu host,migratable=on,hv-time=on,hv-relaxed=on,hv-vapic=on,hv-spinlocks=0x1fff \
-m 12384 \
-object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":12985565184}' \
-overcommit mem-lock=off \
-smp 8,sockets=8,cores=1,threads=1 \
-uuid e723c7e4-d6df-45a7-a0aa-bb9e21570326 \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=33,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot strict=on \
-device '{"driver":"pcie-root-port","port":16,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x2"}' \
-device '{"driver":"pcie-root-port","port":17,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x2.0x1"}' \
-device '{"driver":"pcie-root-port","port":18,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x2.0x2"}' \
-device '{"driver":"pcie-root-port","port":19,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x2.0x3"}' \
-device '{"driver":"pcie-root-port","port":20,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x2.0x4"}' \
-device '{"driver":"pcie-root-port","port":21,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x2.0x5"}' \
-device '{"driver":"pcie-root-port","port":22,"chassis":7,"id":"pci.7","bus":"pcie.0","addr":"0x2.0x6"}' \
-device '{"driver":"pcie-root-port","port":23,"chassis":8,"id":"pci.8","bus":"pcie.0","addr":"0x2.0x7"}' \
-device '{"driver":"pcie-root-port","port":24,"chassis":9,"id":"pci.9","bus":"pcie.0","multifunction":true,"addr":"0x3"}' \
-device '{"driver":"pcie-root-port","port":25,"chassis":10,"id":"pci.10","bus":"pcie.0","addr":"0x3.0x1"}' \
-device '{"driver":"pcie-root-port","port":26,"chassis":11,"id":"pci.11","bus":"pcie.0","addr":"0x3.0x2"}' \
-device '{"driver":"pcie-root-port","port":27,"chassis":12,"id":"pci.12","bus":"pcie.0","addr":"0x3.0x3"}' \
-device '{"driver":"pcie-root-port","port":28,"chassis":13,"id":"pci.13","bus":"pcie.0","addr":"0x3.0x4"}' \
-device '{"driver":"pcie-root-port","port":29,"chassis":14,"id":"pci.14","bus":"pcie.0","addr":"0x3.0x5"}' \
-device '{"driver":"qemu-xhci","p2":15,"p3":15,"id":"usb","bus":"pci.2","addr":"0x0"}' \
-device '{"driver":"virtio-serial-pci","id":"virtio-serial0","bus":"pci.3","addr":"0x0"}' \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/Microsoft-Windows.img","node-name":"libvirt-1-storage","cache":{"direct":false,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":false,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \
-device '{"driver":"virtio-blk-pci","bus":"pci.4","addr":"0x0","drive":"libvirt-1-format","id":"virtio-disk0","bootindex":1,"write-cache":"on"}' \
-netdev tap,fd=34,id=hostnet0 \
-device '{"driver":"e1000e","netdev":"hostnet0","id":"net0","mac":"52:54:00:65:47:3f","bus":"pci.1","addr":"0x0"}' \
-chardev pty,id=charserial0 \
-device '{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \
-chardev spicevmc,id=charchannel0,name=vdagent \
-device '{"driver":"virtserialport","bus":"virtio-serial0.0","nr":1,"chardev":"charchannel0","id":"channel0","name":"com.redhat.spice.0"}' \
-chardev socket,id=chrtpm,path=/run/libvirt/qemu/swtpm/3-Windows-swtpm.sock \
-tpmdev emulator,id=tpm-tpm0,chardev=chrtpm \
-device '{"driver":"tpm-crb","tpmdev":"tpm-tpm0","id":"tpm0"}' \
-device '{"driver":"usb-tablet","id":"input0","bus":"usb.0","port":"1"}' \
-audiodev '{"id":"audio1","driver":"spice"}' \
-spice port=5900,addr=127.0.0.1,disable-ticketing=on,image-compression=off,seamless-migration=on \
-device '{"driver":"qxl-vga","id":"video0","max_outputs":1,"ram_size":67108864,"vram_size":67108864,"vram64_size_mb":0,"vgamem_mb":16,"bus":"pcie.0","addr":"0x1"}' \
-device '{"driver":"ich9-intel-hda","id":"sound0","bus":"pcie.0","addr":"0x1b"}' \
-device '{"driver":"hda-duplex","id":"sound0-codec0","bus":"sound0.0","cad":0,"audiodev":"audio1"}' \
-chardev spicevmc,id=charredir0,name=usbredir \
-device '{"driver":"usb-redir","chardev":"charredir0","id":"redir0","bus":"usb.0","port":"2"}' \
-chardev spicevmc,id=charredir1,name=usbredir \
-device '{"driver":"usb-redir","chardev":"charredir1","id":"redir1","bus":"usb.0","port":"3"}' \
-device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.5","addr":"0x0"}' \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
Steps to reproduce
- Create Windows 11 guest, SWTPM, SECBOOT (haven't tested without since this is not an option for installing Windows 11)
- Use OS
- Will eventually crash. Have tried across multiple kernels 5.17, 5.18, 5.19
Additional information
Stack trace of thread 76223:
#0 0x00007f24072d44dc n/a (libc.so.6 + 0x884dc)
#1 0x00007f2407284998 raise (libc.so.6 + 0x38998)
#2 0x00007f240726e53d abort (libc.so.6 + 0x2253d)
#3 0x00007f240726e45c n/a (libc.so.6 + 0x2245c)
#4 0x00007f240727d4c6 __assert_fail (libc.so.6 + 0x314c6)
#5 0x0000555681a35101 cpu_asidx_from_attrs (qemu-system-x86_64 + 0x572101)
#6 0x0000555681c6531e cpu_memory_rw_debug (qemu-system-x86_64 + 0x7a231e)
#7 0x0000555681bfb54a x86_cpu_dump_state (qemu-system-x86_64 + 0x73854a)
#8 0x0000555681d84a65 kvm_cpu_exec (qemu-system-x86_64 + 0x8c1a65)
#9 0x0000555681d85e48 kvm_vcpu_thread_fn (qemu-system-x86_64 + 0x8c2e48)
#10 0x0000555681fed0a8 qemu_thread_start (qemu-system-x86_64 + 0xb2a0a8)
#11 0x00007f24072d278d n/a (libc.so.6 + 0x8678d)
#12 0x00007f24073538e4 __clone (libc.so.6 + 0x1078e4)
KVM: entry failed, hardware error 0x80000021
If you're running a guest on an Intel machine without unrestricted mode
support, the failure can be most likely due to the guest entering an invalid
state for Intel VT. For example, the guest maybe running in big real mode
which is not supported on less recent Intel processors.
EAX=00000000 EBX=00000000 ECX=00000000 EDX=04c6d3e0
ESI=12af7eb0 EDI=9e55d420 EBP=821b5aa0 ESP=10db0fb0
EIP=00008000 EFL=00000002 [-------] CPL=0 II=0 A20=1 SMM=1 HLT=0
ES =0000 00000000 ffffffff 00809300
CS =b500 7ffb5000 ffffffff 00809300
SS =0000 00000000 ffffffff 00809300
DS =0000 00000000 ffffffff 00809300
FS =0000 00000000 ffffffff 00809300
GS =0000 00000000 ffffffff 00809300
LDT=0000 00000000 000fffff 00000000
TR =0040 10d97000 00000067 00008b00
GDT= 10d98fb0 00000057
IDT= 00000000 00000000
CR0=00050032 CR2=f80ff80c CR3=e47e7000 CR4=00000000
DR0=0000000000000000 DR1=0000000000000000 DR2=0000000000000000 DR3=0000000000000000
DR6=00000000ffff0ff0 DR7=0000000000000400
EFER=0000000000000000
Code=qemu-system-x86_64: ../qemu-7.0.0/hw/core/cpu-sysemu.c:77: cpu_asidx_from_attrs: Assertion `ret < cpu->num_ases && ret >= 0' failed.
2022-09-06 14:48:15.392+0000: shutting down, reason=crashed
Edited by Thomas Huth