Skip to content

"DMAR: DRHD: handling fault status reg 2" with vfio on kernel 6.13.11-200.fc41.x86_64, works with 6.13.9-200.fc41.x86_64

Host environment

  • Operating system: Fedora
  • OS/kernel version: 6.13.11-200.fc41.x86_64 (and beyond)
  • Architecture: x86_64
  • QEMU flavor: qemu-system-x86_64
  • QEMU version: qemu-9.1.3-2.fc41
  • QEMU command line:
/usr/bin/qemu-system-x86_64 \
-name guest=win11-ableton,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-1-win11-ableton/master-key.aes"}' \
-blockdev '{"driver":"file","filename":"/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/win11-ableton_VARS.fd","node-name":"libvirt-pflash1-storage","read-only":false}' \
-machine pc-q35-7.0,usb=off,vmport=off,smm=on,kernel_irqchip=on,dump-guest-core=off,mem-merge=off,memory-backend=pc.ram,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-storage,hpet=off,acpi=on \
-accel kvm \
-cpu host,migratable=off,invtsc=on,hv-time=on,hv-passthrough=on,kvm=off,kvm-hint-dedicated=on,host-cache-info=on,l3-cache=off \
-global driver=cfi.pflash01,property=secure,value=on \
-m size=13631488k \
-object '{"qom-type":"memory-backend-file","id":"pc.ram","mem-path":"/dev/hugepages/libvirt/qemu/1-win11-ableton","x-use-canonical-path-for-ramblock-id":false,"prealloc":true,"prealloc-threads":8,"size":13958643712}' \
-overcommit mem-lock=off \
-smp 6,sockets=1,dies=1,clusters=1,cores=3,threads=2 \
-object '{"qom-type":"iothread","id":"iothread1"}' \
-object '{"qom-type":"iothread","id":"iothread2"}' \
-object '{"qom-type":"iothread","id":"iothread3"}' \
-object '{"qom-type":"iothread","id":"iothread4"}' \
-object '{"qom-type":"iothread","id":"iothread5"}' \
-object '{"qom-type":"iothread","id":"iothread6"}' \
-uuid 94b90c3c-6431-4a6c-ab74-11910b8ceff3 \
-smbios 'type=0,vendor=xxx,version=xxx,date=xxx,release=xxx' \
-smbios 'type=1,manufacturer=xxx,product=xxx,version=Not Applicable,serial=Not Applicable,sku=Not Applicable,family=Not Applicable' \
-smbios 'type=2,manufacturer=xxx,product=xxx,version=Not Applicable,serial=Not Applicable,asset=Tag xxx,location=Not Specified' \
-smbios 'type=3,manufacturer=xxx,version=N/A,serial=None,asset=No Asset Tag,sku=xxx' \
-smbios 'type=11,value=123,value=OEM String,value=xxx,value=xxx,value=BIOS:1.07.12NTSAS               ' \
-display none \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=23,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot strict=on \
-device '{"driver":"pcie-root-port","port":16,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x2"}' \
-device '{"driver":"pcie-root-port","port":17,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x2.0x1"}' \
-device '{"driver":"pcie-root-port","port":18,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x2.0x2"}' \
-device '{"driver":"pcie-root-port","port":19,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x2.0x3"}' \
-device '{"driver":"pcie-root-port","port":20,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x2.0x4"}' \
-device '{"driver":"pcie-root-port","port":21,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x2.0x5"}' \
-device '{"driver":"pcie-root-port","port":22,"chassis":7,"id":"pci.7","bus":"pcie.0","addr":"0x2.0x6"}' \
-device '{"driver":"pcie-root-port","port":23,"chassis":8,"id":"pci.8","bus":"pcie.0","addr":"0x2.0x7"}' \
-device '{"driver":"pcie-root-port","port":24,"chassis":9,"id":"pci.9","bus":"pcie.0","multifunction":true,"addr":"0x3"}' \
-device '{"driver":"pcie-root-port","port":25,"chassis":10,"id":"pci.10","bus":"pcie.0","addr":"0x3.0x1"}' \
-device '{"driver":"pcie-root-port","port":26,"chassis":11,"id":"pci.11","bus":"pcie.0","addr":"0x3.0x2"}' \
-device '{"driver":"pcie-root-port","port":27,"chassis":12,"id":"pci.12","bus":"pcie.0","addr":"0x3.0x3"}' \
-device '{"driver":"pcie-root-port","port":28,"chassis":13,"id":"pci.13","bus":"pcie.0","addr":"0x3.0x4"}' \
-device '{"driver":"pcie-root-port","port":29,"chassis":14,"id":"pci.14","bus":"pcie.0","addr":"0x3.0x5"}' \
-device '{"driver":"qemu-xhci","p2":15,"p3":15,"id":"usb","bus":"pci.2","addr":"0x0"}' \
-device '{"driver":"virtio-serial-pci","id":"virtio-serial0","bus":"pci.3","addr":"0x0"}' \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/win11-ableton.qcow2","aio":"native","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap","cache":{"direct":true,"no-flush":false}}' \
-blockdev '{"node-name":"libvirt-2-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":"libvirt-2-storage","backing":null}' \
-device '{"driver":"virtio-blk-pci","num-queues":6,"iothread-vq-mapping":[{"iothread":"iothread1"},{"iothread":"iothread2"},{"iothread":"iothread3"},{"iothread":"iothread4"},{"iothread":"iothread5"},{"iothread":"iothread6"}],"bus":"pci.4","addr":"0x0","drive":"libvirt-2-format","id":"virtio-disk0","bootindex":1,"write-cache":"on"}' \
-blockdev '{"driver":"file","filename":"/usr/share/virtio-win/virtio-win.iso","node-name":"libvirt-1-storage","read-only":true}' \
-device '{"driver":"ide-cd","bus":"ide.3","drive":"libvirt-1-storage","id":"sata0-0-3"}' \
-netdev '{"type":"tap","fd":"24","vhost":true,"vhostfd":"26","id":"hostnet0"}' \
-device '{"driver":"virtio-net-pci","netdev":"hostnet0","id":"net0","mac":"c4:23:60:0c:4f:ec","bus":"pci.1","addr":"0x0"}' \
-chardev pty,id=charserial0 \
-device '{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \
-chardev socket,id=charchannel0,fd=22,server=on,wait=off \
-device '{"driver":"virtserialport","bus":"virtio-serial0.0","nr":1,"chardev":"charchannel0","id":"channel0","name":"org.qemu.guest_agent.0"}' \
-chardev socket,id=chrtpm,path=/run/libvirt/qemu/swtpm/1-win11-ableton-swtpm.sock \
-tpmdev emulator,id=tpm-tpm0,chardev=chrtpm \
-device '{"driver":"tpm-crb","tpmdev":"tpm-tpm0","id":"tpm0"}' \
-device '{"driver":"usb-tablet","id":"input2","bus":"usb.0","port":"1"}' \
-audiodev '{"id":"audio1","driver":"none"}' \
-global ICH9-LPC.noreboot=off \
-watchdog-action reset \
-device '{"driver":"vfio-pci","host":"0000:36:00.0","id":"hostdev0","bus":"pci.5","multifunction":true,"addr":"0x0"}' \
-device '{"driver":"vfio-pci","host":"0000:36:00.1","id":"hostdev1","bus":"pci.5","addr":"0x0.0x1"}' \
-object '{"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"}' \
-device '{"driver":"virtio-rng-pci","rng":"objrng0","id":"rng0","bus":"pci.6","addr":"0x0"}' \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on

Emulated/Virtualized environment

  • Operating system: Windows 11
  • Architecture: x86_64

Description of problem

Since kernel 6.13.11-200.fc41.x86_64, I cannot use VFIO to pass an NVIDIA GeForce GTX 1070 card to a Windows guest. The same setup works just fine in 6.13.9-200.fc41.x86_64. The issue symptoms are the same regardless if I use kernel command line arguments to isolate cpus or not.

Symptoms:

  • qemu logs show:
2025-05-07T09:59:49.957891Z qemu-system-x86_64: vfio: Cannot reset device 0000:36:00.1, no available reset mechanism.
2025-05-07T09:59:49.958444Z qemu-system-x86_64: vfio: Cannot reset device 0000:36:00.0, no available reset mechanism.
2025-05-07T09:59:49.959119Z qemu-system-x86_64: vfio: Cannot reset device 0000:36:00.1, no available reset mechanism.
2025-05-07T09:59:49.959635Z qemu-system-x86_64: vfio: Cannot reset device 0000:36:00.0, no available reset mechanism.
  • in dmesg I see:
kernel: DMAR: DRHD: handling fault status reg 2
kernel: DMAR: [INTR-REMAP] Request device [36:00.0] fault index 0x50 [fault reason 0x22] Present field in the IRTE entry is clear
  • the VM hangs at boot (please see the notes below (*)).

Steps to reproduce

Boot the same libvirt domain in kernel 6.13.9-200.fc41.x86_64 (works) and any other more recent kernel (>= 6.13.11-200.fc41.x86_64).

Additional information

(*) Note that in a working kernel, the boot process is in any case finicky, and it shows these phases:

  1. tianocore logo shows, and one single cpu is fully utilized by the guest
  2. slowly, the loader find the Windows bootloader, and prints a message that it is loading and running it
  3. some time passes, while cpus seem idle
  4. finally the spinning wheel of the Windows bootloader appears

Phase 1-3 can take anywhere from 0 to 60 seconds, in an apparently random manner.

When running on the faulty kernels, it seems that the virtual machine gets stuck in phase 1, and I must use virsh destroy to interrupt it.

lspci output:

-[0000:00]-+-00.0  Intel Corporation Tiger Lake-UP3/H35 4 cores Host Bridge/DRAM Registers
           +-02.0  Intel Corporation TigerLake-LP GT2 [Iris Xe Graphics]
           +-04.0  Intel Corporation TigerLake-LP Dynamic Tuning Processor Participant
           +-06.0-[01]----00.0  Samsung Electronics Co Ltd NVMe SSD Controller SM981/PM981/PM983
           +-07.0-[02-33]--
           +-0a.0  Intel Corporation Tigerlake Telemetry Aggregator Driver
           +-0d.0  Intel Corporation Tiger Lake-LP Thunderbolt 4 USB Controller
           +-0d.2  Intel Corporation Tiger Lake-LP Thunderbolt 4 NHI #0
           +-14.0  Intel Corporation Tiger Lake-LP USB 3.2 Gen 2x1 xHCI Host Controller
           +-14.2  Intel Corporation Tiger Lake-LP Shared SRAM
           +-15.0  Intel Corporation Tiger Lake-LP Serial IO I2C Controller #0
           +-15.1  Intel Corporation Tiger Lake-LP Serial IO I2C Controller #1
           +-15.2  Intel Corporation Tiger Lake-LP Serial IO I2C Controller #2
           +-16.0  Intel Corporation Tiger Lake-LP Management Engine Interface
           +-1c.0-[34]----00.0  Intel Corporation Wi-Fi 6 AX200
           +-1c.5-[35]----00.0  Realtek Semiconductor Co., Ltd. RTS522A PCI Express Card Reader
           +-1d.0-[36]--+-00.0  NVIDIA Corporation GP104 [GeForce GTX 1070]
           |            \-00.1  NVIDIA Corporation GP104 High Definition Audio Controller
           +-1f.0  Intel Corporation Tiger Lake-LP LPC Controller
           +-1f.3  Intel Corporation Tiger Lake-LP Smart Sound Technology Audio Controller
           +-1f.4  Intel Corporation Tiger Lake-LP SMBus Controller
           \-1f.5  Intel Corporation Tiger Lake-LP SPI Controller

kernel command line arguments (optimized with cpu isolation):

intel_pstate=per_cpu_perf_limits rd.driver.blacklist=nouveau modprobe.blacklist=nouveau module_blacklist=nouveau default_hugepagesz=1G hugepagesz=1G hugepages=13 i2c_i801.disable_features=0x10 rd.driver.pre=vfio_pci,vfio,vfio_iommu_type1 vfio-pci.ids=10de:1b81,10de:10f0 modprobe.blacklist=xpad systemd.unit=multi-user.target systemd.wants=bluetooth.service isolcpus=domain,managed_irq,1-3,5-7 rcu_nocbs=1-3,5-7 irqaffinity=0,4 nospectre_v2

kernel command line arguments (without cpu isolation, same symptoms):

intel_pstate=per_cpu_perf_limits rd.driver.blacklist=nouveau modprobe.blacklist=nouveau module_blacklist=nouveau default_hugepagesz=1G hugepagesz=1G hugepages=13 rd.driver.pre=vfio_pci,vfio,vfio_iommu_type1 vfio-pci.ids=10de:1b81,10de:10f0 modprobe.blacklist=xpad systemd.unit=multi-user.target systemd.wants=bluetooth.service
To upload designs, you'll need to enable LFS and have an admin enable hashed storage. More information