qemu got sigabrt when using vpp in guest and dpdk for qemu

Host environment

Operating system: Rockylinux 8
OS/kernel version: 4.18.0-477.27.1.el8.x86_64
Architecture: x86
QEMU flavor: qemu-system-x86_64
QEMU version: qemu-kvm-7.2.0-14.el9 (in a container)
QEMU command line:


/usr/libexec/qemu-kvm
-name guest=mec-images_upf-test,debug-threads=on
-S
-object {"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/run/kubevirt-private/libvirt/qemu/lib/domain-1-mec-images_upf-test/master-key.aes"}
-machine pc-q35-rhel9.2.0,usb=off,dump-guest-core=off
-accel kvm
-cpu host,migratable=on,kvm-hint-dedicated=on
-m 4096
-overcommit mem-lock=off
-smp 2,sockets=1,dies=1,cores=2,threads=1
-object {"qom-type":"iothread","id":"iothread1"}
-object {"qom-type":"thread-context","id":"tc-ram-node0","node-affinity":[0]}
-object {"qom-type":"memory-backend-memfd","id":"ram-node0","hugetlb":true,"hugetlbsize":2097152,"prealloc":true,"size":4294967296,"host-nodes":[0],"policy":"bind","prealloc-context":"tc-ram-node0"}
-numa node,nodeid=0,cpus=0-1,memdev=ram-node0
-uuid 920344c1-017b-51f7-b693-b063ffa66883
-smbios type=1,manufacturer=KubeVirt,product=None,uuid=920344c1-017b-51f7-b693-b063ffa66883,family=KubeVirt
-no-user-config
-nodefaults
-chardev socket,id=charmonitor,fd=19,server=on,wait=off
-mon chardev=charmonitor,id=monitor,mode=control
-rtc base=utc
-no-shutdown
-boot strict=on
-device {"driver":"pcie-root-port","port":17,"chassis":1,"id":"pci.1","bus":"pcie.0","addr":"0x2.0x1"}
-device {"driver":"pcie-root-port","port":18,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x2.0x2"}
-device {"driver":"pcie-root-port","port":19,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x2.0x3"}
-device {"driver":"pcie-root-port","port":20,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x2.0x4"}
-device {"driver":"pcie-root-port","port":21,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x2.0x5"}
-device {"driver":"pcie-root-port","port":22,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x2.0x6"}
-device {"driver":"pcie-root-port","port":23,"chassis":7,"id":"pci.7","bus":"pcie.0","addr":"0x2.0x7"}
-device {"driver":"pcie-root-port","port":16,"chassis":8,"id":"pci.8","bus":"pcie.0","multifunction":true,"addr":"0x2"}
-device {"driver":"pcie-root-port","port":25,"chassis":9,"id":"pci.9","bus":"pcie.0","addr":"0x3.0x1"}
-device {"driver":"pcie-root-port","port":26,"chassis":10,"id":"pci.10","bus":"pcie.0","addr":"0x3.0x2"}
-device {"driver":"pcie-root-port","port":27,"chassis":11,"id":"pci.11","bus":"pcie.0","addr":"0x3.0x3"}
-device {"driver":"pcie-root-port","port":28,"chassis":12,"id":"pci.12","bus":"pcie.0","addr":"0x3.0x4"}
-device {"driver":"pcie-root-port","port":29,"chassis":13,"id":"pci.13","bus":"pcie.0","addr":"0x3.0x5"}
-device {"driver":"pcie-root-port","port":30,"chassis":14,"id":"pci.14","bus":"pcie.0","addr":"0x3.0x6"}
-device {"driver":"pcie-root-port","port":31,"chassis":15,"id":"pci.15","bus":"pcie.0","addr":"0x3.0x7"}
-device {"driver":"pcie-root-port","port":24,"chassis":16,"id":"pci.16","bus":"pcie.0","multifunction":true,"addr":"0x3"}
-device {"driver":"pcie-root-port","port":33,"chassis":17,"id":"pci.17","bus":"pcie.0","addr":"0x4.0x1"}
-device {"driver":"pcie-root-port","port":34,"chassis":18,"id":"pci.18","bus":"pcie.0","addr":"0x4.0x2"}
-device {"driver":"pcie-root-port","port":35,"chassis":19,"id":"pci.19","bus":"pcie.0","addr":"0x4.0x3"}
-device {"driver":"pcie-root-port","port":36,"chassis":20,"id":"pci.20","bus":"pcie.0","addr":"0x4.0x4"}
-device {"driver":"pcie-root-port","port":37,"chassis":21,"id":"pci.21","bus":"pcie.0","addr":"0x4.0x5"}
-device {"driver":"pcie-root-port","port":38,"chassis":22,"id":"pci.22","bus":"pcie.0","addr":"0x4.0x6"}
-device {"driver":"pcie-root-port","port":39,"chassis":23,"id":"pci.23","bus":"pcie.0","addr":"0x4.0x7"}
-device {"driver":"pcie-root-port","port":32,"chassis":24,"id":"pci.24","bus":"pcie.0","multifunction":true,"addr":"0x4"}
-device {"driver":"pcie-root-port","port":41,"chassis":25,"id":"pci.25","bus":"pcie.0","addr":"0x5.0x1"}
-device {"driver":"pcie-root-port","port":42,"chassis":26,"id":"pci.26","bus":"pcie.0","addr":"0x5.0x2"}
-device {"driver":"pcie-root-port","port":43,"chassis":27,"id":"pci.27","bus":"pcie.0","addr":"0x5.0x3"}
-device {"driver":"pcie-root-port","port":44,"chassis":28,"id":"pci.28","bus":"pcie.0","addr":"0x5.0x4"}
-device {"driver":"pcie-root-port","port":45,"chassis":29,"id":"pci.29","bus":"pcie.0","addr":"0x5.0x5"}
-device {"driver":"pcie-root-port","port":46,"chassis":30,"id":"pci.30","bus":"pcie.0","addr":"0x5.0x6"}
-device {"driver":"pcie-root-port","port":47,"chassis":31,"id":"pci.31","bus":"pcie.0","addr":"0x5.0x7"}
-device {"driver":"pcie-root-port","port":40,"chassis":32,"id":"pci.32","bus":"pcie.0","multifunction":true,"addr":"0x5"}
-device {"driver":"virtio-scsi-pci-non-transitional","id":"scsi0","bus":"pci.5","addr":"0x0"}
-device {"driver":"virtio-serial-pci-non-transitional","id":"virtio-serial0","bus":"pci.6","addr":"0x0"}
-blockdev {"driver":"host_device","filename":"/dev/bootdisk","aio":"native","node-name":"libvirt-2-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}
-blockdev {"node-name":"libvirt-2-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-2-storage"}
-device {"driver":"virtio-blk-pci-non-transitional","bus":"pci.7","addr":"0x0","drive":"libvirt-2-format","id":"ua-bootdisk","bootindex":1,"write-cache":"on","werror":"stop","rerror":"stop"}
-blockdev {"driver":"file","filename":"/var/run/kubevirt-ephemeral-disks/cloud-init-data/mec-images/upf-test/configdrive.iso","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}
-blockdev {"node-name":"libvirt-1-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}
-device {"driver":"virtio-blk-pci-non-transitional","bus":"pci.8","addr":"0x0","drive":"libvirt-1-format","id":"ua-cloudinitdisk","write-cache":"on","werror":"stop","rerror":"stop"}
-chardev socket,id=charua-attachnet4,path=/var/run/openvswitch/vhostuser-sockets/pode33e8d449e8,server=on
-netdev {"type":"vhost-user","chardev":"charua-attachnet4","queues":2,"id":"hostua-attachnet4"}
-device {"driver":"virtio-net-pci-non-transitional","mq":true,"vectors":6,"netdev":"hostua-attachnet4","id":"ua-attachnet4","mac":"00:00:00:e1:bc:11","bus":"pci.1","addr":"0x0"}
-chardev socket,id=charserial0,fd=17,server=on,wait=off
-device {"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}
-chardev socket,id=charchannel0,fd=18,server=on,wait=off
-device {"driver":"virtserialport","bus":"virtio-serial0.0","nr":1,"chardev":"charchannel0","id":"channel0","name":"org.qemu.guest_agent.0"}
-add-fd set=0,fd=20,opaque=channel1-source
-chardev file,id=charchannel1,path=/dev/fdset/0,append=on
-device {"driver":"virtserialport","bus":"virtio-serial0.0","nr":2,"chardev":"charchannel1","id":"channel1","name":"org.monitor.0"}
-audiodev {"id":"audio1","driver":"none"}
-vnc vnc=unix:/var/run/kubevirt-private/2dda8080-baa8-4798-a69d-ce97c2b9cf83/virt-vnc,audiodev=audio1
-device {"driver":"VGA","id":"video0","vgamem_mb":16,"bus":"pcie.0","addr":"0x1"}
-device {"driver":"virtio-balloon-pci-non-transitional","id":"balloon0","free-page-reporting":false,"bus":"pci.9","addr":"0x0"}
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny
-msg timestamp=on

Emulated/Virtualized environment

Operating system: Ubuntu 22.04.3 LTS
OS/kernel version: 5.15.0-86-generic
Architecture: x86
VPP: 22.06.0-6~g1513b381d~b23

Description of problem

When set the interface up in vpp, the qemu process is crashed with signal sigabrt.

After some debug, i have identified that the problem lies in the following function.

static int setup_routing_entry(struct kvm *kvm,
			       struct kvm_irq_routing_table *rt,
			       struct kvm_kernel_irq_routing_entry *e,
			       const struct kvm_irq_routing_entry *ue)
{
	struct kvm_kernel_irq_routing_entry *ei;
	int r;
	u32 gsi = array_index_nospec(ue->gsi, KVM_MAX_IRQ_ROUTES);

	/*
	 * Do not allow GSI to be mapped to the same irqchip more than once.
	 * Allow only one to one mapping between GSI and non-irqchip routing.
	 */
	hlist_for_each_entry(ei, &rt->map[gsi], link)
		if (ei->type != KVM_IRQ_ROUTING_IRQCHIP ||
		    ue->type != KVM_IRQ_ROUTING_IRQCHIP ||
		    ue->u.irqchip.irqchip == ei->irqchip.irqchip)
			return -EINVAL;

I added some debug printk like following

        hlist_for_each_entry(ei, &rt->map[gsi], link)
                if (ei->type != KVM_IRQ_ROUTING_IRQCHIP ||
                    ue->type != KVM_IRQ_ROUTING_IRQCHIP ||
                    ue->u.irqchip.irqchip == ei->irqchip.irqchip){
                        printk("ei->type: %u, KVM_IRQ_ROUTING_IRQCHIP: %u, ue->type: %u, ue->u.irqchip.irqchip: %u , ei->irqchip.irqchip: %u",  ei->type, KVM_IRQ_ROUTING_IRQCHIP , ue->type, ue->u.irqchip.irqchip , ei->irqchip.irqchip);
                        return -EINVAL;
        }

Then i got following in dmesg

[Thu Nov 23 09:29:10 2023] ei->type: 2, KVM_IRQ_ROUTING_IRQCHIP: 1, ue->type: 1, ue->u.irqchip.irqchip: 2 , ei->irqchip.irqchip: 4276097024
[Thu Nov 23 09:29:10 2023] ei->type: 2, KVM_IRQ_ROUTING_IRQCHIP: 1, ue->type: 1, ue->u.irqchip.irqchip: 2 , ei->irqchip.irqchip: 4276097024

Steps to reproduce

This is a kube-ovn + dpdk env, not easy to reproduce now..

Additional information

I also file a bug on kernel.org: https://bugzilla.kernel.org/show_bug.cgi?id=218177
the libvirt xml file is also attached instance.xml

Edited Nov 23, 2023 by Jeffrey4l