Initial PoC of Xen on Cortex-R82 (MPU) platform
CC: @Weichen81 @BertrandMarquis @Pennyzct
This merge request contains the patch series of the initial PoC of Xen on R82 platform. The patch series contains 71 patches starting from 51ef3acb7b docs, xen/arm: Introduce reserved Xenheap memory to db94100401 xen/arm: Add a workaround to use ns-physical-timer on armv8r FVP, and this patch series is based on 686f13cfce CI/Coverity: Do not build QEMU, SeaBIOS or OVMF (Note that the base of integration/mpu is a little bit behind the upstream staging branch on Feb 19, 2022).
Below is the guidance on how to run this PoC:
Get FVP
The current Xen R82 PoC implementation is for FVP_BaseR_AEMV8R only. The FVP can be downloaded for free at the Armv8-R AEM FVP section of [1]. The FVP binary can be found at the AEMv8R_base_pkg/models/Linux64_GCC-6.4 directory after unpacking the tarball.
Device Tree
Some new device tree configurations in chosen node is required to run the Xen R82 PoC.
(1) r82.dtb (The dtb for Xen)
Below is the device tree for Xen on the FVP_BaseR_AEMV8R platform (domU2 is showing the configuration for a Zephyr domain, feel free to delete):
/dts-v1/;
/ {
#address-cells = <0x02>;
#size-cells = <0x02>;
interrupt-parent = <0x01>;
model = "Generated";
compatible = "arm,base";
memory@0 {
#address-cells = <0x02>;
#size-cells = <0x02>;
device_type = "memory";
reg = <0x00 0x00 0x00 0x7ffff000>;
};
chosen {
xen,xen-bootargs = "console=dtuart dtuart=serial0";
#mpu,address-cells = <0x01>;
#mpu,size-cells = <0x01>;
mpu,boot-module-section = <0x10000000 0x10000000>;
mpu,guest-memory-section = <0x20000000 0x30000000>;
mpu,device-memory-section = <0x80000000 0x7ffff000>;
#xen,static-mem-address-cells = <0x01>;
#xen,static-mem-size-cells = <0x01>;
xen,static-mem = <0x50000000 0x20000000>;
domU1 {
compatible = "xen,domain";
#address-cells = <0x01>;
#size-cells = <0x01>;
cpus = <0x02>;
memory = <0x00 0x7c000>;
direct-map;
vpl011;
#xen,static-mem-address-cells = <0x01>;
#xen,static-mem-size-cells = <0x01>;
xen,static-mem = <0x30000000 0x1f000000>;
module@11000000 {
compatible = "multiboot,kernel\0multiboot,module";
reg = <0x11000000 0x5000000>;
bootargs = "earlycon=pl011,0x9c090000,0x9c0b0000 console=ttyAMA0 console=ttyAMA1";
};
module@10FF0000 {
compatible = "multiboot,device-tree\0multiboot,module";
reg = <0x10ff0000 0x10000>;
};
};
domU2 {
compatible = "xen,domain";
#address-cells = <0x01>;
#size-cells = <0x01>;
cpus = <0x01>;
memory = <0x00 0x40000>;
vpl011;
direct-map;
#xen,static-mem-address-cells = <0x01>;
#xen,static-mem-size-cells = <0x01>;
xen,static-mem = <0x20000000 0x10000000>;
mpu;
module@15000000 {
compatible = "multiboot,kernel\0multiboot,module";
reg = <0x15000000 0x3000000>;
};
module@19FF0000 {
compatible = "multiboot,device-tree\0multiboot,module";
reg = <0x19ff0000 0x10000>;
};
};
};
cpus {
#address-cells = <0x02>;
#size-cells = <0x00>;
cpu-map {
cluster0 {
core0 {
thread0 {
cpu = <0x02>;
};
};
};
};
cpu@0 {
device_type = "cpu";
compatible = "arm,armv8";
reg = <0x00 0x00>;
enable-method = "spin-table";
cpu-release-addr = <0x00 0x7f800>;
phandle = <0x02>;
};
};
interrupt-controller@af000000 {
compatible = "arm,gic-v3";
#interrupt-cells = <0x03>;
#address-cells = <0x02>;
#size-cells = <0x02>;
ranges;
interrupt-controller;
#redistributor-regions = <0x01>;
reg = <0x00 0xaf000000 0x00 0x10000 0x00 0xaf100000 0x00 0x200000 0x00 0xac000000 0x00 0x2000 0x00 0xac010000 0x00 0x2000 0x00 0xac02f000 0x00 0x2000>;
interrupts = <0x01 0x09 0x04>;
phandle = <0x01>;
};
uart@9c090000 {
compatible = "arm,pl011\0arm,primecell";
reg = <0x00 0x9c090000 0x00 0x10000>;
interrupts = <0x00 0x05 0x04>;
clock-names = "uartclk\0apb_pclk";
clocks = <0x06 0x07>;
};
uart@9c0a0000 {
compatible = "arm,pl011\0arm,primecell";
reg = <0x00 0x9c0a0000 0x00 0x10000>;
interrupts = <0x00 0x06 0x04>;
clock-names = "uartclk\0apb_pclk";
clocks = <0x06 0x07>;
xen,force-assign-without-iommu = "/uart@9c0a0000";
};
uart@9c0b0000 {
compatible = "arm,pl011\0arm,primecell";
reg = <0x00 0x9c0b0000 0x00 0x10000>;
interrupts = <0x00 0x07 0x04>;
clock-names = "uartclk\0apb_pclk";
clocks = <0x06 0x07>;
};
uart@9c0c0000 {
compatible = "arm,pl011\0arm,primecell";
reg = <0x00 0x9c0c0000 0x00 0x10000>;
interrupts = <0x00 0x08 0x04>;
clock-names = "uartclk\0apb_pclk";
clocks = <0x06 0x07>;
};
virtio-block@9c130000 {
compatible = "virtio,mmio";
reg = <0x00 0x9c130000 0x00 0x200>;
interrupts = <0x00 0x2a 0x04>;
};
virtio-net@9c150000 {
compatible = "virtio,mmio";
reg = <0x00 0x9c150000 0x00 0x200>;
interrupts = <0x00 0x2c 0x04>;
};
timer {
compatible = "arm,armv8-timer";
interrupts = <0x01 0x0d 0xf08 0x01 0x0e 0xf08 0x01 0x0b 0xf08 0x01 0x0a 0xf08>;
clock-frequency = <0x5f5e100>;
};
clk0 {
compatible = "fixed-clock";
#clock-cells = <0x00>;
clock-output-names = "uartclk";
clock-frequency = <0x708000>;
phandle = <0x06>;
};
clk1 {
compatible = "fixed-clock";
#clock-cells = <0x00>;
clock-output-names = "apb_pclk";
clock-frequency = <0x5f5e100>;
phandle = <0x07>;
};
aliases {
serial0 = "/uart@9c090000";
serial1 = "/uart@9c0a0000";
serial2 = "/uart@9c0b0000";
serial3 = "/uart@9c0c0000";
};
};(2) dom_linux_pass.dtb (The dtb to passthrough the FVP UART to the linux domain)
/dts-v1/;
/ {
#address-cells = <0x02>;
#size-cells = <0x02>;
gic {
#interrupt-cells = <0x03>;
interrupt-controller;
phandle = <0x01>;
};
passthrough {
compatible = "simple-bus";
ranges;
#address-cells = <0x02>;
#size-cells = <0x02>;
clk0 {
compatible = "fixed-clock";
#clock-cells = <0x00>;
clock-output-names = "uartclk";
clock-frequency = <0x708000>;
phandle = <0x06>;
};
clk1 {
compatible = "fixed-clock";
#clock-cells = <0x00>;
clock-output-names = "apb_pclk";
clock-frequency = <0x5f5e100>;
phandle = <0x07>;
};
uart@9c0b0000 {
compatible = "arm,pl011\0arm,primecell";
reg = <0x00 0x9c0b0000 0x00 0x10000>;
interrupt-parent = <0x01>;
interrupts = <0x00 0x07 0x04>;
clock-names = "uartclk\0apb_pclk";
clocks = <0x06 0x07>;
xen,path = "/uart@9c0b0000";
xen,reg = <0x00 0x9c0b0000 0x00 0x10000 0x00 0x9c0b0000>;
xen,force-assign-without-iommu;
};
};
};(3) dom_zephyr_pass.dtb (The dtb to passthrough the FVP UART to the Zephyr domain)
/dts-v1/;
/ {
#address-cells = <0x02>;
#size-cells = <0x02>;
gic {
#interrupt-cells = <0x03>;
interrupt-controller;
phandle = <0x01>;
};
passthrough {
compatible = "simple-bus";
ranges;
#address-cells = <0x02>;
#size-cells = <0x02>;
clk0 {
compatible = "fixed-clock";
#clock-cells = <0x00>;
clock-output-names = "uartclk";
clock-frequency = <0x708000>;
phandle = <0x06>;
};
clk1 {
compatible = "fixed-clock";
#clock-cells = <0x00>;
clock-output-names = "apb_pclk";
clock-frequency = <0x5f5e100>;
phandle = <0x07>;
};
uart@9c0a0000 {
compatible = "arm,pl011\0arm,primecell";
reg = <0x00 0x9c0a0000 0x00 0x10000>;
interrupt-parent = <0x01>;
interrupts = <0x00 0x06 0x04>;
clock-names = "uartclk\0apb_pclk";
clocks = <0x06 0x07>;
xen,path = "/uart@9c0a0000";
xen,reg = <0x00 0x9c0a0000 0x00 0x10000 0x00 0x9c0a0000>;
xen,force-assign-without-iommu;
};
};
};Xen Config
Some new Kconfig options have been introduced for the R82 support in this patch series.
An example of Xen config file is provided here (Please make sure to change the CONFIG_DTB_FILE to your own path before building Xen):
#
# Automatically generated file; DO NOT EDIT.
# Xen/arm 4.17-unstable Configuration
#
CONFIG_CC_IS_GCC=y
CONFIG_GCC_VERSION=100201
CONFIG_CLANG_VERSION=0
CONFIG_CC_HAS_VISIBILITY_ATTRIBUTE=y
CONFIG_ARM_64=y
CONFIG_ARM=y
CONFIG_ARCH_DEFCONFIG="arch/arm/configs/arm64_defconfig"
#
# Architecture Features
#
CONFIG_64BIT=y
CONFIG_NR_CPUS=128
# CONFIG_ACPI is not set
CONFIG_GICV3=y
# CONFIG_HAS_ITS is not set
CONFIG_HVM=y
# CONFIG_NEW_VGIC is not set
CONFIG_SBSA_VUART_CONSOLE=y
CONFIG_HARDEN_BRANCH_PREDICTOR=y
# CONFIG_TEE is not set
CONFIG_ARM_MPU=y
CONFIG_ARM64_V8R=y
CONFIG_SECURE_HYP_TIMER_WORKAROUND=y
CONFIG_XEN_START_ADDRESS=0x0
CONFIG_ARM_MPU_EL1_PROTECTION_REGIONS=32
CONFIG_ARM_SECURE_STATE=y
# end of Architecture Features
CONFIG_ARM64_HARDEN_BRANCH_PREDICTOR=y
CONFIG_ALL_PLAT=y
# CONFIG_QEMU is not set
# CONFIG_RCAR3 is not set
# CONFIG_MPSOC is not set
# CONFIG_NO_PLAT is not set
CONFIG_ALL64_PLAT=y
CONFIG_MPSOC_PLATFORM=y
#
# Common Features
#
CONFIG_HAS_DEVICE_TREE=y
CONFIG_HAS_FAST_MULTIPLY=y
CONFIG_HAS_PDX=y
# CONFIG_MEM_ACCESS is not set
CONFIG_STATIC_MEMORY=y
#
# Speculative hardening
#
CONFIG_SPECULATIVE_HARDEN_ARRAY=y
# end of Speculative hardening
CONFIG_HYPFS=y
CONFIG_HYPFS_CONFIG=y
CONFIG_XSM=y
CONFIG_XSM_FLASK=y
CONFIG_XSM_FLASK_AVC_STATS=y
CONFIG_XSM_FLASK_POLICY=y
CONFIG_XSM_SILO=y
# CONFIG_XSM_DUMMY_DEFAULT is not set
# CONFIG_XSM_FLASK_DEFAULT is not set
CONFIG_XSM_SILO_DEFAULT=y
# CONFIG_ARGO is not set
CONFIG_SCHED_CREDIT=y
CONFIG_SCHED_CREDIT2=y
CONFIG_SCHED_RTDS=y
CONFIG_SCHED_ARINC653=y
CONFIG_SCHED_NULL=y
CONFIG_SCHED_DEFAULT="credit2"
# CONFIG_LIVEPATCH is not set
# CONFIG_ENFORCE_UNIQUE_SYMBOLS is not set
# CONFIG_SUPPRESS_DUPLICATE_SYMBOL_WARNINGS is not set
CONFIG_CMDLINE=""
CONFIG_DOM0_MEM=""
CONFIG_DTB_FILE="/home/henry/run_fvp_scripts/r82/r82.dtb"
CONFIG_TRACEBUFFER=y
# end of Common Features
#
# Device Drivers
#
# CONFIG_HAS_NS16550 is not set
CONFIG_HAS_CADENCE_UART=y
CONFIG_HAS_MVEBU=y
CONFIG_HAS_MESON=y
CONFIG_HAS_PL011=y
CONFIG_HAS_SCIF=y
CONFIG_HAS_PASSTHROUGH=y
CONFIG_IOMMU_FORCE_PT_SHARE=y
# end of Device Drivers
# CONFIG_EXPERT is not set
CONFIG_UNSUPPORTED=y
#
# Debugging Options
#
CONFIG_DEBUG=y
CONFIG_DEBUG_INFO=y
CONFIG_FRAME_POINTER=y
# CONFIG_COVERAGE is not set
# CONFIG_DEBUG_LOCK_PROFILE is not set
CONFIG_DEBUG_LOCKS=y
# CONFIG_PERF_COUNTERS is not set
CONFIG_VERBOSE_DEBUG=y
CONFIG_DEVICE_TREE_DEBUG=y
CONFIG_SCRUB_DEBUG=y
# CONFIG_DEBUG_TRACE is not set
CONFIG_XMEM_POOL_POISON=y
# CONFIG_EARLY_UART_CHOICE_8250 is not set
# CONFIG_EARLY_UART_CHOICE_CADENCE is not set
# CONFIG_EARLY_UART_CHOICE_MESON is not set
# CONFIG_EARLY_UART_CHOICE_MVEBU is not set
# CONFIG_EARLY_UART_CHOICE_PL011 is not set
# CONFIG_EARLY_UART_CHOICE_SCIF is not set
# CONFIG_EARLY_PRINTK_BRCM is not set
# CONFIG_EARLY_PRINTK_DRA7 is not set
# CONFIG_EARLY_PRINTK_FASTMODEL is not set
CONFIG_EARLY_PRINTK_FASTMODEL_BASER=y
# CONFIG_EARLY_PRINTK_HIKEY960 is not set
# CONFIG_EARLY_PRINTK_JUNO is not set
# CONFIG_EARLY_PRINTK_LAGER is not set
# CONFIG_EARLY_PRINTK_MIDWAY is not set
# CONFIG_EARLY_PRINTK_MVEBU is not set
# CONFIG_EARLY_PRINTK_OMAP5432 is not set
# CONFIG_EARLY_PRINTK_RCAR3 is not set
# CONFIG_EARLY_PRINTK_SEATTLE is not set
# CONFIG_EARLY_PRINTK_SUN6I is not set
# CONFIG_EARLY_PRINTK_SUN7I is not set
# CONFIG_EARLY_PRINTK_THUNDERX is not set
# CONFIG_EARLY_PRINTK_VEXPRESS is not set
# CONFIG_EARLY_PRINTK_XGENE_MCDIVITT is not set
# CONFIG_EARLY_PRINTK_XGENE_STORM is not set
# CONFIG_EARLY_PRINTK_ZYNQMP is not set
CONFIG_EARLY_UART_PL011=y
CONFIG_EARLY_PRINTK=y
CONFIG_EARLY_UART_BASE_ADDRESS=0x9c090000
CONFIG_EARLY_UART_PL011_BAUD_RATE=115200
CONFIG_EARLY_UART_INIT=y
CONFIG_EARLY_PRINTK_INC="debug-pl011.inc"
# end of Debugging OptionsShell script to run FVP
Note that currently there is no PV block device in the guest, and therefore the Linux domain should use a kernel which contains an initramfs.
#!/bin/bash
set -ex
PLATFORM=BaseR_AEMv8R
FVP_PATH=/home/henry/FVPs
DEMO_PATH=/home/henry/run_fvp_scripts/r82
MODEL=${FVP_PATH}/FVP_${PLATFORM}
KERNEL_LINUX=${DEMO_PATH}/Image-initramfs-5.14
KERNEL_ZEPHYR=${DEMO_PATH}/zephyr.bin
DOM_LINUX_DTB=${DEMO_PATH}/dom_linux_pass.dtb
DOM_ZEPHYR_DTB=${DEMO_PATH}/dom_zephyr_pass.dtb
XEN=${DEMO_PATH}/xen/xen/xen
CLUSTER0_NUM_CORES=${CLUSTER0_NUM_CORES:-1}
CLUSTER1_NUM_CORES=${CLUSTER1_NUM_CORES:-1}
SECURE_MEMORY=${SECURE_MEMORY:-0}
### Prepare network
#FVP_IF=FVP_TAP0
#FVP_SWITCH=FVP_vSwtich
#FVP_IP=192.168.200.1
#FVP_IF_STATUS=$(ifconfig -a | grep ${FVP_IF})
#FVP_SWITCH_STATUS=$(ifconfig -a | grep ${FVP_SWITCH})
#if [ -z "${FVP_IF_STATUS}" ] || [ -z "${FVP_SWITCH_STATUS}" ]
#then
# sudo -S ip tuntap add dev ${FVP_IF} mode tap user henry
# sudo -S brctl addbr ${FVP_SWITCH}
# sudo -S brctl addif ${FVP_SWITCH} ${FVP_IF}
# sudo -S ifconfig ${FVP_SWITCH} ${FVP_IP} netmask 255.255.255.0
# sudo -S ifconfig ${FVP_IF} up
#fi
AEMv8A_CLUSTER0_CFG="-C cluster0.NUM_CORES=$CLUSTER0_NUM_CORES -C cluster0.has_nested_virtualization=1"
AEMv8A_CLUSTER1_CFG="-C cluster1.NUM_CORES=$CLUSTER1_NUM_CORES -C cluster1.has_nested_virtualization=1"
CADI_DEBUG="-S -C REMOTE_CONNECTION.CADIServer.enable_remote_cadi=true -C REMOTE_CONNECTION.CADIServer.listen_address=0.0.0.0 -C REMOTE_CONNECTION.CADIServer.port=18888"
IRIS_DEBUG="--iris-server --iris-allow-remote"
UART0_CFG="-C bp.pl011_uart0.untimed_fifos=1 -C bp.pl011_uart0.unbuffered_output=1"
TERM0_CFG="-C bp.terminal_0.mode=telnet -C bp.terminal_0.quiet=0 -C bp.terminal_0.start_port=5000 -C bp.terminal_0.start_telnet=0"
UART1_CFG="-C bp.pl011_uart1.untimed_fifos=1 -C bp.pl011_uart1.unbuffered_output=1"
TERM1_CFG="-C bp.terminal_1.mode=telnet -C bp.terminal_1.quiet=0 -C bp.terminal_1.start_port=5001 -C bp.terminal_1.start_telnet=0"
UART2_CFG="-C bp.pl011_uart2.untimed_fifos=1 -C bp.pl011_uart2.unbuffered_output=1"
TERM2_CFG="-C bp.terminal_2.mode=telnet -C bp.terminal_2.quiet=0 -C bp.terminal_2.start_port=5002 -C bp.terminal_2.start_telnet=0"
UART3_CFG="-C bp.pl011_uart3.untimed_fifos=1 -C bp.pl011_uart3.unbuffered_output=1"
TERM3_CFG="-C bp.terminal_3.mode=telnet -C bp.terminal_3.quiet=0 -C bp.terminal_3.start_port=5003 -C bp.terminal_3.start_telnet=0"
VIRTIO_LAN_CFG="-C bp.virtio_net.enabled=1 \
-C bp.virtio_net.hostbridge.interfaceName=FVP_TAP0 \
-C bp.virtio_net.mac_address=52:54:aa:51:ff:f1 \
-C bp.virtio_net.transport=legacy"
SMSC_LAN_CFG="-C bp.smsc_91c111.enabled=1 \
-C bp.smsc_91c111.cache_size=65536 \
-C bp.smsc_91c111.mac_address=52:54:aa:51:ff:f1 \
-C bp.hostbridge.interfaceName=FVP_TAP0 \
-C bp.smsc_91c111.promiscuous=0"
${MODEL} \
-C semihosting-enable=0 \
-C cache_state_modelled=1 \
-C pctl.startup=0.0.0.0 \
-C bp.vis.disable_visualisation=1 \
-C bp.refcounter.base_frequency=100000000 \
-C bp.refcounter.use_real_time=0 \
-C bp.refcounter.non_arch_start_at_default=1 \
-C bp.ve_sysregs.mmbSiteDefault=0 \
-C bp.ve_sysregs.exit_on_shutdown=1 \
-C gic_distributor.GICD_CTLR-DS-1-means-secure-only=1 \
-C gic_distributor.has-two-security-states=0 \
-C cluster0.has_pl2=1 \
-C cluster0.has_aarch64=1 \
-C cluster0.VMSA_supported=1 \
${UART0_CFG} ${UART1_CFG} ${UART2_CFG} ${UART3_CFG} \
${TERM0_CFG} ${TERM1_CFG} ${TERM2_CFG} ${TERM3_CFG} \
--data cluster0.cpu0=${XEN}@0x0 \
--data cluster0.cpu0=${KERNEL_LINUX}@0x11000000 \
--data cluster0.cpu0=${DOM_LINUX_DTB}@0x10FF0000 \
--data cluster0.cpu0=${KERNEL_ZEPHYR}@0x15000000 \
--data cluster0.cpu0=${DOM_ZEPHYR_DTB}@0x19FF0000Start FVP
Assuming that the shell script is named as r82_fvp_run.sh, use below command to start the FVP:
$ cd <DEMO_DIR_NAME>
$ ./r82_fvp_run.shThe r82_fvp_run.sh will start 4 telnet ports (500[0123]):
5000: Output of Xen
5001: Output of DomU2
5002: Output of DomU1Users can connect these ports using the command below:
$ telnet localhost 500[0123]The default username of the Linux guest is root without password.