target/sh4: invalidating a 64 KiB or 1 MiB TLB entry only flushes its first 4 KiB

Host environment

  • Operating system: macOS 14.8.7 (build 23J520)
  • OS/kernel version: Darwin 23.6.0 arm64
  • Architecture: arm64 (Apple Silicon, TCG)
  • QEMU flavor: qemu-system-sh4
  • QEMU version: 11.1.50 (v11.1.0-1860-gefa3b9d5ac), built from master; also 11.0.2 (Homebrew)
  • QEMU command line: 
    ./qemu-system-sh4 -M r2d -display none -serial null -serial stdio \
    -kernel tlbstale.bin

Emulated/Virtualized environment

  • Operating system: none, bare-metal reproducer; also seen with a Linux guest
  • OS/kernel version: n/a
  • Architecture: sh4 (r2d, SH7751R)

Description of problem

Disclosure: AI-assisted tooling helped prepare this report. I ran and verified the tests below.

When a valid UTLB or ITLB entry is invalidated or overwritten, target/sh4 flushes only the softmmu page at the entry's VPN. For a 64 KiB or 1 MiB entry the other 4 KiB pages keep translating to the old physical page. All seven invalidation sites in target/sh4/helper.c do this, including ldtlb and the associative UTLB address array write that Linux uses for TLB flushes.

In a Linux guest with 64 KiB HugeTLB pages, a process can still read a huge page after munmap() instead of getting SIGSEGV. That guest needs two SH kernel fixes that are not upstream yet, so the reproducer below is bare-metal.

Steps to reproduce

  1. Save these as start.S, tlbstale.c and link.ld:

    	.section .text.start
	.globl _start
_start:
	mov.l	stack, r15
	mov.l	main_addr, r0
	jsr	@r0
	nop
1:	bra	1b
	nop
	.align 2
stack:	.long 0xacf00000
main_addr: .long main
	.section .note.GNU-stack,"",%progbits
    #define MMUCR	(*(volatile unsigned int *)0xff000010)
#define PTEH	(*(volatile unsigned int *)0xff000000)
#define PTEL	(*(volatile unsigned int *)0xff000004)
/* UTLB address array, associative write (what Linux's local_flush_tlb_one() uses) */
#define UTLB_ADDR_ASSOC	(*(volatile unsigned int *)0xf6000080)
#define SCIF_FTDR	(*(volatile unsigned char *)0xffe8000c)

#define MMUCR_AT	0x001
#define MMUCR_TI	0x004
#define MMUCR_URC(n)	((n) << 10)

#define PTEL_V		0x100
#define PTEL_SZ1	0x080	/* SZ1:SZ0 = 10 -> 64 KiB */
#define PTEL_PR_RW	0x060
#define PTEL_SZ0	0x010	/* SZ1:SZ0 = 01 -> 4 KiB */
#define PTEL_D		0x004

#define VA	0x10000000u
#define PA_A	0x0d000000u	/* SDRAM, 64 KiB aligned */
#define PA_B	0x0d100000u
#define P2(pa)	((volatile unsigned int *)((pa) | 0xa0000000u))

static void putc(char c)
{
	SCIF_FTDR = c;
}

static void puts(const char *s)
{
	while (*s)
		putc(*s++);
}

static void puthex(unsigned int v)
{
	for (int i = 28; i >= 0; i -= 4)
		putc("0123456789abcdef"[(v >> i) & 0xf]);
}

static void ldtlb(unsigned int urc, unsigned int va, unsigned int pa,
		  unsigned int sz)
{
	MMUCR = MMUCR_AT | MMUCR_URC(urc);
	PTEH = va;			/* ASID 0 */
	PTEL = pa | PTEL_V | PTEL_PR_RW | PTEL_D | sz;
	__asm__ volatile("ldtlb\n\tnop" ::: "memory");
}

static unsigned int rd(unsigned int off)
{
	return *(volatile unsigned int *)(VA + off);
}

static void check(unsigned int off, unsigned int want)
{
	unsigned int got = rd(off);

	puts("  VA+0x");
	puthex(off);
	puts(" = ");
	puthex(got);
	puts(got == want ? "  ok\r\n" : "  STALE (still page A)\r\n");
}

/* invalidate: 0 = associative UTLB address array write, 1 = ldtlb overwrite */
static void run(const char *name, unsigned int sz, int how)
{
	MMUCR = MMUCR_AT | MMUCR_TI;	/* start from an empty TLB */
	puts(name);
	puts("\r\n");

	ldtlb(5, VA, PA_A, sz);
	rd(0x0000);
	if (sz == PTEL_SZ1)
		rd(0x1000);		/* fill the softmmu TLB for a second 4 KiB page */

	if (how == 0) {
		UTLB_ADDR_ASSOC = VA;	/* V=0, ASID 0: invalidate the entry */
		ldtlb(9, VA, PA_B, sz);
	} else {
		ldtlb(5, VA, PA_B, sz);	/* overwrite the same UTLB slot */
	}
	check(0x0000, 0xb0000000);
	if (sz == PTEL_SZ1)
		check(0x1000, 0xb0001000);
}

void main(void)
{
	for (unsigned int off = 0; off < 0x10000; off += 0x1000) {
		*P2(PA_A + off) = 0xa0000000 | off;
		*P2(PA_B + off) = 0xb0000000 | off;
	}

	puts("sh4 UTLB large-page invalidation test\r\n");
	run("64K entry, invalidated by associative UTLB write:", PTEL_SZ1, 0);
	run("64K entry, replaced by ldtlb into the same slot:", PTEL_SZ1, 1);
	run("4K entry, invalidated by associative UTLB write (control):", PTEL_SZ0, 0);
	puts("done\r\n");
	for (;;)
		__asm__ volatile("sleep");
}
    OUTPUT_ARCH(sh)
ENTRY(_start)
SECTIONS {
	. = 0xac800000;
	.text : { *(.text.start) *(.text*) }
	.rodata : { *(.rodata*) }
	.data : { *(.data*) }
	.bss : { *(.bss*) *(COMMON) }
}

  2. Build it with an sh4 cross compiler (I used gcc 15.2.0 from kernel.org crosstool):

    sh4-linux-gcc -m4 -ml -O2 -Wall -ffreestanding -nostdlib -nostartfiles \
    -fno-builtin -Wl,-T,link.ld -o tlbstale.elf start.S tlbstale.c -lgcc
sh4-linux-objcopy -O binary tlbstale.elf tlbstale.bin

  3. Run it with the command line above and observe:

    sh4 UTLB large-page invalidation test
64K entry, invalidated by associative UTLB write:
  VA+0x00000000 = b0000000  ok
  VA+0x00001000 = a0001000  STALE (still page A)
64K entry, replaced by ldtlb into the same slot:
  VA+0x00000000 = b0000000  ok
  VA+0x00001000 = a0001000  STALE (still page A)
4K entry, invalidated by associative UTLB write (control):
  VA+0x00000000 = b0000000  ok
done

    Expected: every read returns page B, as the 4 KiB control case does.