Commit 2bbe2e75 authored by Josh Bleecher Snyder's avatar Josh Bleecher Snyder
Browse files

acquire slab regions in batches of 16 

Aligning a fresh mapping to a pageSize boundary costs up to two munmap
calls for the trimmed-off ends, so a cold allocator paid roughly three
syscalls per 64 KiB slab region. When a single pageSize region is
needed and the pool is empty, acquire slabBatch of them in one mapping
and put the rest into the pool: three syscalls amortized over sixteen
regions, and the pooled remainder immediately serves the other size
classes, which all share the pageSize region class.

Unix only: VirtualFree with MEM_RELEASE frees only whole VirtualAlloc
allocations, never a sub-range, so on Windows a carved region could not
be returned individually (canCarve).

This trims the mapping churn of a cold or recently Trimmed allocator -
a process that touches eleven size classes now issues one mmap instead
of eleven mmaps and up to twenty-two trim munmaps. Steady-state
throughput of long benchmark runs is unchanged: once the pool is warm,
fresh mappings are rare either way.

TestTrim and TestTrimClasses asserted exactly one mapping per class
and now assert stability of the mapping count while cycling, plus the
batch-rounded count; TestSlabBatch covers the batch acquisition
itself.
parent e8bac908

all_test.go

+89 −7
Original line number Diff line number Diff line
@@ -660,11 +660,13 @@ func TestTrim(t *testing.T) { 
		t.Fatal(err)

	}



	regs := len(alloc.regs)

	mmaps := alloc.Mmaps

	if err := alloc.UintptrFree(p); err != nil {

		t.Fatal(err)

	}



	if g, e := len(alloc.regs), 1; g != e {

	if g, e := len(alloc.regs), regs; g != e {

		t.Fatalf("mapped pages %v, expected %v", g, e)

	}
@@ -678,12 +680,12 @@ func TestTrim(t *testing.T) { 
			t.Fatal(err)

		}

	}

	if g, e := len(alloc.regs), 1; g != e {

	if g, e := len(alloc.regs), regs; g != e {

		t.Fatalf("mapped pages %v, expected %v", g, e)

	}



	if counters && alloc.Mmaps != 1 {

		t.Fatalf("mmaps %v, expected 1", alloc.Mmaps)

	if counters && alloc.Mmaps != mmaps {

		t.Fatalf("mmaps %v, expected %v", alloc.Mmaps, mmaps)

	}



	// A page with a live slot is not released by Trim.
@@ -747,8 +749,14 @@ func TestTrimClasses(t *testing.T) { 
		t.Fatalf("retained classes %v, expected %v", g, e)

	}



	if g, e := len(alloc.regs), classes; g != e {

		t.Fatalf("mapped pages %v, retained classes %v", g, e)

	e := classes

	if canCarve {

		// Slab pages are acquired slabBatch at a time; the classes

		// carve their pages from whole batches.

		e = roundup(classes, slabBatch)

	}

	if g := len(alloc.regs); g != e {

		t.Fatalf("mapped pages %v, expected %v (retained classes %v)", g, e, classes)

	}



	if err := alloc.Trim(); err != nil {
@@ -776,11 +784,12 @@ func TestTrimBigPage(t *testing.T) { 
		t.Fatal("not a dedicated page")

	}



	regs := len(alloc.regs)

	if err := alloc.UintptrFree(p); err != nil {

		t.Fatal(err)

	}



	if g, e := len(alloc.regs), 1; g != e {

	if g, e := len(alloc.regs), regs; g != e {

		t.Fatalf("regs %v, want %v: freed region was unmapped, not retained", g, e)

	}
@@ -1044,3 +1053,76 @@ func TestFreedPool(t *testing.T) { 
		t.Fatalf("%+v", alloc)

	}

}



func TestSlabBatch(t *testing.T) {

	if !canCarve {

		t.Skip("carving not supported on this platform")

	}



	var alloc Allocator



	defer alloc.Close()



	p, err := alloc.UintptrMalloc(1)

	if err != nil {

		t.Fatal(err)

	}



	if g, e := len(alloc.regs), slabBatch; g != e {

		t.Fatalf("regs %v, want %v: pageSize regions were not acquired in a batch", g, e)

	}



	if err := alloc.UintptrFree(p); err != nil {

		t.Fatal(err)

	}



	if err := alloc.Trim(); err != nil {

		t.Fatal(err)

	}



	if alloc.Allocs != 0 || alloc.Mmaps != 0 || alloc.Bytes != 0 || len(alloc.regs) != 0 {

		t.Fatalf("%+v", alloc)

	}

}



func TestSlabBatchFreedPoolBound(t *testing.T) {

	if !canCarve {

		t.Skip("carving not supported on this platform")

	}



	var alloc Allocator



	defer alloc.Close()



	for _, size := range []int{16 << 20, 4 << 20} {

		p, err := alloc.UintptrMalloc(size - int(headerSize))

		if err != nil {

			t.Fatal(err)

		}

		if err := alloc.UintptrFree(p); err != nil {

			t.Fatal(err)

		}

	}



	if room := alloc.maxFreedSize() - alloc.freedSize; room >= (slabBatch-1)*pageSize {

		t.Fatalf("test setup left %v bytes in pool, enough for a slab batch", room)

	}



	p, err := alloc.UintptrMalloc(1)

	if err != nil {

		t.Fatal(err)

	}

	if alloc.freedSize > alloc.maxFreedSize() {

		t.Fatalf("freed pool size %v exceeds bound %v", alloc.freedSize, alloc.maxFreedSize())

	}



	if err := alloc.UintptrFree(p); err != nil {

		t.Fatal(err)

	}

	if err := alloc.Trim(); err != nil {

		t.Fatal(err)

	}

	if alloc.Allocs != 0 || alloc.Mmaps != 0 || alloc.Bytes != 0 || len(alloc.regs) != 0 {

		t.Fatalf("%+v", alloc)

	}

}

memory.go

+41 −6
Original line number Diff line number Diff line
@@ -138,6 +138,9 @@ func (a *Allocator) addLive(n int) { 
	}

}



// canRetain reports whether a.freed has room to retain size more bytes.

func (a *Allocator) canRetain(size int) bool { return a.freedSize+size <= a.maxFreedSize() }



// mmapSize returns the region size class that backs a request for size

// bytes: requests up to pageSize share the single pageSize class, and

// on 64-bit larger requests are rounded up to a power of two. The
@@ -155,6 +158,10 @@ func mmapSize(size int) int { 
	}

}



// slabBatch is how many pageSize regions a single mapping acquires

// when a pageSize region is needed and the pool is empty.

const slabBatch = 16



func (a *Allocator) mmap(size int) (uintptr /* *page */, error) {

	size = mmapSize(size)

	if s := a.freed[size]; len(s) != 0 {
@@ -168,6 +175,29 @@ func (a *Allocator) mmap(size int) (uintptr /* *page */, error) { 
		return p, nil

	}



	// Aligning a fresh mapping to a pageSize boundary costs up to two

	// munmap calls for the trimmed-off ends, so when a single pageSize

	// region is needed, acquire a batch in one mapping and put the

	// rest into the pool: three syscalls amortized over slabBatch

	// regions instead of per region.

	if canCarve && size == pageSize {

		if p, n, err := mmap(slabBatch * pageSize); err == nil {

			if n%pageSize == 0 && a.canRetain(n-pageSize) {

				for off := pageSize; off < n; off += pageSize {

					q := p + uintptr(off)

					a.reg(q, pageSize)

					a.retain(q, pageSize)

				}

				return a.reg(p, pageSize), nil

			}



			if err := unmap(p, n); err != nil {

				return 0, err

			}

		}

		// Fall through and map a single region.

	}



	p, size, err := mmap(size)

	if err != nil {

		return 0, err
@@ -239,18 +269,23 @@ func (a *Allocator) unmap(p uintptr /* *page */) error { 
// the counters; Trim and Close return them to the OS.

func (a *Allocator) release(p uintptr /* *page */) error {

	size := (*page)(unsafe.Pointer(p)).size

	if a.freedSize+size <= a.maxFreedSize() {

	if a.canRetain(size) {

		a.retain(p, size)

		return nil

	}



	a.live -= size

	return a.unmap(p)

}



// retain puts the empty size-byte region at p into a.freed.

func (a *Allocator) retain(p uintptr, size int) {

	if a.freed == nil {

		a.freed = map[int][]uintptr{}

	}

	a.freed[size] = append(a.freed[size], p)

	a.freedSize += size

	a.live -= size

		return nil

	}



	a.live -= size

	return a.unmap(p)

}



// UintptrCalloc is like Calloc except it returns an uintptr.

mmap_unix.go

+5 −0
Original line number Diff line number Diff line
@@ -16,6 +16,11 @@ import ( 


const pageSizeLog = 16



// canCarve reports that unmap can release any page-aligned sub-range

// of a mapping, so a large mapping can be carved into independently

// releasable pageSize regions.

const canCarve = true



var (

	osPageMask = osPageSize - 1

	osPageSize = os.Getpagesize()

mmap_windows.go

+4 −0
Original line number Diff line number Diff line
@@ -21,6 +21,10 @@ const ( 


const pageSizeLog = 16



// canCarve is false on Windows: VirtualFree with MEM_RELEASE frees

// only whole VirtualAlloc allocations, never a sub-range.

const canCarve = false



var (

	modkernel32      = syscall.NewLazySystemDLL("kernel32.dll")

	osPageMask       = osPageSize - 1