retain freed regions in a bounded pool for reuse (e8bac908) · Commits · cznic / memory

all_test.go

+74 −2

Original line number	Diff line number	Diff line
		@@ -353,6 +353,23 @@ func TestUMalloc(t *testing.T) {
		}
		}

		func TestUMallocSizeOverflow(t *testing.T) {
		var alloc Allocator

		defer alloc.Close()

		p, err := alloc.UintptrMalloc(int(^uint(0) >> 1))
		if err == nil {
		t.Fatalf("UintptrMalloc returned %#x, want an error", p)
		}
		if p != 0 {
		t.Fatalf("UintptrMalloc returned %#x with error %v", p, err)
		}
		if alloc.Allocs != 0 \|\| alloc.Mmaps != 0 \|\| alloc.Bytes != 0 \|\| len(alloc.regs) != 0 {
		t.Fatalf("%+v", alloc)
		}
		}

		func test1(t *testing.T, max int) {
		var alloc Allocator

		@@ -743,12 +760,13 @@ func TestTrimClasses(t *testing.T) {
		}
		}

		func TestNoTrimBigPage(t *testing.T) {
		func TestTrimBigPage(t *testing.T) {
		var alloc Allocator

		defer alloc.Close()

		// A page backing a single big allocation is not retained, no Trim needed.
		// A page backing a single big allocation is retained for reuse;
		// Trim returns it.
		p, err := alloc.UintptrMalloc(maxSlotSize + 1)
		if err != nil {
		t.Fatal(err)
		@@ -762,6 +780,14 @@ func TestNoTrimBigPage(t *testing.T) {
		t.Fatal(err)
		}

		if g, e := len(alloc.regs), 1; g != e {
		t.Fatalf("regs %v, want %v: freed region was unmapped, not retained", g, e)
		}

		if err := alloc.Trim(); err != nil {
		t.Fatal(err)
		}

		if alloc.Allocs != 0 \|\| alloc.Mmaps != 0 \|\| alloc.Bytes != 0 \|\| len(alloc.regs) != 0 {
		t.Fatalf("%+v", alloc)
		}
		@@ -972,3 +998,49 @@ func benchmarkUintptrMalloc(b *testing.B, size int) {
		func BenchmarkUintptrMalloc16(b *testing.B) { benchmarkUintptrMalloc(b, 1<<4) }
		func BenchmarkUintptrMalloc32(b *testing.B) { benchmarkUintptrMalloc(b, 1<<5) }
		func BenchmarkUintptrMalloc64(b *testing.B) { benchmarkUintptrMalloc(b, 1<<6) }

		func TestFreedPool(t *testing.T) {
		var alloc Allocator

		defer alloc.Close()

		p, err := alloc.UintptrMalloc(1 << 20)
		if err != nil {
		t.Fatal(err)
		}

		mmaps := alloc.Mmaps
		regs := len(alloc.regs)
		if err := alloc.UintptrFree(p); err != nil {
		t.Fatal(err)
		}

		if g, e := len(alloc.regs), regs; g != e {
		t.Fatalf("regs %v, want %v: freed region was unmapped, not retained", g, e)
		}

		q, err := alloc.UintptrMalloc(1 << 20)
		if err != nil {
		t.Fatal(err)
		}

		if q != p {
		t.Fatalf("got %#x, want %#x: allocation did not reuse the retained region", q, p)
		}

		if g, e := alloc.Mmaps, mmaps; g != e {
		t.Fatalf("Mmaps %v, want %v", g, e)
		}

		if err := alloc.UintptrFree(q); err != nil {
		t.Fatal(err)
		}

		if err := alloc.Trim(); err != nil {
		t.Fatal(err)
		}

		if alloc.Allocs != 0 \|\| alloc.Mmaps != 0 \|\| alloc.Bytes != 0 \|\| len(alloc.regs) != 0 {
		t.Fatalf("%+v", alloc)
		}
		}

memory.go

+106 −12

Original line number	Diff line number	Diff line
		@@ -20,11 +20,16 @@
		//
		// # Page retention
		//
		// Memory is acquired from the OS in 64 KiB units. An Allocator retains at most
		// one empty page per size class instead of unmapping it as soon as its last
		// slot is freed, so an allocation pattern that repeatedly drains a size class
		// no longer pays a mmap/munmap round trip per turnover. Use Allocator.Trim to
		// hand the retained pages back.
		// Memory is acquired from the OS in 64 KiB units or, for larger
		// allocations, in units rounded up to a power of two on 64-bit hosts.
		// An Allocator retains empty regions for reuse instead of unmapping
		// them as soon as their last slot is freed: each size class keeps its
		// current carve-target page, and beyond that, retired regions of any
		// size go into a pool bounded by 4 MiB plus the high-water mark of the
		// live mapping. An allocation pattern that repeatedly drains a size
		// class, or repeatedly frees and reallocates large buffers, no longer
		// pays a mmap/munmap round trip per turnover. Use Allocator.Trim to
		// hand the retained regions back.
		//
		// # Benchmarks
		//
		@@ -102,7 +107,7 @@ type page struct {
		// exported counters are updated only when build tag memory.counters is
		// present.
		//
		// An Allocator retains at most one empty page per size class, see Trim.
		// An Allocator retains empty regions for reuse, see Trim.
		type Allocator struct {
		Allocs int // # of allocs.
		Bytes int // Asked from OS.
		@@ -111,9 +116,58 @@ type Allocator struct {
		Mmaps int // Asked from OS.
		pages [64]uintptr // *page
		regs map[uintptr]struct{} // map[*page]struct{}
		freed map[int][]uintptr // empty regions retained for reuse, keyed by region size
		freedSize int // total bytes in freed
		live int // bytes mapped and not in freed
		hiLive int // high-water mark of live
		}

		// maxFreedSize returns the bound on the total bytes retained in
		// a.freed: a fixed floor plus the high-water mark of the live mapping.
		// The high-water mark rather than the current live size matters for
		// large transient buffers: when one is freed, live has already shrunk
		// by the buffer, and it is exactly the region most worth retaining.
		func (a *Allocator) maxFreedSize() int { return 4<<20 + a.hiLive }

		// addLive adds n bytes to the live mapping size, tracking its
		// high-water mark.
		func (a *Allocator) addLive(n int) {
		a.live += n
		if a.live > a.hiLive {
		a.hiLive = a.live
		}
		}

		// mmapSize returns the region size class that backs a request for size
		// bytes: requests up to pageSize share the single pageSize class, and
		// on 64-bit larger requests are rounded up to a power of two. The
		// extra address space is never touched and so costs nothing, and
		// collapsing sizes into classes makes reuse via a.freed much more
		// likely.
		func mmapSize(size int) int {
		switch {
		case size <= pageSize && pageSize%osPageSize == 0:
		return pageSize
		case size > pageSize && bits.UintSize == 64:
		return roundup(1<<bits.Len(uint(size-1)), osPageSize)
		default:
		return roundup(size, osPageSize)
		}
		}

		func (a Allocator) mmap(size int) (uintptr / page /, error) {
		size = mmapSize(size)
		if s := a.freed[size]; len(s) != 0 {
		p := s[len(s)-1]
		a.freed[size] = s[:len(s)-1]
		a.freedSize -= size
		a.addLive(size)
		pg := (*page)(unsafe.Pointer(p))
		pg.brk = 0
		pg.used = 0
		return p, nil
		}

		p, size, err := mmap(size)
		if err != nil {
		return 0, err
		@@ -125,6 +179,12 @@ func (a Allocator) mmap(size int) (uintptr / page /, error) {
		//
		// Related: This is a consequence of fixing the bigsort.test failures on
		// linux/s390x, see: https://gitlab.com/cznic/sqlite/-/issues/207
		return a.reg(p, size), nil
		}

		// reg registers the freshly mapped size-byte region at p.
		func (a *Allocator) reg(p uintptr, size int) uintptr {
		a.addLive(size)
		if counters {
		a.Mmaps++
		a.Bytes += size
		@@ -134,7 +194,7 @@ func (a Allocator) mmap(size int) (uintptr / page /, error) {
		}
		(*page)(unsafe.Pointer(p)).size = size
		a.regs[p] = struct{}{}
		return p, nil
		return p
		}

		func (a Allocator) newPage(size int) (uintptr / page /, error) {
		@@ -173,6 +233,26 @@ func (a Allocator) unmap(p uintptr / page /) error {
		return unmap(p, size)
		}

		// release retires the empty region at p: it is retained in a.freed for
		// reuse, unless the retained total is at its bound, in which case the
		// region is returned to the OS. Retained regions stay in a.regs and in
		// the counters; Trim and Close return them to the OS.
		func (a Allocator) release(p uintptr / page /) error {
		size := (*page)(unsafe.Pointer(p)).size
		if a.freedSize+size <= a.maxFreedSize() {
		if a.freed == nil {
		a.freed = map[int][]uintptr{}
		}
		a.freed[size] = append(a.freed[size], p)
		a.freedSize += size
		a.live -= size
		return nil
		}

		a.live -= size
		return a.unmap(p)
		}

		// UintptrCalloc is like Calloc except it returns an uintptr.
		func (a *Allocator) UintptrCalloc(size int) (r uintptr, err error) {
		if trace {
		@@ -208,7 +288,7 @@ func (a *Allocator) UintptrFree(p uintptr) (err error) {
		pg := p &^ uintptr(pageMask)
		log := (*page)(unsafe.Pointer(pg)).log
		if log == 0 {
		return a.unmap(pg)
		return a.release(pg)
		}

		(*node)(unsafe.Pointer(p)).prev = 0
		@@ -250,7 +330,7 @@ func (a *Allocator) UintptrFree(p uintptr) (err error) {
		return nil
		}

		return a.unmap(pg)
		return a.release(pg)
		}

		// UintptrMalloc is like Malloc except it returns an uinptr.
		@@ -267,6 +347,9 @@ func (a *Allocator) UintptrMalloc(size int) (r uintptr, err error) {
		if size == 0 {
		return 0, nil
		}
		if size > int(^uint(0)>>1)-int(headerSize) {
		return 0, fmt.Errorf("memory: allocation size %d is too large", size)
		}

		if counters {
		a.Allocs++
		@@ -434,9 +517,10 @@ func (a *Allocator) Realloc(b []byte, size int) (r []byte, err error) {
		return (*rawmem)(unsafe.Pointer(p))[:size:usableSize(p)], nil
		}

		// Trim returns to the OS the empty pages a retains for reuse, at most one per
		// size class. Pages still having live allocations are not affected and a stays
		// ready for use.
		// Trim returns to the OS the empty regions a retains for reuse: each
		// size class's empty carve-target page and the bounded pool of retired
		// regions. Pages still having live allocations are not affected and a
		// stays ready for use.
		//
		// Trim is never necessary for correctness. It trades a smaller resident set
		// now for the cost of mapping those pages again later.
		@@ -453,10 +537,20 @@ func (a *Allocator) Trim() (err error) {
		}

		a.pages[log] = 0
		a.live -= (*page)(unsafe.Pointer(pg)).size
		if e := a.unmap(pg); e != nil && err == nil {
		err = e
		}
		}
		for _, s := range a.freed {
		for _, pg := range s {
		if e := a.unmap(pg); e != nil && err == nil {
		err = e
		}
		}
		}
		a.freed = nil
		a.freedSize = 0
		return err
		}