Commit 697300ff authored by cznic's avatar cznic
Browse files

Merge branch 'dbstatus-binding' into 'master' 

sqlite: add DBStatus wrapper for sqlite3_db_status

See merge request !132
parents 6a28fe7d 759639fa

CHANGELOG.md

+2 −0
Original line number Diff line number Diff line
@@ -13,6 +13,8 @@ 
     - See [GitLab merge request #130](https://gitlab.com/cznic/sqlite/-/merge_requests/130), thanks Ian Chechin!

     - Make `modernc.org/sqlite/pcache` `-race`-clean under SQLite's `cache=shared` mode. The pool already runs correctly under shared-cache because every callback into a given `Cache` is serialised internally by SQLite's `sqlite3BtreeEnter` on the `BtShared` mutex; verified empirically with a lock-free in-flight probe (max-in-flight = 1 on the canonical two-connection workload, 4 on a positive control with goroutines hitting the cache directly). However the Go race detector does not recognise SQLite's libc mutex as a happens-before edge and reports false-positive races on `Fetch` vs `Unpin` reads/writes of the per-cache state, which surfaces as `DATA RACE` failures for any user who registers the pool and runs their suite under `-race`. A `sync.Mutex` on the `cache` type is now taken on every public method (`SetSize`, `PageCount`, `Fetch`, `Unpin`, `Rekey`, `Truncate`, `Destroy`, `Shrink`), always. On the common non-shared-cache path the lock is uncontended (one atomic CAS per Lock/Unlock pair, negligible next to the SQLite work it bookends); on the shared-cache path it just rubber-stamps the order SQLite's `BtShared` mutex already established. A new `e2e_test.go` `TestSharedCacheTwoConns_Integrity` drives two `sql.Conn` against the same `cache=shared` URI with concurrent writers and asserts `PRAGMA integrity_check = ok` under `-race`; passes cleanly with the lock, would surface the false-positive without it. Design notes live in `pcache/sharing.go`.

     - See [GitLab merge request #131](https://gitlab.com/cznic/sqlite/-/merge_requests/131), thanks Ian Chechin!

     - Add a Go wrapper for `sqlite3_db_status`, the per-connection runtime counters (cache hit/miss/write/spill rates, schema and prepared-statement memory, lookaside usage, deferred foreign keys). `DBStatus` is an interface implemented by the driver connection and reached through the `database/sql` escape hatch `(*sql.Conn).Raw()`, mirroring the existing `FileControl` surface; `DBStatusOp` is a distinct typed enum of the `SQLITE_DBSTATUS_*` verbs so a counter from a different op family will not compile in its place. `Status(op, reset)` returns the `(current, high)` pair and optionally resets the counter. This also lets `modernc.org/sqlite/pcache` measure real I/O instead of the `EasyRefusals` proxy: the new `BenchmarkPoolSpillIO` reads the pager-level `SQLITE_DBSTATUS_CACHE_SPILL`/`_CACHE_WRITE` counters, which the pager maintains identically for pcache1 and the pool, making the pcache1-vs-pool comparison cznic raised on the !127 review a genuine apples-to-apples measurement. On the rotating-residue eviction-churn workload at `cache_size=16` the pool spills ~3.5x more than pcache1 (cache-spill/op 31.96 vs 8.96) for ~3% more page writes (cache-write/op 450 vs 436) at identical hit/miss, quantifying the I/O cost of the strict Easy contract that `EasyRefusals` only proxied.

     - See [GitLab merge request #132](https://gitlab.com/cznic/sqlite/-/merge_requests/132), thanks Ian Chechin!

     - Add an opt-in `_dqs` DSN query parameter that disables SQLite's double-quoted string literal compatibility quirk on a per-connection basis. When `_dqs=0` (or any `strconv.ParseBool` false value) is supplied, the driver calls `sqlite3_db_config` with `SQLITE_DBCONFIG_DQS_DDL` and `SQLITE_DBCONFIG_DQS_DML` set to off before any statement is prepared, so a double-quoted identifier that fails to resolve raises a parse error instead of silently falling back to a string literal. Absence of the parameter, or `_dqs=1`, leaves SQLite's default behavior unchanged; existing DSNs continue to work byte-for-byte. Resolves [GitLab issue #61](https://gitlab.com/cznic/sqlite/-/issues/61).

     - See [GitLab merge request #128](https://gitlab.com/cznic/sqlite/-/merge_requests/128), thanks Ian Chechin!

     - Add an opt-in `_error_rc` DSN query parameter for clearer error reporting on open-time failures. When `_error_rc=1` (or any `strconv.ParseBool` true value) is supplied, error strings synthesised from a `(rc, db)` pair only append `sqlite3_errmsg(db)` when `sqlite3_extended_errcode(db)` is consistent with the operation rc (full match first, primary code `&0xff` as fallback). On mismatch the canonical `sqlite3_errstr(rc)` is used alone, so an open-time `SQLITE_CANTOPEN` no longer carries the temporary handle's stale "out of memory" errmsg. Absence of the parameter, or `_error_rc=0`, preserves the legacy "errstr: errmsg" form byte-for-byte; existing callers that parse error strings are unaffected. The driver's `*Error.Code()` returns the same SQLite result code in both modes. Parsed before `sqlite3_open_v2` so open-time errors are covered. Resolves [GitLab issue #230](https://gitlab.com/cznic/sqlite/-/issues/230).

dbstatus.go

0 → 100644
+98 −0
Original line number Diff line number Diff line 
// Copyright 2026 The Sqlite Authors. All rights reserved.

// Use of this source code is governed by a BSD-style

// license that can be found in the LICENSE file.



package sqlite // import "modernc.org/sqlite"



import (

	"unsafe"



	sqlite3 "modernc.org/sqlite/lib"

)



// DBStatusOp identifies a per-connection runtime counter readable through

// [DBStatus.Status]. The values mirror the SQLITE_DBSTATUS_* verbs of the C

// API; the distinct type keeps a counter from a different family (for example

// a file-control or db-config op) from compiling in its place.

//

// See https://www.sqlite.org/c3ref/c_dbstatus_options.html for the per-op

// semantics.

type DBStatusOp int32



// DBStatus* are the operations accepted by [DBStatus.Status]. They report

// their value differently depending on the op:

//

//   - DBStatusLookasideUsed: current is the lookaside memory in use now; high

//     is its high-water mark. The reset flag rebases the high-water mark to

//     current. This is the only op that maintains a high-water mark.

//   - Memory-usage ops (DBStatusCacheUsed, DBStatusSchemaUsed,

//     DBStatusStmtUsed, DBStatusCacheUsedShared): current is the bytes in use

//     now; high is always 0; the reset flag is ignored.

//   - Running-counter ops (DBStatusCacheHit, DBStatusCacheMiss,

//     DBStatusCacheWrite, DBStatusCacheSpill, DBStatusTempbufSpill): current

//     is the cumulative count (bytes spilled, for DBStatusTempbufSpill); high

//     is always 0. The reset flag zeroes current.

//   - Lookaside event ops (DBStatusLookasideHit, DBStatusLookasideMissSize,

//     DBStatusLookasideMissFull): the count is reported in high, not current

//     (current is always 0). The reset flag zeroes high.

//   - DBStatusDeferredFKs: current is 1 if the connection has unresolved

//     deferred foreign-key constraints, else 0; high is always 0; the reset

//     flag is ignored.

const (

	DBStatusLookasideUsed     = DBStatusOp(sqlite3.SQLITE_DBSTATUS_LOOKASIDE_USED)

	DBStatusCacheUsed         = DBStatusOp(sqlite3.SQLITE_DBSTATUS_CACHE_USED)

	DBStatusSchemaUsed        = DBStatusOp(sqlite3.SQLITE_DBSTATUS_SCHEMA_USED)

	DBStatusStmtUsed          = DBStatusOp(sqlite3.SQLITE_DBSTATUS_STMT_USED)

	DBStatusLookasideHit      = DBStatusOp(sqlite3.SQLITE_DBSTATUS_LOOKASIDE_HIT)

	DBStatusLookasideMissSize = DBStatusOp(sqlite3.SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE)

	DBStatusLookasideMissFull = DBStatusOp(sqlite3.SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL)

	DBStatusCacheHit          = DBStatusOp(sqlite3.SQLITE_DBSTATUS_CACHE_HIT)

	DBStatusCacheMiss         = DBStatusOp(sqlite3.SQLITE_DBSTATUS_CACHE_MISS)

	DBStatusCacheWrite        = DBStatusOp(sqlite3.SQLITE_DBSTATUS_CACHE_WRITE)

	DBStatusDeferredFKs       = DBStatusOp(sqlite3.SQLITE_DBSTATUS_DEFERRED_FKS)

	DBStatusCacheUsedShared   = DBStatusOp(sqlite3.SQLITE_DBSTATUS_CACHE_USED_SHARED)

	DBStatusCacheSpill        = DBStatusOp(sqlite3.SQLITE_DBSTATUS_CACHE_SPILL)

	DBStatusTempbufSpill      = DBStatusOp(sqlite3.SQLITE_DBSTATUS_TEMPBUF_SPILL)

)



// DBStatus exposes sqlite3_db_status, the per-connection runtime counters

// (cache hit/miss/write/spill rates, schema and prepared-statement memory,

// lookaside usage, deferred foreign keys). Reach it through the

// database/sql escape hatch, the same way as [FileControl]:

//

//	err := sqlConn.Raw(func(dc any) error {

//		cur, _, err := dc.(sqlite.DBStatus).Status(sqlite.DBStatusCacheSpill, false)

//		if err != nil {

//			return err

//		}

//		// use cur

//		return nil

//	})

type DBStatus interface {

	// Status returns the current and high-water values of the per-connection

	// counter identified by op. When reset is true the counter is reset after

	// the read; which value the reset affects depends on the op's family, see

	// the DBStatus* constant documentation. The returned error is non-nil only

	// when SQLite rejects op as out of range.

	Status(op DBStatusOp, reset bool) (current, high int, err error)

}



var _ DBStatus = (*conn)(nil)



func (c *conn) Status(op DBStatusOp, reset bool) (current, high int, err error) {

	// Two int32 out-params: pCurrent, pHighwater. sqlite3_db_status writes

	// C int (32-bit) through both, so a single 8-byte buffer holds the pair.

	p := c.tls.Alloc(8)

	defer c.tls.Free(8)



	pCur, pHi := p, p+4

	resetFlag := int32(0)

	if reset {

		resetFlag = 1

	}

	if rc := sqlite3.Xsqlite3_db_status(c.tls, c.db, int32(op), pCur, pHi, resetFlag); rc != sqlite3.SQLITE_OK {

		return 0, 0, c.errstr(rc)

	}



	return int(*(*int32)(unsafe.Pointer(pCur))), int(*(*int32)(unsafe.Pointer(pHi))), nil

}

dbstatus_test.go

0 → 100644
+151 −0
Original line number Diff line number Diff line 
// Copyright 2026 The Sqlite Authors. All rights reserved.

// Use of this source code is governed by a BSD-style

// license that can be found in the LICENSE file.



package sqlite // import "modernc.org/sqlite"



import (

	"context"

	"database/sql"

	"fmt"

	"path/filepath"

	"testing"

)



// dbStatusConn pins a single connection to a fresh on-disk database and

// returns it along with a helper that runs op through the DBStatus escape

// hatch on that exact connection.

func dbStatusConn(t *testing.T) (*sql.Conn, func(op DBStatusOp, reset bool) (cur, high int)) {

	t.Helper()

	name := filepath.Join(t.TempDir(), "dbstatus.db")

	db, err := sql.Open(driverName, fmt.Sprintf("file:%s", name))

	if err != nil {

		t.Fatal(err)

	}

	t.Cleanup(func() { db.Close() })



	conn, err := db.Conn(context.TODO())

	if err != nil {

		t.Fatal(err)

	}

	t.Cleanup(func() { conn.Close() })



	status := func(op DBStatusOp, reset bool) (cur, high int) {

		t.Helper()

		if err := conn.Raw(func(driverConn any) error {

			s, ok := driverConn.(DBStatus)

			if !ok {

				return fmt.Errorf("driver connection didn't implement DBStatus")

			}

			c, h, err := s.Status(op, reset)

			cur, high = c, h

			return err

		}); err != nil {

			t.Fatalf("Status(%d, %v): %v", op, reset, err)

		}

		return cur, high

	}

	return conn, status

}



// TestDBStatusSchemaUsedGrows exercises a memory high-water op: schema memory

// in use rises after new tables are added to the connection's schema.

func TestDBStatusSchemaUsedGrows(t *testing.T) {

	conn, status := dbStatusConn(t)



	before, _ := status(DBStatusSchemaUsed, false)

	for i := 0; i < 8; i++ {

		stmt := fmt.Sprintf("create table t%d (a int, b text, c blob, d real)", i)

		if _, err := conn.ExecContext(context.TODO(), stmt); err != nil {

			t.Fatal(err)

		}

	}

	after, _ := status(DBStatusSchemaUsed, false)



	if after <= before {

		t.Errorf("SchemaUsed did not grow after adding tables: before=%d after=%d", before, after)

	}

}



// TestDBStatusCacheHitReset exercises a running-counter op: cache hits

// accumulate as pages are re-read, and the reset flag zeroes the counter so a

// subsequent read sees a fresh, smaller count.

func TestDBStatusCacheHitReset(t *testing.T) {

	conn, status := dbStatusConn(t)



	if _, err := conn.ExecContext(context.TODO(),

		`create table t (k integer primary key, v blob)`); err != nil {

		t.Fatal(err)

	}

	blob := make([]byte, 256)

	for i := 0; i < 500; i++ {

		if _, err := conn.ExecContext(context.TODO(),

			`insert into t(v) values (?)`, blob); err != nil {

			t.Fatalf("insert[%d]: %v", i, err)

		}

	}

	// Re-read the same pages several times so the page cache records hits.

	for i := 0; i < 5; i++ {

		var n int

		if err := conn.QueryRowContext(context.TODO(),

			`select count(*) from t`).Scan(&n); err != nil {

			t.Fatalf("count[%d]: %v", i, err)

		}

	}



	// Read-and-reset, then read again with no intervening SQL. The counter is

	// a cumulative total with a 0 high-water; the reset zeroes current, so the

	// immediate re-read must be strictly smaller than the value we reset.

	hit, high := status(DBStatusCacheHit, true)

	if hit <= 0 {

		t.Fatalf("CacheHit current = %d after a re-reading workload, want > 0", hit)

	}

	if high != 0 {

		t.Errorf("CacheHit high = %d, want 0 (running-counter ops report 0 high-water)", high)

	}

	afterReset, _ := status(DBStatusCacheHit, false)

	if afterReset >= hit {

		t.Errorf("CacheHit after reset = %d, want < pre-reset value %d", afterReset, hit)

	}

}



// TestDBStatusLookasideHitInHighwater pins the documented quirk that the

// lookaside event ops report their count in the high-water out-param, not

// current.

func TestDBStatusLookasideHitInHighwater(t *testing.T) {

	conn, status := dbStatusConn(t)



	if _, err := conn.ExecContext(context.TODO(),

		`create table t (a int, b int, c int)`); err != nil {

		t.Fatal(err)

	}

	for i := 0; i < 50; i++ {

		if _, err := conn.ExecContext(context.TODO(),

			`insert into t values (?, ?, ?)`, i, i*2, i*3); err != nil {

			t.Fatalf("insert[%d]: %v", i, err)

		}

	}



	cur, high := status(DBStatusLookasideHit, false)

	if cur != 0 {

		t.Errorf("LookasideHit current = %d, want 0 (the value is reported in high-water)", cur)

	}

	if high <= 0 {

		t.Errorf("LookasideHit high = %d, want > 0 after a statement workload", high)

	}

}



// TestDBStatusUnknownOpErrors confirms an out-of-range op surfaces an error

// rather than silently returning zero values.

func TestDBStatusUnknownOpErrors(t *testing.T) {

	conn, _ := dbStatusConn(t)



	err := conn.Raw(func(driverConn any) error {

		s := driverConn.(DBStatus)

		_, _, e := s.Status(DBStatusOp(9999), false)

		return e

	})

	if err == nil {

		t.Fatal("expected an error from an out-of-range DBStatusOp, got nil")

	}

}

pcache/bench_test.go

+113 −0
Original line number Diff line number Diff line
@@ -5,11 +5,13 @@ 
package pcache_test



import (

	"context"

	"database/sql"

	"path/filepath"

	"runtime"

	"testing"



	"modernc.org/sqlite"

	"modernc.org/sqlite/pcache"

)
@@ -182,6 +184,117 @@ func BenchmarkPoolEvictionChurn(b *testing.B) { 
	b.Logf("eviction-churn delta over %d cycles: %+v", b.N, delta)

}



// dbStatus reads one per-connection counter through the parent package's

// [sqlite.DBStatus] escape hatch. The benchmark uses it to read the real

// pager-level CACHE_SPILL / CACHE_WRITE counters instead of the in-package

// EasyRefusals proxy.

func dbStatus(b *testing.B, conn *sql.Conn, op sqlite.DBStatusOp) int {

	b.Helper()

	var cur int

	if err := conn.Raw(func(dc any) error {

		s, ok := dc.(sqlite.DBStatus)

		if !ok {

			b.Fatalf("driver conn does not implement sqlite.DBStatus")

		}

		c, _, err := s.Status(op, false)

		cur = c

		return err

	}); err != nil {

		b.Fatalf("db_status(%d): %v", op, err)

	}

	return cur

}



// BenchmarkPoolSpillIO measures the pool's real I/O pressure under the

// rotating-residue eviction-churn workload by reading SQLite's pager-level

// counters (SQLITE_DBSTATUS_CACHE_SPILL, _CACHE_WRITE, _CACHE_HIT,

// _CACHE_MISS) through the parent package's sqlite.DBStatus API, rather than

// the in-package EasyRefusals proxy that BenchmarkPoolEvictionChurn reports.

//

// cache-spill/op and cache-write/op are the numbers that actually answer the

// "does the strict Easy contract cost extra I/O vs pcache1" question cznic

// raised on the !127 review: they are maintained identically by the pager for

// pcache1 and the pool, so they are a genuine apples-to-apples measure.

//

// To produce the pcache1 baseline, run this same workload in a sibling

// working tree whose test binary does not import this package, so the parent

// driver falls back to the in-engine pcache1; registration is process-global

// and one-shot, so the two caches cannot be compared in a single process. The

// MR description carries the side-by-side numbers; this benchmark is the

// reproducer for the pool side.

//

// The workload runs on a single pinned *sql.Conn so PRAGMA cache_size and the

// per-connection db_status counters refer to the same connection.

func BenchmarkPoolSpillIO(b *testing.B) {

	path := filepath.Join(b.TempDir(), "spillio.db")

	db, err := sql.Open("sqlite", path)

	if err != nil {

		b.Fatalf("Open: %v", err)

	}

	defer db.Close()



	conn, err := db.Conn(context.Background())

	if err != nil {

		b.Fatalf("Conn: %v", err)

	}

	defer conn.Close()



	exec := func(q string, args ...any) {

		if _, err := conn.ExecContext(context.Background(), q, args...); err != nil {

			b.Fatalf("exec %q: %v", q, err)

		}

	}

	exec(`PRAGMA cache_size=16`)

	exec(`PRAGMA auto_vacuum=incremental`)

	exec(`CREATE TABLE r (k INTEGER PRIMARY KEY, v BLOB)`)



	const batchPerCycle = 200



	blob := make([]byte, 256)

	var seedKey int64

	for residue := int64(0); residue < 3; residue++ {

		for j := 0; j < batchPerCycle; j++ {

			seedKey++

			for seedKey%3 != residue {

				seedKey++

			}

			exec(`INSERT INTO r(k, v) VALUES (?, ?)`, seedKey, blob)

		}

	}



	spill0 := dbStatus(b, conn, sqlite.DBStatusCacheSpill)

	write0 := dbStatus(b, conn, sqlite.DBStatusCacheWrite)

	hit0 := dbStatus(b, conn, sqlite.DBStatusCacheHit)

	miss0 := dbStatus(b, conn, sqlite.DBStatusCacheMiss)



	nextKey := seedKey + 1

	b.ResetTimer()

	for i := 0; i < b.N; i++ {

		residue := int64(i % 3)

		exec(`DELETE FROM r WHERE k % 3 = ?`, residue)

		exec(`PRAGMA incremental_vacuum`)

		for j := 0; j < batchPerCycle; j++ {

			for nextKey%3 != residue {

				nextKey++

			}

			exec(`INSERT INTO r(k, v) VALUES (?, ?)`, nextKey, blob)

			nextKey += 3

		}

	}

	b.StopTimer()



	spill := dbStatus(b, conn, sqlite.DBStatusCacheSpill) - spill0

	write := dbStatus(b, conn, sqlite.DBStatusCacheWrite) - write0

	hit := dbStatus(b, conn, sqlite.DBStatusCacheHit) - hit0

	miss := dbStatus(b, conn, sqlite.DBStatusCacheMiss) - miss0

	b.ReportMetric(float64(spill)/float64(b.N), "cache-spill/op")

	b.ReportMetric(float64(write)/float64(b.N), "cache-write/op")

	b.ReportMetric(float64(hit)/float64(b.N), "cache-hit/op")

	b.ReportMetric(float64(miss)/float64(b.N), "cache-miss/op")

	b.Logf("pager-counter deltas over %d cycles: spill=%d write=%d hit=%d miss=%d",

		b.N, spill, write, hit, miss)

}



// statsDelta is defined in e2e_test.go; same package.



var _ = pcache.Stats{}