conn: bind statement arguments in linear time (b122d041) · Commits · cznic / sqlite

CHANGELOG.md

+3 −0

Changes for CHANGELOG.md: 3 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -2,6 +2,9 @@

		Entries for v1.38.1 through v1.44.1 and for v1.49.1 were added on 2026-09-05, reconstructed from the git history and the merge requests they cite; they were missing at release time.

		- 2026-09-29 v1.60.1:
		- Binding arguments to a statement is no longer quadratic in the number of its parameters, which made multi-row `INSERT`s with thousands of `?` parameters slow. Resolves [GitHub issue #8](https://github.com/modernc-org/sqlite/issues/8), thanks wencycool!

		- 2026-09-28 v1.60.0:
		- A fault while reading the memory-mapped `-shm` file of a WAL database no longer crashes the process. The statement fails with a disk I/O error, extended code `SQLITE_IOERR_IN_PAGE` (8714), and the connection stays usable, as in MSVC builds of SQLite. On by default on every platform and not switchable; `lib.SehInject` and `lib.SehPending` inject such a fault for tests. Resolves [GitLab issue #221](https://gitlab.com/cznic/sqlite/-/issues/221), thanks Roman (@requilence) for the report, and supersedes [libsqlite3!4](https://gitlab.com/cznic/libsqlite3/-/merge_requests/4) and [GitHub pull request #7](https://github.com/modernc-org/sqlite/pull/7), thanks hazyhaar for the two rounds, the `ccgo` finding and the Windows Server runs!
		- Re-vendor `lib/` from [modernc.org/libsqlite3 v1.15.0](https://gitlab.com/cznic/libsqlite3/-/tags/v1.15.0) and `vec/` from [modernc.org/libsqlite_vec v0.6.0](https://gitlab.com/cznic/libsqlite_vec/-/tags/v0.6.0); SQLite stays 3.53.4 and sqlite-vec v0.1.9. Go 1.26 is now required, and the pinned `modernc.org/libc` becomes [v1.77.1](https://gitlab.com/cznic/libc/-/tags/v1.77.1); as always, downstream `go.mod` files must pin the same `modernc.org/libc` version this repository's `go.mod` does, see [GitLab issue #177](https://gitlab.com/cznic/sqlite/-/issues/177).

bind_test.go

0 → 100644

+164 −0

Changes for bind_test.go: 164 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		// Copyright 2026 The Sqlite Authors. All rights reserved.
		// Use of this source code is governed by a BSD-style
		// license that can be found in the LICENSE file.

		package sqlite

		import (
		"database/sql"
		"database/sql/driver"
		"fmt"
		"math/rand"
		"strings"
		"testing"
		)

		// TestArgFinder checks that argFinder.find returns what scanArgs, the linear
		// scan it replaces, returns, for arguments ordered as database/sql passes
		// them, where find takes its fast path, and for arguments that are not.
		func TestArgFinder(t *testing.T) {
		names := []string{
		"", "?1", "?2", "?3", "?17", "?40", "?41", "?05", "?0",
		"?99999999999999999999", "?9223372036854775807",
		"$1", "$3", "$20", "$01", "$0", "$1a", "$a", "$b", "$05", "$x",
		":a", ":b", ":1", ":05", ":1a", ":missing", "@a", "@x",
		}
		argNames := []string{"", "", "", "a", "b", "1", "3", "17", "20", "05", "1a", "x"}
		rng := rand.New(rand.NewSource(42))
		for _, n := range []int{0, 1, 2, 3, 16, 17, 40} {
		for round := 0; round < 50; round++ {
		args := make([]driver.NamedValue, n)
		for j := range args {
		args[j] = driver.NamedValue{Name: argNames[rng.Intn(len(argNames))], Ordinal: j + 1, Value: int64(j)}
		}
		if round%2 == 1 && n != 0 {
		// Not how database/sql passes arguments: find must fall
		// back to the scan.
		switch rng.Intn(3) {
		case 0:
		rng.Shuffle(n, func(i, j int) { args[i].Ordinal, args[j].Ordinal = args[j].Ordinal, args[i].Ordinal })
		case 1:
		args[rng.Intn(n)].Ordinal = rng.Intn(n + 2)
		case 2:
		for j := range args {
		args[j].Ordinal = j
		}
		}
		}
		f := newArgFinder(args)
		for i := 1; i <= n+2; i++ {
		for _, name := range names {
		gotV, gotOK := f.find(name, i)
		wantV, wantOK := scanArgs(args, name, i)
		if gotV != wantV \|\| gotOK != wantOK {
		t.Fatalf("n=%d round=%d find(%q, %d) = %+v, %v, want %+v, %v\nargs: %+v", n, round, name, i, gotV, gotOK, wantV, wantOK, args)
		}
		}
		}
		}
		}
		}

		// TestBindManyParameters binds statements with many parameters of each kind
		// and checks every argument arrives at its parameter.
		//
		// https://github.com/modernc-org/sqlite/issues/8
		func TestBindManyParameters(t *testing.T) {
		db, err := sql.Open(driverName, "file::memory:")
		if err != nil {
		t.Fatal(err)
		}
		defer db.Close()

		const n = 2000
		for _, kind := range []string{"?", "?N", "$N", ":name", "@name", "$name"} {
		t.Run(kind, func(t *testing.T) {
		args := make([]any, n)
		rows := make([]string, n)
		for i := range args {
		args[i] = int64(3*i + 1)
		var param string
		switch kind {
		case "?":
		param = "?"
		case "?N", "$N":
		param = fmt.Sprintf("%c%d", kind[0], i+1)
		default:
		param = fmt.Sprintf("%cp%d", kind[0], i)
		args[i] = sql.Named(fmt.Sprintf("p%d", i), args[i])
		}
		// Except for ?, whose position is its index, list the
		// parameters backwards, so SQLite numbers them in the
		// reverse order of their arguments.
		k := i
		if kind != "?" {
		k = n - 1 - i
		}
		rows[k] = fmt.Sprintf("(%d, %s)", i, param)
		}
		r, err := db.Query("select column1, column2 from (values "+strings.Join(rows, ",")+")", args...)
		if err != nil {
		t.Fatal(err)
		}
		defer r.Close()

		seen := 0
		for r.Next() {
		var i, v int64
		if err := r.Scan(&i, &v); err != nil {
		t.Fatal(err)
		}
		if v != 3*i+1 {
		t.Fatalf("parameter %d bound to %d, want %d", i, v, 3*i+1)
		}
		seen++
		}
		if err := r.Err(); err != nil {
		t.Fatal(err)
		}
		if seen != n {
		t.Fatalf("got %d rows, want %d", seen, n)
		}
		})
		}
		}

		// BenchmarkBindManyParameters measures binding a multi-row INSERT with about
		// 20k positional parameters, which was quadratic in the number of parameters.
		//
		// https://github.com/modernc-org/sqlite/issues/8
		func BenchmarkBindManyParameters(b *testing.B) {
		for _, n := range []int{10, 1000, 20000} {
		b.Run(fmt.Sprint(n), func(b *testing.B) {
		db, err := sql.Open(driverName, "file::memory:")
		if err != nil {
		b.Fatal(err)
		}
		defer db.Close()

		if _, err := db.Exec("create table t(a, b, c, d, e, f, g, h, i, j)"); err != nil {
		b.Fatal(err)
		}

		q := "insert into t values " + strings.TrimSuffix(strings.Repeat("(?,?,?,?,?,?,?,?,?,?),", n/10), ",")
		stmt, err := db.Prepare(q)
		if err != nil {
		b.Fatal(err)
		}
		defer stmt.Close()

		args := make([]any, n)
		for i := range args {
		args[i] = int64(i)
		}
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
		if _, err := stmt.Exec(args...); err != nil {
		b.Fatal(err)
		}
		}
		b.StopTimer()
		b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N)/float64(n), "ns/param")
		})
		}
		}

conn.go

+112 −31

Changes for conn.go: 112 added lines, 31 removed lines.
Original line number	Diff line number	Diff line
		@@ -484,43 +484,14 @@ func (c *conn) bind(pstmt uintptr, n int, args []driver.NamedValue) (allocs []ui
		allocs = nil
		}()

		f := newArgFinder(args)
		for i := 1; i <= n; i++ {
		name, err := c.bindParameterName(pstmt, i)
		if err != nil {
		return allocs, err
		}

		var found bool
		var v driver.NamedValue
		for _, v = range args {
		if name != "" {
		// For ?NNN and $NNN params, match if NNN == v.Ordinal.
		//
		// Supporting this for $NNN is a special case that makes eg
		// `select $1, $2, $3 ...` work without needing to use
		// sql.Named.
		if (name[0] == '?' \|\| name[0] == '$') && name[1:] == strconv.Itoa(v.Ordinal) {
		found = true
		break
		}

		// sqlite supports '$', '@' and ':' prefixes for string
		// identifiers and '?' for numeric, so we cannot
		// combine different prefixes with the same name
		// because `database/sql` requires variable names
		// to start with a letter
		if name[1:] == v.Name[:] {
		found = true
		break
		}
		} else {
		if v.Ordinal == i {
		found = true
		break
		}
		}
		}

		v, found := f.find(name, i)
		if !found {
		if name != "" {
		return allocs, fmt.Errorf("missing named argument %q", name[1:])
		@@ -593,6 +564,116 @@ func (c *conn) bind(pstmt uintptr, n int, args []driver.NamedValue) (allocs []ui
		return allocs, nil
		}

		// argFinder finds the argument bound to a statement parameter.
		type argFinder struct {
		args []driver.NamedValue
		byName map[string]int // index of the first argument with a name, built on first use
		ordered bool // args[j].Ordinal == j+1 for every j
		named bool // some argument has a name
		}

		func newArgFinder(args []driver.NamedValue) argFinder {
		f := argFinder{args: args, ordered: true}
		for j := range args {
		f.ordered = f.ordered && args[j].Ordinal == j+1
		f.named = f.named \|\| args[j].Name != ""
		}
		return f
		}

		// find returns the argument scanArgs(f.args, name, i) returns. database/sql
		// passes args ordered by Ordinal, starting at 1, so the argument with ordinal
		// k is args[k-1] and find does not have to scan args for every parameter,
		// which made binding quadratic in the number of parameters.
		//
		// https://github.com/modernc-org/sqlite/issues/8
		func (f *argFinder) find(name string, i int) (driver.NamedValue, bool) {
		if !f.ordered {
		return scanArgs(f.args, name, i)
		}

		ord := i
		if name != "" {
		// scanArgs matches ?NNN and $NNN if NNN == strconv.Itoa(v.Ordinal),
		// which for an ordinal >= 1 has no sign and no leading zero.
		ord = 0
		if (name[0] == '?' \|\| name[0] == '$') && len(name) > 1 && name[1] >= '1' && name[1] <= '9' {
		if k, err := strconv.Atoi(name[1:]); err == nil {
		ord = k
		}
		}
		}
		j := -1 // index of the first matching argument
		if ord >= 1 && ord <= len(f.args) {
		j = ord - 1
		}
		if name != "" && f.named {
		if k, ok := f.nameIndex(name[1:]); ok && (j < 0 \|\| k < j) {
		j = k
		}
		}
		if j < 0 {
		return driver.NamedValue{}, false
		}

		return f.args[j], true
		}

		// nameIndex returns the index of the first argument with the given name.
		func (f *argFinder) nameIndex(name string) (int, bool) {
		// Up to a few dozen arguments, scanning beats building the map.
		if len(f.args) <= 32 {
		for j := range f.args {
		if f.args[j].Name == name {
		return j, true
		}
		}
		return 0, false
		}

		if f.byName == nil {
		f.byName = make(map[string]int, len(f.args))
		for j := len(f.args) - 1; j >= 0; j-- {
		if f.args[j].Name != "" {
		f.byName[f.args[j].Name] = j
		}
		}
		}
		j, ok := f.byName[name]
		return j, ok
		}

		// scanArgs returns the first argument in args matching the parameter at index
		// i named name, "" for an anonymous parameter. It is linear in len(args).
		func scanArgs(args []driver.NamedValue, name string, i int) (driver.NamedValue, bool) {
		for _, v := range args {
		if name != "" {
		// For ?NNN and $NNN params, match if NNN == v.Ordinal.
		//
		// Supporting this for $NNN is a special case that makes eg
		// `select $1, $2, $3 ...` work without needing to use
		// sql.Named.
		if (name[0] == '?' \|\| name[0] == '$') && name[1:] == strconv.Itoa(v.Ordinal) {
		return v, true
		}

		// sqlite supports '$', '@' and ':' prefixes for string
		// identifiers and '?' for numeric, so we cannot
		// combine different prefixes with the same name
		// because `database/sql` requires variable names
		// to start with a letter
		if name[1:] == v.Name[:] {
		return v, true
		}
		} else {
		if v.Ordinal == i {
		return v, true
		}
		}
		}
		return driver.NamedValue{}, false
		}

		// C documentation
		//
		// int sqlite3_bind_null(sqlite3_stmt*, int);