Merge branch 'perf/cache-parse-time-format' into 'master' (44857934) · Commits · cznic / sqlite

conn.go

+33 −13

Original line number	Diff line number	Diff line
		@@ -102,29 +102,49 @@ func newConn(dsn string) (*conn, error) {
		return c, nil
		}

		// Attempt to parse s as a time. Return (s, false) if s is not
		// recognized as a valid time encoding.
		func (c *conn) parseTime(s string) (interface{}, bool) {
		// parseTime attempts to parse s as a time encoding. If hintIdx is a valid
		// index into parseTimeFormats, that format is tried before the rest of the
		// list; otherwise the search runs in declaration order. The returned int is
		// the index of the format that matched, or -1 if the parseTimeString
		// (t.String()) branch matched or all formats failed. Callers that scan many
		// rows of a same-format column can feed the previous match back as hintIdx
		// to skip the redundant time.Parse attempts that would otherwise run for
		// every row.
		//
		// Return value contract is preserved: (parsed-value, ok). On failure the
		// value is the original input string and ok is false.
		func (c *conn) parseTime(s string, hintIdx int) (interface{}, bool, int) {
		if v, ok := c.parseTimeString(s, strings.Index(s, "m=")); ok {
		return v, true
		return v, true, -1
		}

		ts, hadZ := strings.CutSuffix(s, "Z")

		for _, f := range parseTimeFormats {
		var t time.Time
		var err error
		tryFormat := func(f string) (time.Time, error) {
		if c.loc != nil && !hadZ {
		t, err = time.ParseInLocation(f, ts, c.loc)
		} else {
		t, err = time.Parse(f, ts)
		return time.ParseInLocation(f, ts, c.loc)
		}
		if err == nil {
		return c.applyTimezone(t), true
		return time.Parse(f, ts)
		}

		// Try the caller's hint first, if any.
		if hintIdx >= 0 && hintIdx < len(parseTimeFormats) {
		if t, err := tryFormat(parseTimeFormats[hintIdx]); err == nil {
		return c.applyTimezone(t), true, hintIdx
		}
		}

		// Sequential fallthrough, skipping the hint we already tried.
		for i, f := range parseTimeFormats {
		if i == hintIdx {
		continue
		}
		if t, err := tryFormat(f); err == nil {
		return c.applyTimezone(t), true, i
		}
		}

		return s, false
		return s, false, -1
		}

		// Attempt to parse s as a time string produced by t.String(). If x > 0 it's

conn_test.go

+130 −0

Original line number	Diff line number	Diff line
		@@ -396,3 +396,133 @@ func TestColumnTypeScanTypeDecltypeCache(t *testing.T) {
		})
		}
		}

		// TestParseTimeFormatCache verifies that the per-rows-per-column format-index
		// hint reused by (*conn).parseTime keeps returning correct parsed time values
		// across many rows of a steady-format column, and that a column whose format
		// switches mid-result-set still parses correctly via the fallthrough path.
		func TestParseTimeFormatCache(t *testing.T) {
		db, err := sql.Open(driverName, "file::memory:")
		if err != nil {
		t.Fatal(err)
		}
		defer db.Close()

		if _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, dt DATETIME)`); err != nil {
		t.Fatal(err)
		}

		// First three rows use the same canonical SQLite TEXT format (matches
		// format index 2 of parseTimeFormats: "2006-01-02 15:04:05.999999999").
		// Row 4 uses the ISO-T format (matches index 3). Row 5 uses the
		// date-only fallback (index 6). After the cache stabilises on row 1, the
		// hinted format helps rows 2 and 3 directly and rows 4-5 fall through.
		values := []string{
		"2025-01-15 10:30:00",
		"2025-01-15 11:00:00",
		"2025-01-15 11:30:00",
		"2025-01-16T08:15:00",
		"2025-01-17",
		}
		for i, v := range values {
		if _, err := db.Exec(`INSERT INTO t(id, dt) VALUES (?, ?)`, i+1, v); err != nil {
		t.Fatalf("insert id=%d: %v", i+1, err)
		}
		}

		rows, err := db.Query(`SELECT dt FROM t ORDER BY id`)
		if err != nil {
		t.Fatal(err)
		}
		defer rows.Close()

		wantTimes := []time.Time{
		time.Date(2025, 1, 15, 10, 30, 0, 0, time.UTC),
		time.Date(2025, 1, 15, 11, 0, 0, 0, time.UTC),
		time.Date(2025, 1, 15, 11, 30, 0, 0, time.UTC),
		time.Date(2025, 1, 16, 8, 15, 0, 0, time.UTC),
		time.Date(2025, 1, 17, 0, 0, 0, 0, time.UTC),
		}
		i := 0
		for rows.Next() {
		if i >= len(wantTimes) {
		t.Fatalf("too many rows; want %d", len(wantTimes))
		}
		var got time.Time
		if err := rows.Scan(&got); err != nil {
		t.Fatalf("row %d scan: %v", i, err)
		}
		if !got.Equal(wantTimes[i]) {
		t.Errorf("row %d: got %v, want %v", i, got, wantTimes[i])
		}
		i++
		}
		if err := rows.Err(); err != nil {
		t.Fatal(err)
		}
		if i != len(wantTimes) {
		t.Fatalf("row count: got %d, want %d", i, len(wantTimes))
		}
		}

		// benchParseTimeScan exercises the rows.Next + Scan path on a DATETIME TEXT
		// column. With the parseTime format-index cache, every row after the first
		// hits the hinted format directly; without the cache, every row re-walks
		// the parseTimeFormats list until it finds a match.
		func benchParseTimeScan(b *testing.B) {
		db, err := sql.Open(driverName, "file::memory:")
		if err != nil {
		b.Fatal(err)
		}
		defer db.Close()

		if _, err := db.Exec(`CREATE TABLE t (dt DATETIME)`); err != nil {
		b.Fatal(err)
		}

		const rows = 1000
		tx, err := db.Begin()
		if err != nil {
		b.Fatal(err)
		}
		stmt, err := tx.Prepare(`INSERT INTO t (dt) VALUES (?)`)
		if err != nil {
		b.Fatal(err)
		}
		for i := 0; i < rows; i++ {
		// Canonical SQLite TEXT datetime format (index 2 of
		// parseTimeFormats).
		if _, err := stmt.Exec("2025-01-15 10:30:00"); err != nil {
		b.Fatal(err)
		}
		}
		stmt.Close()
		if err := tx.Commit(); err != nil {
		b.Fatal(err)
		}

		b.ReportAllocs()
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
		r, err := db.Query(`SELECT dt FROM t`)
		if err != nil {
		b.Fatal(err)
		}
		var got time.Time
		for r.Next() {
		if err := r.Scan(&got); err != nil {
		b.Fatal(err)
		}
		}
		if err := r.Err(); err != nil {
		b.Fatal(err)
		}
		r.Close()
		}
		}

		// BenchmarkParseTimeScan measures the rows.Next DATETIME TEXT path with the
		// format-index cache active.
		func BenchmarkParseTimeScan(b *testing.B) {
		benchParseTimeScan(b)
		}

rows.go

+19 −2

Original line number	Diff line number	Diff line
		@@ -27,6 +27,17 @@ type rows struct {
		// libc.GoString + strings.ToUpper from the Next() hot path for callers
		// that hit the time-conversion branches (_texttotime, _time_format).
		decltypes []string
		// parseFmtIdx caches, per column, the index into parseTimeFormats that
		// matched the first successful (*conn).parseTime call on that column.
		// Subsequent rows reuse the saved index as the first attempt instead of
		// re-walking the format list from the top. Slot value -1 means no match
		// has been recorded yet (either parseTime has not run on this column, or
		// the parseTimeString / m= branch matched, which is not in
		// parseTimeFormats). The cache is sticky: once a successful index is
		// stored it is not overwritten if a later row happens to match a
		// different format, so mixed-format columns still pay only the original
		// fallthrough cost and a steady column wins on every subsequent row.
		parseFmtIdx []int8
		pstmt uintptr

		doStep bool
		@@ -56,11 +67,13 @@ func newRows(c conn, pstmt uintptr, allocs []uintptr, empty bool) (r *rows, er

		r.columns = make([]string, n)
		r.decltypes = make([]string, n)
		r.parseFmtIdx = make([]int8, n)
		for i := range r.columns {
		if r.columns[i], err = r.c.columnName(pstmt, i); err != nil {
		return nil, err
		}
		r.decltypes[i] = strings.ToUpper(r.c.columnDeclType(pstmt, i))
		r.parseFmtIdx[i] = -1
		}

		return r, nil
		@@ -179,7 +192,11 @@ func (r *rows) Next(dest []driver.Value) (err error) {

		switch r.ColumnTypeDatabaseTypeName(i) {
		case "DATE", "DATETIME", "TIMESTAMP":
		dest[i], _ = r.c.parseTime(v)
		val, ok, idx := r.c.parseTime(v, int(r.parseFmtIdx[i]))
		if ok && r.parseFmtIdx[i] < 0 && idx >= 0 {
		r.parseFmtIdx[i] = int8(idx)
		}
		dest[i] = val
		default:
		dest[i] = v
		}