Commit 6efd3c70 authored by cznic's avatar cznic
Browse files

v1.1.0-alpha.2

parent 87170804
......@@ -12,9 +12,11 @@ import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"testing"
......@@ -353,10 +355,10 @@ func TestMemDB(t *testing.T) {
}
}
func TestConcurrentInserts(t *testing.T) {
func TestConcurrentGoroutines(t *testing.T) {
const (
ngoroutines = 8
nrows = 2500
nrows = 5000
)
dir, err := ioutil.TempDir("", "sqlite-test-")
......@@ -456,3 +458,85 @@ func TestConcurrentInserts(t *testing.T) {
t.Logf("%d goroutines concurrently inserted %d rows in %v", ngoroutines, ngoroutines*nrows, d)
}
func TestConcurrentProcesses(t *testing.T) {
dir, err := ioutil.TempDir("", "sqlite-test-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
m, err := filepath.Glob(filepath.FromSlash("internal/mptest/*"))
if err != nil {
t.Fatal(err)
}
for _, v := range m {
if s := filepath.Ext(v); s != ".test" && s != ".subtest" {
continue
}
b, err := ioutil.ReadFile(v)
if err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(dir, filepath.Base(v)), b, 0666); err != nil {
t.Fatal(err)
}
}
out, err := exec.Command("go", "build", "-o", filepath.Join(dir, "mptest"), "modernc.org/sqlite/internal/mptest").CombinedOutput()
if err != nil {
t.Fatalf("%s\n%v", out, err)
}
wd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
defer os.Chdir(wd)
if err := os.Chdir(dir); err != nil {
t.Fatal(err)
}
outer:
for _, script := range m {
script = filepath.Base(script)
if filepath.Ext(script) != ".test" {
continue
}
out, err := exec.Command(filepath.FromSlash("./mptest"), "db", "--trace", "2", script).CombinedOutput()
if err != nil {
t.Fatalf("%s\n%v", out, err)
}
a := strings.Split(string(out), "\n")
for _, v := range a {
if strings.HasPrefix(v, "Summary:") {
b := strings.Fields(v)
if len(b) < 2 {
t.Fatalf("unexpected format of %q", v)
}
n, err := strconv.Atoi(b[1])
if err != nil {
t.Fatalf("unexpected format of %q", v)
}
if n != 0 {
t.Errorf("%s", out)
}
t.Logf("%v: %v", script, v)
continue outer
}
}
t.Fatalf("%s\nerror: summary line not found", out)
}
}
......@@ -7,10 +7,15 @@
//
// Changelog
//
// 2019-12-18 v1.1.0 First alpha release using the new cc/v3, gocc, qbe
// toolchain. Some primitive tests pass on linux_{amd64,386}. Not yet safe for
// concurrent access by multiple goroutines. Next alpha release is planed to
// arrive before the end of this year.
// 2019-12-26 v1.1.0-alpha.2: Second alpha release adds support for accessing a
// database concurrently by multiple goroutines and/or processes. v1.1.0 is now
// considered feature-complete. Next planed release should be a beta with a
// proper test suite.
//
// 2019-12-18 v1.1.0-alpha.1: First alpha release using the new cc/v3, gocc,
// qbe toolchain. Some primitive tests pass on linux_{amd64,386}. Not yet safe
// for concurrent access by multiple goroutines. Next alpha release is planed
// to arrive before the end of this year.
//
// 2017-06-10 Windows/Intel no more uses the VM (thanks Steffen Butzer).
//
......
......@@ -21,22 +21,24 @@ import (
var (
config = []string{
"-DHAVE_USLEEP",
"-DLONGDOUBLE_TYPE=double",
"-DSQLITE_DEBUG", //TODO-
"-DSQLITE_DEFAULT_MEMSTATUS=0",
"-DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1",
"-DSQLITE_DQS=0",
"-DSQLITE_LIKE_DOESNT_MATCH_BLOBS",
"-DSQLITE_MAX_EXPR_DEPTH=0",
"-DSQLITE_MEMDEBUG", //TODO-
"-DSQLITE_MAX_MMAP_SIZE=0", // mmap somehow fails on linux/386
"-DSQLITE_MUTEX_APPDEF=1",
"-DSQLITE_MUTEX_NOOP",
"-DSQLITE_OMIT_DECLTYPE",
"-DSQLITE_OMIT_DEPRECATED",
"-DSQLITE_OMIT_PROGRESS_CALLBACK",
"-DSQLITE_OMIT_SHARED_CACHE",
"-DSQLITE_THREADSAFE=2", // Multi-thread
"-DSQLITE_USE_ALLOCA",
// "-DSQLITE_OMIT_DEPRECATED", // mptest needs deprecated sqlite3_trace.
//TODO- "-DSQLITE_DEBUG", //TODO-
//TODO- "-DSQLITE_MEMDEBUG", //TODO-
}
downloads = []struct {
......@@ -44,10 +46,12 @@ var (
sz int
dev bool
}{
{sqliteDir, "https://www.sqlite.org/2019/sqlite-amalgamation-3300100.zip", 2400, false},
{sqliteDir, "https://www.sqlite.org/2019/sqlite-amalgamation-3300100.zip", 2240, false},
{sqliteSrcDir, "https://www.sqlite.org/2019/sqlite-src-3300100.zip", 12060, false},
}
sqliteDir = filepath.FromSlash("testdata/sqlite-amalgamation-3300100")
sqliteDir = filepath.FromSlash("testdata/sqlite-amalgamation-3300100")
sqliteSrcDir = filepath.FromSlash("testdata/sqlite-src-3300100")
)
func download() {
......@@ -203,4 +207,20 @@ func main() {
if err != nil {
fail("%s\n%s\n", out, err)
}
out, err = exec.Command(
"gocc",
append(
[]string{
filepath.Join(sqliteSrcDir, "mptest", "mptest.c"),
"-o", filepath.FromSlash(fmt.Sprintf("internal/mptest/main_%s_%s.go", runtime.GOOS, runtime.GOARCH)),
fmt.Sprintf("-I%s", sqliteDir),
"-qbec-dot-import", "modernc.org/sqlite/internal/bin",
"-qbec-static-prefix", "s_",
},
config...)...,
).CombinedOutput()
if err != nil {
fail("%s\n%s\n", out, err)
}
}
......@@ -3,6 +3,6 @@ module modernc.org/sqlite
go 1.13
require (
modernc.org/crt/v2 v2.0.0-20191219143825-5728f219e36a
modernc.org/crt/v2 v2.0.0-alpha.2
modernc.org/mathutil v1.0.0
)
// Code generated by `gocc /tmp/go-generate-081920477/x.c -o internal/bin/h_linux_386.go -Itestdata/sqlite-amalgamation-3300100 -qbec-defines -qbec-enumconsts -qbec-import <none> -qbec-pkgname bin -qbec-structs -DLONGDOUBLE_TYPE=double -DSQLITE_DEBUG -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 -DSQLITE_DQS=0 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DSQLITE_MAX_EXPR_DEPTH=0 -DSQLITE_MEMDEBUG -DSQLITE_MUTEX_APPDEF=1 -DSQLITE_MUTEX_NOOP -DSQLITE_OMIT_DECLTYPE -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_THREADSAFE=2 -DSQLITE_USE_ALLOCA`, DO NOT EDIT.
// Code generated by `gocc /tmp/go-generate-873789291/x.c -o internal/bin/h_linux_386.go -Itestdata/sqlite-amalgamation-3300100 -qbec-defines -qbec-enumconsts -qbec-import <none> -qbec-pkgname bin -qbec-structs -DHAVE_USLEEP -DLONGDOUBLE_TYPE=double -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 -DSQLITE_DQS=0 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DSQLITE_MAX_EXPR_DEPTH=0 -DSQLITE_MAX_MMAP_SIZE=0 -DSQLITE_MUTEX_APPDEF=1 -DSQLITE_MUTEX_NOOP -DSQLITE_OMIT_DECLTYPE -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_THREADSAFE=2 -DSQLITE_USE_ALLOCA`, DO NOT EDIT.
package bin
......@@ -9,6 +9,7 @@ const (
DFTS5_TOKENIZE_QUERY = 1
DFTS5_TOKEN_COLOCATED = 1
DFULLY_WITHIN = 2
DHAVE_USLEEP = 1
DNOT_WITHIN = 0
DPARTLY_WITHIN = 1
DSQLITE3_H = 0
......@@ -125,7 +126,6 @@ const (
DSQLITE_DBSTATUS_MAX = 12
DSQLITE_DBSTATUS_SCHEMA_USED = 2
DSQLITE_DBSTATUS_STMT_USED = 3
DSQLITE_DEBUG = 1
DSQLITE_DEFAULT_MEMSTATUS = 0
DSQLITE_DEFAULT_WAL_SYNCHRONOUS = 1
DSQLITE_DELETE = 9
......@@ -286,7 +286,7 @@ const (
DSQLITE_LOCK_RESERVED = 2
DSQLITE_LOCK_SHARED = 1
DSQLITE_MAX_EXPR_DEPTH = 0
DSQLITE_MEMDEBUG = 1
DSQLITE_MAX_MMAP_SIZE = 0
DSQLITE_MISMATCH = 20
DSQLITE_MISUSE = 21
DSQLITE_MUTEX_APPDEF = 1
......@@ -318,7 +318,6 @@ const (
DSQLITE_OK = 0
DSQLITE_OK_LOAD_PERMANENTLY = 256
DSQLITE_OMIT_DECLTYPE = 1
DSQLITE_OMIT_DEPRECATED = 1
DSQLITE_OMIT_PROGRESS_CALLBACK = 1
DSQLITE_OMIT_SHARED_CACHE = 1
DSQLITE_OPEN_AUTOPROXY = 32
......
// Code generated by `gocc /tmp/go-generate-795667064/x.c -o internal/bin/h_linux_amd64.go -Itestdata/sqlite-amalgamation-3300100 -qbec-defines -qbec-enumconsts -qbec-import <none> -qbec-pkgname bin -qbec-structs -DLONGDOUBLE_TYPE=double -DSQLITE_DEBUG -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 -DSQLITE_DQS=0 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DSQLITE_MAX_EXPR_DEPTH=0 -DSQLITE_MEMDEBUG -DSQLITE_MUTEX_APPDEF=1 -DSQLITE_MUTEX_NOOP -DSQLITE_OMIT_DECLTYPE -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_THREADSAFE=2 -DSQLITE_USE_ALLOCA`, DO NOT EDIT.
// Code generated by `gocc /tmp/go-generate-218625601/x.c -o internal/bin/h_linux_amd64.go -Itestdata/sqlite-amalgamation-3300100 -qbec-defines -qbec-enumconsts -qbec-import <none> -qbec-pkgname bin -qbec-structs -DHAVE_USLEEP -DLONGDOUBLE_TYPE=double -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 -DSQLITE_DQS=0 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DSQLITE_MAX_EXPR_DEPTH=0 -DSQLITE_MAX_MMAP_SIZE=0 -DSQLITE_MUTEX_APPDEF=1 -DSQLITE_MUTEX_NOOP -DSQLITE_OMIT_DECLTYPE -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_THREADSAFE=2 -DSQLITE_USE_ALLOCA`, DO NOT EDIT.
package bin
......@@ -9,6 +9,7 @@ const (
DFTS5_TOKENIZE_QUERY = 1
DFTS5_TOKEN_COLOCATED = 1
DFULLY_WITHIN = 2
DHAVE_USLEEP = 1
DNOT_WITHIN = 0
DPARTLY_WITHIN = 1
DSQLITE3_H = 0
......@@ -125,7 +126,6 @@ const (
DSQLITE_DBSTATUS_MAX = 12
DSQLITE_DBSTATUS_SCHEMA_USED = 2
DSQLITE_DBSTATUS_STMT_USED = 3
DSQLITE_DEBUG = 1
DSQLITE_DEFAULT_MEMSTATUS = 0
DSQLITE_DEFAULT_WAL_SYNCHRONOUS = 1
DSQLITE_DELETE = 9
......@@ -286,7 +286,7 @@ const (
DSQLITE_LOCK_RESERVED = 2
DSQLITE_LOCK_SHARED = 1
DSQLITE_MAX_EXPR_DEPTH = 0
DSQLITE_MEMDEBUG = 1
DSQLITE_MAX_MMAP_SIZE = 0
DSQLITE_MISMATCH = 20
DSQLITE_MISUSE = 21
DSQLITE_MUTEX_APPDEF = 1
......@@ -318,7 +318,6 @@ const (
DSQLITE_OK = 0
DSQLITE_OK_LOAD_PERMANENTLY = 256
DSQLITE_OMIT_DECLTYPE = 1
DSQLITE_OMIT_DEPRECATED = 1
DSQLITE_OMIT_PROGRESS_CALLBACK = 1
DSQLITE_OMIT_SHARED_CACHE = 1
DSQLITE_OPEN_AUTOPROXY = 32
......
This diff is collapsed.
This diff is collapsed.
/*
** Configure five tasks in different ways, then run tests.
*/
--if vfsname() GLOB 'unix'
PRAGMA page_size=8192;
--task 1
PRAGMA journal_mode=PERSIST;
PRAGMA mmap_size=0;
--end
--task 2
PRAGMA journal_mode=TRUNCATE;
PRAGMA mmap_size=28672;
--end
--task 3
PRAGMA journal_mode=MEMORY;
--end
--task 4
PRAGMA journal_mode=OFF;
--end
--task 4
PRAGMA mmap_size(268435456);
--end
--source multiwrite01.test
--wait all
PRAGMA page_size=16384;
VACUUM;
CREATE TABLE pgsz(taskid, sz INTEGER);
--task 1
INSERT INTO pgsz VALUES(1, eval('PRAGMA page_size'));
--end
--task 2
INSERT INTO pgsz VALUES(2, eval('PRAGMA page_size'));
--end
--task 3
INSERT INTO pgsz VALUES(3, eval('PRAGMA page_size'));
--end
--task 4
INSERT INTO pgsz VALUES(4, eval('PRAGMA page_size'));
--end
--task 5
INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size'));
--end
--source multiwrite01.test
--wait all
SELECT sz FROM pgsz;
--match 16384 16384 16384 16384 16384
/*
** Configure five tasks in different ways, then run tests.
*/
PRAGMA page_size=512;
--task 1
PRAGMA mmap_size=0;
--end
--task 2
PRAGMA mmap_size=28672;
--end
--task 3
PRAGMA mmap_size=8192;
--end
--task 4
PRAGMA mmap_size=65536;
--end
--task 5
PRAGMA mmap_size=268435456;
--end
--source multiwrite01.test
--source crash02.subtest
PRAGMA page_size=1024;
VACUUM;
CREATE TABLE pgsz(taskid, sz INTEGER);
--task 1
INSERT INTO pgsz VALUES(1, eval('PRAGMA page_size'));
--end
--task 2
INSERT INTO pgsz VALUES(2, eval('PRAGMA page_size'));
--end
--task 3
INSERT INTO pgsz VALUES(3, eval('PRAGMA page_size'));
--end
--task 4
INSERT INTO pgsz VALUES(4, eval('PRAGMA page_size'));
--end
--task 5
INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size'));
--end
--source multiwrite01.test
--source crash02.subtest
--wait all
SELECT sz FROM pgsz;
--match 1024 1024 1024 1024 1024
PRAGMA page_size=2048;
VACUUM;
DELETE FROM pgsz;
--task 1
INSERT INTO pgsz VALUES(1, eval('PRAGMA page_size'));
--end
--task 2
INSERT INTO pgsz VALUES(2, eval('PRAGMA page_size'));
--end
--task 3
INSERT INTO pgsz VALUES(3, eval('PRAGMA page_size'));
--end
--task 4
INSERT INTO pgsz VALUES(4, eval('PRAGMA page_size'));
--end
--task 5
INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size'));
--end
--source multiwrite01.test
--source crash02.subtest
--wait all
SELECT sz FROM pgsz;
--match 2048 2048 2048 2048 2048
PRAGMA page_size=8192;
VACUUM;
DELETE FROM pgsz;
--task 1
INSERT INTO pgsz VALUES(1, eval('PRAGMA page_size'));
--end
--task 2
INSERT INTO pgsz VALUES(2, eval('PRAGMA page_size'));
--end
--task 3
INSERT INTO pgsz VALUES(3, eval('PRAGMA page_size'));
--end
--task 4
INSERT INTO pgsz VALUES(4, eval('PRAGMA page_size'));
--end
--task 5
INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size'));
--end
--source multiwrite01.test
--source crash02.subtest
--wait all
SELECT sz FROM pgsz;
--match 8192 8192 8192 8192 8192
PRAGMA page_size=16384;
VACUUM;
DELETE FROM pgsz;
--task 1
INSERT INTO pgsz VALUES(1, eval('PRAGMA page_size'));
--end
--task 2
INSERT INTO pgsz VALUES(2, eval('PRAGMA page_size'));
--end
--task 3
INSERT INTO pgsz VALUES(3, eval('PRAGMA page_size'));
--end
--task 4
INSERT INTO pgsz VALUES(4, eval('PRAGMA page_size'));
--end
--task 5
INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size'));
--end
--source multiwrite01.test
--source crash02.subtest
--wait all
SELECT sz FROM pgsz;
--match 16384 16384 16384 16384 16384
PRAGMA auto_vacuum=FULL;
VACUUM;
--source multiwrite01.test
--source crash02.subtest
--wait all
PRAGMA auto_vacuum=FULL;
PRAGMA page_size=512;
VACUUM;
--source multiwrite01.test
--source crash02.subtest
/* Test cases involving incomplete transactions that must be rolled back.
*/
--task 1
DROP TABLE IF EXISTS t1;
CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
--sleep 1
INSERT INTO t1 VALUES(1, randomblob(2000));
INSERT INTO t1 VALUES(2, randomblob(1000));
--sleep 1
INSERT INTO t1 SELECT a+2, randomblob(1500) FROM t1;
INSERT INTO t1 SELECT a+4, randomblob(1500) FROM t1;
INSERT INTO t1 SELECT a+8, randomblob(1500) FROM t1;
--sleep 1
INSERT INTO t1 SELECT a+16, randomblob(1500) FROM t1;
--sleep 1
INSERT INTO t1 SELECT a+32, randomblob(1500) FROM t1;
SELECT count(*) FROM t1;
--match 64
SELECT avg(length(b)) FROM t1;
--match 1500.0
--sleep 2
UPDATE t1 SET b='x'||a||'y';
SELECT sum(length(b)) FROM t1;
--match 247
SELECT a FROM t1 WHERE b='x17y';
--match 17
CREATE INDEX t1b ON t1(b);
SELECT a FROM t1 WHERE b='x17y';
--match 17
SELECT a FROM t1 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5;
--match 29 28 27 26 25
--end
--wait 1
--task 2
DROP TABLE IF EXISTS t2;
CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
INSERT INTO t2 SELECT a, b FROM t1;
UPDATE t1 SET b='x'||a||'y';
SELECT sum(length(b)) FROM t2;
--match 247
SELECT a FROM t2 WHERE b='x17y';
--match 17
CREATE INDEX t2b ON t2(b);
SELECT a FROM t2 WHERE b='x17y';
--match 17
SELECT a FROM t2 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5;
--match 29 28 27 26 25
--end
--task 3
DROP TABLE IF EXISTS t3;
CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
INSERT INTO t3 SELECT a, b FROM t1;
UPDATE t1 SET b='x'||a||'y';
SELECT sum(length(b)) FROM t3;
--match 247
SELECT a FROM t3 WHERE b='x17y';
--match 17
CREATE INDEX t3b ON t3(b);
SELECT a FROM t3 WHERE b='x17y';
--match 17
SELECT a FROM t3 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5;
--match 29 28 27 26 25
--end
--task 4
DROP TABLE IF EXISTS t4;
CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
INSERT INTO t4 SELECT a, b FROM t1;
UPDATE t1 SET b='x'||a||'y';
SELECT sum(length(b)) FROM t4;
--match 247
SELECT a FROM t4 WHERE b='x17y';
--match 17
CREATE INDEX t4b ON t4(b);
SELECT a FROM t4 WHERE b='x17y';
--match 17
SELECT a FROM t4 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5;
--match 29 28 27 26 25
--end
--task 5
DROP TABLE IF EXISTS t5;
CREATE TABLE t5(a INTEGER PRIMARY KEY, b);
INSERT INTO t5 SELECT a, b FROM t1;
UPDATE t1 SET b='x'||a||'y';
SELECT sum(length(b)) FROM t5;
--match 247
SELECT a FROM t5 WHERE b='x17y';
--match 17
CREATE INDEX t5b ON t5(b);
SELECT a FROM t5 WHERE b='x17y';
--match 17
SELECT a FROM t5 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5;
--match 29 28 27 26 25
--end
--wait all
/* After the database file has been set up, run the crash2 subscript
** multiple times. */
--source crash02.subtest
--source crash02.subtest
--source crash02.subtest
--source crash02.subtest
--source crash02.subtest
--source crash02.subtest
--source crash02.subtest
--source crash02.subtest
--source crash02.subtest
/*
** This script is called from crash01.test and config02.test and perhaps other
** script. After the database file has been set up, make a big rollback
** journal in client 1, then crash client 1.
** Then in the other clients, do an integrity check.
*/
--task 1 leave-hot-journal
--sleep 5
--finish
PRAGMA cache_size=10;
BEGIN;
UPDATE t1 SET b=randomblob(20000);
UPDATE t2 SET b=randomblob(20000);
UPDATE t3 SET b=randomblob(20000);
UPDATE t4 SET b=randomblob(20000);
UPDATE t5 SET b=randomblob(20000);
UPDATE t1 SET b=NULL;
UPDATE t2 SET b=NULL;
UPDATE t3 SET b=NULL;
UPDATE t4 SET b=NULL;
UPDATE t5 SET b=NULL;
--print Task one crashing an incomplete transaction
--exit 1
--end
--task 2 integrity_check-2
SELECT count(*) FROM t1;
--match 64
--sleep 100
PRAGMA integrity_check(10);