Commit 0bd6d67f authored by cznic's avatar cznic

prepare pre-alpha release

parent 8afe3d5b
......@@ -5,18 +5,30 @@
.PHONY: all clean cover cpu editor internalError later mem nuke todo edit
grep=--include=*.go --include=*.l --include=*.y --include=*.yy
ngrep='TODOOK\|parser\.go\|scanner\.go\|.*_string\.go'
ngrep='TODOOK\|internal\/bin'
all: editor
go test 2>&1 | tee log
date
go version 2>&1 | tee log
./unconvert.sh
gofmt -l -s -w *.go
go test -i
go test 2>&1 -timeout 1h | tee -a log
#TODO GOOS=linux GOARCH=arm go build
#TODO GOOS=linux GOARCH=arm64 go build
GOOS=linux GOARCH=386 go build
GOOS=linux GOARCH=amd64 go build
#TODO GOOS=windows GOARCH=386 go build
#TODO GOOS=windows GOARCH=amd64 go build
go vet 2>&1 | grep -v $(ngrep) || true
golint 2>&1 | grep -v $(ngrep) || true
make todo
unused . || true
misspell *.go
gosimple || true
staticcheck | grep -v 'lexer\.go\|parser\.go' || true
maligned || true
unconvert -apply
grep -n 'FAIL\|PASS' log
go version
date 2>&1 | tee -a log
clean:
go clean
......@@ -30,12 +42,11 @@ cpu: clean
go tool pprof -lines *.test cpu.out
edit:
@ 1>/dev/null 2>/dev/null gvim -p Makefile main.c *.go
gvim -p Makefile *.go &
editor:
gofmt -l -s -w *.go
indent -linux *.c
go test -i
go install -v ./...
internalError:
egrep -ho '"internal error.*"' *.go | sort | cat -n
......
......@@ -58,7 +58,6 @@ func init() {
// ============================================================================
var (
memTrace = flag.Bool("memory.trace", false, "")
recsPerSec = flag.Bool("recs_per_sec_as_mbps", false, "Show records per second as MB/s.")
)
......@@ -89,7 +88,7 @@ func TestScalar(t *testing.T) {
t2 := time.Date(2018, 5, 21, 2, 3, 4, 98765, time.UTC)
r, err := db.Exec(`
create table t(i int, f double, b bool, s text, t time);
insert into t values(12, 3.14, ?, "foo", ?), (34, 2.78, ?, "bar", ?);
insert into t values(12, 3.14, ?, 'foo', ?), (34, 2.78, ?, 'bar', ?);
`,
true, t1,
false, t2,
......
......@@ -7,6 +7,11 @@
//
// Changelog
//
// 2019-12-18 v1.1.0 Pre-alpha release using the new cc/v3, gocc, qbe
// toolchain. Some primitive tests pass on linux_{amd64,386}. Not yet safe for
// concurrent access by multiple goroutines. Alpha release is planed to arrive
// before the end of this year.
//
// 2017-06-10 Windows/Intel no more uses the VM (thanks Steffen Butzer).
//
// 2017-06-05 Linux/Intel no more uses the VM (cznic/virtual).
......@@ -31,15 +36,17 @@
//
// Do not use in production
//
// This is an experimental, pre-alpha, technology preview package.
// Supported platforms and architectures
//
// The alpha release is due when the C runtime support of SQLite in cznic/crt
// will be complete.
// linux 386
// linux amd64
//
// Supported platforms and architectures
// Planned platforms and architectures
// linux arm
// linux arm64
// windows 386
// windows amd64
//
// See http://modernc.org/ccir. To add a newly supported os/arch
// combination to this package try running 'go generate'.
//
// Sqlite documentation
//
......
// Copyright 2017 The Sqlite Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:generate go run generator.go
package sqlite // import "modernc.org/sqlite"
......@@ -7,691 +7,198 @@
package main
import (
"bytes"
"flag"
"archive/zip"
"fmt"
"go/format"
"go/scanner"
"go/token"
"io"
"io/ioutil"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"sort"
"strings"
"log"
"modernc.org/cc"
"modernc.org/ccgo"
"modernc.org/ccir"
"modernc.org/internal/buffer"
"modernc.org/strutil"
"modernc.org/xc"
)
var (
cpp = flag.Bool("cpp", false, "")
dict = xc.Dict
errLimit = flag.Int("errlimit", 10, "")
filter = flag.String("re", "", "")
ndebug = flag.Bool("ndebug", false, "")
noexec = flag.Bool("noexec", false, "")
oLog = flag.Bool("log", false, "")
trace = flag.Bool("trc", false, "")
unconvertBin string
yydebug = flag.Int("yydebug", 0, "")
config = []string{
"-DLONGDOUBLE_TYPE=double",
"-DSQLITE_DEBUG", //TODO-
"-DSQLITE_DEFAULT_MEMSTATUS=0",
"-DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1",
"-DSQLITE_DQS=0",
"-DSQLITE_LIKE_DOESNT_MATCH_BLOBS",
"-DSQLITE_MAX_EXPR_DEPTH=0",
"-DSQLITE_MEMDEBUG", //TODO-
"-DSQLITE_OMIT_DECLTYPE",
"-DSQLITE_OMIT_DEPRECATED",
"-DSQLITE_OMIT_PROGRESS_CALLBACK",
"-DSQLITE_OMIT_SHARED_CACHE",
"-DSQLITE_THREADSAFE=0",
"-DSQLITE_USE_ALLOCA",
}
downloads = []struct {
dir, url string
sz int
dev bool
}{
{sqliteDir, "https://www.sqlite.org/2019/sqlite-amalgamation-3300100.zip", 2400, false},
}
sqliteDir = filepath.FromSlash("testdata/sqlite-amalgamation-3300100")
)
const (
sqliteRepo = "sqlite.org"
version = "3190300"
prologueSqlite = `// Code generated by ccgo. DO NOT EDIT.
/*
%s
*/
// Package sqlite is an in-process implementation of a self-contained,
// serverless, zero-configuration, transactional SQL database engine. (Work In Progress)
%s
package bin
import (
"fmt"
"math"
"os"
"path"
"runtime"
"unsafe"
"modernc.org/ccgo/crt"
)
func ftrace(s string, args ...interface{}) {
_, fn, fl, _ := runtime.Caller(1)
fmt.Fprintf(os.Stderr, "# %%s:%%d: %%v\n", path.Base(fn), fl, fmt.Sprintf(s, args...))
os.Stderr.Sync()
}
`
prologueTest = `// Code generated by ccgo. DO NOT EDIT.
// %s
%s
package main
import (
"math"
"os"
"unsafe"
"modernc.org/ccgo/crt"
"modernc.org/sqlite/internal/bin"
)
var argv []*int8
func main() {
for _, v := range os.Args {
argv = append(argv, (*int8)(crt.CString(v)))
func download() {
tmp, err := ioutil.TempDir("", "")
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
return
}
argv = append(argv, nil)
X_start(crt.NewTLS(), int32(len(os.Args)), &argv[0])
}
`
defines = `
#define HAVE_MALLOC_H 1
#define HAVE_MALLOC_USABLE_SIZE 1
#define HAVE_USLEEP 1
#define SQLITE_DEBUG 1
#define SQLITE_ENABLE_API_ARMOR 1
#define SQLITE_USE_URI 1
#define SQLITE_WITHOUT_MSIZE 1
int sqlite3PendingByte;
`
)
defer os.RemoveAll(tmp)
func findRepo(s string) string {
s = filepath.FromSlash(s)
for _, v := range strings.Split(strutil.Gopath(), string(os.PathListSeparator)) {
p := filepath.Join(v, "src", s)
fi, err := os.Lstat(p)
if err != nil {
for _, v := range downloads {
dir := filepath.FromSlash(v.dir)
root := filepath.Dir(v.dir)
fi, err := os.Stat(dir)
switch {
case err == nil:
if !fi.IsDir() {
fmt.Fprintf(os.Stderr, "expected %s to be a directory\n", dir)
}
continue
default:
if !os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "%s", err)
continue
}
}
if fi.IsDir() {
wd, err := os.Getwd()
if err := func() error {
fmt.Printf("Downloading %v MB from %s\n", float64(v.sz)/1000, v.url)
resp, err := http.Get(v.url)
if err != nil {
log.Fatal(err)
}
if p, err = filepath.Rel(wd, p); err != nil {
log.Fatal(err)
return err
}
return p
}
}
return ""
}
defer resp.Body.Close()
func errStr(err error) string {
switch x := err.(type) {
case scanner.ErrorList:
if len(x) != 1 {
x.RemoveMultiples()
}
var b bytes.Buffer
for i, v := range x {
if i != 0 {
b.WriteByte('\n')
}
b.WriteString(v.Error())
if i == 9 {
fmt.Fprintf(&b, "\n\t... and %v more errors", len(x)-10)
break
base := filepath.Base(v.url)
name := filepath.Join(tmp, base)
f, err := os.Create(name)
if err != nil {
return err
}
}
return b.String()
default:
return err.Error()
}
}
func build(predef string, tus [][]string, ccgoOpts []ccgo.Option, opts ...cc.Opt) ([]*cc.TranslationUnit, []byte) {
ndbg := ""
if *ndebug {
ndbg = "#define NDEBUG 1"
}
defer os.Remove(name)
var lpos token.Position
if *cpp {
opts = append(opts, cc.Cpp(func(toks []xc.Token) {
if len(toks) != 0 {
p := toks[0].Position()
if p.Filename != lpos.Filename {
fmt.Fprintf(os.Stderr, "# %d %q\n", p.Line, p.Filename)
}
lpos = p
}
for _, v := range toks {
os.Stderr.WriteString(cc.TokSrc(v))
n, err := io.Copy(f, resp.Body)
if err != nil {
return err
}
os.Stderr.WriteString("\n")
}))
}
var build []*cc.TranslationUnit
tus = append(tus, []string{ccir.CRT0Path})
for _, src := range tus {
model, err := ccir.NewModel()
if err != nil {
log.Fatal(err)
}
ast, err := cc.Parse(
fmt.Sprintf(`
%s
#define _CCGO 1
#define __arch__ %s
#define __os__ %s
#include <builtin.h>
%s
`, ndbg, runtime.GOARCH, runtime.GOOS, predef),
src,
model,
append([]cc.Opt{
cc.AllowCompatibleTypedefRedefinitions(),
cc.EnableEmptyStructs(),
cc.EnableImplicitFuncDef(),
cc.EnableNonConstStaticInitExpressions(),
cc.EnableWideBitFieldTypes(),
cc.ErrLimit(*errLimit),
cc.KeepComments(),
cc.SysIncludePaths([]string{ccir.LibcIncludePath}),
}, opts...)...,
)
if err != nil {
log.Fatal(errStr(err))
}
build = append(build, ast)
}
var out buffer.Bytes
if err := ccgo.New(build, &out, ccgoOpts...); err != nil {
log.Fatal(err)
}
return build, out.Bytes()
}
func macros(buf io.Writer, ast *cc.TranslationUnit) {
fmt.Fprintf(buf, `const (
`)
var a []string
for k, v := range ast.Macros {
if v.Value != nil && v.Type.Kind() != cc.Bool {
switch fn := v.DefTok.Position().Filename; {
case
fn == "builtin.h",
fn == "<predefine>",
strings.HasPrefix(fn, "predefined_"):
// ignore
default:
a = append(a, string(dict.S(k)))
if _, err := f.Seek(0, io.SeekStart); err != nil {
return err
}
}
}
sort.Strings(a)
for _, v := range a {
m := ast.Macros[dict.SID(v)]
if m.Value == nil {
log.Fatal("TODO")
}
switch t := m.Type; t.Kind() {
case
cc.Int, cc.UInt, cc.Long, cc.ULong, cc.LongLong, cc.ULongLong,
cc.Float, cc.Double, cc.LongDouble, cc.Bool:
fmt.Fprintf(buf, "X%s = %v\n", v, m.Value)
case cc.Ptr:
switch t := t.Element(); t.Kind() {
case cc.Char:
fmt.Fprintf(buf, "X%s = %q\n", v, dict.S(int(m.Value.(cc.StringLitID))))
default:
log.Fatalf("%v", t.Kind())
}
default:
log.Fatalf("%v", t.Kind())
}
}
switch {
case strings.HasSuffix(base, ".zip"):
r, err := zip.NewReader(f, n)
if err != nil {
return err
}
a = a[:0]
for _, v := range ast.Declarations.Identifiers {
switch x := v.Node.(type) {
case *cc.DirectDeclarator:
d := x.TopDeclarator()
id, _ := d.Identifier()
if x.EnumVal == nil {
break
for _, f := range r.File {
fi := f.FileInfo()
if fi.IsDir() {
if err := os.MkdirAll(filepath.Join(root, f.Name), 0770); err != nil {
return err
}
continue
}
if err := func() error {
rc, err := f.Open()
if err != nil {
return err
}
defer rc.Close()
dname := filepath.Join(root, f.Name)
g, err := os.Create(dname)
if err != nil {
return err
}
defer g.Close()
n, err = io.Copy(g, rc)
return err
}(); err != nil {
return err
}
}
return nil
}
a = append(a, string(dict.S(id)))
default:
log.Fatalf("%T", x)
}
}
sort.Strings(a)
for _, v := range a {
dd := ast.Declarations.Identifiers[dict.SID(v)].Node.(*cc.DirectDeclarator)
fmt.Fprintf(buf, "X%s = %v\n", v, dd.EnumVal)
}
fmt.Fprintf(buf, ")\n")
}
func unconvert(pth string) {
wd, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
defer func() {
if err := os.Chdir(wd); err != nil {
log.Fatal(err)
panic("internal error") //TODOOK
}(); err != nil {
fmt.Fprintln(os.Stderr, err)
}
}()
if err := os.Chdir(filepath.Dir(pth)); err != nil {
log.Fatal(err)
}
if out, err := exec.Command(unconvertBin, "-apply").CombinedOutput(); err != nil {
log.Fatalf("unconvert: %s\n%s", err, out)
}
}
func cp(dst, src, glob string) {
pat := filepath.Join(filepath.FromSlash(src), glob)
m, err := filepath.Glob(pat)
if err != nil {
log.Fatal(err)
}
if len(m) == 0 {
log.Fatalf("cp(%q, %q, %q): no files for %q", dst, src, glob, pat)
}
dst = filepath.FromSlash(dst)
for _, v := range m {
f, err := ioutil.ReadFile(v)
if err != nil {
log.Fatal(err)
}
_, nm := filepath.Split(v)
if err := ioutil.WriteFile(filepath.Join(dst, nm), f, 0664); err != nil {
log.Fatal(err)
}
}
func fail(s string, args ...interface{}) {
fmt.Fprintf(os.Stderr, s, args...)
os.Exit(1)
}
func header(f string) []byte {
b, err := ioutil.ReadFile(f)
if err != nil {
log.Fatal(err)
}
var s scanner.Scanner
s.Init(token.NewFileSet().AddFile(f, -1, len(b)), b, nil, scanner.ScanComments)
var buf buffer.Bytes
for {
_, tok, lit := s.Scan()
switch tok {
case token.COMMENT:
buf.WriteString(lit)
buf.WriteByte('\n')
default:
return buf.Bytes()
}
}
}
func tidyComment(s string) string {
switch {
case strings.HasPrefix(s, "/*"):
a := strings.Split("/"+s[1:len(s)-1], "\n")
for i, v := range a {
a[i] = "// " + v
}
return strings.Join(a, "\n") + "/\n"
case strings.HasPrefix(s, "//"):
return "// " + s[2:] + "\n"
default:
panic("internal error")
}
}
func tidyComments(b []byte) string {
var s scanner.Scanner
s.Init(token.NewFileSet().AddFile("", -1, len(b)), b, nil, scanner.ScanComments)
var a []string
for {
_, tok, lit := s.Scan()
if tok == token.EOF {