generator.go

// Copyright 2017 The Sqlite Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/format"
	"go/scanner"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"sort"
	"strings"

	"log"

	"github.com/cznic/cc"
	"github.com/cznic/ccgo"
	"github.com/cznic/ccir"
	"github.com/cznic/internal/buffer"
	"github.com/cznic/strutil"
	"github.com/cznic/xc"
)

var (
	cpp          = flag.Bool("cpp", false, "")
	dict         = xc.Dict
	errLimit     = flag.Int("errlimit", 10, "")
	filter       = flag.String("re", "", "")
	ndebug       = flag.Bool("ndebug", false, "")
	noexec       = flag.Bool("noexec", false, "")
	oLog         = flag.Bool("log", false, "")
	trace        = flag.Bool("trc", false, "")
	unconvertBin string
	yydebug      = flag.Int("yydebug", 0, "")
)

const (
	sqliteRepo = "sqlite.org"
	version    = "3190300"

	prologueSqlite = `// Code generated by ccgo. DO NOT EDIT.

/*

%s
*/

// Package sqlite is an in-process implementation of a self-contained,
// serverless, zero-configuration, transactional SQL database engine. (Work In Progress)
%s
package bin

import (
	"fmt"
	"math"
	"os"
	"path"
	"runtime"
	"unsafe"

	"github.com/cznic/ccgo/crt"
)

func ftrace(s string, args ...interface{}) {
	_, fn, fl, _ := runtime.Caller(1)
	fmt.Fprintf(os.Stderr, "# %%s:%%d: %%v\n", path.Base(fn), fl, fmt.Sprintf(s, args...))
	os.Stderr.Sync()
}
`

	prologueTest = `// Code generated by ccgo. DO NOT EDIT.

// %s
%s
package main

import (
	"math"
	"os"
	"unsafe"

	"github.com/cznic/ccgo/crt"
	"github.com/cznic/sqlite/internal/bin"
)

var argv []*int8

func main() {
	for _, v := range os.Args {
		argv = append(argv, (*int8)(crt.CString(v)))
	}
	argv = append(argv, nil)
	X_start(crt.NewTLS(), int32(len(os.Args)), &argv[0])
}

`

	defines = `
		#define HAVE_MALLOC_H 1
		#define HAVE_MALLOC_USABLE_SIZE 1
		#define HAVE_USLEEP 1
		#define SQLITE_DEBUG 1
		#define SQLITE_ENABLE_API_ARMOR 1
		#define SQLITE_USE_URI 1
		#define SQLITE_WITHOUT_MSIZE 1

		int sqlite3PendingByte;
		`
)

func findRepo(s string) string {
	s = filepath.FromSlash(s)
	for _, v := range strings.Split(strutil.Gopath(), string(os.PathListSeparator)) {
		p := filepath.Join(v, "src", s)
		fi, err := os.Lstat(p)
		if err != nil {
			continue
		}

		if fi.IsDir() {
			wd, err := os.Getwd()
			if err != nil {
				log.Fatal(err)
			}

			if p, err = filepath.Rel(wd, p); err != nil {
				log.Fatal(err)
			}

			return p
		}
	}
	return ""
}

func errStr(err error) string {
	switch x := err.(type) {
	case scanner.ErrorList:
		if len(x) != 1 {
			x.RemoveMultiples()
		}
		var b bytes.Buffer
		for i, v := range x {
			if i != 0 {
				b.WriteByte('\n')
			}
			b.WriteString(v.Error())
			if i == 9 {
				fmt.Fprintf(&b, "\n\t... and %v more errors", len(x)-10)
				break
			}
		}
		return b.String()
	default:
		return err.Error()
	}
}

func build(predef string, tus [][]string, ccgoOpts []ccgo.Option, opts ...cc.Opt) ([]*cc.TranslationUnit, []byte) {
	ndbg := ""
	if *ndebug {
		ndbg = "#define NDEBUG 1"
	}

	var lpos token.Position
	if *cpp {
		opts = append(opts, cc.Cpp(func(toks []xc.Token) {
			if len(toks) != 0 {
				p := toks[0].Position()
				if p.Filename != lpos.Filename {
					fmt.Fprintf(os.Stderr, "# %d %q\n", p.Line, p.Filename)
				}
				lpos = p
			}
			for _, v := range toks {
				os.Stderr.WriteString(cc.TokSrc(v))
			}
			os.Stderr.WriteString("\n")
		}))
	}

	var build []*cc.TranslationUnit
	tus = append(tus, []string{ccir.CRT0Path})
	for _, src := range tus {
		model, err := ccir.NewModel()
		if err != nil {
			log.Fatal(err)
		}

		ast, err := cc.Parse(
			fmt.Sprintf(`
%s
#define _CCGO 1
#define __arch__ %s
#define __os__ %s
#include <builtin.h>
%s
`, ndbg, runtime.GOARCH, runtime.GOOS, predef),
			src,
			model,
			append([]cc.Opt{
				cc.AllowCompatibleTypedefRedefinitions(),
				cc.EnableEmptyStructs(),
				cc.EnableImplicitFuncDef(),
				cc.EnableNonConstStaticInitExpressions(),
				cc.EnableWideBitFieldTypes(),
				cc.ErrLimit(*errLimit),
				cc.KeepComments(),
				cc.SysIncludePaths([]string{ccir.LibcIncludePath}),
			}, opts...)...,
		)
		if err != nil {
			log.Fatal(errStr(err))
		}

		build = append(build, ast)
	}

	var out buffer.Bytes
	if err := ccgo.New(build, &out, ccgoOpts...); err != nil {
		log.Fatal(err)
	}

	return build, out.Bytes()
}

func macros(buf io.Writer, ast *cc.TranslationUnit) {
	fmt.Fprintf(buf, `const (
`)
	var a []string
	for k, v := range ast.Macros {
		if v.Value != nil && v.Type.Kind() != cc.Bool {
			switch fn := v.DefTok.Position().Filename; {
			case
				fn == "builtin.h",
				fn == "<predefine>",
				strings.HasPrefix(fn, "predefined_"):
				// ignore
			default:
				a = append(a, string(dict.S(k)))
			}
		}
	}
	sort.Strings(a)
	for _, v := range a {
		m := ast.Macros[dict.SID(v)]
		if m.Value == nil {
			log.Fatal("TODO")
		}

		switch t := m.Type; t.Kind() {
		case
			cc.Int, cc.UInt, cc.Long, cc.ULong, cc.LongLong, cc.ULongLong,
			cc.Float, cc.Double, cc.LongDouble, cc.Bool:
			fmt.Fprintf(buf, "X%s = %v\n", v, m.Value)
		case cc.Ptr:
			switch t := t.Element(); t.Kind() {
			case cc.Char:
				fmt.Fprintf(buf, "X%s = %q\n", v, dict.S(int(m.Value.(cc.StringLitID))))
			default:
				log.Fatalf("%v", t.Kind())
			}
		default:
			log.Fatalf("%v", t.Kind())
		}
	}

	a = a[:0]
	for _, v := range ast.Declarations.Identifiers {
		switch x := v.Node.(type) {
		case *cc.DirectDeclarator:
			d := x.TopDeclarator()
			id, _ := d.Identifier()
			if x.EnumVal == nil {
				break
			}

			a = append(a, string(dict.S(id)))
		default:
			log.Fatalf("%T", x)
		}
	}
	sort.Strings(a)
	for _, v := range a {
		dd := ast.Declarations.Identifiers[dict.SID(v)].Node.(*cc.DirectDeclarator)
		fmt.Fprintf(buf, "X%s = %v\n", v, dd.EnumVal)
	}
	fmt.Fprintf(buf, ")\n")
}

func unconvert(pth string) {
	wd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	defer func() {
		if err := os.Chdir(wd); err != nil {
			log.Fatal(err)
		}
	}()

	if err := os.Chdir(filepath.Dir(pth)); err != nil {
		log.Fatal(err)
	}

	if out, err := exec.Command(unconvertBin, "-apply").CombinedOutput(); err != nil {
		log.Fatalf("unconvert: %s\n%s", err, out)
	}
}

func cp(dst, src, glob string) {
	pat := filepath.Join(filepath.FromSlash(src), glob)
	m, err := filepath.Glob(pat)
	if err != nil {
		log.Fatal(err)
	}

	if len(m) == 0 {
		log.Fatalf("cp(%q, %q, %q): no files for %q", dst, src, glob, pat)
	}

	dst = filepath.FromSlash(dst)
	for _, v := range m {
		f, err := ioutil.ReadFile(v)
		if err != nil {
			log.Fatal(err)
		}

		_, nm := filepath.Split(v)
		if err := ioutil.WriteFile(filepath.Join(dst, nm), f, 0664); err != nil {
			log.Fatal(err)
		}
	}
}

func header(f string) []byte {
	b, err := ioutil.ReadFile(f)
	if err != nil {
		log.Fatal(err)
	}

	var s scanner.Scanner
	s.Init(token.NewFileSet().AddFile(f, -1, len(b)), b, nil, scanner.ScanComments)
	var buf buffer.Bytes
	for {
		_, tok, lit := s.Scan()
		switch tok {
		case token.COMMENT:
			buf.WriteString(lit)
			buf.WriteByte('\n')
		default:
			return buf.Bytes()
		}
	}
}

func tidyComment(s string) string {
	switch {
	case strings.HasPrefix(s, "/*"):
		a := strings.Split("/"+s[1:len(s)-1], "\n")
		for i, v := range a {
			a[i] = "//  " + v
		}
		return strings.Join(a, "\n") + "/\n"
	case strings.HasPrefix(s, "//"):
		return "//  " + s[2:] + "\n"
	default:
		panic("internal error")
	}
}

func tidyComments(b []byte) string {
	var s scanner.Scanner
	s.Init(token.NewFileSet().AddFile("", -1, len(b)), b, nil, scanner.ScanComments)
	var a []string
	for {
		_, tok, lit := s.Scan()
		if tok == token.EOF {
			return strings.Join(a, "\n")
		}

		a = append(a, tidyComment(lit))
	}
}

func sqlite() {
	repo := findRepo(sqliteRepo)
	if repo == "" {
		log.Fatalf("repository not found: %v", sqliteRepo)
		return
	}

	pth := filepath.Join(repo, "sqlite-amalgamation-"+version)
	ast, _ := build(
		defines,
		[][]string{
			{filepath.Join(pth, "sqlite3.h")},
			{"main.c"},
		},
		[]ccgo.Option{ccgo.Library(), ccgo.LibcTypes()},
		cc.EnableAnonymousStructFields(),
		cc.IncludePaths([]string{pth}),
	)
	sqlite3 := filepath.Join(pth, "sqlite3.c")
	_, src := build(
		defines,
		[][]string{
			{sqlite3},
			{"main.c"},
		},
		[]ccgo.Option{ccgo.Library(), ccgo.LibcTypes()},
		cc.EnableAnonymousStructFields(),
		cc.IncludePaths([]string{pth}),
	)

	var b bytes.Buffer
	lic, err := ioutil.ReadFile("SQLITE-LICENSE")
	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintf(&b, prologueSqlite, lic, strings.TrimSpace(tidyComments(header(sqlite3))))
	macros(&b, ast[0])
	b.Write(src)
	b2, err := format.Source(b.Bytes())
	if err != nil {
		b2 = b.Bytes()
	}
	if err := os.MkdirAll(filepath.Join("internal", "bin"), 0775); err != nil {
		log.Fatal(err)
	}

	dst := fmt.Sprintf(filepath.Join("internal", "bin", "bin_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
	b2 = bytes.Replace(b2, []byte("var Xsqlite3PendingByte int32"), []byte("func Xsqlite3PendingByte() int32 { return _sqlite3PendingByte }"), 1)
	if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
		log.Fatal(err)
	}

	unconvert(dst)
}

func mpTest() {
	repo := findRepo(sqliteRepo)
	if repo == "" {
		log.Fatalf("repository not found: %v", sqliteRepo)
		return
	}

	sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version)
	pth := filepath.Join(repo, "sqlite-src-"+version, "mptest")
	tag := "mptest"
	test := filepath.Join(pth, tag+".c")
	_, src := build(
		defines,
		[][]string{
			{filepath.Join(sqlitePth, "sqlite3.c")},
			{test},
		},
		[]ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()},
		cc.EnableAnonymousStructFields(),
		cc.IncludePaths([]string{sqlitePth}),
	)

	var b bytes.Buffer
	fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test))))
	b.Write(src)
	b2, err := format.Source(b.Bytes())
	if err != nil {
		b2 = b.Bytes()
	}
	if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil {
		log.Fatal(err)
	}

	if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil {
		log.Fatal(err)
	}

	dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
	if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
		log.Fatal(err)
	}

	unconvert(dst)
	cp(filepath.Join("testdata", tag), pth, "*.test")
	cp(filepath.Join("testdata", tag), pth, "*.subtest")
}

func threadTest1() {
	repo := findRepo(sqliteRepo)
	if repo == "" {
		log.Fatalf("repository not found: %v", sqliteRepo)
		return
	}

	sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version)
	tag := "threadtest1"
	test := filepath.Join("internal", "sqlite.org", "sqlite-src-3190300", "test", "threadtest1.c")
	_, src := build(
		defines,
		[][]string{
			{filepath.Join(sqlitePth, "sqlite3.c")},
			{test},
		},
		[]ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()},
		cc.EnableAnonymousStructFields(),
		cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}),
	)

	var b bytes.Buffer
	fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test))))
	b.Write(src)
	b2, err := format.Source(b.Bytes())
	if err != nil {
		b2 = b.Bytes()
	}
	if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil {
		log.Fatal(err)
	}

	if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil {
		log.Fatal(err)
	}

	dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
	if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
		log.Fatal(err)
	}

	unconvert(dst)
}

func threadTest2() {
	repo := findRepo(sqliteRepo)
	if repo == "" {
		log.Fatalf("repository not found: %v", sqliteRepo)
		return
	}

	sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version)
	pth := filepath.Join(repo, "sqlite-src-"+version, "test")
	tag := "threadtest2"
	test := filepath.Join(pth, tag+".c")
	_, src := build(
		defines,
		[][]string{
			{filepath.Join(sqlitePth, "sqlite3.c")},
			{test},
		},
		[]ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()},
		cc.EnableAnonymousStructFields(),
		cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}),
	)

	var b bytes.Buffer
	fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test))))
	b.Write(src)
	b2, err := format.Source(b.Bytes())
	if err != nil {
		b2 = b.Bytes()
	}
	if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil {
		log.Fatal(err)
	}

	if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil {
		log.Fatal(err)
	}

	dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
	if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
		log.Fatal(err)
	}

	unconvert(dst)
}

func threadTest3() {
	n := 3
	repo := findRepo(sqliteRepo)
	if repo == "" {
		log.Fatalf("repository not found: %v", sqliteRepo)
		return
	}

	sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version)
	pth := filepath.Join(repo, "sqlite-src-"+version, "test")
	tag := fmt.Sprintf("threadtest%v", n)
	test := filepath.Join(pth, tag+".c")
	_, src := build(
		defines,
		[][]string{
			{filepath.Join(sqlitePth, "sqlite3.c")},
			{filepath.Join(repo, "sqlite-src-"+version, "src", "test_multiplex.c")},
			{test},
		},
		[]ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()},
		cc.EnableAnonymousStructFields(),
		cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}),
	)

	var b bytes.Buffer
	fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test))))
	b.Write(src)
	b2, err := format.Source(b.Bytes())
	if err != nil {
		b2 = b.Bytes()
	}
	if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil {
		log.Fatal(err)
	}

	if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil {
		log.Fatal(err)
	}

	dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
	b2 = bytes.Replace(b2, []byte("Xsqlite3PendingByte"), []byte("bin.Xsqlite3PendingByte()"), -1)
	if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
		log.Fatal(err)
	}

	unconvert(dst)
}

func threadTest4() {
	repo := findRepo(sqliteRepo)
	if repo == "" {
		log.Fatalf("repository not found: %v", sqliteRepo)
		return
	}

	sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version)
	tag := "threadtest4"
	test := filepath.Join("internal", "sqlite.org", "sqlite-src-3190300", "test", "threadtest4.c")
	_, src := build(
		defines,
		[][]string{
			{filepath.Join(sqlitePth, "sqlite3.c")},
			{test},
		},
		[]ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()},
		cc.EnableAnonymousStructFields(),
		cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}),
	)

	var b bytes.Buffer
	fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test))))
	b.Write(src)
	b2, err := format.Source(b.Bytes())
	if err != nil {
		b2 = b.Bytes()
	}
	if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil {
		log.Fatal(err)
	}

	if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil {
		log.Fatal(err)
	}

	dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
	if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
		log.Fatal(err)
	}

	unconvert(dst)
}

func main() {
	log.SetFlags(log.Lshortfile | log.Lmicroseconds)
	var err error
	if unconvertBin, err = exec.LookPath("unconvert"); err != nil {
		log.Fatal("Please install the unconvert tool (go get -u github.com/mdempsky/unconvert)")
	}

	flag.Parse()

	sqlite()
	mpTest()
	threadTest1()
	threadTest2()
	threadTest3()
	threadTest4()
}