Postgres database + migrations

README.md

@@ -16,6 +16,7 @@ Toolbox and building blocks for new Go projects, to get started quickly and righ
   * Using https://pkg.go.dev/github.com/go-chi/chi/v5 for routing
   * [Urfave](https://cli.urfave.org/) for cli args
 * https://github.com/uber-go/nilaway
+* Postgres database with migrations
 * See also:
   * Public project setup: https://github.com/flashbots/flashbots-repository-template
   * Repository for common Go utilities: https://github.com/flashbots/go-utils
@@ -55,3 +56,19 @@ make lint
 make test
 make fmt
 ```
+
+
+**Database tests (using a live Postgres instance)**
+
+Database tests will be run if the `RUN_DB_TESTS` environment variable is set to `1`.
+
+```bash
+# start the database
+docker run -d --name postgres-test -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres postgres
+
+# run the tests
+RUN_DB_TESTS=1 make test
+
+# stop the database
+docker rm -f postgres-test
+```

common/utils.go

@@ -0,0 +1,10 @@
+package common
+
+import "os"
+
+func GetEnv(key, defaultValue string) string {
+	if value, ok := os.LookupEnv(key); ok {
+		return value
+	}
+	return defaultValue
+}

database/database.go

@@ -0,0 +1,54 @@
+// Package database exposes the postgres database
+package database
+
+import (
+	"os"
+
+	"github.com/flashbots/go-template/database/migrations"
+	"github.com/flashbots/go-template/database/vars"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq"
+	migrate "github.com/rubenv/sql-migrate"
+)
+
+type DatabaseService struct {
+	DB *sqlx.DB
+}
+
+func NewDatabaseService(dsn string) (*DatabaseService, error) {
+	db, err := sqlx.Connect("postgres", dsn)
+	if err != nil {
+		return nil, err
+	}
+
+	db.DB.SetMaxOpenConns(50)
+	db.DB.SetMaxIdleConns(10)
+	db.DB.SetConnMaxIdleTime(0)
+
+	if os.Getenv("DB_DONT_APPLY_SCHEMA") == "" {
+		migrate.SetTable(vars.TableMigrations)
+		_, err := migrate.Exec(db.DB, "postgres", migrations.Migrations, migrate.Up)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	dbService := &DatabaseService{DB: db} //nolint:exhaustruct
+	err = dbService.prepareNamedQueries()
+	return dbService, err
+}
+
+func (s *DatabaseService) prepareNamedQueries() (err error) {
+	return nil
+}
+
+func (s *DatabaseService) Close() error {
+	return s.DB.Close()
+}
+
+func (s *DatabaseService) SomeQuery() (count uint64, err error) {
+	query := `SELECT COUNT(*) FROM ` + vars.TableTest + `;`
+	row := s.DB.QueryRow(query)
+	err = row.Scan(&count)
+	return count, err
+}

database/database_test.go

@@ -0,0 +1,50 @@
+package database
+
+import (
+	"os"
+	"testing"
+
+	"github.com/flashbots/go-template/common"
+	"github.com/flashbots/go-template/database/migrations"
+	"github.com/flashbots/go-template/database/vars"
+	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	runDBTests = os.Getenv("RUN_DB_TESTS") == "1" //|| true
+	testDBDSN  = common.GetEnv("TEST_DB_DSN", "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable")
+)
+
+func resetDatabase(t *testing.T) *DatabaseService {
+	t.Helper()
+	if !runDBTests {
+		t.Skip("Skipping database tests")
+	}
+
+	// Wipe test database
+	_db, err := sqlx.Connect("postgres", testDBDSN)
+	require.NoError(t, err)
+	_, err = _db.Exec(`DROP SCHEMA public CASCADE; CREATE SCHEMA public;`)
+	require.NoError(t, err)
+
+	db, err := NewDatabaseService(testDBDSN)
+	require.NoError(t, err)
+	return db
+}
+
+func TestMigrations(t *testing.T) {
+	db := resetDatabase(t)
+	query := `SELECT COUNT(*) FROM ` + vars.TableMigrations + `;`
+	rowCount := 0
+	err := db.DB.QueryRow(query).Scan(&rowCount)
+	require.NoError(t, err)
+	require.Len(t, migrations.Migrations.Migrations, rowCount)
+}
+
+func Test_DB1(t *testing.T) {
+	db := resetDatabase(t)
+	x, err := db.SomeQuery()
+	require.NoError(t, err)
+	require.Equal(t, uint64(0), x)
+}

database/migrations/001_init_database.go

@@ -0,0 +1,21 @@
+package migrations
+
+import (
+	"github.com/flashbots/go-template/database/vars"
+	migrate "github.com/rubenv/sql-migrate"
+)
+
+var Migration001InitDatabase = &migrate.Migration{
+	Id: "001-init-database",
+	Up: []string{`
+		CREATE TABLE IF NOT EXISTS ` + vars.TableTest + ` (
+			id          bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+			inserted_at timestamp NOT NULL default current_timestamp
+		);
+	`},
+	Down: []string{`
+		DROP TABLE IF EXISTS ` + vars.TableTest + `;
+	`},
+	DisableTransactionUp:   false,
+	DisableTransactionDown: false,
+}

database/migrations/migration.go

@@ -0,0 +1,12 @@
+// Package migrations contains all the migration files
+package migrations
+
+import (
+	migrate "github.com/rubenv/sql-migrate"
+)
+
+var Migrations = migrate.MemoryMigrationSource{
+	Migrations: []*migrate.Migration{
+		Migration001InitDatabase,
+	},
+}

database/types.go

@@ -0,0 +1,29 @@
+package database
+
+import (
+	"database/sql"
+	"time"
+)
+
+func NewNullInt64(i int64) sql.NullInt64 {
+	return sql.NullInt64{
+		Int64: i,
+		Valid: true,
+	}
+}
+
+func NewNullString(s string) sql.NullString {
+	return sql.NullString{
+		String: s,
+		Valid:  true,
+	}
+}
+
+// NewNullTime returns a sql.NullTime with the given time.Time. If the time is
+// the zero value, the NullTime is invalid.
+func NewNullTime(t time.Time) sql.NullTime {
+	return sql.NullTime{
+		Time:  t,
+		Valid: t != time.Time{},
+	}
+}

database/types_test.go

@@ -0,0 +1,19 @@
+package database
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewNullTime(t *testing.T) {
+	var t1 time.Time
+	nt1 := NewNullTime(t1)
+	require.False(t, nt1.Valid)
+
+	t1 = time.Now()
+	nt1 = NewNullTime(t1)
+	require.True(t, nt1.Valid)
+	require.Equal(t, t1, nt1.Time)
+}

database/vars/tables.go

@@ -0,0 +1,11 @@
+// Package vars contains the database variables such as dynamic table names
+package vars
+
+import "github.com/flashbots/go-template/common"
+
+var (
+	tablePrefix = common.GetEnv("DB_TABLE_PREFIX", "dev")
+
+	TableMigrations = tablePrefix + "_migrations"
+	TableTest       = tablePrefix + "_test"
+)

go.mod

@@ -6,7 +6,10 @@ require (
 	github.com/flashbots/go-utils v0.6.1-0.20240610084140-4461ab748667
 	github.com/go-chi/chi/v5 v5.0.12
 	github.com/google/uuid v1.6.0
+	github.com/jmoiron/sqlx v1.4.0
+	github.com/lib/pq v1.10.9
 	github.com/prometheus/client_golang v1.19.1
+	github.com/rubenv/sql-migrate v1.7.0
 	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.27.2
 	go.opentelemetry.io/contrib/instrumentation/runtime v0.46.1
@@ -22,6 +25,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/ethereum/go-ethereum v1.13.14 // indirect
+	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/holiman/uint256 v1.2.4 // indirect

go.sum

@@ -1,3 +1,5 @@
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -15,23 +17,35 @@ github.com/flashbots/go-utils v0.6.1-0.20240610084140-4461ab748667 h1:Zpdah3TPNH
 github.com/flashbots/go-utils v0.6.1-0.20240610084140-4461ab748667/go.mod h1:6ZfgrAI+ApKSBF4QghFO06VfRJGGAOOyG4DO0siN2ow=
 github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s=
 github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs=
+github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU=
 github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E=
+github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
+github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY=
+github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg=
 github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
 github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
 github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
@@ -42,6 +56,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI=
+github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=