diff --git a/README.md b/README.md index 0c92c85..c3b11a6 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Toolbox and building blocks for new Go projects, to get started quickly and righ * Using https://pkg.go.dev/github.com/go-chi/chi/v5 for routing * [Urfave](https://cli.urfave.org/) for cli args * https://github.com/uber-go/nilaway +* Postgres database with migrations * See also: * Public project setup: https://github.com/flashbots/flashbots-repository-template * Repository for common Go utilities: https://github.com/flashbots/go-utils @@ -55,3 +56,19 @@ make lint make test make fmt ``` + + +**Database tests (using a live Postgres instance)** + +Database tests will be run if the `RUN_DB_TESTS` environment variable is set to `1`. + +```bash +# start the database +docker run -d --name postgres-test -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres postgres + +# run the tests +RUN_DB_TESTS=1 make test + +# stop the database +docker rm -f postgres-test +``` diff --git a/common/utils.go b/common/utils.go new file mode 100644 index 0000000..788b3f0 --- /dev/null +++ b/common/utils.go @@ -0,0 +1,10 @@ +package common + +import "os" + +func GetEnv(key, defaultValue string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return defaultValue +} diff --git a/database/database.go b/database/database.go new file mode 100644 index 0000000..3ab1601 --- /dev/null +++ b/database/database.go @@ -0,0 +1,54 @@ +// Package database exposes the postgres database +package database + +import ( + "os" + + "github.com/flashbots/go-template/database/migrations" + "github.com/flashbots/go-template/database/vars" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + migrate "github.com/rubenv/sql-migrate" +) + +type DatabaseService struct { + DB *sqlx.DB +} + +func NewDatabaseService(dsn string) (*DatabaseService, error) { + db, err := sqlx.Connect("postgres", dsn) + if err != nil { + return nil, err + } + + db.DB.SetMaxOpenConns(50) + db.DB.SetMaxIdleConns(10) + db.DB.SetConnMaxIdleTime(0) + + if os.Getenv("DB_DONT_APPLY_SCHEMA") == "" { + migrate.SetTable(vars.TableMigrations) + _, err := migrate.Exec(db.DB, "postgres", migrations.Migrations, migrate.Up) + if err != nil { + return nil, err + } + } + + dbService := &DatabaseService{DB: db} //nolint:exhaustruct + err = dbService.prepareNamedQueries() + return dbService, err +} + +func (s *DatabaseService) prepareNamedQueries() (err error) { + return nil +} + +func (s *DatabaseService) Close() error { + return s.DB.Close() +} + +func (s *DatabaseService) SomeQuery() (count uint64, err error) { + query := `SELECT COUNT(*) FROM ` + vars.TableTest + `;` + row := s.DB.QueryRow(query) + err = row.Scan(&count) + return count, err +} diff --git a/database/database_test.go b/database/database_test.go new file mode 100644 index 0000000..4982939 --- /dev/null +++ b/database/database_test.go @@ -0,0 +1,50 @@ +package database + +import ( + "os" + "testing" + + "github.com/flashbots/go-template/common" + "github.com/flashbots/go-template/database/migrations" + "github.com/flashbots/go-template/database/vars" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" +) + +var ( + runDBTests = os.Getenv("RUN_DB_TESTS") == "1" //|| true + testDBDSN = common.GetEnv("TEST_DB_DSN", "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable") +) + +func resetDatabase(t *testing.T) *DatabaseService { + t.Helper() + if !runDBTests { + t.Skip("Skipping database tests") + } + + // Wipe test database + _db, err := sqlx.Connect("postgres", testDBDSN) + require.NoError(t, err) + _, err = _db.Exec(`DROP SCHEMA public CASCADE; CREATE SCHEMA public;`) + require.NoError(t, err) + + db, err := NewDatabaseService(testDBDSN) + require.NoError(t, err) + return db +} + +func TestMigrations(t *testing.T) { + db := resetDatabase(t) + query := `SELECT COUNT(*) FROM ` + vars.TableMigrations + `;` + rowCount := 0 + err := db.DB.QueryRow(query).Scan(&rowCount) + require.NoError(t, err) + require.Len(t, migrations.Migrations.Migrations, rowCount) +} + +func Test_DB1(t *testing.T) { + db := resetDatabase(t) + x, err := db.SomeQuery() + require.NoError(t, err) + require.Equal(t, uint64(0), x) +} diff --git a/database/migrations/001_init_database.go b/database/migrations/001_init_database.go new file mode 100644 index 0000000..7ab713b --- /dev/null +++ b/database/migrations/001_init_database.go @@ -0,0 +1,21 @@ +package migrations + +import ( + "github.com/flashbots/go-template/database/vars" + migrate "github.com/rubenv/sql-migrate" +) + +var Migration001InitDatabase = &migrate.Migration{ + Id: "001-init-database", + Up: []string{` + CREATE TABLE IF NOT EXISTS ` + vars.TableTest + ` ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + inserted_at timestamp NOT NULL default current_timestamp + ); + `}, + Down: []string{` + DROP TABLE IF EXISTS ` + vars.TableTest + `; + `}, + DisableTransactionUp: false, + DisableTransactionDown: false, +} diff --git a/database/migrations/migration.go b/database/migrations/migration.go new file mode 100644 index 0000000..a990cca --- /dev/null +++ b/database/migrations/migration.go @@ -0,0 +1,12 @@ +// Package migrations contains all the migration files +package migrations + +import ( + migrate "github.com/rubenv/sql-migrate" +) + +var Migrations = migrate.MemoryMigrationSource{ + Migrations: []*migrate.Migration{ + Migration001InitDatabase, + }, +} diff --git a/database/types.go b/database/types.go new file mode 100644 index 0000000..85b6593 --- /dev/null +++ b/database/types.go @@ -0,0 +1,29 @@ +package database + +import ( + "database/sql" + "time" +) + +func NewNullInt64(i int64) sql.NullInt64 { + return sql.NullInt64{ + Int64: i, + Valid: true, + } +} + +func NewNullString(s string) sql.NullString { + return sql.NullString{ + String: s, + Valid: true, + } +} + +// NewNullTime returns a sql.NullTime with the given time.Time. If the time is +// the zero value, the NullTime is invalid. +func NewNullTime(t time.Time) sql.NullTime { + return sql.NullTime{ + Time: t, + Valid: t != time.Time{}, + } +} diff --git a/database/types_test.go b/database/types_test.go new file mode 100644 index 0000000..8f4dfed --- /dev/null +++ b/database/types_test.go @@ -0,0 +1,19 @@ +package database + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestNewNullTime(t *testing.T) { + var t1 time.Time + nt1 := NewNullTime(t1) + require.False(t, nt1.Valid) + + t1 = time.Now() + nt1 = NewNullTime(t1) + require.True(t, nt1.Valid) + require.Equal(t, t1, nt1.Time) +} diff --git a/database/vars/tables.go b/database/vars/tables.go new file mode 100644 index 0000000..5e47c64 --- /dev/null +++ b/database/vars/tables.go @@ -0,0 +1,11 @@ +// Package vars contains the database variables such as dynamic table names +package vars + +import "github.com/flashbots/go-template/common" + +var ( + tablePrefix = common.GetEnv("DB_TABLE_PREFIX", "dev") + + TableMigrations = tablePrefix + "_migrations" + TableTest = tablePrefix + "_test" +) diff --git a/go.mod b/go.mod index c021ba6..c0ba625 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,10 @@ require ( github.com/flashbots/go-utils v0.6.1-0.20240610084140-4461ab748667 github.com/go-chi/chi/v5 v5.0.12 github.com/google/uuid v1.6.0 + github.com/jmoiron/sqlx v1.4.0 + github.com/lib/pq v1.10.9 github.com/prometheus/client_golang v1.19.1 + github.com/rubenv/sql-migrate v1.7.0 github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.2 go.opentelemetry.io/contrib/instrumentation/runtime v0.46.1 @@ -22,6 +25,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/ethereum/go-ethereum v1.13.14 // indirect + github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/holiman/uint256 v1.2.4 // indirect diff --git a/go.sum b/go.sum index 662cbf7..bafb283 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -15,23 +17,35 @@ github.com/flashbots/go-utils v0.6.1-0.20240610084140-4461ab748667 h1:Zpdah3TPNH github.com/flashbots/go-utils v0.6.1-0.20240610084140-4461ab748667/go.mod h1:6ZfgrAI+ApKSBF4QghFO06VfRJGGAOOyG4DO0siN2ow= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= +github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= +github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= @@ -42,6 +56,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI= +github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=