diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index caaac78..e6d2424 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -5,27 +5,38 @@ name: Node.js CI on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] + +env: + cache-name: postgres-migrations jobs: build: - runs-on: ubuntu-latest strategy: matrix: node-version: [12.x, 14.x, 16.x] - # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: - - uses: actions/checkout@v2 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - run: npm ci - - run: npm run - - run: docker pull postgres:9.4 - - run: npm test + - uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node-version }} + - name: Cache node modules + uses: actions/cache@v2 + env: + cache-name: cache-node-modules + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + - run: npm ci + - run: docker pull postgres:13.2 + - run: npm test diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..d8223b5 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,36 @@ +name: Publish package + +on: + push: + tags: + - "v*" + +env: + cache-name: postgres-migrations + +jobs: + test-and-publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup Nodejs + uses: actions/setup-node@v2 + with: + node-version: 14.x + registry-url: "https://registry.npmjs.org" + - name: Cache node modules + uses: actions/cache@v2 + env: + cache-name: cache-node-modules + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + - run: npm ci + - run: docker pull postgres:13.2 + - env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public diff --git a/.gitignore b/.gitignore index e315e6c..77a8e56 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ node_modules *.log test-reports dist/ +.vscode +.env \ No newline at end of file diff --git a/.prettierignore b/.prettierignore index 1521c8b..db4c6d9 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1 +1,2 @@ dist +node_modules \ No newline at end of file diff --git a/README.md b/README.md index 5ff1947..0fcb7ef 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ # Postgres migrations -![GitHub Actions](https://github.com/ThomWright/postgres-migrations/actions/workflows/node.js.yml/badge.svg) -[![npm](https://img.shields.io/npm/v/postgres-migrations.svg)](https://www.npmjs.com/package/postgres-migrations) -[![David](https://img.shields.io/david/ThomWright/postgres-migrations.svg)](https://david-dm.org/ThomWright/postgres-migrations) -[![David](https://img.shields.io/david/dev/ThomWright/postgres-migrations.svg)](https://david-dm.org/ThomWright/postgres-migrations) +![GitHub Actions](https://github.com/nektarai/postgres-migrations/actions/workflows/node.js.yml/badge.svg) +[![npm](https://img.shields.io/npm/v/@nektarai/postgres-migrations.svg)](https://www.npmjs.com/package/@nektarai/postgres-migrations) A PostgreSQL migration library inspired by the Stack Overflow system described in [Nick Craver's blog](http://nickcraver.com/blog/2016/05/03/stack-overflow-how-we-do-deployment-2016-edition/#database-migrations). @@ -68,7 +66,7 @@ async function() { const client = new pg.Client(dbConfig) // or a Pool, or a PoolClient await client.connect() try { - await migrate({client}, "path/to/migration/files") + await migrate({client}, "path/to/migration/files", { migrationTableName: "my_schema.migrations" }) } finally { await client.end() } diff --git a/ava.config.cjs b/ava.config.cjs index 07d463e..ecb4740 100644 --- a/ava.config.cjs +++ b/ava.config.cjs @@ -1,4 +1,5 @@ module.exports = { extensions: ["ts"], require: ["ts-node/register"], + verbose: true, } diff --git a/package-lock.json b/package-lock.json index 00e8dd9..37628a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { - "name": "postgres-migrations", - "version": "5.3.0", + "name": "@nektarai/postgres-migrations", + "version": "5.3.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index f373daa..770fcca 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,13 @@ { - "name": "postgres-migrations", - "version": "5.3.0", + "name": "@nektarai/postgres-migrations", + "version": "5.3.1", "description": "Stack Overflow style database migrations for PostgreSQL", "main": "dist/index.js", "types": "dist/index.d.ts", "bin": { "pg-validate-migrations": "./dist/bin/validate.js" }, - "author": "Thom Wright", + "author": "Nektar AI", "keywords": [ "postgres", "postgresql", @@ -17,25 +17,24 @@ "database", "db" ], - "homepage": "https://github.com/thomwright/postgres-migrations#readme", + "homepage": "https://github.com/nektarai/postgres-migrations#readme", "license": "MIT", "repository": { "type": "git", - "url": "git@github.com:thomwright/postgres-migrations.git" + "url": "git@github.com:nektarai/postgres-migrations.git" }, "bugs": { - "url": "https://github.com/thomwright/postgres-migrations/issues" + "url": "https://github.com/nektarai/postgres-migrations/issues" }, "engines": { "node": ">10.17.0" }, "scripts": { - "checkPushed": "[ \"$(git rev-list --count @{upstream}..HEAD)\" -eq 0 ] || (echo You have unpushed commits && exit 1)", - "prepublishOnly": "npm run checkPushed && npm test && npm run build", - "check-formatting": "./node_modules/.bin/prettier '**/*.ts' --list-different", - "fix-formatting": "./node_modules/.bin/prettier '**/*.ts' --write", + "prepublishOnly": "npm test && npm run build", + "check-formatting": "prettier '**/*.ts' --list-different", + "fix-formatting": "prettier '**/*.ts' --write", "lint": "npm run tslint && npm run check-formatting", - "tslint": "tslint 'src/**/*.ts' --type-check --project tsconfig.json --format verbose", + "tslint": "tslint 'src/**/*.ts' --project tsconfig.json --format verbose", "test-integration": "ava --config ava.config.integration.cjs", "test-unit": "ava --config ava.config.unit.cjs", "test": "npm run test-unit && npm run lint && npm run test-integration", @@ -68,4 +67,4 @@ "typescript": "^4.3.4", "typescript-tslint-plugin": "^1.0.1" } -} +} \ No newline at end of file diff --git a/src/__tests__/fixtures/docker-postgres.ts b/src/__tests__/fixtures/docker-postgres.ts index 0c83dac..e698cd4 100644 --- a/src/__tests__/fixtures/docker-postgres.ts +++ b/src/__tests__/fixtures/docker-postgres.ts @@ -55,7 +55,7 @@ export const startPostgres = (containerName: string, t: CbExecutionContext) => { --health-interval=1s \ --health-retries=30 \ --health-timeout=1s \ - postgres:9.4`) + postgres:13.2`) const portMapping = execSync(`docker port ${containerName} 5432`).toString() const port = parseInt(portMapping.split(":")[1], 10) diff --git a/src/__tests__/fixtures/success-existing-db/1_success.sql b/src/__tests__/fixtures/success-existing-db/1_success.sql new file mode 100644 index 0000000..598172f --- /dev/null +++ b/src/__tests__/fixtures/success-existing-db/1_success.sql @@ -0,0 +1,3 @@ +CREATE TABLE success ( + id integer +); diff --git a/src/__tests__/fixtures/success-existing-table/1_success.sql b/src/__tests__/fixtures/success-existing-table/1_success.sql new file mode 100644 index 0000000..598172f --- /dev/null +++ b/src/__tests__/fixtures/success-existing-table/1_success.sql @@ -0,0 +1,3 @@ +CREATE TABLE success ( + id integer +); diff --git a/src/__tests__/migrate.ts b/src/__tests__/migrate.ts index 0503395..c040cfc 100644 --- a/src/__tests__/migrate.ts +++ b/src/__tests__/migrate.ts @@ -3,6 +3,7 @@ import test from "ava" import * as pg from "pg" import SQL from "sql-template-strings" import {createDb, migrate, MigrateDBConfig} from "../" +import {loadInitialMigration} from "../initial-migration" import {PASSWORD, startPostgres, stopPostgres} from "./fixtures/docker-postgres" const CONTAINER_NAME = "pg-migrations-test-migrate" @@ -688,35 +689,141 @@ test("rollback", (t) => { }) }) +test("with custom migration table name", async (t) => { + const databaseName = "migration-test-custom-migration-table" + const dbConfig = { + database: databaseName, + user: "postgres", + password: PASSWORD, + host: "localhost", + port, + } + + const migrateWithCustomMigrationTable = () => + migrate(dbConfig, "src/__tests__/fixtures/success-first", { + migrationTableName: "my_migrations", + }) + + await createDb(databaseName, dbConfig) + await migrateWithCustomMigrationTable() + + t.truthy(await doesTableExist(dbConfig, "my_migrations")) + t.truthy(await doesTableExist(dbConfig, "success")) + + await migrateWithCustomMigrationTable() +}) + +test("with custom migration table name in a custom schema", async (t) => { + const databaseName = "migration-test-custom-schema-custom-migration-table" + const dbConfig = { + database: databaseName, + user: "postgres", + password: PASSWORD, + host: "localhost", + port, + } + + const migrateWithCustomMigrationTable = () => + migrate(dbConfig, "src/__tests__/fixtures/success-first", { + migrationTableName: "my_schema.my_migrations", + }) + + const pool = new pg.Pool(dbConfig) + + try { + await createDb(databaseName, dbConfig) + await pool.query("CREATE SCHEMA IF NOT EXISTS my_schema") + await migrateWithCustomMigrationTable() + + t.truthy(await doesTableExist(dbConfig, "my_schema.my_migrations")) + t.truthy(await doesTableExist(dbConfig, "success")) + + await migrateWithCustomMigrationTable() + } finally { + await pool.end() + } +}) + +test("with custom migration table name in a custom schema with same table name in another schema", async (t) => { + const databaseName = "migration-test-success-existing-table" + const dbConfig = { + database: databaseName, + user: "postgres", + password: PASSWORD, + host: "localhost", + port, + } + + const pool = new pg.Pool(dbConfig) + + try { + await createDb(databaseName, dbConfig) + await pool.query(` + CREATE SCHEMA IF NOT EXISTS existing_schema; + + CREATE TABLE existing_schema.migrations ( + id integer + ); + `) + await migrate(dbConfig, "src/__tests__/fixtures/success-existing-table") + t.truthy(await doesTableExist(dbConfig, "success")) + } finally { + await pool.end() + } +}) + +test("successful migration on an existing database", async (t) => { + const databaseName = "migration-test-success-existing-db" + const dbConfig = { + database: databaseName, + user: "postgres", + password: PASSWORD, + host: "localhost", + port, + } + + const pool = new pg.Pool(dbConfig) + + try { + await createDb(databaseName, dbConfig) + const initSql = await loadInitialMigration("migrations") + await pool.query(` + ${initSql.sql} + INSERT INTO migrations ("id","name","hash","executed_at") VALUES (${initSql.id},'${initSql.fileName}','${initSql.hash}','2020-06-29 18:38:05.064546'); + `) + await migrate(dbConfig, "src/__tests__/fixtures/success-existing-db") + t.truthy(await doesTableExist(dbConfig, "success")) + } finally { + await pool.end() + } +}) + function doesTableExist(dbConfig: pg.ClientConfig, tableName: string) { const client = new pg.Client(dbConfig) client.on("error", (err) => console.log("doesTableExist on error", err)) return client .connect() .then(() => - client.query(SQL` - SELECT EXISTS ( - SELECT 1 - FROM pg_catalog.pg_class c - WHERE c.relname = ${tableName} - AND c.relkind = 'r' - ); - `), + client.query(SQL`SELECT to_regclass(${tableName}) as matching_tables;`), ) .then((result) => { try { return client .end() .then(() => { - return result.rows.length > 0 && result.rows[0].exists + return ( + result.rows.length > 0 && result.rows[0].matching_tables !== null + ) }) .catch((error) => { console.log("Async error in 'doesTableExist", error) - return result.rows.length > 0 && result.rows[0].exists + return ( + result.rows.length > 0 && result.rows[0].matching_tables !== null + ) }) } catch (error) { console.log("Sync error in 'doesTableExist", error) - return result.rows.length > 0 && result.rows[0].exists + return result.rows.length > 0 && result.rows[0].matching_tables !== null } }) } diff --git a/src/__unit__/migration-file-validation/validate.ts b/src/__unit__/migration-file-validation/validate.ts index 50e0055..9cb93b9 100644 --- a/src/__unit__/migration-file-validation/validate.ts +++ b/src/__unit__/migration-file-validation/validate.ts @@ -9,6 +9,8 @@ test("two migrations with the same id", async (t) => { const error = await t.throwsAsync(async () => loadMigrationFiles( "src/__unit__/migration-file-validation/fixtures/conflict", + console.log, + "migrations", ), ) t.regex(error.message, /non-consecutive/) diff --git a/src/bin/validate.ts b/src/bin/validate.ts index 545b183..32bbd03 100755 --- a/src/bin/validate.ts +++ b/src/bin/validate.ts @@ -6,8 +6,13 @@ import {loadMigrationFiles} from "../files-loader" async function main(args: Array) { const directory = args[0] + const migrationTableName = args[1] ?? "migrations" - await loadMigrationFiles(directory, (x) => console.error(x)) + await loadMigrationFiles( + directory, + (x) => console.error(x), + migrationTableName, + ) } main(argv.slice(2)).catch((e) => { diff --git a/src/files-loader.ts b/src/files-loader.ts index 013dd2f..c714e2a 100644 --- a/src/files-loader.ts +++ b/src/files-loader.ts @@ -1,6 +1,7 @@ import * as fs from "fs" import * as path from "path" import {promisify} from "util" +import {loadInitialMigration} from "./initial-migration" import {loadMigrationFile} from "./migration-file" import {Logger, Migration} from "./types" import {validateMigrationOrdering} from "./validation" @@ -21,6 +22,7 @@ export const loadMigrationFiles = async ( directory: string, // tslint:disable-next-line no-empty log: Logger = () => {}, + migrationTableName: string, ): Promise> => { log(`Loading migrations from: ${directory}`) @@ -31,17 +33,19 @@ export const loadMigrationFiles = async ( return [] } - const migrationFiles = [ - path.join(__dirname, "migrations/0_create-migrations-table.sql"), - ...fileNames.map((fileName) => path.resolve(directory, fileName)), - ].filter(isValidFile) + const migrationFiles = fileNames + .map((fileName) => path.resolve(directory, fileName)) + .filter(isValidFile) const unorderedMigrations = await Promise.all( migrationFiles.map(loadMigrationFile), ) // Arrange in ID order - const orderedMigrations = unorderedMigrations.sort((a, b) => a.id - b.id) + const orderedMigrations = [ + await loadInitialMigration(migrationTableName), + ...unorderedMigrations.sort((a, b) => a.id - b.id), + ] validateMigrationOrdering(orderedMigrations) diff --git a/src/initial-migration.ts b/src/initial-migration.ts new file mode 100644 index 0000000..ba08075 --- /dev/null +++ b/src/initial-migration.ts @@ -0,0 +1,29 @@ +import {hashString} from "./migration-file" + +export const loadInitialMigration = async (migrationTableName: string) => { + // Since the hash of the initial migration is distributed across users' databases + // the values `fileName` and `sql` must NEVER change! + const fileName = "0_create-migrations-table.sql" + const sql = getInitialMigrationSql(migrationTableName) + const hash = hashString(fileName + sql) + + return { + id: 0, + name: "create-migrations-table", + contents: sql, + fileName, + hash, + sql, + } +} + +// Formatting must not change to ensure content hash remains the same +export const getInitialMigrationSql = ( + migrationTableName: string, +) => `CREATE TABLE IF NOT EXISTS ${migrationTableName} ( + id integer PRIMARY KEY, + name varchar(100) UNIQUE NOT NULL, + hash varchar(40) NOT NULL, -- sha1 hex encoded hash of the file name and contents, to ensure it hasn't been altered since applying the migration + executed_at timestamp DEFAULT current_timestamp +); +` diff --git a/src/migrate.ts b/src/migrate.ts index e00807b..6090795 100644 --- a/src/migrate.ts +++ b/src/migrate.ts @@ -38,6 +38,8 @@ export async function migrate( // } + const migrationTableName = config.migrationTableName ?? "migrations" + if (dbConfig == null) { throw new Error("No config object") } @@ -45,13 +47,17 @@ export async function migrate( if (typeof migrationsDirectory !== "string") { throw new Error("Must pass migrations directory as a string") } - const intendedMigrations = await loadMigrationFiles(migrationsDirectory, log) + const intendedMigrations = await loadMigrationFiles( + migrationsDirectory, + log, + migrationTableName, + ) if ("client" in dbConfig) { // we have been given a client to use, it should already be connected return withAdvisoryLock( log, - runMigrations(intendedMigrations, log), + runMigrations(intendedMigrations, log, migrationTableName), )(dbConfig.client) } @@ -99,18 +105,23 @@ export async function migrate( const runWith = withConnection( log, - withAdvisoryLock(log, runMigrations(intendedMigrations, log)), + withAdvisoryLock( + log, + runMigrations(intendedMigrations, log, migrationTableName), + ), ) return runWith(client) } } -function runMigrations(intendedMigrations: Array, log: Logger) { +function runMigrations( + intendedMigrations: Array, + log: Logger, + migrationTableName: string, +) { return async (client: BasicPgClient) => { try { - const migrationTableName = "migrations" - log("Starting migrations") const appliedMigrations = await fetchAppliedMigrationFromDB( @@ -202,12 +213,9 @@ function logResult(completedMigrations: Array, log: Logger) { /** Check whether table exists in postgres - http://stackoverflow.com/a/24089729 */ async function doesTableExist(client: BasicPgClient, tableName: string) { - const result = await client.query(SQL`SELECT EXISTS ( - SELECT 1 - FROM pg_catalog.pg_class c - WHERE c.relname = ${tableName} - AND c.relkind = 'r' -);`) - - return result.rows.length > 0 && result.rows[0].exists + const result = await client.query(SQL` + SELECT to_regclass(${tableName}) as matching_tables; + `) + + return result.rows.length > 0 && result.rows[0].matching_tables !== null } diff --git a/src/migration-file.ts b/src/migration-file.ts index a766b57..eb3fd9f 100644 --- a/src/migration-file.ts +++ b/src/migration-file.ts @@ -11,7 +11,7 @@ const getFileName = (filePath: string) => path.basename(filePath) const getFileContents = async (filePath: string) => readFile(filePath, "utf8") -const hashString = (s: string) => +export const hashString = (s: string) => crypto.createHash("sha1").update(s, "utf8").digest("hex") const getSqlStringLiteral = ( diff --git a/src/types.ts b/src/types.ts index 5e16f9a..fb7841e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -59,6 +59,11 @@ export type Config = Partial export interface FullConfig { readonly logger: Logger + /** + * The default migrations table. + * @default migrations + */ + readonly migrationTableName: string } export class MigrationError extends Error {