Use paste.deploy to parse alembic config files

Alembic uses configparser to parse its config files, which is
compatible with the paste.deploy ini config files in terms of
encoding, but paste.deploy has additional features such as
`user = config:` lines. This reduces the amount of places that
alembic config files are generated throughout the codebase and
uses paste.deploy to extract the config values for the `app:conduit`
section before directly inserting them into an alembic config.

This allows users to omit `script_location` from test.ini, and moreover
to omit any parameters required by alembic that are set in a parent
of the relevant app:conduit block.

etc/production.ini

@@ -21,10 +21,6 @@ pyramid_force_https.structlog = true
 
 sqlalchemy.url = ${DATABASE_URL}
 
-# TODO: this here shouldn't be necessary, alembic should inherit
-# script_location from development.ini, but it doesn't
-script_location = src/conduit/migrations
-
 # secrets
 jwt.secret = ${JWT_SECRET}
 

etc/test.ini

@@ -5,10 +5,6 @@ pipeline =
 [app:conduit]
 use = config:development.ini#conduit
 
-# TODO: this here shouldn't be necessary, alembic should inherit
-# script_location from development.ini, but it doesn't
-script_location = src/conduit/migrations
-
 sqlalchemy.url = postgresql+psycopg2://conduit_test@localhost/conduit_test
 
 [server:main]

mypy.ini

@@ -16,6 +16,9 @@ ignore_missing_imports = True
 [mypy-passlib.*]
 ignore_missing_imports = True
 
+[mypy-paste.*]
+ignore_missing_imports = True
+
 [mypy-pyramid.*]
 ignore_missing_imports = True
 

src/conduit/__init__.py

@@ -4,6 +4,8 @@
 from alembic.config import Config
 from alembic.migration import MigrationContext
 from alembic.script import ScriptDirectory
+from paste.deploy.loadwsgi import APP
+from paste.deploy.loadwsgi import ConfigLoader
 from pyramid.config import Configurator
 from pyramid.router import Router
 from pyramid_heroku import expandvars_dict
@@ -37,6 +39,25 @@ def configure_logging() -> None:
     )
 
 
+def get_alembic_config(ini_path: str) -> Config:
+    """Build an alembic config file that supports use= lines.
+
+    The paste.deploy config files that we are deriving alembic configuration from
+    support extensions to the configparser format. This uses paste.deploy to parse the
+    config and then creates a new alembic configuration that uses the extracted values
+    directly.
+    """
+    # Use paste.deploy to parse config file and build an alembic config
+    test_ini = ConfigLoader(ini_path)
+    configuration = test_ini.get_context(APP, "conduit").config()
+
+    # The alembic config only cares about app:conduit
+    alembic_config = Config(ini_section="app:conduit")
+    for key, value in configuration.items():
+        alembic_config.set_section_option("app:conduit", key, value)
+    return alembic_config
+
+
 def check_db_migrated(config: Configurator, global_config: t.Dict[str, str]) -> None:
     """Check if db is migrated to the latest alembic step.
 
@@ -50,7 +71,8 @@ def check_db_migrated(config: Configurator, global_config: t.Dict[str, str]) ->
         return
 
     # get latest migration file
-    alembic_config = Config(global_config["__file__"], "app:conduit")
+    alembic_config = get_alembic_config(global_config["__file__"])
+
     script = ScriptDirectory.from_config(alembic_config)
     head = EnvironmentContext(alembic_config, script).get_head_revision()
 

src/conduit/conftest.py

@@ -7,7 +7,7 @@
 """
 
 from alembic import command
-from alembic.config import Config
+from conduit import get_alembic_config
 from conduit.scripts.populate import add_articles
 from conduit.scripts.populate import add_users
 from pyramid.paster import bootstrap
@@ -40,8 +40,7 @@ def app_env(ini_path: str) -> AppEnvType:
     """Initialize WSGI application from INI file given on the command line."""
     env = bootstrap(ini_path, options={"SKIP_CHECK_DB_MIGRATED": "true"})
 
-    # build schema
-    alembic_cfg = Config("etc/test.ini", "app:conduit")
+    alembic_cfg = get_alembic_config(ini_path)
     command.upgrade(alembic_cfg, "head")
     return env
 

src/conduit/scripts/tests/test_drop_tables.py

@@ -4,8 +4,6 @@
 simple testing here.
 """
 
-from alembic import command
-from alembic.config import Config
 from conduit.conftest import AppEnvType
 from conduit.scripts.drop_tables import main
 from pyramid.registry import Registry
@@ -32,6 +30,7 @@ def test_drop_tables(argparse: mock.MagicMock, app_env: AppEnvType) -> None:
     )
     assert len(list(tables)) == 0
 
-    # re-create the dropped tables so that other tests work
-    alembic_cfg = Config("etc/test.ini", "app:conduit")
-    command.upgrade(alembic_cfg, "head")
+    # re-create the dropped tables so that other tests work by re-running a fixture
+    from conduit.conftest import app_env as app_env_fixture
+
+    app_env_fixture.__wrapped__("etc/test.ini")

src/conduit/tests/test_check_db_migrated.py

@@ -49,7 +49,7 @@ def test_database_outdated(
     config.registry.settings[
         "sqlalchemy.engine"
     ].connect.return_value.__exit__ = mock.Mock(return_value=None)
-    global_config = {"__file__": "foofile"}
+    global_config = {"__file__": "etc/test.ini"}
 
     EnvironmentContext.return_value.get_head_revision.return_value = "foo"
     MigrationContext.configure.return_value.get_current_revision.return_value = "bar"
@@ -83,7 +83,7 @@ def test_database_up_to_date(
     config.registry.settings[
         "sqlalchemy.engine"
     ].connect.return_value.__exit__ = mock.Mock(return_value=None)
-    global_config = {"__file__": "foofile"}
+    global_config = {"__file__": "etc/test.ini"}
 
     EnvironmentContext.return_value.get_head_revision.return_value = "foo"
     MigrationContext.configure.return_value.get_current_revision.return_value = "foo"
@@ -101,7 +101,7 @@ def test_SKIP_CHECK_DB_MIGRATED(
 ) -> None:
     """Support skipping the check with a config flag."""
     main(  # type: ignore
-        {"__file__": "foofile", "SKIP_CHECK_DB_MIGRATED": "true"}, **{}
+        {"__file__": "etc/test.ini", "SKIP_CHECK_DB_MIGRATED": "true"}, **{}
     )
     check_db_migrated.assert_not_called()
 
@@ -115,5 +115,5 @@ def test_not_SKIP_CHECK_DB_MIGRATED(
     check_db_migrated: mock.MagicMock,
 ) -> None:
     """Support skipping the check with a config flag."""
-    main({"__file__": "foofile"}, **{})  # type: ignore
-    check_db_migrated.assert_called_with(Configurator(), {"__file__": "foofile"})
+    main({"__file__": "etc/test.ini"}, **{})  # type: ignore
+    check_db_migrated.assert_called_with(Configurator(), {"__file__": "etc/test.ini"})