task/WG-396: sqlachemy-update follow on (#231)

* Add migration to update to newer PostGis

* Revert "Add migration to update to newer PostGis"

This reverts commit 74977c7.

* Include models so that autogenerate works in sqlachemy2

* Omit PostGIS-related migrations as those handled by postgis

PostGIS manages its own schema

* Add migraiton of spatial index of feature's the_geom

The default behavior for spatial indexing in GeoAlchemy2. This migration
should have been added earlier in projects development

* Update README now that we try to omit all PostGIS-related migrations

See commit 3a641cb

* Fix linting/formatting issues

README.md

@@ -73,7 +73,7 @@ docker exec -it geoapi /bin/bash
 # determine a description for the migration like 'add_user_email_column'
 alembic revision --autogenerate -m "add_user_email_column"
 # Then:
-# - remove drop table commands for postgis
+# - check to make sure no postgis table commands (we try to ommit them in env.py)
 # - add/commit migrations
 ```
 

geoapi/migrations/env.py

@@ -1,6 +1,9 @@
 from logging.config import fileConfig
-from geoapi.db import Base
+from geoapi.log import logger
 from alembic import context
+from geoapi.models import *  # noqa: F401, F403 important to include all models for autogenerate support; do not delete
+from geoapi.db import Base
+
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -10,11 +13,7 @@
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
 
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-
+# add your model's MetaData object
 target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,
@@ -23,6 +22,105 @@
 # ... etc.
 
 
+def include_object(object, name, type_, reflected, compare_to):
+    """Filter objects for Alembic migrations, specifically handling PostGIS tables/schemas.
+
+    PostGIS manages its own schema updates through the extension system. This prevents our
+    application migrations from conflicting with PostGIS's internal version management and
+    schema updates. When PostGIS is upgraded, it will handle its own schema migrations.
+
+    """
+
+    # Get schema info
+    schema = getattr(object, "schema", None)
+    if schema is None and type_ == "index" and hasattr(object, "table"):
+        schema = object.table.schema
+
+    # Tiger/PostGIS related tables and sequences to exclude
+    excluded_tables = [
+        "layer",
+        "topology",
+        "geocode_settings_default",
+        "street_type_lookup",
+        "county",
+        "zip_lookup_base",
+        "bg",
+        "zip_lookup",
+        "direction_lookup",
+        "state_lookup",
+        "featnames",
+        "tract",
+        "addrfeat",
+        "loader_platform",
+        "loader_lookuptables",
+        "zip_state_loc",
+        "tabblock",
+        "cousub",
+        "addr",
+        "county_lookup",
+        "loader_variables",
+        "place_lookup",
+        "zip_lookup_all",
+        "geocode_settings",
+        "place",
+        "secondary_unit_lookup",
+        "faces",
+        "pagc_lex",
+        "countysub_lookup",
+        "tabblock20",
+        "pagc_rules",
+        "edges",
+        "state",
+        "zcta5",
+        "zip_state",
+        "pagc_gaz",
+    ]
+
+    excluded_sequences = [
+        "county_gid_seq",
+        "state_gid_seq",
+        "addr_gid_seq",
+        "edges_gid_seq",
+        "pagc_lex_id_seq",
+        "cousub_gid_seq",
+        "addrfeat_gid_seq",
+        "place_gid_seq",
+        "bg_gid_seq",
+        "faces_gid_seq",
+        "pagc_rules_id_seq",
+        "tabblock_gid_seq",
+        "tract_gid_seq",
+        "featnames_gid_seq",
+        "zcta5_gid_seq",
+        "pagc_gaz_id_seq",
+    ]
+
+    # Check schemas first
+    if schema in ["tiger", "topology"]:
+        logger.info(f"Excluding {type_} {name} due to schema {schema}")
+        return False
+
+    # Check type-specific exclusions
+    if type_ == "table":
+        if name == "spatial_ref_sys" or name.lower() in excluded_tables:
+            logger.info(f"Excluding table {name}")
+            return False
+
+    elif type_ == "index" and hasattr(object, "table"):
+        table_name = object.table.name.lower()
+        if table_name in excluded_tables:
+            logger.info(f"Excluding index {name} for excluded table {table_name}")
+            return False
+
+    elif type_ == "sequence":
+        if name.lower() in excluded_sequences:
+            logger.info(f"Excluding sequence {name}")
+            return False
+
+    logger.info(f"Including {type_} {name}")
+    return True
+
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -41,6 +139,7 @@ def run_migrations_offline():
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
+        include_object=include_object,
     )
 
     with context.begin_transaction():
@@ -63,7 +162,11 @@ def run_migrations_online():
     # )
 
     with engine.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            include_object=include_object,
+        )
 
         with context.begin_transaction():
             context.run_migrations()

geoapi/migrations/versions/20241211_0351-f2d7f797c82a_add_migration_of_spatial_index.py

@@ -0,0 +1,36 @@
+"""Add migration of spatial index
+
+Revision ID: f2d7f797c82a
+Revises: 968f358e102a
+Create Date: 2024-12-11 03:51:58.509951
+
+"""
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "f2d7f797c82a"
+down_revision = "968f358e102a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index(
+        "idx_features_the_geom",
+        "features",
+        ["the_geom"],
+        unique=False,
+        postgresql_using="gist",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        "idx_features_the_geom", table_name="features", postgresql_using="gist"
+    )
+    # ### end Alembic commands ###

geoapi/models/feature.py

@@ -21,7 +21,9 @@ class Feature(Base):
     project_id = Column(
         ForeignKey("projects.id", ondelete="CASCADE", onupdate="CASCADE"), index=True
     )
-    the_geom = Column(Geometry(geometry_type="GEOMETRY", srid=4326), nullable=False)
+    the_geom = Column(
+        Geometry(geometry_type="GEOMETRY", srid=4326), nullable=False
+    )  # Spatial index included by default
     properties = Column(JSONB, default={})
     styles = Column(JSONB, default={})
     created_date = Column(DateTime(timezone=True), server_default=func.now())