From 4f0b165879ddc5385856891bb945ddb487fd4c75 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Wed, 11 Dec 2024 20:44:47 +0100 Subject: [PATCH] task/WG-396: sqlachemy-update follow on (#231) * Add migration to update to newer PostGis * Revert "Add migration to update to newer PostGis" This reverts commit 74977c754296b44c1b13446acda9390932bc872c. * Include models so that autogenerate works in sqlachemy2 * Omit PostGIS-related migrations as those handled by postgis PostGIS manages its own schema * Add migraiton of spatial index of feature's the_geom The default behavior for spatial indexing in GeoAlchemy2. This migration should have been added earlier in projects development * Update README now that we try to omit all PostGIS-related migrations See commit 3a641cbaa623bbf60149ba7f6d19351de57a5ea2 * Fix linting/formatting issues --- README.md | 2 +- geoapi/migrations/env.py | 117 ++++++++++++++++-- ...f797c82a_add_migration_of_spatial_index.py | 36 ++++++ geoapi/models/feature.py | 4 +- 4 files changed, 150 insertions(+), 9 deletions(-) create mode 100644 geoapi/migrations/versions/20241211_0351-f2d7f797c82a_add_migration_of_spatial_index.py diff --git a/README.md b/README.md index f3d3cf46..0580702d 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ docker exec -it geoapi /bin/bash # determine a description for the migration like 'add_user_email_column' alembic revision --autogenerate -m "add_user_email_column" # Then: -# - remove drop table commands for postgis +# - check to make sure no postgis table commands (we try to ommit them in env.py) # - add/commit migrations ``` diff --git a/geoapi/migrations/env.py b/geoapi/migrations/env.py index 95798793..33c39bb1 100644 --- a/geoapi/migrations/env.py +++ b/geoapi/migrations/env.py @@ -1,6 +1,9 @@ from logging.config import fileConfig -from geoapi.db import Base +from geoapi.log import logger from alembic import context +from geoapi.models import * # noqa: F401, F403 important to include all models for autogenerate support; do not delete +from geoapi.db import Base + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -10,11 +13,7 @@ # This line sets up loggers basically. fileConfig(config.config_file_name) -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata - +# add your model's MetaData object target_metadata = Base.metadata # other values from the config, defined by the needs of env.py, @@ -23,6 +22,105 @@ # ... etc. +def include_object(object, name, type_, reflected, compare_to): + """Filter objects for Alembic migrations, specifically handling PostGIS tables/schemas. + + PostGIS manages its own schema updates through the extension system. This prevents our + application migrations from conflicting with PostGIS's internal version management and + schema updates. When PostGIS is upgraded, it will handle its own schema migrations. + + """ + + # Get schema info + schema = getattr(object, "schema", None) + if schema is None and type_ == "index" and hasattr(object, "table"): + schema = object.table.schema + + # Tiger/PostGIS related tables and sequences to exclude + excluded_tables = [ + "layer", + "topology", + "geocode_settings_default", + "street_type_lookup", + "county", + "zip_lookup_base", + "bg", + "zip_lookup", + "direction_lookup", + "state_lookup", + "featnames", + "tract", + "addrfeat", + "loader_platform", + "loader_lookuptables", + "zip_state_loc", + "tabblock", + "cousub", + "addr", + "county_lookup", + "loader_variables", + "place_lookup", + "zip_lookup_all", + "geocode_settings", + "place", + "secondary_unit_lookup", + "faces", + "pagc_lex", + "countysub_lookup", + "tabblock20", + "pagc_rules", + "edges", + "state", + "zcta5", + "zip_state", + "pagc_gaz", + ] + + excluded_sequences = [ + "county_gid_seq", + "state_gid_seq", + "addr_gid_seq", + "edges_gid_seq", + "pagc_lex_id_seq", + "cousub_gid_seq", + "addrfeat_gid_seq", + "place_gid_seq", + "bg_gid_seq", + "faces_gid_seq", + "pagc_rules_id_seq", + "tabblock_gid_seq", + "tract_gid_seq", + "featnames_gid_seq", + "zcta5_gid_seq", + "pagc_gaz_id_seq", + ] + + # Check schemas first + if schema in ["tiger", "topology"]: + logger.info(f"Excluding {type_} {name} due to schema {schema}") + return False + + # Check type-specific exclusions + if type_ == "table": + if name == "spatial_ref_sys" or name.lower() in excluded_tables: + logger.info(f"Excluding table {name}") + return False + + elif type_ == "index" and hasattr(object, "table"): + table_name = object.table.name.lower() + if table_name in excluded_tables: + logger.info(f"Excluding index {name} for excluded table {table_name}") + return False + + elif type_ == "sequence": + if name.lower() in excluded_sequences: + logger.info(f"Excluding sequence {name}") + return False + + logger.info(f"Including {type_} {name}") + return True + + def run_migrations_offline(): """Run migrations in 'offline' mode. @@ -41,6 +139,7 @@ def run_migrations_offline(): target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}, + include_object=include_object, ) with context.begin_transaction(): @@ -63,7 +162,11 @@ def run_migrations_online(): # ) with engine.connect() as connection: - context.configure(connection=connection, target_metadata=target_metadata) + context.configure( + connection=connection, + target_metadata=target_metadata, + include_object=include_object, + ) with context.begin_transaction(): context.run_migrations() diff --git a/geoapi/migrations/versions/20241211_0351-f2d7f797c82a_add_migration_of_spatial_index.py b/geoapi/migrations/versions/20241211_0351-f2d7f797c82a_add_migration_of_spatial_index.py new file mode 100644 index 00000000..b3197645 --- /dev/null +++ b/geoapi/migrations/versions/20241211_0351-f2d7f797c82a_add_migration_of_spatial_index.py @@ -0,0 +1,36 @@ +"""Add migration of spatial index + +Revision ID: f2d7f797c82a +Revises: 968f358e102a +Create Date: 2024-12-11 03:51:58.509951 + +""" + +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "f2d7f797c82a" +down_revision = "968f358e102a" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index( + "idx_features_the_geom", + "features", + ["the_geom"], + unique=False, + postgresql_using="gist", + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "idx_features_the_geom", table_name="features", postgresql_using="gist" + ) + # ### end Alembic commands ### diff --git a/geoapi/models/feature.py b/geoapi/models/feature.py index 34db63e5..831e4843 100644 --- a/geoapi/models/feature.py +++ b/geoapi/models/feature.py @@ -21,7 +21,9 @@ class Feature(Base): project_id = Column( ForeignKey("projects.id", ondelete="CASCADE", onupdate="CASCADE"), index=True ) - the_geom = Column(Geometry(geometry_type="GEOMETRY", srid=4326), nullable=False) + the_geom = Column( + Geometry(geometry_type="GEOMETRY", srid=4326), nullable=False + ) # Spatial index included by default properties = Column(JSONB, default={}) styles = Column(JSONB, default={}) created_date = Column(DateTime(timezone=True), server_default=func.now())