From 7d4fc046722f1e4ac2782aee91f9bb149cfded2b Mon Sep 17 00:00:00 2001 From: David Bitner Date: Thu, 10 Nov 2022 17:33:54 -0600 Subject: [PATCH 1/2] initial functions working --- tifeatures/dbmodel.py | 227 +++++++++++++++++++++++++++++++++++++----- tifeatures/factory.py | 16 ++- tifeatures/layer.py | 57 ++++++++--- tifeatures/main.py | 8 +- 4 files changed, 267 insertions(+), 41 deletions(-) diff --git a/tifeatures/dbmodel.py b/tifeatures/dbmodel.py index 6e8e088..748afe1 100644 --- a/tifeatures/dbmodel.py +++ b/tifeatures/dbmodel.py @@ -65,9 +65,16 @@ class DatetimeColumn(Column): max: Optional[str] +class Parameter(Column): + """Model for PostGIS function parameters.""" + + default: Optional[str] = None + + class Table(BaseModel): """Model for DB Table.""" + type: str id: str table: str dbschema: str = Field(..., alias="schema") @@ -78,6 +85,7 @@ class Table(BaseModel): datetime_columns: List[DatetimeColumn] = [] geometry_column: Optional[GeometryColumn] datetime_column: Optional[DatetimeColumn] + parameters: List[Parameter] = [] def get_datetime_column(self, name: Optional[str] = None) -> Optional[Column]: """Return the Column for either the passed in tstz column or the first tstz column.""" @@ -144,6 +152,7 @@ async def get_table_index( db_pool: asyncpg.BuildPgPool, schemas: Optional[List[str]] = ["public"], tables: Optional[List[str]] = None, + functions: Optional[List[str]] = None, spatial: bool = True, ) -> Database: """Fetch Table index.""" @@ -158,7 +167,7 @@ async def get_table_index( obj_description(c.oid, 'pg_class') as description, attname, atttypmod, - replace(replace(replace(replace(format_type(atttypid, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp') as "type", + replace(replace(replace(replace(format_type(atttypid, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp') as typ, col_description(attrelid, attnum) FROM pg_class c @@ -175,10 +184,10 @@ async def get_table_index( ), grouped as (SELECT - nspname, - relname, + 'Table' as entity, + nspname as dbschema, + relname as tbl, id, - t_oid, description, ( SELECT attname @@ -202,10 +211,10 @@ async def get_table_index( coalesce(jsonb_agg( jsonb_build_object( 'name', attname, - 'type', "type", + 'type', typ, 'geometry_type', postgis_typmod_type(atttypmod), 'srid', postgis_typmod_srid(atttypmod), - 'description', description, + 'description', col_description, 'bounds', CASE WHEN postgis_typmod_srid(atttypmod) IS NOT NULL AND postgis_typmod_srid(atttypmod) != 0 THEN ( @@ -233,35 +242,110 @@ async def get_table_index( ELSE ARRAY[-180,-90,180,90] END ) - ) FILTER (WHERE "type" IN ('geometry','geography')), '[]'::jsonb) as geometry_columns, + ) FILTER (WHERE typ IN ('geometry','geography')), '[]'::jsonb) as geometry_columns, coalesce(jsonb_agg( jsonb_build_object( 'name', attname, - 'type', "type", - 'description', description + 'type', typ, + 'description', col_description ) - ) FILTER (WHERE type LIKE 'timestamp%'), '[]'::jsonb) as datetime_columns, + ) FILTER (WHERE typ LIKE 'timestamp%'), '[]'::jsonb) as datetime_columns, coalesce(jsonb_agg( jsonb_build_object( 'name', attname, - 'type', "type", - 'description', description + 'type', typ, + 'description', col_description ) - ),'[]'::jsonb) as properties + ),'[]'::jsonb) as properties, + '[]'::jsonb as parameters FROM table_columns - GROUP BY 1,2,3,4,5,6 ORDER BY 1,2 - ) + GROUP BY 1,2,3,4,5,6 ORDER BY 1,2,3,4 + ), + f AS ( + SELECT + 'Function' as entity, + nspname as dbschema, + proname as tbl, + format('%s.%s',nspname, proname) as id, + d.description as description, + CASE WHEN pronargdefaults > 0 AND pronargs > 0 THEN + array_fill( + NULL::text, + ARRAY[pronargs-pronargdefaults] + ) || string_to_array(pg_get_expr(p.proargdefaults, 0::OID),',') + ELSE array_fill(null::text, ARRAY[pronargs]) + END as defaults, + p.proallargtypes, + p.proargmodes, + p.proargnames + + FROM + pg_proc p + JOIN + pg_namespace n on (p.pronamespace=n.oid) + LEFT JOIN pg_description d ON (p.oid = d.objoid) + WHERE + proretset + AND nspname='public' + AND prokind='f' + AND proargnames is not null + AND '' != ANY(proargnames) + AND has_function_privilege(p.oid, 'execute') + AND has_schema_privilege(n.oid, 'usage') + AND provariadic=0 + AND cardinality(p.proargnames) = cardinality(p.proargmodes) + AND cardinality(p.proargmodes) = cardinality(p.proallargtypes) + ), functions as ( SELECT + entity, + dbschema, + tbl, id, - relname as table, - nspname as dbschema, description, - id_column, - geometry_columns, - datetime_columns, - properties - FROM grouped + null::text as id_column, + coalesce(jsonb_agg( + jsonb_strip_nulls(jsonb_build_object( + 'name', a.argname, + 'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp'), + 'geometry_type', 'Geometry' + )) + ORDER BY a.argnum + ) FILTER (WHERE argmode IN ('t', 'b', 'o') AND format_type(argtype, null) LIKE 'geo%'),'[]'::jsonb) as geometry_columns, + coalesce(jsonb_agg( + jsonb_strip_nulls(jsonb_build_object( + 'name', a.argname, + 'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp') + )) + ORDER BY a.argnum + ) FILTER (WHERE argmode IN ('t', 'b', 'o') AND format_type(argtype, null) LIKE 'timestamp%'),'[]'::jsonb) as datetime_columns, + coalesce(jsonb_agg( + jsonb_build_object( + 'name', a.argname, + 'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp') + ) + ORDER BY a.argnum + ) FILTER (WHERE argmode IN ('t', 'b', 'o')),'[]'::jsonb) as properties, + coalesce(jsonb_agg( + jsonb_strip_nulls(jsonb_build_object( + 'name', a.argname, + 'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp') + ,'default', regexp_replace(def, '''([a-zA-Z0-9_\-\.]+)''::\w+', '\1', 'g') + )) + ORDER BY a.argnum + ) FILTER (WHERE argmode IN ('i', 'b')),'[]'::jsonb) as parameters + + FROM + f + LEFT JOIN LATERAL unnest(f.proallargtypes,f.proargmodes,f.proargnames,f.defaults) WITH ORDINALITY AS a(argtype,argmode,argname,def,argnum) ON TRUE + GROUP BY 1,2,3,4,5,6 ORDER BY 1,2,3,4 + ), + unioned as ( + SELECT * FROM grouped + UNION ALL + SELECT * FROM functions + ) + SELECT * FROM unioned WHERE :spatial = FALSE OR jsonb_array_length(geometry_columns)>=1 ; @@ -272,6 +356,7 @@ async def get_table_index( query, schemas=schemas, tables=tables, + functions=functions, spatial=spatial, ) catalog = {} @@ -326,8 +411,9 @@ async def get_table_index( geometry_column = geometry_columns[0] catalog[id] = { + "type": table["entity"], "id": id, - "table": table["table"], + "table": table["tbl"], "schema": table["dbschema"], "description": table["description"], "id_column": id_column, @@ -336,6 +422,101 @@ async def get_table_index( "properties": properties, "datetime_column": datetime_column, "geometry_column": geometry_column, + "parameters": table["parameters"], } return catalog + + +async def get_function_index( + db_pool: asyncpg.BuildPgPool, + schemas: Optional[List[str]] = ["public"], + functions: Optional[List[str]] = None, + spatial: bool = True, +) -> Database: + """Fetch Function index.""" + + query = """ + WITH f AS ( + SELECT + p.oid, + proname, + nspname, + d.description as description, + pg_get_function_arguments(p.oid) args, + CASE WHEN pronargdefaults > 0 AND pronargs > 0 THEN + array_fill( + NULL::text, + ARRAY[pronargs-pronargdefaults] + ) || string_to_array(pg_get_expr(p.proargdefaults, 0::OID),',') + ELSE array_fill(null::text, ARRAY[pronargs]) + END as defaults, + p.proallargtypes, + p.proargmodes, + p.proargnames + + FROM + pg_proc p + JOIN + pg_namespace n on (p.pronamespace=n.oid) + LEFT JOIN pg_description d ON (p.oid = d.objoid) + WHERE + proretset + AND nspname='public' + AND prokind='f' + AND proargnames is not null + AND '' != ANY(proargnames) + AND has_function_privilege(p.oid, 'execute') + AND has_schema_privilege(n.oid, 'usage') + AND provariadic=0 + AND cardinality(p.proargnames) = cardinality(p.proargmodes) + AND cardinality(p.proargmodes) = cardinality(p.proallargtypes) + ) + SELECT + proname as table, + nspname as dbschema, + format('%s.%s',nspname, proname) as id, + '' as description, + jsonb_agg( + jsonb_strip_nulls(jsonb_build_object( + 'name', a.argname, + 'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp'), + 'default', regexp_replace(def, '''([a-zA-Z0-9_\-\.]+)''::\w+', '\1', 'g') + )) + ORDER BY a.argnum + ) FILTER (WHERE argmode IN ('i', 'b')) as parameters, + jsonb_agg( + jsonb_build_object( + 'name', a.argname, + 'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp') + ) + ORDER BY a.argnum + ) FILTER (WHERE argmode IN ('t', 'b', 'o')) as properties + + FROM + f + LEFT JOIN LATERAL unnest(f.proallargtypes,f.proargmodes,f.proargnames,f.defaults) WITH ORDINALITY AS a(argtype,argmode,argname,def,argnum) ON TRUE + GROUP BY 1,2,3,4 + + ; + """ + + async with db_pool.acquire() as conn: + rows = await conn.fetch_b( + query, + schemas=schemas, + functions=functions, + spatial=spatial, + ) + catalog = {} + for row in rows: + func = dict(row) + id = func["id"] + func["geometry_columns"] = [ + c for c in func["properties"] if c["type"].startswith("geo") + ] + func["datetime_columns"] = [ + c for c in func["properties"] if c["type"].startswith("timestamp") + ] + catalog[id] = func + return catalog diff --git a/tifeatures/factory.py b/tifeatures/factory.py index e9b391c..577d5cb 100644 --- a/tifeatures/factory.py +++ b/tifeatures/factory.py @@ -618,12 +618,17 @@ async def items( "simplify", "sortby", ] + properties_filter = [] + function_parameters = {} table_property = [prop.name for prop in collection.properties] - properties_filter = [ - (key, value) - for (key, value) in request.query_params.items() - if key.lower() not in exclude and key.lower() in table_property - ] + function_parameter_names = [p.name for p in collection.parameters] + + for k, v in request.query_params.items(): + k = k.lower() + if k in function_parameter_names: + function_parameters[k] = v + elif k in table_property and k not in exclude: + properties_filter.append((k, v)) items, matched_items = await collection.features( request.app.state.pool, @@ -640,6 +645,7 @@ async def items( dt=datetime_column, bbox_only=bbox_only, simplify=simplify, + function_parameters=function_parameters, ) if output_type in ( diff --git a/tifeatures/layer.py b/tifeatures/layer.py index 5f47a4e..ea0bc04 100644 --- a/tifeatures/layer.py +++ b/tifeatures/layer.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from typing import Any, ClassVar, Dict, List, Optional, Tuple -from buildpg import asyncpg, clauses +from buildpg import RawDangerous, asyncpg, clauses from buildpg import funcs as pg_funcs from buildpg import logic, render from ciso8601 import parse_rfc3339 @@ -106,8 +106,6 @@ class Table(CollectionLayer, DBTable): """ - type: str = "Table" - @root_validator def bounds_default(cls, values): """Get default bounds from the first geometry columns.""" @@ -126,8 +124,25 @@ def _select(self, properties: Optional[List[str]]): def _select_count(self): return clauses.Select(pg_funcs.count("*")) - def _from(self): - return clauses.From(self.id) + def _from(self, function_parameters: Optional[Dict[str, str]]): + print("parsing from", function_parameters, self.parameters) + if self.type == "Function": + if not function_parameters: + print("no parameters") + return clauses.From(self.id) + RawDangerous("()") + params = [] + for p in self.parameters: + print(p) + if p.name in function_parameters: + print("appending", p.name, function_parameters[p.name]) + params.append( + pg_funcs.cast( + pg_funcs.cast(function_parameters[p.name], "text"), p.type + ) + ) + return clauses.From(logic.Func(self.id, *params)) + else: + return clauses.From(self.id) def _geom( self, @@ -309,11 +324,12 @@ def _features_query( dt: str = None, limit: Optional[int] = None, offset: Optional[int] = None, + function_parameters: Optional[Dict[str, str]], ): """Build Features query.""" - return ( + q = ( self._select(properties) - + self._from() + + self._from(function_parameters) + self._where( ids=ids_filter, datetime=datetime_filter, @@ -327,6 +343,8 @@ def _features_query( + clauses.Limit(limit or 10) + clauses.Offset(offset or 0) ) + print("features", q) + return q def _features_count_query( self, @@ -338,11 +356,12 @@ def _features_count_query( cql_filter: Optional[AstType] = None, geom: str = None, dt: str = None, + function_parameters: Optional[Dict[str, str]], ): """Build features COUNT query.""" - return ( + q = ( self._select_count() - + self._from() + + self._from(function_parameters) + self._where( ids=ids_filter, datetime=datetime_filter, @@ -353,6 +372,8 @@ def _features_count_query( dt=dt, ) ) + print("features_count", q) + return q async def query( self, @@ -362,11 +383,12 @@ async def query( bbox_filter: Optional[List[float]] = None, datetime_filter: Optional[List[str]] = None, properties_filter: Optional[List[Tuple[str, str]]] = None, + function_parameters: Optional[Dict[str, str]] = None, cql_filter: Optional[AstType] = None, sortby: Optional[str] = None, properties: Optional[List[str]] = None, - geom: str = None, - dt: str = None, + geom: Optional[str] = None, + dt: Optional[str] = None, limit: Optional[int] = None, offset: Optional[int] = None, bbox_only: Optional[bool] = None, @@ -376,6 +398,8 @@ async def query( if geom and geom.lower() != "none" and not self.get_geometry_column(geom): raise InvalidGeometryColumnName(f"Invalid Geometry Column: {geom}.") + print(self._from(function_parameters)) + sql_query = """ WITH features AS ( @@ -406,8 +430,14 @@ async def query( ) ; """ - id_column = logic.V(self.id_column) or pg_funcs.cast(None, "text") + + if self.id_column: + id_column = logic.V(self.id_column) + else: + id_column = pg_funcs.cast(None, "text") + geom_columns = [g.name for g in self.geometry_columns] + q, p = render( sql_query, features_q=self._features_query( @@ -422,6 +452,7 @@ async def query( dt=dt, limit=limit, offset=offset, + function_parameters=function_parameters, ), count_q=self._features_count_query( ids_filter=ids_filter, @@ -431,6 +462,7 @@ async def query( cql_filter=cql_filter, geom=geom, dt=dt, + function_parameters=function_parameters, ), id_column=id_column, geometry_q=self._geom( @@ -441,6 +473,7 @@ async def query( geom_columns=geom_columns, ) async with pool.acquire() as conn: + print(q, p) items = await conn.fetchval(q, *p) return ( diff --git a/tifeatures/main.py b/tifeatures/main.py index 8ab5df9..adb6768 100644 --- a/tifeatures/main.py +++ b/tifeatures/main.py @@ -6,7 +6,7 @@ from tifeatures import __version__ as tifeatures_version from tifeatures.db import close_db_connection, connect_to_db, register_table_catalog -from tifeatures.dbmodel import Table +from tifeatures.dbmodel import Table, get_function_index from tifeatures.errors import DEFAULT_STATUS_CODES, add_exception_handlers from tifeatures.factory import Endpoints from tifeatures.layer import FunctionRegistry @@ -99,3 +99,9 @@ def raw_catalog(request: Request): for k, v in cat.items(): ret[k] = Table(**v) return ret + + @app.get("/rawfunccatalog") + async def raw_func_catalog(request: Request): + """Return parsed catalog data for testing.""" + cat = await get_function_index(request.app.state.pool, spatial=False) + return cat From 3e8ea17cf40cb92b3199b119873c5af57a510ebe Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 11 Nov 2022 07:47:22 +0100 Subject: [PATCH 2/2] make it pass --- tifeatures/dbmodel.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tifeatures/dbmodel.py b/tifeatures/dbmodel.py index 8755ade..addd45c 100644 --- a/tifeatures/dbmodel.py +++ b/tifeatures/dbmodel.py @@ -353,8 +353,7 @@ async def get_table_index( SELECT * FROM unioned WHERE :spatial = FALSE OR jsonb_array_length(geometry_columns)>=1 ; - - """ + """ # noqa: W605 async with db_pool.acquire() as conn: rows = await conn.fetch_b( @@ -509,7 +508,7 @@ async def get_function_index( GROUP BY 1,2,3,4 ; - """ + """ # noqa: W605 async with db_pool.acquire() as conn: rows = await conn.fetch_b(