Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Funcs #70

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

Funcs #70

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7d4fc04
initial functions working
bitner Nov 10, 2022
6e695a3
Merge branch 'master' of https://github.com/developmentseed/tifeature…
vincentsarago Nov 11, 2022
3e8ea17
make it pass
vincentsarago Nov 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
230 changes: 205 additions & 25 deletions tifeatures/dbmodel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,16 @@ class DatetimeColumn(Column):
max: Optional[str]


class Parameter(Column):
"""Model for PostGIS function parameters."""

default: Optional[str] = None


class Table(BaseModel):
"""Model for DB Table."""

type: str
id: str
table: str
dbschema: str = Field(..., alias="schema")
Expand All @@ -77,6 +84,7 @@ class Table(BaseModel):
datetime_columns: List[DatetimeColumn] = []
geometry_column: Optional[GeometryColumn]
datetime_column: Optional[DatetimeColumn]
parameters: List[Parameter] = []

def get_datetime_column(self, name: Optional[str] = None) -> Optional[Column]:
"""Return the Column for either the passed in tstz column or the first tstz column."""
Expand Down Expand Up @@ -149,6 +157,7 @@ async def get_table_index(
db_pool: asyncpg.BuildPgPool,
schemas: Optional[List[str]] = ["public"],
tables: Optional[List[str]] = None,
functions: Optional[List[str]] = None,
spatial: bool = True,
) -> Database:
"""Fetch Table index."""
Expand All @@ -163,7 +172,7 @@ async def get_table_index(
obj_description(c.oid, 'pg_class') as description,
attname,
atttypmod,
replace(replace(replace(replace(format_type(atttypid, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp') as "type",
replace(replace(replace(replace(format_type(atttypid, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp') as typ,
col_description(attrelid, attnum)
FROM
pg_class c
Expand All @@ -180,10 +189,10 @@ async def get_table_index(
),
grouped as
(SELECT
nspname,
relname,
'Table' as entity,
nspname as dbschema,
relname as tbl,
id,
t_oid,
description,
(
SELECT attname
Expand All @@ -207,10 +216,10 @@ async def get_table_index(
coalesce(jsonb_agg(
jsonb_build_object(
'name', attname,
'type', "type",
'type', typ,
'geometry_type', postgis_typmod_type(atttypmod),
'srid', postgis_typmod_srid(atttypmod),
'description', description,
'description', col_description,
'bounds',
CASE WHEN postgis_typmod_srid(atttypmod) IS NOT NULL AND postgis_typmod_srid(atttypmod) != 0 THEN
(
Expand Down Expand Up @@ -238,45 +247,120 @@ async def get_table_index(
ELSE ARRAY[-180,-90,180,90]
END
)
) FILTER (WHERE "type" IN ('geometry','geography')), '[]'::jsonb) as geometry_columns,
) FILTER (WHERE typ IN ('geometry','geography')), '[]'::jsonb) as geometry_columns,
coalesce(jsonb_agg(
jsonb_build_object(
'name', attname,
'type', "type",
'description', description
'type', typ,
'description', col_description
)
) FILTER (WHERE type LIKE 'timestamp%'), '[]'::jsonb) as datetime_columns,
) FILTER (WHERE typ LIKE 'timestamp%'), '[]'::jsonb) as datetime_columns,
coalesce(jsonb_agg(
jsonb_build_object(
'name', attname,
'type', "type",
'description', description
'type', typ,
'description', col_description
)
),'[]'::jsonb) as properties
),'[]'::jsonb) as properties,
'[]'::jsonb as parameters
FROM
table_columns
GROUP BY 1,2,3,4,5,6 ORDER BY 1,2
)
GROUP BY 1,2,3,4,5,6 ORDER BY 1,2,3,4
),
f AS (
SELECT
'Function' as entity,
nspname as dbschema,
proname as tbl,
format('%s.%s',nspname, proname) as id,
d.description as description,
CASE WHEN pronargdefaults > 0 AND pronargs > 0 THEN
array_fill(
NULL::text,
ARRAY[pronargs-pronargdefaults]
) || string_to_array(pg_get_expr(p.proargdefaults, 0::OID),',')
ELSE array_fill(null::text, ARRAY[pronargs])
END as defaults,
p.proallargtypes,
p.proargmodes,
p.proargnames

FROM
pg_proc p
JOIN
pg_namespace n on (p.pronamespace=n.oid)
LEFT JOIN pg_description d ON (p.oid = d.objoid)
WHERE
proretset
AND nspname='public'
AND prokind='f'
AND proargnames is not null
AND '' != ANY(proargnames)
AND has_function_privilege(p.oid, 'execute')
AND has_schema_privilege(n.oid, 'usage')
AND provariadic=0
AND cardinality(p.proargnames) = cardinality(p.proargmodes)
AND cardinality(p.proargmodes) = cardinality(p.proallargtypes)
), functions as (
SELECT
entity,
dbschema,
tbl,
id,
relname as table,
nspname as dbschema,
description,
id_column,
geometry_columns,
datetime_columns,
properties
FROM grouped
null::text as id_column,
coalesce(jsonb_agg(
jsonb_strip_nulls(jsonb_build_object(
'name', a.argname,
'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp'),
'geometry_type', 'Geometry'
))
ORDER BY a.argnum
) FILTER (WHERE argmode IN ('t', 'b', 'o') AND format_type(argtype, null) LIKE 'geo%'),'[]'::jsonb) as geometry_columns,
coalesce(jsonb_agg(
jsonb_strip_nulls(jsonb_build_object(
'name', a.argname,
'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp')
))
ORDER BY a.argnum
) FILTER (WHERE argmode IN ('t', 'b', 'o') AND format_type(argtype, null) LIKE 'timestamp%'),'[]'::jsonb) as datetime_columns,
coalesce(jsonb_agg(
jsonb_build_object(
'name', a.argname,
'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp')
)
ORDER BY a.argnum
) FILTER (WHERE argmode IN ('t', 'b', 'o')),'[]'::jsonb) as properties,
coalesce(jsonb_agg(
jsonb_strip_nulls(jsonb_build_object(
'name', a.argname,
'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp')
,'default', regexp_replace(def, '''([a-zA-Z0-9_\-\.]+)''::\w+', '\1', 'g')
))
ORDER BY a.argnum
) FILTER (WHERE argmode IN ('i', 'b')),'[]'::jsonb) as parameters

FROM
f
LEFT JOIN LATERAL unnest(f.proallargtypes,f.proargmodes,f.proargnames,f.defaults) WITH ORDINALITY AS a(argtype,argmode,argname,def,argnum) ON TRUE
GROUP BY 1,2,3,4,5,6 ORDER BY 1,2,3,4
),
unioned as (
SELECT * FROM grouped
UNION ALL
SELECT * FROM functions
)
SELECT * FROM unioned
WHERE :spatial = FALSE OR jsonb_array_length(geometry_columns)>=1
;

"""
""" # noqa: W605

async with db_pool.acquire() as conn:
rows = await conn.fetch_b(
query,
schemas=schemas,
tables=tables,
functions=functions,
spatial=spatial,
)

Expand Down Expand Up @@ -336,8 +420,9 @@ async def get_table_index(
geometry_column = geometry_columns[0]

catalog[id] = {
"type": table["entity"],
"id": id,
"table": table["table"],
"table": table["tbl"],
"schema": table["dbschema"],
"description": table["description"],
"id_column": id_column,
Expand All @@ -346,6 +431,101 @@ async def get_table_index(
"properties": properties,
"datetime_column": datetime_column,
"geometry_column": geometry_column,
"parameters": table["parameters"],
}

return catalog


async def get_function_index(
db_pool: asyncpg.BuildPgPool,
schemas: Optional[List[str]] = ["public"],
functions: Optional[List[str]] = None,
spatial: bool = True,
) -> Database:
"""Fetch Function index."""

query = """
WITH f AS (
SELECT
p.oid,
proname,
nspname,
d.description as description,
pg_get_function_arguments(p.oid) args,
CASE WHEN pronargdefaults > 0 AND pronargs > 0 THEN
array_fill(
NULL::text,
ARRAY[pronargs-pronargdefaults]
) || string_to_array(pg_get_expr(p.proargdefaults, 0::OID),',')
ELSE array_fill(null::text, ARRAY[pronargs])
END as defaults,
p.proallargtypes,
p.proargmodes,
p.proargnames

FROM
pg_proc p
JOIN
pg_namespace n on (p.pronamespace=n.oid)
LEFT JOIN pg_description d ON (p.oid = d.objoid)
WHERE
proretset
AND nspname='public'
AND prokind='f'
AND proargnames is not null
AND '' != ANY(proargnames)
AND has_function_privilege(p.oid, 'execute')
AND has_schema_privilege(n.oid, 'usage')
AND provariadic=0
AND cardinality(p.proargnames) = cardinality(p.proargmodes)
AND cardinality(p.proargmodes) = cardinality(p.proallargtypes)
)
SELECT
proname as table,
nspname as dbschema,
format('%s.%s',nspname, proname) as id,
'' as description,
jsonb_agg(
jsonb_strip_nulls(jsonb_build_object(
'name', a.argname,
'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp'),
'default', regexp_replace(def, '''([a-zA-Z0-9_\-\.]+)''::\w+', '\1', 'g')
))
ORDER BY a.argnum
) FILTER (WHERE argmode IN ('i', 'b')) as parameters,
jsonb_agg(
jsonb_build_object(
'name', a.argname,
'type', replace(replace(replace(replace(format_type(argtype, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp')
)
ORDER BY a.argnum
) FILTER (WHERE argmode IN ('t', 'b', 'o')) as properties

FROM
f
LEFT JOIN LATERAL unnest(f.proallargtypes,f.proargmodes,f.proargnames,f.defaults) WITH ORDINALITY AS a(argtype,argmode,argname,def,argnum) ON TRUE
GROUP BY 1,2,3,4

;
""" # noqa: W605

async with db_pool.acquire() as conn:
rows = await conn.fetch_b(
query,
schemas=schemas,
functions=functions,
spatial=spatial,
)
catalog = {}
for row in rows:
func = dict(row)
id = func["id"]
func["geometry_columns"] = [
c for c in func["properties"] if c["type"].startswith("geo")
]
func["datetime_columns"] = [
c for c in func["properties"] if c["type"].startswith("timestamp")
]
catalog[id] = func
return catalog
16 changes: 11 additions & 5 deletions tifeatures/factory.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -618,12 +618,17 @@ async def items(
"simplify",
"sortby",
]
properties_filter = []
function_parameters = {}
table_property = [prop.name for prop in collection.properties]
properties_filter = [
(key, value)
for (key, value) in request.query_params.items()
if key.lower() not in exclude and key.lower() in table_property
]
function_parameter_names = [p.name for p in collection.parameters]

for k, v in request.query_params.items():
k = k.lower()
if k in function_parameter_names:
function_parameters[k] = v
elif k in table_property and k not in exclude:
properties_filter.append((k, v))

items, matched_items = await collection.features(
request.app.state.pool,
Expand All @@ -640,6 +645,7 @@ async def items(
dt=datetime_column,
bbox_only=bbox_only,
simplify=simplify,
function_parameters=function_parameters,
)

if output_type in (
Expand Down
Loading