diff --git a/README.md b/README.md index 794e27a..7af058b 100644 --- a/README.md +++ b/README.md @@ -189,5 +189,65 @@ Wraparound action! Handy for testing. Connect your database back to your databas OPTIONS ( layer 'typetest' ); SELECT * FROM typetest_fdw; - -Enjoy! \ No newline at end of file + +### Using IMPORT FOREIGN SCHEMA (for PostgreSQL 9.5+ only) + +## Importing links to all tables +If you want to import all tables use the special schema called *ogr_all* + + CREATE SCHEMA fgdball; + IMPORT FOREIGN SCHEMA ogr_all + FROM server fgdbtest INTO fgdball; + +## Importing subset of tables using prefixes +Not all ogr data sources have a concept of schema, so we use the remote_schema as a prefix. +Note this is case sensitive, so make sure casing matches your layer names. + +For example the following will only import tables that start with *CitiesIn*. As long as you quote, you can handle +true schemaed databases such as SQL server or PostgreSQL by using something like *"dbo."* + + CREATE SCHEMA fgdbcityinf; + IMPORT FOREIGN SCHEMA "CitiesIn" + FROM server fgdbtest INTO fgdbcityinf; + +## Preserving case and special characters in column names and table names +By default, when IMPORT FOREIGN SCHEMA is run on an ogr foreign data server, the table names and column names are laundered +(meaning all upper case is converted to lowercase and special characters such as spaces are replaced with _). + +This is not desirable in all cases. You can override this behavior with 2 IMPORT FOREIGN SCHEMA options specific to ogr fdw servers. + +These are `launder_column_names` and `launder_tables_names`. + +To preserve casing and other funky characters in both column names and table names you can do the following: + + CREATE SCHEMA fgdbcitypreserve; + IMPORT FOREIGN SCHEMA ogr_all + FROM server fgdbtest INTO fgdbpreserve + OPTIONS(launder_table_names 'false', launder_column_names 'false') ; + + +## Importing subset of layers using LIMIT and EXCEPT +Note: LIMIT TO /EXCEPT should contain resulting table names (NOT the layer names) +In the default case, the table names are laundered should not have mixed case or weird characters. + + CREATE SCHEMA fgdbcitysub; + -- import only layer called Cities + IMPORT FOREIGN SCHEMA ogr_all + LIMIT TO(cities) + FROM server fgdbtest INTO fgdbcitysub ; + + -- import only layers not called Cities or Countries + IMPORT FOREIGN SCHEMA ogr_all + EXCEPT (cities, countries) + FROM server fgdbtest INTO fgdbcitysub; + + -- With table laundering turned off, need to use exact layer names + DROP SCHEMA IF EXISTS fgdbcitysub CASCADE; + + IMPORT FOREIGN SCHEMA ogr_all + LIMIT TO("Cities") + FROM server fgdbtest INTO fgdbcitysub OPTIONS(launder_table_names 'false') ; + + + +Enjoy! diff --git a/ogr_fdw.c b/ogr_fdw.c index 709ed7f..c4e326a 100644 --- a/ogr_fdw.c +++ b/ogr_fdw.c @@ -104,6 +104,16 @@ static TupleTableSlot *ogrIterateForeignScan(ForeignScanState *node); static void ogrReScanForeignScan(ForeignScanState *node); static void ogrEndForeignScan(ForeignScanState *node); +static void strTableColumnLaunder (char *str); + +#if PG_VERSION_NUM >= 90500 +/* +* Require PostgreSQL >= 9.5 +*/ +static List *ogrImportForeignSchema(ImportForeignSchemaStmt *stmt, + Oid serverOid); +#endif + /* * Helper functions */ @@ -115,6 +125,8 @@ static void ogr_fdw_exit(int code, Datum arg); /* Global to hold GEOMETRYOID */ Oid GEOMETRYOID = InvalidOid; +#define STR_MAX_LEN 256 + void _PG_init(void) @@ -180,6 +192,12 @@ ogr_fdw_handler(PG_FUNCTION_ARGS) fdwroutine->IterateForeignScan = ogrIterateForeignScan; fdwroutine->ReScanForeignScan = ogrReScanForeignScan; fdwroutine->EndForeignScan = ogrEndForeignScan; + +#if PG_VERSION_NUM >= 90500 + /* PostgreSQL 9.5+ + Support functions for IMPORT FOREIGN SCHEMA */ + fdwroutine->ImportForeignSchema = ogrImportForeignSchema; +#endif PG_RETURN_POINTER(fdwroutine); } @@ -1265,3 +1283,288 @@ ogrEndForeignScan(ForeignScanState *node) #endif /* PostgreSQL 9.3 version check */ + +#if PG_VERSION_NUM >= 90500 +/* + * PostgreSQL 9.5 or above. Import a foreign schema + */ +static List * +ogrImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid) +{ + List *commands = NIL; + ForeignServer *server; + + List *options; + ListCell *cell; + char *sGeomType; + bool check_schema = false; + bool launder_column_names = true; + bool launder_table_names = true; + StringInfoData buf; + OgrConnection ogr; + int i; + int j; + int k; + char layer_name[STR_MAX_LEN]; + char table_name[STR_MAX_LEN]; + ListCell *lc; + bool include_item = false; + OGRDataSourceH ogr_ds = NULL; + OGRSFDriverH ogr_dr = NULL; + OGRFeatureDefnH ogr_fd = NULL; + OGRLayerH ogr_lyr = NULL; + + /** check table prefix if remote_schema asked for is not ogr_all **/ + check_schema = !( strcmp(stmt->remote_schema, "ogr_all") == 0 ); + + elog(NOTICE, "Check schema %d %s", check_schema, stmt->remote_schema); + if ( GEOMETRYOID == BYTEAOID){ /* postgis is not in search path */ + sGeomType = "bytea"; + } + else { + sGeomType = "geometry"; + } + +#if GDAL_VERSION_MAJOR >= 2 || GDAL_VERSION_MINOR >= 11 + int geom_field_count; +#endif + /* Null all values */ + memset(&ogr, 0, sizeof(OgrConnection)); + + /* + * Get connection to the foreign server. Connection manager will + * establish new connection if necessary. + */ + server = GetForeignServer(serverOid); + + /* Read server druver and data source connection string + */ + options = NIL; + options = list_concat(options, server->options); + foreach(cell, options) + { + DefElem *def = (DefElem *) lfirst(cell); + if (strcmp(def->defname, OPT_SOURCE) == 0) + ogr.ds_str = defGetString(def); + if (strcmp(def->defname, OPT_DRIVER) == 0) + ogr.dr_str = defGetString(def); + } + + /* Parse statement laundering options */ + foreach(lc, stmt->options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, "launder_column_names") == 0) + launder_column_names = defGetBoolean(def); + else if (strcmp(def->defname, "launder_table_names") == 0) + launder_table_names = defGetBoolean(def); + else + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid option \"%s\"", def->defname))); + } + + OGRRegisterAll(); + ogr_ds = OGROpen(ogr.ds_str, FALSE, &ogr_dr); + + + /* Create workspace for strings */ + initStringInfo(&buf); + + for ( i = 0; i < OGR_DS_GetLayerCount(ogr_ds); i++ ) + { + include_item = false; + ogr_lyr = OGR_DS_GetLayer(ogr_ds, i); + /* we have a table */ + if ( ogr_lyr ) + { + /* layer name is never laundered, since it's link back to foreign data */ + strncpy(layer_name, OGR_L_GetName(ogr_lyr), STR_MAX_LEN); + + /* We need to compare against created table names + * because postgres does an extra check on create foriegn table + * and removes statements not in limit + */ + /* having this as separate variable since we may choose to launder it */ + strncpy(table_name, OGR_L_GetName(ogr_lyr), STR_MAX_LEN); + if (launder_table_names){ + strTableColumnLaunder(table_name); + } + + /* only include if layer prefix starts with remote schema + or remote schema is ogr_all */ + include_item = (!check_schema || + ( strncmp(layer_name, stmt->remote_schema, strlen(stmt->remote_schema) ) == 0 ) ); + /* Apply restrictions for LIMIT TO and EXCEPT */ + if (include_item && ( stmt->list_type == FDW_IMPORT_SCHEMA_LIMIT_TO || + stmt->list_type == FDW_IMPORT_SCHEMA_EXCEPT ) ) + { + /* Check if current table is in list of except/include tables */ + /* default state is only true if type is EXCEPT */ + include_item = ( stmt->list_type == FDW_IMPORT_SCHEMA_EXCEPT ); + foreach(lc, stmt->table_list) + { + RangeVar *rv = (RangeVar *) lfirst(lc); + if ( strcmp(rv->relname, table_name) == 0 ){ + //elog(NOTICE, "MATCH layer %s, table %s", layer_name, rv->relname ); + /* bit is true on match only if limit to */ + include_item = ( stmt->list_type == FDW_IMPORT_SCHEMA_LIMIT_TO ); + break; + } + + } + } + } + + if (include_item){ + resetStringInfo(&buf); + + if (launder_table_names){ + strTableColumnLaunder(table_name); + } + ogr_fd = OGR_L_GetLayerDefn(ogr_lyr); + if ( !ogr_fd ) + { + /** TODO raise error **/ + elog(NOTICE, "Error in layer def load %s", layer_name); + } + + appendStringInfo(&buf, "CREATE FOREIGN TABLE %s (\n", + quote_identifier(table_name)); + appendStringInfo(&buf, " fid integer"); + +#if GDAL_VERSION_MAJOR >= 2 || GDAL_VERSION_MINOR >= 11 + geom_field_count = OGR_FD_GetGeomFieldCount(ogr_fd); + if( geom_field_count == 1 ) + { + appendStringInfo(&buf, " ,geom %s", sGeomType); + } + else + { + for ( j = 0; j < geom_field_count; j++ ) + { + appendStringInfo(&buf, " ,geom%d %s", j + 1, sGeomType); + } + } +#else + if( OGR_L_GetGeomType(ogr_lyr) != wkbNone ) + appendStringInfo(&buf, " ,geom %s", sGeomType); +#endif + + for ( k = 0; k < OGR_FD_GetFieldCount(ogr_fd); k++ ) + { + char field_name[STR_MAX_LEN]; + OGRFieldDefnH ogr_fld = OGR_FD_GetFieldDefn(ogr_fd, k); + strncpy(field_name, OGR_Fld_GetNameRef(ogr_fld), STR_MAX_LEN); + if (launder_column_names){ + strTableColumnLaunder(field_name); + } + appendStringInfo(&buf, " , %s ", quote_identifier(field_name)); + switch( OGR_Fld_GetType(ogr_fld) ) + { + case OFTInteger: +#if GDAL_VERSION_MAJOR >= 2 + if( OGR_Fld_GetSubType(ogr_fld) == OFSTBoolean ) + appendStringInfoString(&buf,"boolean"); + else +#endif + appendStringInfoString(&buf,"integer"); + break; + case OFTReal: + appendStringInfoString(&buf,"real"); + break; + case OFTString: + appendStringInfoString(&buf,"varchar"); + break; + case OFTBinary: + appendStringInfoString(&buf,"bytea"); + break; + case OFTDate: + appendStringInfoString(&buf,"date"); + break; + case OFTTime: + appendStringInfoString(&buf,"time"); + break; + case OFTDateTime: + appendStringInfoString(&buf,"timestamp"); + break; + case OFTIntegerList: + appendStringInfoString(&buf,"integer[]"); + break; + case OFTRealList: + appendStringInfoString(&buf,"real[]"); + break; + case OFTStringList: + appendStringInfoString(&buf,"varchar[]"); + break; + +#if GDAL_VERSION_MAJOR >= 2 + case OFTInteger64: + appendStringInfoString(&buf,"bigint"); + break; +#endif + default: + elog(NOTICE, "Unsupported GDAL type '%s'", OGR_GetFieldTypeName(OGR_Fld_GetType(ogr_fld)) ); + //CPLError(CE_Failure, CPLE_AppDefined, "Unsupported GDAL type '%s'", OGR_GetFieldTypeName(OGR_Fld_GetType(ogr_fld))); + //return OGRERR_FAILURE; + } + } + + /* + * Add server name and layer-level options. We specify remote + * layer name as option + */ + appendStringInfo(&buf, "\n) SERVER %s\nOPTIONS (", + quote_identifier(server->servername)); + + appendStringInfoString(&buf, "layer "); + ogrDeparseStringLiteral(&buf, layer_name); + + appendStringInfoString(&buf, ");"); + + commands = lappend(commands, pstrdup(buf.data)); + } + } + OGR_DS_Destroy(ogr_ds); + elog(NOTICE, "Number of tables to be created %d", list_length(commands) ); + //elog(NOTICE, "The nth item %s", list_nth(commands,0) ); + + /* Clean up */ + pfree(buf.data); + /** returns list of create foreign table statements to run **/ + return commands; +} +#endif /*end import foreign schema **/ + +static void strTableColumnLaunder (char *str) +{ + int i, j = 0; + for(i = 0; str[i]; i++) + { + char c = tolower(str[i]); + + /* First character is a numeral, prefix with 'n' */ + if ( i == 0 && (c >= 48 && c <= 57) ) + { + str[j++] = 'n'; + } + + /* Replace non-safe characters w/ _ */ + if ( (c >= 48 && c <= 57) || /* 0-9 */ + (c >= 65 && c <= 90) || /* A-Z */ + (c >= 97 && c <= 122 ) /* a-z */ ) + { + /* Good character, do nothing */ + } + else + { + c = '_'; + } + str[j++] = c; + + /* Avoid mucking with data beyond the end of our stack-allocated strings */ + if ( j >= STR_MAX_LEN ) + j = STR_MAX_LEN - 1; + } +} diff --git a/ogr_fdw.h b/ogr_fdw.h index 9bc753f..2bfb952 100644 --- a/ogr_fdw.h +++ b/ogr_fdw.h @@ -123,6 +123,8 @@ typedef struct OgrFdwExecState /* Shared function signatures */ bool ogrDeparse(StringInfo buf, PlannerInfo *root, RelOptInfo *foreignrel, List *exprs, List **param); +void ogrDeparseStringLiteral(StringInfo buf, const char *val); + /* Shared global value of the Geometry OId */ extern Oid GEOMETRYOID; diff --git a/ogr_fdw_deparse.c b/ogr_fdw_deparse.c index ef65989..b78bd61 100644 --- a/ogr_fdw_deparse.c +++ b/ogr_fdw_deparse.c @@ -512,4 +512,32 @@ ogrDeparse(StringInfo buf, PlannerInfo *root, RelOptInfo *foreignrel, List *expr } +/* + * Append a SQL string literal representing "val" to buf. + */ +void +ogrDeparseStringLiteral(StringInfo buf, const char *val) +{ + const char *valptr; + + /* + * Rather than making assumptions about the remote server's value of + * standard_conforming_strings, always use E'foo' syntax if there are any + * backslashes. This will fail on remote servers before 8.1, but those + * are long out of support. + */ + if (strchr(val, '\\') != NULL) + appendStringInfoChar(buf, ESCAPE_STRING_SYNTAX); + appendStringInfoChar(buf, '\''); + for (valptr = val; *valptr; valptr++) + { + char ch = *valptr; + + if (SQL_STR_DOUBLE(ch, true)) + appendStringInfoChar(buf, ch); + appendStringInfoChar(buf, ch); + } + appendStringInfoChar(buf, '\''); +} +