diff --git a/R/dm-from-src.R b/R/dm-from-src.R index b07d96b4fc..9092b7c6d6 100644 --- a/R/dm-from-src.R +++ b/R/dm-from-src.R @@ -62,6 +62,8 @@ dm_from_src <- function(src = NULL, table_names = NULL, learn_keys = NULL, src <- src_from_src_or_con(src) con <- con_from_src_or_con(src) + # FIXME: Get rid of legacy method once it works for all + if (is.null(learn_keys) || isTRUE(learn_keys)) { dm_learned <- dm_learn_from_db(src, ...) @@ -76,12 +78,12 @@ dm_from_src <- function(src = NULL, table_names = NULL, learn_keys = NULL, inform("Keys queried successfully, use `learn_keys = TRUE` to mute this message.") } - tbls_in_dm <- src_tbls_impl(dm_learned) - if (is_null(table_names)) { return(dm_learned) } + tbls_in_dm <- src_tbls_impl(dm_learned) + if (!all(table_names %in% tbls_in_dm)) { abort_tbl_access(setdiff(table_names, tbls_in_dm)) } @@ -123,7 +125,9 @@ dm_from_src <- function(src = NULL, table_names = NULL, learn_keys = NULL, } quote_ids <- function(x, con, schema = NULL) { - if (is.null(con)) return(x) + if (is.null(con)) { + return(x) + } if (is_null(schema)) { map( diff --git a/R/dm.R b/R/dm.R index 825699c385..c65a44de1c 100644 --- a/R/dm.R +++ b/R/dm.R @@ -94,23 +94,19 @@ new_dm <- function(tables = list()) { } new_dm2 <- function(tables = list(), - pks = structure(list(), names = character()), - fks = structure(list(), names = character()), + pks_df = tibble(table = character(), pks = list()), + fks_df = tibble(table = character(), fks = list()), validate = TRUE) { # Legacy data <- unname(tables) table <- names2(tables) - stopifnot(!is.null(names(pks)), all(names(pks) %in% table)) - stopifnot(!is.null(names(fks)), all(names(fks) %in% table)) + stopifnot(all(pks_df$table %in% table)) + stopifnot(all(fks_df$table %in% table)) zoom <- new_zoom() col_tracker_zoom <- new_col_tracker_zoom() - pks_df <- enframe(pks, "table", "pks") - - fks_df <- enframe(fks, "table", "fks") - filters <- tibble( table = table, diff --git a/R/global.R b/R/global.R index 0210982b49..1a8adc8020 100644 --- a/R/global.R +++ b/R/global.R @@ -113,6 +113,37 @@ utils::globalVariables(c( "orders", "trans", # + # information_schema + "catalog", + "catalog_name", + "column_default", + "column_id", + "column_name", + "con", + "constraint_catalog", + "constraint_column_id", + "constraint_column_usage", + "constraint_name", + "constraint_schema", + "constraint_type", + "dbname", + "FIXME", + "is_nullable", + "key_column_usage", + "object_id", + "ordinal_position", + "schema_id", + "schemata", + "table_catalog", + "table_constraints", + "table_schema", + "table_type", + "tables", + "constraint_column_usage.column_name", + "constraint_column_usage.dm_name", + "key_column_usage.column_name", + "key_column_usage.dm_name", + # # pixarfilms "pixar_films", "pixar_people", diff --git a/R/learn.R b/R/learn.R index b696a883a9..015f656222 100644 --- a/R/learn.R +++ b/R/learn.R @@ -32,7 +32,7 @@ #' # the `dm` from the SQLite DB #' iris_dm_learned <- dm_learn_from_db(src_sqlite) #' } -dm_learn_from_db <- function(dest, dbname = NULL, ...) { +dm_learn_from_db <- function(dest, dbname = NA, ...) { # assuming that we will not try to learn from (globally) temporary tables, which do not appear in sys.table con <- con_from_src_or_con(dest) src <- src_from_src_or_con(dest) @@ -41,6 +41,130 @@ dm_learn_from_db <- function(dest, dbname = NULL, ...) { return() } + if (!is_mssql(con)) { + return(dm_learn_from_db_legacy(con, dbname, ...)) + } + + dm_learn_from_db_meta(con, catalog = dbname, ...) +} + +dm_learn_from_db_meta <- function(con, catalog = NULL, schema = NULL, name_format = "{table}") { + info <- dm_meta(con, catalog = catalog, schema = schema) + + df_info <- + info %>% + dm_select_tbl(-schemata) %>% + collect() + + dm_name <- + df_info$tables %>% + select(catalog = table_catalog, schema = table_schema, table = table_name) %>% + mutate(name = glue(!!name_format)) %>% + pull() %>% + unclass() %>% + vec_as_names(repair = "unique") + + from <- + df_info$tables %>% + select(catalog = table_catalog, schema = table_schema, table = table_name) %>% + pmap_chr(~ DBI::dbQuoteIdentifier(con, DBI::Id(...))) + + df_key_info <- + df_info %>% + dm_zoom_to(tables) %>% + mutate(dm_name = !!dm_name, from = !!from) %>% + dm_update_zoomed() %>% + dm_zoom_to(columns) %>% + arrange(ordinal_position) %>% + select(-ordinal_position) %>% + left_join(tables) %>% + dm_update_zoomed() %>% + dm_select_tbl(constraint_column_usage, key_column_usage, columns) + + table_info <- + df_key_info %>% + dm_zoom_to(columns) %>% + group_by(dm_name, from) %>% + summarize(vars = list(column_name)) %>% + ungroup() %>% + pull_tbl() + + tables <- map2(table_info$from, table_info$vars, ~ tbl(con, dbplyr::ident_q(.x), vars = .y)) + names(tables) <- table_info$dm_name + + pks_df <- + df_key_info %>% + dm_zoom_to(key_column_usage) %>% + anti_join(constraint_column_usage) %>% + arrange(ordinal_position) %>% + dm_update_zoomed() %>% + dm_squash_to_tbl(key_column_usage) %>% + select(constraint_catalog, constraint_schema, constraint_name, dm_name, column_name) %>% + group_by(constraint_catalog, constraint_schema, constraint_name, dm_name) %>% + summarize(pks = list(tibble(column = list(column_name)))) %>% + ungroup() %>% + select(table = dm_name, pks) + + fks_df <- + df_key_info %>% + dm_zoom_to(key_column_usage) %>% + left_join(columns, select = c(column_name, dm_name, table_catalog, table_schema, table_name)) %>% + dm_update_zoomed() %>% + dm_zoom_to(constraint_column_usage) %>% + left_join(columns, select = c(column_name, dm_name, table_catalog, table_schema, table_name)) %>% + dm_update_zoomed() %>% + dm_select_tbl(-columns) %>% + dm_rename(constraint_column_usage, constraint_column_usage.table_catalog = table_catalog) %>% + dm_rename(constraint_column_usage, constraint_column_usage.table_schema = table_schema) %>% + dm_rename(constraint_column_usage, constraint_column_usage.table_name = table_name) %>% + dm_rename(constraint_column_usage, constraint_column_usage.column_name = column_name) %>% + dm_rename(constraint_column_usage, constraint_column_usage.dm_name = dm_name) %>% + dm_rename(key_column_usage, key_column_usage.table_catalog = table_catalog) %>% + dm_rename(key_column_usage, key_column_usage.table_schema = table_schema) %>% + dm_rename(key_column_usage, key_column_usage.table_name = table_name) %>% + dm_rename(key_column_usage, key_column_usage.column_name = column_name) %>% + dm_rename(key_column_usage, key_column_usage.dm_name = dm_name) %>% + dm_flatten_to_tbl(constraint_column_usage) %>% + select( + constraint_catalog, + constraint_schema, + constraint_name, + ordinal_position, + ref_table = constraint_column_usage.dm_name, + ref_column = constraint_column_usage.column_name, + table = key_column_usage.dm_name, + column = key_column_usage.column_name, + ) %>% + arrange( + constraint_catalog, + constraint_schema, + constraint_name, + ordinal_position, + ) %>% + select(-ordinal_position) %>% + # FIXME: Where to learn this in INFORMATION_SCHEMA? + group_by( + constraint_catalog, + constraint_schema, + constraint_name, + ref_table, + ) %>% + summarize(fks = list(tibble( + ref_column = list(ref_column), + table = if (length(table) > 0) table[[1]] else NA_character_, + column = list(column), + on_delete = "no_action" + ))) %>% + ungroup() %>% + select(-(1:3)) %>% + group_by(table = ref_table) %>% + summarize(fks = list(bind_rows(fks))) %>% + ungroup() + + new_dm2(tables, pks_df, fks_df) +} + +dm_learn_from_db_legacy <- function(con, dbname, ...) { sql <- db_learn_query(con, dbname = dbname, ...) if (is.null(sql)) { return() @@ -74,7 +198,7 @@ dm_learn_from_db <- function(dest, dbname = NULL, ...) { schema_if <- function(schema, table, con, dbname = NULL) { table_sql <- DBI::dbQuoteIdentifier(con, table) - if (is_null(dbname) || dbname == "") { + if (is_null(dbname) || is.na(dbname) || dbname == "") { if_else( are_na(schema), table_sql, @@ -91,63 +215,11 @@ schema_if <- function(schema, table, con, dbname = NULL) { } db_learn_query <- function(dest, dbname, ...) { - if (is_mssql(dest)) { - return(mssql_learn_query(dest, dbname = dbname, ...)) - } if (is_postgres(dest)) { return(postgres_learn_query(dest, ...)) } } -mssql_learn_query <- function(con, schema = "dbo", dbname = NULL) { # taken directly from {datamodelr} and subsequently tweaked a little - dbname_sql <- if (is_null(dbname)) { - "" - } else { - paste0(DBI::dbQuoteIdentifier(con, dbname), ".") - } - glue::glue( - "select - schemas.name as [schema], - tabs.name as [table], - cols.name as [column], - isnull(ind_col.column_id, 0) as [key], - ref_tabs.name AS ref, - ref_cols.name AS ref_col, - 1 - cols.is_nullable as mandatory, - types.name as [type], - cols.max_length, - cols.precision, - cols.scale - from - {dbname_sql}sys.all_columns cols - inner join {dbname_sql}sys.tables tabs on - cols.object_id = tabs.object_id - inner join {dbname_sql}sys.schemas schemas on - tabs.schema_id = schemas.schema_id - left outer join {dbname_sql}sys.foreign_key_columns ref on - ref.parent_object_id = tabs.object_id - and ref.parent_column_id = cols.column_id - left outer join {dbname_sql}sys.indexes ind on - ind.object_id = tabs.object_id - and ind.is_primary_key = 1 - left outer join {dbname_sql}sys.index_columns ind_col on - ind_col.object_id = ind.object_id - and ind_col.index_id = ind.index_id - and ind_col.column_id = cols.column_id - left outer join {dbname_sql}sys.systypes [types] on - types.xusertype = cols.system_type_id - left outer join {dbname_sql}sys.tables ref_tabs on - ref_tabs.object_id = ref.referenced_object_id - left outer join {dbname_sql}sys.all_columns ref_cols on - ref_cols.object_id = ref.referenced_object_id - and ref_cols.column_id = ref.referenced_column_id - where schemas.name = {DBI::dbQuoteString(con, schema)} - order by - tabs.create_date, - cols.column_id" - ) -} - postgres_learn_query <- function(con, schema = "public", table_type = "BASE TABLE") { sprintf( "SELECT diff --git a/R/meta.R b/R/meta.R new file mode 100644 index 0000000000..3826c94514 --- /dev/null +++ b/R/meta.R @@ -0,0 +1,179 @@ +dm_meta <- function(con, catalog = NA, schema = NULL) { + need_collect <- FALSE + + if (is_mssql(con)) { + if (is.null(catalog)) { + # FIXME: Classed error message? + abort("SQL server only supports learning from one database.") + } + + if (!is.na(catalog)) { + message("Temporarily switching to database ", tick(catalog), ".") + old_dbname <- dbGetQuery(con, "SELECT DB_NAME()")[[1]] + sql <- paste0("USE ", dbQuoteIdentifier(con, catalog)) + old_sql <- paste0("USE ", dbQuoteIdentifier(con, old_dbname)) + dbExecute(con, sql, immediate = TRUE) + withr::defer({ + dbExecute(con, old_sql, immediate = TRUE) + }) + need_collect <- TRUE + } + } + + out <- + con %>% + dm_meta_raw(catalog) %>% + select_dm_meta() %>% + filter_dm_meta(catalog, schema) + + if (need_collect) { + out <- + out %>% + collect() + } + + out +} + +dm_meta_raw <- function(con, catalog) { + src <- src_from_src_or_con(con) + + local_options(digits.secs = 6) + + schemata <- tbl_lc(src, "information_schema.schemata", vars = c( + "catalog_name", "schema_name", "schema_owner", "default_character_set_catalog", + "default_character_set_schema", "default_character_set_name" + )) + tables <- tbl_lc(src, "information_schema.tables", vars = c( + "table_catalog", "table_schema", "table_name", "table_type" + )) + columns <- tbl_lc(src, "information_schema.columns", vars = c( + "table_catalog", "table_schema", "table_name", "column_name", + "ordinal_position", "column_default", "is_nullable", "data_type", + "character_maximum_length", "character_octet_length", "numeric_precision", + "numeric_precision_radix", "numeric_scale", "datetime_precision", + "character_set_catalog", "character_set_schema", "character_set_name", + "collation_catalog", "collation_schema", "collation_name", "domain_catalog", + "domain_schema", "domain_name" + )) + table_constraints <- tbl_lc(src, "information_schema.table_constraints", vars = c( + "constraint_catalog", "constraint_schema", "constraint_name", + "table_catalog", "table_schema", "table_name", "constraint_type", + "is_deferrable", "initially_deferred" + )) + key_column_usage <- tbl_lc(src, "information_schema.key_column_usage", vars = c( + "constraint_catalog", "constraint_schema", "constraint_name", + "table_catalog", "table_schema", "table_name", "column_name", + "ordinal_position" + )) + + if (is_postgres(src)) { + info_fkc <- + table_constraints %>% + select(constraint_catalog, constraint_schema, constraint_name, constraint_type) %>% + filter(constraint_type == "FOREIGN KEY") + + constraint_column_usage <- + tbl_lc(src, dbplyr::ident_q("information_schema.constraint_column_usage")) %>% + group_by(constraint_catalog, constraint_schema, constraint_name) %>% + mutate(ordinal_position = row_number()) %>% + ungroup() %>% + semi_join(info_fkc, by = c("constraint_catalog", "constraint_schema", "constraint_name")) + + # FIXME: Also has `position_in_unique_constraint`, used elsewhere? + } else if (is_mssql(src)) { + constraint_column_usage <- mssql_constraint_column_usage(src, table_constraints, catalog) + } + + dm(schemata, tables, columns, table_constraints, key_column_usage, constraint_column_usage) %>% + dm_meta_add_keys() +} + +dm_meta_add_keys <- function(dm_meta) { + dm_meta %>% + dm_add_pk(schemata, c(catalog_name, schema_name)) %>% + dm_add_pk(tables, c(table_catalog, table_schema, table_name)) %>% + dm_add_fk(tables, c(table_catalog, table_schema), schemata) %>% + dm_add_pk(columns, c(table_catalog, table_schema, table_name, column_name)) %>% + dm_add_fk(columns, c(table_catalog, table_schema, table_name), tables) %>% + # dm_add_fk(table_constraints, table_schema, schemata) %>% + dm_add_pk(table_constraints, c(constraint_catalog, constraint_schema, constraint_name)) %>% + dm_add_fk(table_constraints, c(table_catalog, table_schema, table_name), tables) %>% + # constraint_schema vs. table_schema? + + # not on mssql: + # dm_add_fk(referential_constraints, c(constraint_schema, table_name), tables) %>% + # dm_add_fk(referential_constraints, c(constraint_schema, referenced_table_name), tables) %>% + + dm_add_pk(key_column_usage, c(constraint_catalog, constraint_schema, constraint_name, ordinal_position)) %>% + dm_add_fk(key_column_usage, c(table_catalog, table_schema, table_name, column_name), columns) %>% + dm_add_fk(key_column_usage, c(constraint_catalog, constraint_schema, constraint_name), table_constraints) %>% + # + # not on mariadb; + dm_add_pk(constraint_column_usage, c(constraint_catalog, constraint_schema, constraint_name, ordinal_position)) %>% + dm_add_fk(constraint_column_usage, c(table_catalog, table_schema, table_name, column_name), columns) %>% + dm_add_fk(constraint_column_usage, c(constraint_catalog, constraint_schema, constraint_name), table_constraints) %>% + dm_add_fk(constraint_column_usage, c(constraint_catalog, constraint_schema, constraint_name, ordinal_position), key_column_usage) %>% + # + dm_set_colors(brown = c(tables, columns), blue = schemata, green4 = ends_with("_constraints"), orange = ends_with("_usage")) +} + +tbl_lc <- function(con, name, vars) { + from <- paste0( + "SELECT ", + paste0(DBI::dbQuoteIdentifier(con_from_src_or_con(con), vars), collapse = ", "), + "\nFROM ", name + ) + + tbl(con, sql(from), vars = vars) +} + +select_dm_meta <- function(dm_meta) { + dm_meta %>% + dm_select(schemata, catalog_name, schema_name) %>% + dm_select(tables, table_catalog, table_schema, table_name, table_type) %>% + dm_select(columns, table_catalog, table_schema, table_name, column_name, ordinal_position, column_default, is_nullable) %>% + dm_select(table_constraints, constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, constraint_type) %>% + dm_select(key_column_usage, constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, column_name, ordinal_position) %>% + dm_select(constraint_column_usage, table_catalog, table_schema, table_name, column_name, constraint_catalog, constraint_schema, constraint_name, ordinal_position) +} + +filter_dm_meta <- function(dm_meta, catalog = NULL, schema = NULL) { + force(catalog) + force(schema) + + schemata <- dm_meta$schemata + tables <- dm_meta$tables + columns <- dm_meta$columns + table_constraints <- dm_meta$table_constraints + key_column_usage <- dm_meta$key_column_usage + constraint_column_usage <- dm_meta$constraint_column_usage + + if (!is.null(catalog) && !is.na(catalog)) { + schemata <- schemata %>% filter(catalog_name %in% !!catalog) + tables <- tables %>% filter(table_catalog %in% !!catalog) + columns <- columns %>% filter(table_catalog %in% !!catalog) + table_constraints <- table_constraints %>% filter(table_catalog %in% !!catalog) + key_column_usage <- key_column_usage %>% filter(table_catalog %in% !!catalog) + constraint_column_usage <- constraint_column_usage %>% filter(table_catalog %in% !!catalog) + } + + if (!is.null(schema)) { + schemata <- schemata %>% filter(schema_name %in% !!schema) + tables <- tables %>% filter(table_schema %in% !!schema) + columns <- columns %>% filter(table_schema %in% !!schema) + table_constraints <- table_constraints %>% filter(table_schema %in% !!schema) + key_column_usage <- key_column_usage %>% filter(table_schema %in% !!schema) + constraint_column_usage <- constraint_column_usage %>% filter(table_schema %in% !!schema) + } + + dm( + schemata, + tables, + columns, + table_constraints, + key_column_usage, + constraint_column_usage + ) %>% + dm_meta_add_keys() +} diff --git a/R/mssql.R b/R/mssql.R new file mode 100644 index 0000000000..8647450a20 --- /dev/null +++ b/R/mssql.R @@ -0,0 +1,66 @@ +mssql_sys_db <- function(con, dbname, name, vars = NULL) { + if (is.na(dbname)) { + fq_name <- name + sql_name <- sql("DB_NAME()") + } else { + fq_name <- paste0(dbname, ".", name) + sql_name <- dbname + } + tbl(con, dbplyr::ident_q(fq_name), vars = vars) %>% + mutate(catalog = !!sql_name) %>% + select(catalog, everything()) +} + +mssql_constraint_column_usage <- function(con, table_constraints, dbname) { + info_fkc <- + table_constraints %>% + select(constraint_catalog, constraint_schema, constraint_name, constraint_type) %>% + filter(constraint_type == "FOREIGN KEY") + + fkc <- mssql_sys_db(con, dbname, "sys.foreign_key_columns", vars = c( + "constraint_object_id", "constraint_column_id", + "referenced_object_id", "referenced_column_id" + )) + + columns <- + mssql_sys_db(con, dbname, "sys.columns", vars = c( + "name", "object_id", "column_id" + )) %>% + rename(column_name = name) + + tables <- + mssql_sys_db(con, dbname, "sys.tables", vars = c( + "schema_id", "name", "object_id" + )) %>% + rename(table_name = name) + + schemas <- + mssql_sys_db(con, dbname, "sys.schemas", vars = c( + "schema_id", "name" + )) %>% + rename(table_schema = name) + + objects <- + mssql_sys_db(con, dbname, "sys.objects", vars = c( + "name", "object_id" + )) %>% + select(constraint_name = name, object_id) + + sys_fkc_column_usage <- + fkc %>% + left_join(columns, by = c("catalog", "referenced_object_id" = "object_id", "referenced_column_id" = "column_id")) %>% + left_join(tables, by = c("catalog", "referenced_object_id" = "object_id")) %>% + left_join(schemas, by = c("catalog", "schema_id")) %>% + left_join(objects, by = c("constraint_object_id" = "object_id")) %>% + # table_schema is used twice + transmute(constraint_catalog = catalog, constraint_schema = table_schema, constraint_name, table_schema, table_name, column_name, ordinal_position = constraint_column_id) + + tbl_lc(con, "information_schema.constraint_column_usage", vars = c( + "table_catalog", "table_schema", "table_name", "column_name", + "constraint_catalog", "constraint_schema", "constraint_name" + )) %>% + semi_join(info_fkc, by = c("constraint_catalog", "constraint_schema", "constraint_name")) %>% + select(-table_schema, -table_name, -column_name) %>% + distinct() %>% + left_join(sys_fkc_column_usage, by = c("constraint_catalog", "constraint_schema", "constraint_name")) +} diff --git a/R/zzz.R b/R/zzz.R index ce0a66b893..1b2dfb800d 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -49,7 +49,7 @@ rigg <- function(fun) { rig <- get("rig", asNamespace("boomer"), mode = "function") - assign(name, rig(fun, ignore = c("~", "{", "(", "<-", "<<-")), getNamespace("dm")) + assign(name, rig(fun, ignore = c("~", "{", "(")), getNamespace("dm")) } check_version_on_load <- function(package, version, reason) { diff --git a/scratch/info.R b/scratch/info.R index bec4d3edff..ea0fffd45b 100644 --- a/scratch/info.R +++ b/scratch/info.R @@ -182,7 +182,7 @@ info_simple %>% dm_get_tables() key_dm <- - info %>% + info_simple %>% dm_zoom_to(table_constraints) %>% filter(constraint_type == "PRIMARY KEY") %>% @@ -196,14 +196,18 @@ key_dm <- dm_update_zoomed() %>% dm_zoom_to(constraint_column_usage) %>% semi_join(table_constraints) %>% - rename(fk_table_catalog = table_catalog, fk_table_schema = table_schema, fk_table_name = table_name, fk_column_name = column_name) %>% + rename(fk_fq_table_name = fq_table_name, fk_column_name = column_name) %>% left_join(key_column_usage) %>% - rename(pk_table_catalog = table_catalog, pk_table_schema = table_schema, pk_table_name = table_name, pk_column_name = column_name) %>% + rename(pk_fq_table_name = fq_table_name, pk_column_name = column_name) %>% # Postgres: Can return int64 here mutate(ordinal_position = as.integer(ordinal_position)) %>% dm_insert_zoomed("fk") %>% - dm_add_fk(fk, c(pk_table_catalog, pk_table_schema, pk_table_name, pk_column_name), columns) %>% + dm_add_fk(fk, c(pk_fq_table_name, pk_column_name), columns) %>% + + dm_zoom_to(columns) %>% + left_join(tables, select = r_table_name) %>% + dm_update_zoomed() %>% dm_select_tbl(columns, pk, fk) diff --git a/tests/testthat/_snaps/learn.md b/tests/testthat/_snaps/learn.md new file mode 100644 index 0000000000..065e3c5b2d --- /dev/null +++ b/tests/testthat/_snaps/learn.md @@ -0,0 +1,40 @@ +# dm_meta() data model + + Code + dm_meta(my_test_src()) %>% dm_paste(options = c("select", "keys", "color")) + Message + dm::dm( + schemata, + tables, + columns, + table_constraints, + key_column_usage, + constraint_column_usage, + ) %>% + dm::dm_select(schemata, catalog_name, schema_name) %>% + dm::dm_select(tables, table_catalog, table_schema, table_name, table_type) %>% + dm::dm_select(columns, table_catalog, table_schema, table_name, column_name, ordinal_position, column_default, is_nullable) %>% + dm::dm_select(table_constraints, constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, constraint_type) %>% + dm::dm_select(key_column_usage, constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, column_name, ordinal_position) %>% + dm::dm_select(constraint_column_usage, table_catalog, table_schema, table_name, column_name, constraint_catalog, constraint_schema, constraint_name, ordinal_position) %>% + dm::dm_add_pk(schemata, c(catalog_name, schema_name)) %>% + dm::dm_add_pk(tables, c(table_catalog, table_schema, table_name)) %>% + dm::dm_add_pk(columns, c(table_catalog, table_schema, table_name, column_name)) %>% + dm::dm_add_pk(table_constraints, c(constraint_catalog, constraint_schema, constraint_name)) %>% + dm::dm_add_pk(key_column_usage, c(constraint_catalog, constraint_schema, constraint_name, ordinal_position)) %>% + dm::dm_add_pk(constraint_column_usage, c(constraint_catalog, constraint_schema, constraint_name, ordinal_position)) %>% + dm::dm_add_fk(tables, c(table_catalog, table_schema), schemata) %>% + dm::dm_add_fk(columns, c(table_catalog, table_schema, table_name), tables) %>% + dm::dm_add_fk(table_constraints, c(table_catalog, table_schema, table_name), tables) %>% + dm::dm_add_fk(key_column_usage, c(table_catalog, table_schema, table_name, column_name), columns) %>% + dm::dm_add_fk(constraint_column_usage, c(table_catalog, table_schema, table_name, column_name), columns) %>% + dm::dm_add_fk(key_column_usage, c(constraint_catalog, constraint_schema, constraint_name), table_constraints) %>% + dm::dm_add_fk(constraint_column_usage, c(constraint_catalog, constraint_schema, constraint_name), table_constraints) %>% + dm::dm_add_fk(constraint_column_usage, c(constraint_catalog, constraint_schema, constraint_name, ordinal_position), key_column_usage) %>% + dm::dm_set_colors(`#0000FFFF` = schemata) %>% + dm::dm_set_colors(`#A52A2AFF` = tables) %>% + dm::dm_set_colors(`#A52A2AFF` = columns) %>% + dm::dm_set_colors(`#008B00FF` = table_constraints) %>% + dm::dm_set_colors(`#FFA500FF` = key_column_usage) %>% + dm::dm_set_colors(`#FFA500FF` = constraint_column_usage) + diff --git a/tests/testthat/test-learn.R b/tests/testthat/test-learn.R index 106c907115..1a617e5598 100644 --- a/tests/testthat/test-learn.R +++ b/tests/testthat/test-learn.R @@ -13,8 +13,8 @@ schema_name <- random_schema() test_that("Standard learning from MSSQL (schema 'dbo') or Postgres (schema 'public') and get_src_tbl_names() works?", { skip_if_src_not(c("mssql", "postgres")) - # FIXME: COMPOUND: Need to fix implementation - skip_if_remote_src() + # FIXME: Enable when fixed + skip_if_src("postgres") # dm_learn_from_mssql() -------------------------------------------------- src_db <- my_test_src() @@ -34,7 +34,12 @@ test_that("Standard learning from MSSQL (schema 'dbo') or Postgres (schema 'publ map_chr( dm_get_tables(dm_for_filter_copied)[order_of_deletion], dbplyr::remote_name - ) + ) %>% + SQL() %>% + DBI::dbUnquoteIdentifier(conn = src_db$con) %>% + map_chr(~ .x@name[["table"]]) + + remote_tbl_map <- set_names(remote_tbl_names, gsub("^(tf_.).*$", "\\1", remote_tbl_names)) # test 'get_src_tbl_names()' src_tbl_names <- sort(unname(gsub("^.*\\.", "", get_src_tbl_names(src_db)))) @@ -44,20 +49,22 @@ test_that("Standard learning from MSSQL (schema 'dbo') or Postgres (schema 'publ sort(dbQuoteIdentifier(src_db$con, remote_tbl_names)) ) - dm_db_learned_all <- expect_message(dm_from_src(src_db)) + expect_message(dm_db_learned_all <- dm_from_src(src_db)) - # in case there happen to be other tables in schema "dbo" or "public" + # Select and fix table names dm_db_learned <- dm_db_learned_all %>% - dm_select_tbl(!!!remote_tbl_names) + dm_select_tbl(!!!remote_tbl_map) expect_equivalent_dm( dm_db_learned, - dm_for_filter()[order_of_deletion] + dm_for_filter()[order_of_deletion], + # FIXME: Enable fetching of on_delete information + ignore_on_delete = TRUE ) # learning without keys: - dm_db_learned_no_keys <- expect_silent(dm_from_src(src_db, learn_keys = FALSE)) + expect_silent(dm_db_learned_no_keys <- dm_from_src(src_db, learn_keys = FALSE)) # for learning from DB without learning the key relations dm_for_filter_no_keys <- @@ -69,10 +76,10 @@ test_that("Standard learning from MSSQL (schema 'dbo') or Postgres (schema 'publ ) %>% new_dm3() - # in case there happen to be other tables in schema "dbo" or "public" + # Select and fix table names dm_db_learned_no_keys <- dm_db_learned_no_keys %>% - dm_select_tbl(!!!remote_tbl_names) + dm_select_tbl(!!!remote_tbl_map) expect_equivalent_dm( dm_db_learned_no_keys, @@ -84,9 +91,6 @@ test_that("Standard learning from MSSQL (schema 'dbo') or Postgres (schema 'publ test_that("Learning from specific schema on MSSQL or Postgres works?", { skip_if_src_not(c("mssql", "postgres")) - # FIXME: COMPOUND: Need to fix implementation - skip_if_remote_src() - src_db <- my_test_src() con_db <- src_db$con @@ -98,10 +102,13 @@ test_that("Learning from specific schema on MSSQL or Postgres works?", { src_db, dm_for_disambiguate(), temporary = FALSE, - table_names = ~ DBI::SQL(paste0(schema_name_q, ".", .x)) + schema = schema_name ) order_of_deletion <- c("iris_3", "iris_2", "iris_1") - remote_tbl_names <- set_names(paste0(schema_name_q, ".\"", order_of_deletion, "\""), order_of_deletion) + remote_tbl_names <- set_names( + paste0(schema_name_q, ".\"", order_of_deletion, "\""), + order_of_deletion + ) withr::defer({ walk( @@ -365,6 +372,11 @@ test_that("Learning from a specific schema in another DB for MSSQL works?", { ) }) -# tests for compound keys ------------------------------------------------- +test_that("dm_meta() data model", { + skip_if_src_not("mssql") -# test is already done in test-dm-from-src.R + expect_snapshot({ + dm_meta(my_test_src()) %>% + dm_paste(options = c("select", "keys", "color")) + }) +})