From e0206e73a9c0eb80dc33fea09ad6ff41f51413ca Mon Sep 17 00:00:00 2001 From: Dat Nguyen <103571964+il-dat@users.noreply.github.com> Date: Thu, 27 Oct 2022 10:08:21 +0700 Subject: [PATCH] #46: Adding FORCE in applying masking policies (#47) * #46: Adding FORCE in applying masking policies * #46: Minor fix after the integration tests * #46: typo error Co-authored-by: Dat Nguyen --- README.md | 8 ++++++++ dbt_project.yml | 1 + integration_tests/dbt_project.yml | 3 ++- .../apply-policy/apply_masking_policy_list_for_models.sql | 4 ++-- .../apply_masking_policy_list_for_sources.sql | 4 ++-- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6890e67..c0ec80b 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,14 @@ vars: masking_policy: mp_encrypt_pii ``` +- Decide you force applying masking policy to avoid unsetting them before re-applying again - it helps to remove handy stuff whenever the masking policy definition is relocated to another database/schema: + **Example** : var block in dbt_project.yml to enable using force + +```yaml +vars: + use_force_applying_masking_policy: "True" +``` + - Create a new `.sql` file with the name `create_masking_policy_.sql` and the sql for masking policy definition. Its important for macro to follow this naming standard. **Example** : create_masking_policy_mp_encrypt_pii.sql diff --git a/dbt_project.yml b/dbt_project.yml index 05f5213..fce8a38 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -10,6 +10,7 @@ macro-paths: ["macros"] log-path: "logs" vars: + use_force_applying_masking_policy: "False" use_common_masking_policy_db: "False" create_masking_policy_schema: "True" common_masking_policy_db: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index deaef22..6932a08 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -18,6 +18,7 @@ clean-targets: - "logs" #vars: +# use_force_applying_masking_policy: "True" # use_common_masking_policy_db: "True" # common_masking_policy_db: "DEMO_DB" # common_masking_policy_schema: "COMPLIANCE" @@ -42,7 +43,7 @@ snapshots: - "{{ dbt_snow_mask.apply_masking_policy('snapshots') }}" dbt_snow_mask_integration_tests: - staging: + pii: database: "DEV_ENTECHLOG_DEMO_DB" schema: staging diff --git a/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql b/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql index a3d5ed9..4cfd701 100644 --- a/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql +++ b/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql @@ -56,9 +56,9 @@ {% for masking_policy_in_db in masking_policy_list['MASKING_POLICY'] %} {% if masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper == masking_policy_in_db %} - {{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to model : " ~ masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ alias ~ '.' ~ column, info=True) }} + {{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to model : " ~ masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ alias ~ '.' ~ column ~ ' [force = ' ~ var('use_force_applying_masking_policy','False') ~ ']', info=True) }} {% set query %} - alter {{materialization}} {{database}}.{{schema}}.{{alias}} modify column {{column}} set masking policy {{masking_policy_db}}.{{masking_policy_schema}}.{{masking_policy_name}}; + alter {{materialization}} {{database}}.{{schema}}.{{alias}} modify column {{column}} set masking policy {{masking_policy_db}}.{{masking_policy_schema}}.{{masking_policy_name}} {% if var('use_force_applying_masking_policy','False')|upper in ['TRUE','YES'] %} force {% endif %}; {% endset %} {% do run_query(query) %} {% endif %} diff --git a/macros/snow-mask/apply-policy/apply_masking_policy_list_for_sources.sql b/macros/snow-mask/apply-policy/apply_masking_policy_list_for_sources.sql index 74636c6..797cb53 100644 --- a/macros/snow-mask/apply-policy/apply_masking_policy_list_for_sources.sql +++ b/macros/snow-mask/apply-policy/apply_masking_policy_list_for_sources.sql @@ -58,10 +58,10 @@ {% for masking_policy_in_db in masking_policy_list['MASKING_POLICY'] %} {% if masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper == masking_policy_in_db %} - {{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to source : " ~ masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ identifier ~ '.' ~ column, info=True) }} + {{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to source : " ~ masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ identifier ~ '.' ~ column ~ ' [force = ' ~ var('use_force_applying_masking_policy','False') ~ ']', info=True) }} {% set query %} {% if operation_type == "apply" %} - alter {{materialization}} {{database}}.{{schema}}.{{identifier}} modify column {{column}} set masking policy {{masking_policy_db}}.{{masking_policy_schema}}.{{masking_policy_name}} + alter {{materialization}} {{database}}.{{schema}}.{{identifier}} modify column {{column}} set masking policy {{masking_policy_db}}.{{masking_policy_schema}}.{{masking_policy_name}} {% if var('use_force_applying_masking_policy','False')|upper in ['TRUE','YES'] %} force {% endif %} {% elif operation_type == "unapply" %} alter {{materialization}} {{database}}.{{schema}}.{{identifier}} modify column {{column}} unset masking policy {% endif %}