#46: Adding FORCE in applying masking policies (#47)

* #46: Adding FORCE in applying masking policies * #46: Minor fix after the integration tests * #46: typo error Co-authored-by: Dat Nguyen <dat@infinitelambda>
entechlog · Oct 27, 2022 · e0206e7 · e0206e7
1 parent 307c8ab
commit e0206e7
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 5 deletions.
diff --git a/README.md b/README.md
@@ -124,6 +124,14 @@ vars:
             masking_policy: mp_encrypt_pii
   ```
 
+- Decide you force applying masking policy to avoid unsetting them before re-applying again - it helps to remove handy stuff whenever the masking policy definition is relocated to another database/schema:
+  **Example** : var block in dbt_project.yml to enable using force
+
+```yaml
+vars:
+  use_force_applying_masking_policy: "True"
+```
+
 - Create a new `.sql` file with the name `create_masking_policy_<masking-policy-name-from-meta>.sql` and the sql for masking policy definition. Its important for macro to follow this naming standard.
 
   **Example** : create_masking_policy_mp_encrypt_pii.sql

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -10,6 +10,7 @@ macro-paths: ["macros"]
 log-path: "logs"
 
 vars:
+  use_force_applying_masking_policy: "False"
   use_common_masking_policy_db: "False"
   create_masking_policy_schema: "True"
   common_masking_policy_db: 

diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -18,6 +18,7 @@ clean-targets:
     - "logs"
 
 #vars:
+#  use_force_applying_masking_policy: "True"
 #  use_common_masking_policy_db: "True"
 #  common_masking_policy_db: "DEMO_DB"
 #  common_masking_policy_schema: "COMPLIANCE"
@@ -42,7 +43,7 @@ snapshots:
     - "{{ dbt_snow_mask.apply_masking_policy('snapshots') }}"
 
   dbt_snow_mask_integration_tests:
-    staging:
+    pii:
       database: "DEV_ENTECHLOG_DEMO_DB"
       schema: staging
 

diff --git a/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql b/macros/snow-mask/apply-policy/apply_masking_policy_list_for_models.sql
@@ -56,9 +56,9 @@
 
                     {% for masking_policy_in_db in masking_policy_list['MASKING_POLICY'] %}
                         {% if masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper == masking_policy_in_db %}
-                            {{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to model  : " ~ masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ alias ~ '.' ~ column, info=True) }}
+                            {{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to model  : " ~ masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ alias ~ '.' ~ column ~ ' [force = ' ~ var('use_force_applying_masking_policy','False') ~ ']', info=True) }}
                             {% set query %}
-                            alter {{materialization}}  {{database}}.{{schema}}.{{alias}} modify column  {{column}} set masking policy {{masking_policy_db}}.{{masking_policy_schema}}.{{masking_policy_name}};
+                            alter {{materialization}}  {{database}}.{{schema}}.{{alias}} modify column  {{column}} set masking policy {{masking_policy_db}}.{{masking_policy_schema}}.{{masking_policy_name}} {% if var('use_force_applying_masking_policy','False')|upper in ['TRUE','YES'] %} force {% endif %};
                             {% endset %}
                             {% do run_query(query) %}
                         {% endif %}

diff --git a/macros/snow-mask/apply-policy/apply_masking_policy_list_for_sources.sql b/macros/snow-mask/apply-policy/apply_masking_policy_list_for_sources.sql
@@ -58,10 +58,10 @@
 
                 {% for masking_policy_in_db in masking_policy_list['MASKING_POLICY'] %}
                     {% if masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper == masking_policy_in_db %}
-                        {{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to source : " ~ masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ identifier ~ '.' ~ column, info=True) }}
+                        {{ log(modules.datetime.datetime.now().strftime("%H:%M:%S") ~ " | " ~ operation_type ~ "ing masking policy to source : " ~ masking_policy_db|upper ~ '.' ~ masking_policy_schema|upper ~ '.' ~ masking_policy_name|upper ~ " on " ~ database ~ '.' ~ schema ~ '.' ~ identifier ~ '.' ~ column ~ ' [force = ' ~ var('use_force_applying_masking_policy','False') ~ ']', info=True) }}
                         {% set query %}
                             {% if operation_type == "apply" %}
-                                alter {{materialization}}  {{database}}.{{schema}}.{{identifier}} modify column  {{column}} set masking policy  {{masking_policy_db}}.{{masking_policy_schema}}.{{masking_policy_name}}
+                                alter {{materialization}}  {{database}}.{{schema}}.{{identifier}} modify column  {{column}} set masking policy  {{masking_policy_db}}.{{masking_policy_schema}}.{{masking_policy_name}} {% if var('use_force_applying_masking_policy','False')|upper in ['TRUE','YES'] %} force {% endif %}
                             {% elif operation_type == "unapply" %}
                                 alter {{materialization}}  {{database}}.{{schema}}.{{identifier}} modify column  {{column}} unset masking policy
                             {% endif %}