cleanup

.github/deploy/steps/25-Provision-Service-Principal.ps1

@@ -29,8 +29,9 @@ $dbSpn = Get-SpnWithSecret -spnName $dbDeploySpnName -keyVaultName $keyVaultName
 
 $mountSpn = Get-SpnWithSecret -spnName $mountSpnName -keyVaultName $keyVaultName
 $secrets.addSecret("Databricks--TenantId", $tenantId)
-$secrets.addSecret("Databricks--ClientId", $mountSpn.clientId)
-$secrets.addSecret("Databricks--ClientSecret", $mountSpn.secretText)
+$secrets.addSecret("DatabricksOauthEndpoint", "https://login.microsoftonline.com/$tenantId/oauth2/token")
+$secrets.addSecret("DatabricksClientId", $mountSpn.clientId)
+$secrets.addSecret("DatabricksClientSecret", $mountSpn.secretText)
 
 # there is a chicken-and-egg problem where we want to save the new SPN secret in the
 # keyvault, but the keyvault may not exist yet. This doesn't matter since the keyvault

.github/deploy/steps/91-Create-SparkConf.ps1

@@ -1,7 +1,6 @@
 
-$repoRoot = git rev-parse --show-toplevel
-
 Write-Host "Write cluster configuration for Direct Access..." -ForegroundColor DarkYellow
+
 $confDirectAccess = [ordered]@{}
 
 $confDirectAccess["spark.databricks.cluster.profile"]= "singleNode"

tests/cluster/config/__init__.py

@@ -9,7 +9,7 @@ def InitConfigurator(*, clear=False):
     if clear:
         tc.clear_all_configurations()
 
-    # This is how you would set yourself up for differnet environments
+    # This is how you would set yourself up for different environments
     # tc.register('ENV','dev')
 
     tc.register("resourceName", values.resourceName())

tests/cluster/etl/test_deltaupsert.py

@@ -69,7 +69,7 @@ def test_02_can_perform_overwrite_over_existing(self):
         """The target table is already filled from before.
         This test does not test .upsert() logic,
         but ensures that test 03 resembles an upsert after a full load.
-        If one needs to make an full load, use the .overwrite() method"""
+        If one needs to make a full load, use the .overwrite() method"""
         self.assertEqual(2, len(self.target_dh_dummy.read().collect()))
 
         df_source = DataframeCreator.make_partial(

tests/cluster/etl/test_loader.py

@@ -11,15 +11,15 @@ class LoaderTests(unittest.TestCase):
     loader: Loader
 
     @classmethod
-    def setUp(self):
+    def setUp(cls):
 
-        self.loader = Loader()
-        self.loader.save = MagicMock()
-        self.loader.save_many = MagicMock()
+        cls.loader = Loader()
+        cls.loader.save = MagicMock()
+        cls.loader.save_many = MagicMock()
 
-        self.df_1 = create_dataframe()
-        self.df_2 = create_dataframe()
-        self.df_3 = create_dataframe()
+        cls.df_1 = create_dataframe()
+        cls.df_2 = create_dataframe()
+        cls.df_3 = create_dataframe()
 
     def test_return_inputs(self):
         # Assert Loader returns ouput

tests/cluster/mount/README.md

@@ -2,7 +2,7 @@ This entire folder and its functions are deprecated.
 Mounting storage accounts is not the recommended way to access them.
 The code is kept here to allow it to be reactivated if needed.
 
-The tests of the class `EventHubCapture` requires mountpoints, or so it seems.
+The tests of the class `EventHubCapture` requires mount-points, or so it seems.
 The external avro SerDe classes do not seem to support direct access. Those tests 
 are skipped together with the deprecation of the class itself. If those tests are 
 removed entirely, this folder should probably disappear, too.

tests/cluster/secrets.py

@@ -24,9 +24,9 @@ def sqlServerUserPassword():
     return getSecret("SqlServer--DatabricksUserPassword")
 
 
-def dbDeployClientSecret():
-    return getSecret("DbDeploy--ClientSecret")
+def clientSecret():
+    return getSecret("DatabricksClientSecret")
 
 
-def dbDeployClientId():
-    return getSecret("DbDeploy--ClientId")
+def clientId():
+    return getSecret("DatabricksClientId")

tests/cluster/sql/DeliverySqlServerSpn.py

@@ -1,5 +1,5 @@
 from atc.sql.SqlServer import SqlServer
-from tests.cluster.secrets import dbDeployClientId, dbDeployClientSecret
+from tests.cluster.secrets import clientId, clientSecret
 
 
 class DeliverySqlServerSpn(SqlServer):
@@ -9,6 +9,6 @@ def __init__(
         super().__init__(
             hostname="{resourceName}test.database.windows.net",
             database="Delivery",
-            spnpassword=dbDeployClientSecret(),
-            spnid=dbDeployClientId(),
+            spnpassword=clientSecret(),
+            spnid=clientId(),
         )

tests/cluster/sql/test_deliverysql.py

@@ -106,7 +106,7 @@ def test09_execute_sql_file(self):
         self.assertTrue(True)
 
     def test10_read_w_id(self):
-        # This might fail if the previous test didnt succeed
+        # This might fail if the previous test didn't succeed
         self.sql_server.read_table("SqlTestTable1")
         self.sql_server.read_table("SqlTestTable2")
         self.assertTrue(True)

tests/cluster/transformations/test_concat_df.py

@@ -189,7 +189,7 @@ def create_df3():
 
 
 def get_number_rows_1(df, id, brand, model, year=None):
-    # When testing the first transformation, theres only "year" as column
+    # When testing the first transformation, there's only "year" as column
     if year is not None:
         return df.filter(
             (f.col("id") == id)
@@ -201,7 +201,7 @@ def get_number_rows_1(df, id, brand, model, year=None):
 
 
 def get_number_rows_2(df, id, brand, model, year=None, size=None):
-    # When testing the second transformation, theres only "year" and "size" as column
+    # When testing the second transformation, there's only "year" and "size" as column
     if year is not None:
         return df.filter(
             (f.col("id") == id)