From 81881e4238056c83e296696606d8e1b1ffb7418c Mon Sep 17 00:00:00 2001 From: Simon Heisterkamp Date: Thu, 3 Nov 2022 14:55:26 +0000 Subject: [PATCH] get storage uri from az cli --- .github/deploy/steps/91-Create-SparkConf.ps1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/deploy/steps/91-Create-SparkConf.ps1 b/.github/deploy/steps/91-Create-SparkConf.ps1 index 2f2d309f..be5dc10e 100644 --- a/.github/deploy/steps/91-Create-SparkConf.ps1 +++ b/.github/deploy/steps/91-Create-SparkConf.ps1 @@ -9,8 +9,13 @@ $confDirectAccess["spark.databricks.delta.preview.enabled"] = $true $confDirectAccess["spark.databricks.io.cache.enabled"] = $true $confDirectAccess["spark.master"]= "local[*, 4]" -Write-Host " Adds Direct Access for $resourceName..." -ForegroundColor DarkYellow -$storageUrl = "$resourceName.dfs.core.windows.net" + +$url = az storage account show --name $dataLakeName --resource-group $resourceGroupName --query "primaryEndpoints.dfs" --out tsv + +$storageUrl = ([System.Uri]$url).Host + +Write-Host " Adds Direct Access for $storageUrl..." -ForegroundColor DarkYellow + $confDirectAccess["fs.azure.account.auth.type.$storageUrl"] = "OAuth" $confDirectAccess["fs.azure.account.oauth.provider.type.$storageUrl"] = "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider" $confDirectAccess["fs.azure.account.oauth2.client.id.$storageUrl"] = "{{secrets/secrets/DatabricksClientId}}"