From e8c55127273937cf8e7e0b0b5d2218ff9d95eb60 Mon Sep 17 00:00:00 2001
From: Aaron Z <aaron.zavora@databricks.com>
Date: Fri, 15 Mar 2024 13:45:05 -0400
Subject: [PATCH] upgrade to spark 3.4.1

---
 build.sbt                                     | 30 +------------------
 project/StableVersionPlugin.scala             |  7 -----
 .../labs/smolder/HL7FileFormat.scala          |  2 +-
 version.sbt                                   |  1 -
 4 files changed, 2 insertions(+), 38 deletions(-)

diff --git a/build.sbt b/build.sbt
index 4bb68a3..6ec7976 100644
--- a/build.sbt
+++ b/build.sbt
@@ -10,7 +10,7 @@ import sbt.nio.Keys._
 lazy val scala212 = "2.12.8"
 lazy val scala211 = "2.11.12"
 
-lazy val sparkVersion = sys.env.getOrElse("SPARK_VERSION", "3.2.1")
+lazy val sparkVersion = sys.env.getOrElse("SPARK_VERSION", "3.4.1")
 
 def majorMinorVersion(version: String): String = {
   StringUtils.ordinalIndexOf(version, ".", 2) match {
@@ -79,9 +79,6 @@ lazy val commonSettings = Seq(
   resolvers += "Apache Snapshots" at "https://repository.apache.org/snapshots/"
 )
 
-lazy val functionsYml = settingKey[File]("functionsYml")
-ThisBuild / functionsYml := (ThisBuild / baseDirectory).value / "functions.yml"
-
 lazy val sparkDependencies = Seq(
   "org.apache.spark" %% "spark-catalyst" % sparkVersion,
   "org.apache.spark" %% "spark-core" % sparkVersion,
@@ -134,7 +131,6 @@ lazy val core = (project in file("."))
     // do so.
     Compile / packageBin / packageOptions +=
     Package.ManifestAttributes("Git-Release-Hash" -> currentGitHash(baseDirectory.value)),
-    bintrayRepository := "smolder",
     libraryDependencies ++= coreDependencies :+ scalaLoggingDependency.value,
     Compile / unmanagedSourceDirectories +=
     baseDirectory.value / "src" / "main" / "shim" / majorMinorVersion(sparkVersion),
@@ -179,29 +175,6 @@ ThisBuild / pomIncludeRepository := { _ =>
 ThisBuild / publishMavenStyle := true
 
 ThisBuild / bintrayOrganization := Some("com.databricks.labs")
-ThisBuild / bintrayRepository := "smolder"
-
-lazy val stableVersion = settingKey[String]("Stable version")
-ThisBuild / stableVersion := IO
-  .read((ThisBuild / baseDirectory).value / "stable-version.txt")
-  .trim()
-
-lazy val stagedRelease = (project in file("src/test"))
-  .settings(
-    commonSettings,
-    Test / resourceDirectory := baseDirectory.value / "resources",
-    Test / scalaSource := baseDirectory.value / "scala",
-    Test / unmanagedSourceDirectories += baseDirectory.value / "shim" / majorMinorVersion(sparkVersion),
-    libraryDependencies ++= testSparkDependencies ++ testCoreDependencies :+
-    "com.databricks.labs" %% "smolder" % stableVersion.value % "test",
-    resolvers := Seq("bintray-staging" at "https://dl.bintray.com/com.databricks.labs/smolder"),
-    org
-      .jetbrains
-      .sbt
-      .extractors
-      .SettingKeys
-      .sbtIdeaIgnoreModule := true // Do not import this SBT project into IDEA
-  )
 
 import ReleaseTransformations._
 
@@ -231,7 +204,6 @@ releaseProcess := Seq[ReleaseStep](
     tagRelease
   ) ++
   crossReleaseStep(publishArtifacts) ++
-  crossReleaseStep(releaseStepCommandAndRemaining("stagedRelease/test")) ++
   Seq(
     setNextVersion,
     commitNextVersion
diff --git a/project/StableVersionPlugin.scala b/project/StableVersionPlugin.scala
index 9dbdc81..fbc86c3 100644
--- a/project/StableVersionPlugin.scala
+++ b/project/StableVersionPlugin.scala
@@ -35,13 +35,6 @@ object StableVersionPlugin extends AutoPlugin {
 
   import autoImport._
 
-  override def projectSettings: Seq[Setting[_]] = {
-    Seq(
-      stableVersionFile := baseDirectory.value / "stable-version.txt",
-      stableCommitMessage := s"Setting stable version to ${runtimeVersion.value}"
-    )
-  }
-
   private def updateStableVersionStep(
       selectVersion: Versions => String,
       fileSettingKey: SettingKey[File]): ReleaseStep = { st: State =>
diff --git a/src/main/scala/com/databricks/labs/smolder/HL7FileFormat.scala b/src/main/scala/com/databricks/labs/smolder/HL7FileFormat.scala
index 2e5d9c2..a42e656 100644
--- a/src/main/scala/com/databricks/labs/smolder/HL7FileFormat.scala
+++ b/src/main/scala/com/databricks/labs/smolder/HL7FileFormat.scala
@@ -73,7 +73,7 @@ class HL7FileFormat extends FileFormat with DataSourceRegister {
     val serializableConf = new SerializableConfiguration(hadoopConf)
     
     partitionedFile => {
-      val path = new Path(partitionedFile.filePath)
+      val path = new Path(partitionedFile.filePath.toUri)
       val hadoopFs = path.getFileSystem(serializableConf.value)
 
       // open file, read fully, and close
diff --git a/version.sbt b/version.sbt
index 2558ec5..e69de29 100644
--- a/version.sbt
+++ b/version.sbt
@@ -1 +0,0 @@
-version in ThisBuild := "0.0.3-SNAPSHOT"