all tests passing after spark3.2.1 upgrade

databrickslabs · Oct 31, 2022 · 6e0927c · 6e0927c
1 parent fe32ab4
commit 6e0927c
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 5 deletions.
diff --git a/build.sbt b/build.sbt
@@ -103,14 +103,14 @@ lazy val testCoreDependencies = Seq(
 )
 
 lazy val coreDependencies = (providedSparkDependencies ++ testCoreDependencies ++ Seq(
-  "log4j" % "log4j" % "1.2.17",
+//  "log4j" % "log4j" % "1.2.17",
   "org.slf4j" % "slf4j-api" % "1.7.25",
   "org.slf4j" % "slf4j-log4j12" % "1.7.25",
   "org.jdbi" % "jdbi" % "2.63.1",
   // Fix versions of libraries that are depended on multiple times
-  "org.apache.hadoop" % "hadoop-client" % "2.7.3",
-  "io.netty" % "netty" % "3.9.9.Final",
-  "io.netty" % "netty-all" % "4.1.17.Final",
+  "org.apache.hadoop" % "hadoop-client" % "3.3.1",
+//  "io.netty" % "netty" % "3.9.9.Final",
+//  "io.netty" % "netty-all" % "4.1.17.Final",
   "org.yaml" % "snakeyaml" % "1.16"
 )).map(_.exclude("com.google.code.findbugs", "jsr305"))
 

diff --git a/src/test/scala/com/databricks/labs/smolder/SmolderBaseTest.scala b/src/test/scala/com/databricks/labs/smolder/SmolderBaseTest.scala
@@ -26,7 +26,8 @@ abstract class SmolderBaseTest
 
   protected implicit def spark: SparkSession = {
     val session = SparkSession.builder()
-      .master("local[2]")
+      .master("local[2]") 
+      .config("spark.driver.bindAddress","127.0.0.1") //Explicitly state this for Spark3.2.1
       .getOrCreate()
 
     SparkSession.setActiveSession(session)