From 1cf34337bc80728106fe9ad9e90e1c3446736ccc Mon Sep 17 00:00:00 2001 From: Johan Nystrom Date: Fri, 9 Feb 2024 17:44:16 +0900 Subject: [PATCH] Upgrade to Spark 3.3.0 (for Java 17 compatibility) as well as sbt, plugins, Scala version --- build.sbt | 16 ++++++++++------ discount-aws.sh | 4 ++-- discount-gcloud.sh | 2 +- discount-shell.sh | 2 +- discount.sh | 2 +- notebooks/Discount k-mer counter.zpln | 2 +- project/build.properties | 2 +- project/plugins.sbt | 4 ++-- 8 files changed, 19 insertions(+), 15 deletions(-) diff --git a/build.sbt b/build.sbt index e2e4c2cf..2b77af84 100644 --- a/build.sbt +++ b/build.sbt @@ -1,10 +1,10 @@ name := "Discount" -version := "3.0.1" +version := "3.1.0" -scalaVersion := "2.12.16" +scalaVersion := "2.12.18" -val sparkVersion = "3.1.0" +val sparkVersion = "3.3.0" //If compiling on JDK 8, the --release 8 flag can be safely removed (needed for backwards compatibility on later JDKs). //Also applies to javacOptions below. @@ -12,8 +12,6 @@ scalacOptions ++= Seq("--feature", "-release", "8") javacOptions ++= Seq("--release=8") -//ThisBuild / scapegoatVersion := "1.4.17" - resolvers += "Spark Packages Repo" at "https://dl.bintray.com/spark-packages/maven" libraryDependencies += "org.rogach" %% "scallop" % "latest.integration" @@ -37,11 +35,17 @@ Compile / unmanagedResourceDirectories += { baseDirectory.value / "resources" } assembly / test := {} //Do not include scala library JARs in assembly (provided by Spark) -assembly / assemblyOption := (assembly / assemblyOption).value.copy(includeScala = false) +assembly / assemblyOption ~= { + _.withIncludeScala(false) +} //Run tests in a separate JVM Test / fork := true Test / javaOptions += "-Xmx4G" +//This option required when running tests on Java 17, as of Spark 3.3.0. +//Can safely be commented out on Java 8 or 11. +Test / javaOptions += "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED" + Test / testOptions += Tests.Argument(TestFrameworks.ScalaCheck, "-verbosity", "1") diff --git a/discount-aws.sh b/discount-aws.sh index b3ff3a14..0ee6e629 100755 --- a/discount-aws.sh +++ b/discount-aws.sh @@ -12,7 +12,7 @@ BUCKET=s3://my-bucket/discount DISCOUNT_HOME="$(dirname -- "$(readlink "${BASH_SOURCE}")")" -aws s3 cp "$DISCOUNT_HOME/target/scala-2.12/Discount-assembly-3.0.1.jar" $BUCKET/ +aws s3 cp "$DISCOUNT_HOME/target/scala-2.12/Discount-assembly-3.1.0.jar" $BUCKET/ #aws s3 sync "$DISCOUNT_HOME/resources/PASHA" $BUCKET/PASHA/ @@ -23,7 +23,7 @@ aws s3 cp "$DISCOUNT_HOME/target/scala-2.12/Discount-assembly-3.0.1.jar" $BUCKET #To set SPLIT or other variables, uncomment below. COMMAND=( \ # --conf $SPLIT \ - --class com.jnpersson.discount.spark.Discount $BUCKET/Discount-assembly-3.0.1.jar $*) + --class com.jnpersson.discount.spark.Discount $BUCKET/Discount-assembly-3.1.0.jar $*) #Turn off paging for output export AWS_PAGER="" diff --git a/discount-gcloud.sh b/discount-gcloud.sh index 87a9977b..2348f8d2 100755 --- a/discount-gcloud.sh +++ b/discount-gcloud.sh @@ -33,6 +33,6 @@ PROPERTIES=$MAXRES DISCOUNT_HOME="$(dirname -- "$(readlink "${BASH_SOURCE}")")" exec gcloud --verbosity=info dataproc jobs submit spark --region $REGION --cluster $CLUSTER \ - --class com.jnpersson.discount.spark.Discount --jars "$DISCOUNT_HOME/target/scala-2.12/Discount-assembly-3.0.1.jar" \ + --class com.jnpersson.discount.spark.Discount --jars "$DISCOUNT_HOME/target/scala-2.12/Discount-assembly-3.1.0.jar" \ --properties $PROPERTIES -- "$@" diff --git a/discount-shell.sh b/discount-shell.sh index 047aac9a..7caeefee 100755 --- a/discount-shell.sh +++ b/discount-shell.sh @@ -25,4 +25,4 @@ exec $SPARK/bin/spark-shell \ --master $MASTER \ --conf $MEMORY \ --conf $LOCAL_DIR \ - --jars "$DISCOUNT_HOME/target/scala-2.12/Discount-assembly-3.0.1.jar" + --jars "$DISCOUNT_HOME/target/scala-2.12/Discount-assembly-3.1.0.jar" diff --git a/discount.sh b/discount.sh index 0629fec7..7cde47b2 100755 --- a/discount.sh +++ b/discount.sh @@ -32,4 +32,4 @@ exec $SPARK/bin/spark-submit \ --conf $MEMORY \ --conf $LOCAL_DIR \ --master $MASTER \ - --class com.jnpersson.discount.spark.Discount "$DISCOUNT_HOME/target/scala-2.12/Discount-assembly-3.0.1.jar" $* + --class com.jnpersson.discount.spark.Discount "$DISCOUNT_HOME/target/scala-2.12/Discount-assembly-3.1.0.jar" $* diff --git a/notebooks/Discount k-mer counter.zpln b/notebooks/Discount k-mer counter.zpln index 99093071..cadceffa 100644 --- a/notebooks/Discount k-mer counter.zpln +++ b/notebooks/Discount k-mer counter.zpln @@ -47,7 +47,7 @@ }, { "title": "Spark settings", - "text": "%spark.conf\n\n#Path to Discount jar\nspark.jars /path/to/Discount/target/scala-2.12/Discount-assembly-3.0.1.jar\n\n#If running a local Spark installation (not in a cloud), increase this setting as much as possible\nspark.driver.memory 16g\n\n#For local Spark, set the path to a Spark 3.0+ installation (if not configured elsewhere)\nSPARK_HOME /path/to/spark-3.1.2-bin-hadoop3.2\n\n#For local spark, it is helpful to set a path to a SSD drive or other fast filesystem (for temporary files)\n#Note that this directory can get large. It will be cleared when you stop or restart the Spark interpreter in Zeppelin.\nspark.local.dir /fast/spark\n\n\n", + "text": "%spark.conf\n\n#Path to Discount jar\nspark.jars /path/to/Discount/target/scala-2.12/Discount-assembly-3.1.0.jar\n\n#If running a local Spark installation (not in a cloud), increase this setting as much as possible\nspark.driver.memory 16g\n\n#For local Spark, set the path to a Spark 3.0+ installation (if not configured elsewhere)\nSPARK_HOME /path/to/spark-3.1.2-bin-hadoop3.2\n\n#For local spark, it is helpful to set a path to a SSD drive or other fast filesystem (for temporary files)\n#Note that this directory can get large. It will be cleared when you stop or restart the Spark interpreter in Zeppelin.\nspark.local.dir /fast/spark\n\n\n", "user": "anonymous", "dateUpdated": "2023-01-13T15:15:55+0900", "progress": 0, diff --git a/project/build.properties b/project/build.properties index 10fd9eee..abbbce5d 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.5.5 +sbt.version=1.9.8 diff --git a/project/plugins.sbt b/project/plugins.sbt index 211dc21e..2c225f3c 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ -//addSbtPlugin("com.sksamuel.scapegoat" %% "sbt-scapegoat" % "1.1.1") -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0") +addSbtPlugin("com.sksamuel.scapegoat" %% "sbt-scapegoat" % "1.2.2") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5")