Merge master

build.sbt

@@ -1,19 +1,17 @@
 name := "Discount"
 
-version := "3.0.1"
+version := "3.1.0"
 
-scalaVersion := "2.13.10"
+scalaVersion := "2.13.12"
 
-val sparkVersion = "3.2.0"
+val sparkVersion = "3.3.0"
 
 //If compiling on JDK 8, the --release 8 flag can be safely removed (needed for backwards compatibility on later JDKs).
 //Also applies to javacOptions below.
 scalacOptions ++= Seq("-deprecation", "--feature", "-release", "8")
 
 javacOptions ++= Seq("--release=8")
 
-//ThisBuild / scapegoatVersion := "1.4.17"
-
 resolvers += "Spark Packages Repo" at "https://dl.bintray.com/spark-packages/maven"
 
 libraryDependencies += "org.rogach" %% "scallop" % "latest.integration"
@@ -37,11 +35,17 @@ Compile / unmanagedResourceDirectories += { baseDirectory.value / "resources" }
 assembly / test := {}
 
 //Do not include scala library JARs in assembly (provided by Spark)
-assembly / assemblyOption := (assembly / assemblyOption).value.copy(includeScala = false)
+assembly / assemblyOption ~= {
+  _.withIncludeScala(false)
+}
 
 //Run tests in a separate JVM
 Test / fork := true
 
 Test / javaOptions += "-Xmx4G"
 
+//This option required when running tests on Java 17, as of Spark 3.3.0.
+//Can safely be commented out on Java 8 or 11.
+Test / javaOptions += "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED"
+
 Test / testOptions += Tests.Argument(TestFrameworks.ScalaCheck, "-verbosity", "1")

discount-aws.sh

@@ -12,8 +12,7 @@ BUCKET=s3://my-bucket/discount
 
 DISCOUNT_HOME="$(dirname -- "$(readlink "${BASH_SOURCE}")")"
 
-aws s3 cp "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.1.jar" $BUCKET/
-
+aws s3 cp "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.1.0.jar" $BUCKET/
 #aws s3 sync "$DISCOUNT_HOME/resources/PASHA" $BUCKET/PASHA/
 
 #Max size of input splits in bytes. A smaller number reduces memory usage but increases the number of 
@@ -23,7 +22,7 @@ aws s3 cp "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.1.jar" $BUCKET
 #To set SPLIT or other variables, uncomment below.
 COMMAND=( \
 #  --conf $SPLIT \
-  --class com.jnpersson.discount.spark.Discount $BUCKET/Discount-assembly-3.0.1.jar $*)
+  --class com.jnpersson.discount.spark.Discount $BUCKET/Discount-assembly-3.1.0.jar $*)
 
 #Turn off paging for output
 export AWS_PAGER=""

discount-gcloud.sh

@@ -33,6 +33,6 @@ PROPERTIES=$MAXRES
 DISCOUNT_HOME="$(dirname -- "$(readlink "${BASH_SOURCE}")")"
 
 exec gcloud --verbosity=info  dataproc jobs submit spark --region $REGION --cluster $CLUSTER \
-  --class com.jnpersson.discount.spark.Discount --jars "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.1.jar" \
+  --class com.jnpersson.discount.spark.Discount --jars "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.1.0.jar" \
   --properties $PROPERTIES -- "$@"
 

discount-shell.sh

@@ -25,4 +25,4 @@ exec $SPARK/bin/spark-shell \
   --master $MASTER \
   --conf $MEMORY \
   --conf $LOCAL_DIR \
-  --jars "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.1.jar"
+  --jars "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.1.0.jar"

discount.sh

@@ -32,4 +32,4 @@ exec $SPARK/bin/spark-submit \
   --conf $MEMORY \
   --conf $LOCAL_DIR \
   --master $MASTER \
-  --class com.jnpersson.discount.spark.Discount "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.0.1.jar" $*
+  --class com.jnpersson.discount.spark.Discount "$DISCOUNT_HOME/target/scala-2.13/Discount-assembly-3.1.0.jar" $*

notebooks/Discount k-mer counter.zpln

@@ -47,7 +47,7 @@
     },
     {
       "title": "Spark settings",
-      "text": "%spark.conf\n\n#Path to Discount jar\nspark.jars /path/to/Discount/target/scala-2.13/Discount-assembly-3.0.1.jar\n\n#If running a local Spark installation (not in a cloud), increase this setting as much as possible\nspark.driver.memory 16g\n\n#For local Spark, set the path to a Spark 3.0+ installation (if not configured elsewhere)\nSPARK_HOME /path/to/spark-3.1.2-bin-hadoop3.2\n\n#For local spark, it is helpful to set a path to a SSD drive or other fast filesystem (for temporary files)\n#Note that this directory can get large. It will be cleared when you stop or restart the Spark interpreter in Zeppelin.\nspark.local.dir /fast/spark\n\n\n",
+      "text": "%spark.conf\n\n#Path to Discount jar\nspark.jars /path/to/Discount/target/scala-2.13/Discount-assembly-3.1.0.jar\n\n#If running a local Spark installation (not in a cloud), increase this setting as much as possible\nspark.driver.memory 16g\n\n#For local Spark, set the path to a Spark 3.0+ installation (if not configured elsewhere)\nSPARK_HOME /path/to/spark-3.1.2-bin-hadoop3.2\n\n#For local spark, it is helpful to set a path to a SSD drive or other fast filesystem (for temporary files)\n#Note that this directory can get large. It will be cleared when you stop or restart the Spark interpreter in Zeppelin.\nspark.local.dir /fast/spark\n\n\n",
       "user": "anonymous",
       "dateUpdated": "2023-01-13T15:15:55+0900",
       "progress": 0,

project/build.properties

@@ -1 +1 @@
-sbt.version=1.5.5
+sbt.version=1.9.8

project/plugins.sbt

@@ -1,3 +1,3 @@
-//addSbtPlugin("com.sksamuel.scapegoat" %% "sbt-scapegoat" % "1.1.1")
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0")
+addSbtPlugin("com.sksamuel.scapegoat" %% "sbt-scapegoat" % "1.2.2")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5")