Upgrade Spark

CHANGELOG.md

@@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## Unreleased
+### Changed
+- Upgrade to Spark 3.5.3
+- Upgrade DataStax Spark Cassandra connector to 3.5.1
+
 ## [24.02.0] 2024-03-04
 ### Fixed
 - excessive logging in container

Dockerfile

@@ -21,8 +21,8 @@ RUN apt-get update && \
 
 # install Spark
 RUN mkdir -p /opt/graphsense && \
-    wget https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-without-hadoop.tgz -O - | tar -xz -C /opt && \
-    ln -s /opt/spark-3.4.2-bin-without-hadoop /opt/spark && \
+    wget https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-without-hadoop.tgz -O - | tar -xz -C /opt && \
+    ln -s /opt/spark-3.5.3-bin-without-hadoop /opt/spark && \
     wget https://archive.apache.org/dist/hadoop/core/hadoop-2.7.7/hadoop-2.7.7.tar.gz -O - | tar -xz -C /opt && \
     ln -s /opt/hadoop-2.7.7 /opt/hadoop && \
     echo "#!/usr/bin/env bash\nexport SPARK_DIST_CLASSPATH=$(/opt/hadoop/bin/hadoop classpath)" >> /opt/spark/conf/spark-env.sh && \

build.sbt

@@ -80,11 +80,11 @@ lazy val root = (project in file(".")).
       "org.scalatest" %% "scalatest" % "3.2.12" % Test,
       "com.github.mrpowers" % "spark-fast-tests_2.12" % "1.0.0" % Test,
       "org.rogach" %% "scallop" % "4.1.0" % Provided,
-      "com.datastax.spark" %% "spark-cassandra-connector" % "3.4.1" % Provided,
+      "com.datastax.spark" %% "spark-cassandra-connector" % "3.5.1" % Provided,
       "joda-time" % "joda-time" % "2.10.10" % Provided,
       "org.web3j" % "core" % "4.8.7" % Provided,
       "org.web3j" % "abi" % "4.8.7" % Provided,
-      "org.apache.spark" %% "spark-sql" % "3.4.2" % Provided,
-      "org.apache.spark" %% "spark-graphx" % "3.4.2" % Provided,
-      "graphframes" % "graphframes" % "0.8.3-spark3.4-s_2.12" % Provided),
+      "org.apache.spark" %% "spark-sql" % "3.5.3" % Provided,
+      "org.apache.spark" %% "spark-graphx" % "3.5.3" % Provided,
+      "graphframes" % "graphframes" % "0.8.3-spark3.5-s_2.12" % Provided),
   )