Upgrade Spark version + dependencies

CHANGELOG.md

@@ -3,24 +3,28 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [Unreleased]
+### Changed
+- Upgrade to Spark 3.4.2
+- Upgrade DataStax Spark Cassandra connector to 3.4.1
+
 ## [24.01.0] 2024-01-08
 ### Added
 - implemented transform for tron currency
-- checkpoints and loading on hdfs
+- checkpoints and loading on HDFS
 ### Changed
 - Upgrade to Spark 3.2.4
 - Change package name graphsense-ethereum-transformation -> graphsense-spark
-- integrated utxo (btc, zec, ltc, bch transform)
+- integrated UTXO (BTC, ZEC, LTC, BCH transform)
 - revised namespace structure (BREAKING: call is different path, new --network parameter needed!)
 
-
 ## [23.09/1.5.1] 2023-10-25
 ### Fixed
-- duplicated txs ids in block_transactions
+- duplicated txs ids in `block_transactions`
 
 ## [23.06/1.5.0] 2023-06-10
 ### Changed
-- Include ethereum internal transactions in address, address_relations tables. [#8](https://github.com/graphsense/graphsense-ethereum-transformation/issues/8)
+- Include Ethereum internal transactions in `address`, `address_relations` tables. [#8](https://github.com/graphsense/graphsense-ethereum-transformation/issues/8)
 
 ## [23.01/1.4.0] 2023-03-29
 ### Changed
@@ -31,9 +35,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [23.01/1.3.0] 2023-01-30
 ### Added
 - Token Support for Ethereum stable coin tokens (WETH, USDT, USDC)
-- Added Parsing of eth-logs to support tokens
+- Added Parsing of ETH-logs to support tokens
 - Compute contracts from traces table
-- Changed schema of address_transactions, address, address_relations to support tokens and their aggregated values.
+- Changed schema of `address_transactions`, `address`, `address_relations` to support tokens and their aggregated values.
 - Balance table now contains on balance per currency (ETH and tokens)
 - New table token configurations containing the supported tokens and their details
 - Added scalafmt and scalastyle sbt plugins

Dockerfile

@@ -23,8 +23,8 @@ RUN apt-get update && \
 
 # install Spark
 RUN mkdir -p /opt/graphsense && \
-    wget https://archive.apache.org/dist/spark/spark-3.2.4/spark-3.2.4-bin-without-hadoop.tgz -O - | tar -xz -C /opt && \
-    ln -s /opt/spark-3.2.4-bin-without-hadoop /opt/spark && \
+    wget https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-without-hadoop.tgz -O - | tar -xz -C /opt && \
+    ln -s /opt/spark-3.4.2-bin-without-hadoop /opt/spark && \
     wget https://archive.apache.org/dist/hadoop/core/hadoop-2.7.7/hadoop-2.7.7.tar.gz -O - | tar -xz -C /opt && \
     ln -s /opt/hadoop-2.7.7 /opt/hadoop && \
     echo "#!/usr/bin/env bash\nexport SPARK_DIST_CLASSPATH=$(/opt/hadoop/bin/hadoop classpath)" >> /opt/spark/conf/spark-env.sh && \

build.sbt

@@ -80,11 +80,11 @@ lazy val root = (project in file(".")).
       "org.scalatest" %% "scalatest" % "3.2.12" % Test,
       "com.github.mrpowers" % "spark-fast-tests_2.12" % "1.0.0" % Test,
       "org.rogach" %% "scallop" % "4.1.0" % Provided,
-      "org.apache.spark" %% "spark-sql" % "3.2.4" % Provided,
-      "com.datastax.spark" %% "spark-cassandra-connector" % "3.2.0" % Provided,
+      "com.datastax.spark" %% "spark-cassandra-connector" % "3.4.1" % Provided,
       "joda-time" % "joda-time" % "2.10.10" % Provided,
       "org.web3j" % "core" % "4.8.7" % Provided,
       "org.web3j" % "abi" % "4.8.7" % Provided,
-      "org.apache.spark" %% "spark-graphx" % "3.2.4" % Provided,
-      "graphframes" % "graphframes" % "0.8.2-spark3.2-s_2.12" % Provided),
+      "org.apache.spark" %% "spark-sql" % "3.4.2" % Provided,
+      "org.apache.spark" %% "spark-graphx" % "3.4.2" % Provided,
+      "graphframes" % "graphframes" % "0.8.3-spark3.4-s_2.12" % Provided),
   )

docker/submit.sh

@@ -14,7 +14,7 @@ FOO="${SPARK_DRIVER_MEMORY:=4g}"
 FOO="${TRANSFORM_BUCKET_SIZE:=10000}"
 FOO="${NETWORK:=ETH}"
 
-FOO="${SPARK_PACKAGES:=com.datastax.spark:spark-cassandra-connector_2.12:3.2.0,org.rogach:scallop_2.12:4.1.0,joda-time:joda-time:2.10.10,org.web3j:core:4.8.7,org.web3j:abi:4.8.7,graphframes:graphframes:0.8.2-spark3.2-s_2.12}"
+FOO="${SPARK_PACKAGES:=com.datastax.spark:spark-cassandra-connector_2.12:3.4.1,org.rogach:scallop_2.12:4.1.0,joda-time:joda-time:2.10.10,org.web3j:core:4.8.7,org.web3j:abi:4.8.7,graphframes:graphframes:0.8.3-spark3.4-s_2.12}"
 
 FOO="${CASSANDRA_HOST:=localhost}"