diff --git a/CHANGELOG.md b/CHANGELOG.md index 235471c..159a11d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,24 +3,28 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] +### Changed +- Upgrade to Spark 3.4.2 +- Upgrade DataStax Spark Cassandra connector to 3.4.1 + ## [24.01.0] 2024-01-08 ### Added - implemented transform for tron currency -- checkpoints and loading on hdfs +- checkpoints and loading on HDFS ### Changed - Upgrade to Spark 3.2.4 - Change package name graphsense-ethereum-transformation -> graphsense-spark -- integrated utxo (btc, zec, ltc, bch transform) +- integrated UTXO (BTC, ZEC, LTC, BCH transform) - revised namespace structure (BREAKING: call is different path, new --network parameter needed!) - ## [23.09/1.5.1] 2023-10-25 ### Fixed -- duplicated txs ids in block_transactions +- duplicated txs ids in `block_transactions` ## [23.06/1.5.0] 2023-06-10 ### Changed -- Include ethereum internal transactions in address, address_relations tables. [#8](https://github.com/graphsense/graphsense-ethereum-transformation/issues/8) +- Include Ethereum internal transactions in `address`, `address_relations` tables. [#8](https://github.com/graphsense/graphsense-ethereum-transformation/issues/8) ## [23.01/1.4.0] 2023-03-29 ### Changed @@ -31,9 +35,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [23.01/1.3.0] 2023-01-30 ### Added - Token Support for Ethereum stable coin tokens (WETH, USDT, USDC) -- Added Parsing of eth-logs to support tokens +- Added Parsing of ETH-logs to support tokens - Compute contracts from traces table -- Changed schema of address_transactions, address, address_relations to support tokens and their aggregated values. +- Changed schema of `address_transactions`, `address`, `address_relations` to support tokens and their aggregated values. - Balance table now contains on balance per currency (ETH and tokens) - New table token configurations containing the supported tokens and their details - Added scalafmt and scalastyle sbt plugins diff --git a/Dockerfile b/Dockerfile index 70d9cc1..ba7bebf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,8 @@ RUN apt-get update && \ # install Spark RUN mkdir -p /opt/graphsense && \ - wget https://archive.apache.org/dist/spark/spark-3.2.4/spark-3.2.4-bin-without-hadoop.tgz -O - | tar -xz -C /opt && \ - ln -s /opt/spark-3.2.4-bin-without-hadoop /opt/spark && \ + wget https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-without-hadoop.tgz -O - | tar -xz -C /opt && \ + ln -s /opt/spark-3.4.2-bin-without-hadoop /opt/spark && \ wget https://archive.apache.org/dist/hadoop/core/hadoop-2.7.7/hadoop-2.7.7.tar.gz -O - | tar -xz -C /opt && \ ln -s /opt/hadoop-2.7.7 /opt/hadoop && \ echo "#!/usr/bin/env bash\nexport SPARK_DIST_CLASSPATH=$(/opt/hadoop/bin/hadoop classpath)" >> /opt/spark/conf/spark-env.sh && \ diff --git a/build.sbt b/build.sbt index 40cc1e3..1c05e03 100644 --- a/build.sbt +++ b/build.sbt @@ -80,11 +80,11 @@ lazy val root = (project in file(".")). "org.scalatest" %% "scalatest" % "3.2.12" % Test, "com.github.mrpowers" % "spark-fast-tests_2.12" % "1.0.0" % Test, "org.rogach" %% "scallop" % "4.1.0" % Provided, - "org.apache.spark" %% "spark-sql" % "3.2.4" % Provided, - "com.datastax.spark" %% "spark-cassandra-connector" % "3.2.0" % Provided, + "com.datastax.spark" %% "spark-cassandra-connector" % "3.4.1" % Provided, "joda-time" % "joda-time" % "2.10.10" % Provided, "org.web3j" % "core" % "4.8.7" % Provided, "org.web3j" % "abi" % "4.8.7" % Provided, - "org.apache.spark" %% "spark-graphx" % "3.2.4" % Provided, - "graphframes" % "graphframes" % "0.8.2-spark3.2-s_2.12" % Provided), + "org.apache.spark" %% "spark-sql" % "3.4.2" % Provided, + "org.apache.spark" %% "spark-graphx" % "3.4.2" % Provided, + "graphframes" % "graphframes" % "0.8.3-spark3.4-s_2.12" % Provided), ) diff --git a/docker/submit.sh b/docker/submit.sh index 3d1a41f..1abe48e 100755 --- a/docker/submit.sh +++ b/docker/submit.sh @@ -14,7 +14,7 @@ FOO="${SPARK_DRIVER_MEMORY:=4g}" FOO="${TRANSFORM_BUCKET_SIZE:=10000}" FOO="${NETWORK:=ETH}" -FOO="${SPARK_PACKAGES:=com.datastax.spark:spark-cassandra-connector_2.12:3.2.0,org.rogach:scallop_2.12:4.1.0,joda-time:joda-time:2.10.10,org.web3j:core:4.8.7,org.web3j:abi:4.8.7,graphframes:graphframes:0.8.2-spark3.2-s_2.12}" +FOO="${SPARK_PACKAGES:=com.datastax.spark:spark-cassandra-connector_2.12:3.4.1,org.rogach:scallop_2.12:4.1.0,joda-time:joda-time:2.10.10,org.web3j:core:4.8.7,org.web3j:abi:4.8.7,graphframes:graphframes:0.8.3-spark3.4-s_2.12}" FOO="${CASSANDRA_HOST:=localhost}"