From 404222f91119d620a61d891904778e6dfbf5294a Mon Sep 17 00:00:00 2001 From: Xu Chen Date: Wed, 19 Jun 2024 14:56:16 +0800 Subject: [PATCH] [Spark] Fix compaction for cdc table (#498) Signed-off-by: chenxu Co-authored-by: chenxu --- .../scala/org/apache/spark/sql/lakesoul/TransactionalWrite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/lakesoul-spark/src/main/scala/org/apache/spark/sql/lakesoul/TransactionalWrite.scala b/lakesoul-spark/src/main/scala/org/apache/spark/sql/lakesoul/TransactionalWrite.scala index 5d9ec0e1a..617545fe2 100644 --- a/lakesoul-spark/src/main/scala/org/apache/spark/sql/lakesoul/TransactionalWrite.scala +++ b/lakesoul-spark/src/main/scala/org/apache/spark/sql/lakesoul/TransactionalWrite.scala @@ -136,6 +136,7 @@ trait TransactionalWrite { val cdcColName = cdcCol.get data.withColumn(cdcColName, when(col(cdcColName) === "update", "insert") + .otherwise(col(cdcColName)) ).where(s"$cdcColName != 'delete'") } else { data