Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enhance Validation Framework #1308

Closed
wants to merge 1 commit into from
Closed

Enhance Validation Framework #1308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions src/main/scala/com/databricks/labs/overwatch/utils/PipelineValidationHelper.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import scala.collection.mutable.ArrayBuffer
* This class contains the utility functions for PipelineValidation.scala.
*/

class PipelineValidationHelper(_etlDB: String ,_allRun: Boolean = true) extends SparkSessionWrapper {
class PipelineValidationHelper(_etlDB: String ,_allRun: Boolean = true,_dateWindow: Array[String] = Array()) extends SparkSessionWrapper {

import spark.implicits._

Expand All @@ -32,6 +32,7 @@ class PipelineValidationHelper(_etlDB: String ,_allRun: Boolean = true) extends
def quarantineID: String = _quarantine_id

def etlDB: String = _etlDB
def dateWindow: Array[String] = _dateWindow
val allRun: Boolean = _allRun

private var _validations: ArrayBuffer[HealthCheckReport] = new ArrayBuffer[HealthCheckReport]()
Expand Down Expand Up @@ -76,9 +77,20 @@ class PipelineValidationHelper(_etlDB: String ,_allRun: Boolean = true) extends
All_Overwatch_RunID
}else {
if (Helpers.pathExists(healthCheckReportPath)) {
val healthCheckDF = spark.read.load(healthCheckReportPath)
val Validated_Overwatch_RunIDs = healthCheckDF.select("Overwatch_RunID").distinct().collect().map(_.getString(0))
All_Overwatch_RunID.diff(Validated_Overwatch_RunIDs)
if (dateWindow.isEmpty) {
val Msg = s"Date Window is not provided. Validation will run for all overwatch_runID which is yet to be validated"
logger.log(Level.INFO, Msg)
val healthCheckDF = spark.read.load(healthCheckReportPath)
val Validated_Overwatch_RunIDs = healthCheckDF.select("Overwatch_RunID").distinct().collect().map(_.getString(0))
All_Overwatch_RunID.diff(Validated_Overwatch_RunIDs)
}else{
val start_date = dateWindow(0)
val end_date = dateWindow(1)
val filtered_overwatch_RunID = spark.read.table(s"$etlDB.pipeline_report").filter(
col("Pipeline_SnapTS") > start_date && col("Pipeline_SnapTS") <= end_date)
.select("Overwatch_RunID").distinct().collect().map(_.getString(0))
filtered_overwatch_RunID
}
} else {
All_Overwatch_RunID
}
Expand Down
14 changes: 7 additions & 7 deletions src/main/scala/com/databricks/labs/overwatch/validation/PipelineValidation.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,27 @@ import org.apache.spark.sql.DataFrame
*/
object PipelineValidation extends SparkSessionWrapper {

def apply(etlDB : String , allRun: Boolean) :Unit = {
new PipelineValidation(etlDB,allRun)
def apply(etlDB : String , allRun: Boolean,_dateWindow: Array[String]) :Unit = {
new PipelineValidation(etlDB,allRun,_dateWindow)
.setPipelineSnapTime()
.process()
}

def apply(etlDB : String, allRun: Boolean, table : Array[String]) :Unit = {
new PipelineValidation(etlDB,allRun)
def apply(etlDB : String, allRun: Boolean,_dateWindow: Array[String], table : Array[String]) :Unit = {
new PipelineValidation(etlDB,allRun,_dateWindow)
.setPipelineSnapTime()
.process(table)
}

def apply(etlDB : String, allRun: Boolean, table : Array[String],crossTableValidation : Boolean) :Unit = {
new PipelineValidation(etlDB,allRun)
def apply(etlDB : String, allRun: Boolean,_dateWindow: Array[String], table : Array[String],crossTableValidation : Boolean) :Unit = {
new PipelineValidation(etlDB,allRun,_dateWindow)
.setPipelineSnapTime()
.process(table,crossTableValidation)
}

}

class PipelineValidation (_etlDB: String, _allRun: Boolean) extends PipelineValidationHelper(_etlDB,_allRun) with SparkSessionWrapper {
class PipelineValidation (_etlDB: String, _allRun: Boolean,_dateWindow: Array[String]) extends PipelineValidationHelper(_etlDB,_allRun,_dateWindow) with SparkSessionWrapper {

import spark.implicits._

Expand Down