Skip to content

Commit

Permalink
reanames for clarity
Browse files Browse the repository at this point in the history
  • Loading branch information
alextheimer committed Feb 23, 2024
1 parent 14a182e commit 262918b
Showing 1 changed file with 3 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -203,12 +203,12 @@ class Downsampler(settings: DownsamplerSettings) extends Serializable {
val exportKeyToRules = settings.exportKeyToRules.map(f => (f._1, f._2)).toSeq
val exportTasks = {
// downsample the data as the first key is exported
val headTask = Seq(() =>
val firstExportTaskWithDs = Seq(() =>
exportForKey(rddWithDs, exportKeyToRules.head._1, exportKeyToRules.head._2, batchExporter, spark))
// export all remaining keys without the downsample step
val tailTasks = exportKeyToRules.tail.map{spec => () =>
val remainingExportTasksWithoutDs = exportKeyToRules.tail.map{spec => () =>
exportForKey(rdd, spec._1, spec._2, batchExporter, spark)}
headTask ++ tailTasks
firstExportTaskWithDs ++ remainingExportTasksWithoutDs
}
// export/downsample RDDs in parallel
exportTasks.par.foreach(_.apply())
Expand Down

0 comments on commit 262918b

Please sign in to comment.