build.gradle

plugins {
    id 'java'
    id 'application'
    id 'com.github.johnrengelman.shadow' version '5.0.0'
}

sourceCompatibility = 1.8
targetCompatibility = 1.8

def scalaBinaryVersion = scalaVersion.take(4)

apply plugin: 'scala'
apply plugin: 'java'
apply plugin: 'idea'

group 			'template.spark'
version 		'1.0-SNAPSHOT'
mainClassName = 'com.deploymentzone.spark.datasource.warc.Main'

configurations {
    provided
}

sourceSets {
    main {
        compileClasspath += configurations.provided
    }
}

repositories {
	jcenter()
    mavenLocal()
    mavenCentral()
}

dependencies {
	compile "org.scala-lang:scala-library:${scalaVersion}"
    compile "org.scala-lang:scala-reflect:${scalaVersion}"
    compile "org.scala-lang:scala-compiler:${scalaVersion}"

    compile "org.apache.spark:spark-sql_${scalaBinaryVersion}:${sparkVersion}"
    compile "org.apache.spark:spark-streaming_${scalaBinaryVersion}:${sparkVersion}"
    compile "org.apache.spark:spark-core_${scalaBinaryVersion}:${sparkVersion}"

    compile("org.netpreserve:jwarc:0.7.0")

    testCompile "org.scalatest:scalatest_${scalaBinaryVersion}:3.0.5"
}

jar {
    classifier = 'all'
    manifest {
        attributes 'Implementation-Title': rootProject.name,
                   'Implementation-Version': version
    }
    include { sourceSets.main.output.classesDir }
    zip64 true
}

idea {
    module {
        scopes.PROVIDED.plus += [ configurations.provided ]
        downloadJavadoc = true
        downloadSources = true
    }
}

shadowJar {
    classifier = 'shadow'
    append 'reference.conf'
    dependencies {
        exclude(dependency('org.scala-lang::'))
        exclude(dependency('org.apache.hadoop::'))
        exclude(dependency('org.apache.parquet::'))
        exclude(dependency('org.apache.avro::'))
        exclude(dependency('org.apache.spark::'))
		// when running in AWS EMR these dependencies are provided by the environment
        //exclude(dependency('com.amazonaws.*::'))
        //exclude(dependency('com.amazon.*::'))
        //exclude(dependency('org.apache.commons.cli.*::'))
    }
    exclude 'scala/*'
    zip64 true
}

task runSpark(type: JavaExec, dependsOn: classes) {
    main = mainClassName
    classpath sourceSets.main.runtimeClasspath
    classpath configurations.runtime
}