barrelsofdata/spark-boilerplate
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix test run
All checks were successful
run-tests
Details

Browse Source
This commit is contained in:
karthik 2023-04-09 18:50:20 +02:00
parent 5b67edca62
commit 7c2fca490a
3 changed files with 25 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
build.gradle
View File

@ -2,8 +2,8 @@ plugins {
id "scala" id "scala"
} }
group "${projectGroup}" group projectGroup
version "${projectVersion}" version projectVersion
repositories { repositories {
mavenCentral() mavenCentral()
@ -12,10 +12,10 @@ repositories {
dependencies { dependencies {
compileOnly group: "org.scala-lang", name:"scala-library", version: "${scalaMajorVersion}.${scalaMinorVersion}" compileOnly group: "org.scala-lang", name:"scala-library", version: "${scalaMajorVersion}.${scalaMinorVersion}"
compileOnly group: "org.apache.spark", name: "spark-core_${scalaMajorVersion}", version: "${apacheSparkVersion}" compileOnly group: "org.apache.spark", name: "spark-core_${scalaMajorVersion}", version: apacheSparkVersion
compileOnly group: "org.apache.spark", name: "spark-sql_${scalaMajorVersion}", version: "${apacheSparkVersion}" compileOnly group: "org.apache.spark", name: "spark-sql_${scalaMajorVersion}", version: apacheSparkVersion
testImplementation group: "org.scalatest", name: "scalatest_${scalaMajorVersion}", version: "${scalaTestVersion}" testImplementation group: "org.scalatest", name: "scalatest_${scalaMajorVersion}", version: scalaTestVersion
} }
configurations { configurations {
@ -26,18 +26,18 @@ tasks.withType(ScalaCompile).configureEach {
scalaCompileOptions.additionalParameters = ["-release:${JavaVersion.current()}".toString()] scalaCompileOptions.additionalParameters = ["-release:${JavaVersion.current()}".toString()]
} }
tasks.register('scalaTest', JavaExec) { tasks.register("scalaTest", JavaExec) {
dependsOn['testClasses'] dependsOn["testClasses"]
mainClass = 'org.scalatest.tools.Runner' mainClass = "org.scalatest.tools.Runner"
args = ['-R', 'build/classes/scala/test', '-o'] args = ["-R", "build/classes/scala/test", "-o"]
jvmArgs = ['--add-exports=java.base/sun.nio.ch=ALL-UNNAMED'] // https://lists.apache.org/thread/p1yrwo126vjx5tht82cktgjbmm2xtpw9 jvmArgs = ["--add-exports=java.base/sun.nio.ch=ALL-UNNAMED"] // https://lists.apache.org/thread/p1yrwo126vjx5tht82cktgjbmm2xtpw9
classpath = sourceSets.test.runtimeClasspath classpath = sourceSets.test.runtimeClasspath
} }
test.dependsOn scalaTest test.dependsOn scalaTest
jar { jar {
manifest { manifest {
attributes "Main-Class": "${mainClass}" attributes "Main-Class": mainClass
} }
from { from {
configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
@ -45,6 +45,12 @@ jar {
archiveFileName.set("${getArchiveBaseName().get()}-${projectVersion}.${getArchiveExtension().get()}") archiveFileName.set("${getArchiveBaseName().get()}-${projectVersion}.${getArchiveExtension().get()}")
} }
java {
toolchain {
languageVersion = JavaLanguageVersion.of(targetJVM)
}
}
clean.doFirst { clean.doFirst {
delete "logs/" delete "logs/"
} }

5
gradle.properties
View File

@ -1,12 +1,13 @@
apacheSparkVersion=3.3.2
scalaMajorVersion=2.13 scalaMajorVersion=2.13
scalaMinorVersion=10 scalaMinorVersion=10
scalaTestVersion=3.2.15 scalaTestVersion=3.2.15
apacheSparkVersion=3.3.2
mainClass=com.barrelsofdata.sparkexamples.Driver mainClass=com.barrelsofdata.sparkexamples.Driver
projectGroup=com.barrelsofdata.sparkexamples projectGroup=com.barrelsofdata.sparkexamples
projectVersion=1.0 projectVersion=1.0
targetJVM=17
org.gradle.daemon=false org.gradle.daemon=false
org.gradle.jvmargs=-Xms128m -Xmx256m -XX:+UseSerialGC -Dfile.encoding=UTF-8
org.gradle.warning.mode=all org.gradle.warning.mode=all

6
src/test/scala/com/barrelsofdata/sparkexamples/DriverTest.scala
View File

@ -25,7 +25,11 @@ class DriverTest extends AnyFunSuite with BeforeAndAfterAll {
override def beforeAll(): Unit = { override def beforeAll(): Unit = {
setupLogger() setupLogger()
LOG.info("Setting up spark session") LOG.info("Setting up spark session")
spark = SparkSession.builder().appName(JOB_NAME).master("local[*]").getOrCreate() spark = SparkSession.builder()
.appName(JOB_NAME)
.master("local[*]")
.config("spark.testing.memory", 471859200)
.getOrCreate()
} }
override def afterAll(): Unit = { override def afterAll(): Unit = {