Initial commit

This commit is contained in:
karthik 2020-07-19 10:19:55 -04:00
parent 9f66e1036c
commit c701d472df
9 changed files with 146 additions and 68 deletions

View File

@ -1,5 +1,5 @@
# Spark Boilerplate
This is a boilerplate project for Apache Spark. The related blog post can be found at [https://www.barrelsofdata.com/spark-boilerplate-using-scala](https://www.barrelsofdata.com/spark-boilerplate-using-scala)
# Spark Structured Streaming Data Deduplication using State Store
This is a project showing how the spark in-built HDFS backed state store can be used to deduplicate data in a stream. The related blog post can be found at [https://www.barrelsofdata.com/data-deduplication-spark-state-store](https://www.barrelsofdata.com/data-deduplication-spark-state-store)
## Build instructions
From the root of the project execute the below commands
@ -21,6 +21,28 @@ From the root of the project execute the below commands
```
## Run
Ensure your local hadoop cluster is running ([hadoop cluster tutorial](https://www.barrelsofdata.com/apache-hadoop-pseudo-distributed-mode)) and start two kafka brokers ([kafka tutorial](https://www.barrelsofdata.com/apache-kafka-setup)).
- Create kafka topic
```shell script
spark-submit --master yarn --deploy-mode cluster build/libs/spark-boilerplate-1.0.jar
kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 2 --partitions 2 --topic streaming-data
```
- Start streaming job
```shell script
spark-submit --master yarn --deploy-mode cluster build/libs/spark-state-store-data-deduplication-1.0.jar <KAFKA_BROKER> <KAFKA_TOPIC> <FULL_OUTPUT_PATH> <DEDUPLICATED_OUTPUT_PATH> <WINDOW_SIZE_SECONDS>
Example: spark-submit --master yarn --deploy-mode client build/libs/spark-state-store-data-deduplication-1.0.jar localhost:9092 streaming-data fullOutput deduplicatedOutput 5
```
- You can feed simulated data to the kafka topic
- Open new terminal and run the shell script located at src/test/resources/dataProducer.sh
- Produces the two instances of the following json structure every 1 second: {"ts":1594307307,"usr":"user1","tmp":98}
```shell script
cd src/test/resources
./dataProducer.sh localhost:9092 streaming-data
```
### View Results
Open a spark-shell and use the following code, do change the paths to where the outputs are stored.
```scala
spark.read.parquet("fullOutput").orderBy("user","eventTime").show(truncate = false)
spark.read.parquet("deduplicatedOutput").orderBy("user","eventTime").show(truncate = false)
```

View File

@ -15,6 +15,7 @@ dependencies {
compileOnly group: "org.apache.spark", name: "spark-core_${scalaMajorVersion}", version: "${apacheSparkVersion}"
compileOnly group: "org.apache.spark", name: "spark-sql_${scalaMajorVersion}", version: "${apacheSparkVersion}"
implementation group: "org.apache.spark", name: "spark-sql-kafka-0-10_${scalaMajorVersion}", version: "${apacheSparkVersion}"
testImplementation group: "org.scalatest", name: "scalatest_${scalaMajorVersion}", version: "${scalaTestVersion}"
}

View File

@ -1 +1 @@
rootProject.name = 'spark-boilerplate'
rootProject.name = 'spark-state-store-data-deduplication'

View File

@ -1,21 +1,72 @@
package com.barrelsofdata.sparkexamples
import org.apache.log4j.Logger
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions.{col, from_json}
import org.apache.spark.sql.streaming.{GroupStateTimeout, OutputMode, StreamingQuery, Trigger}
import org.apache.spark.sql.types.{IntegerType, StringType, StructType, TimestampType}
import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
object Driver {
val JOB_NAME: String = "Boilerplate"
val JOB_NAME: String = "Structured Streaming Data Deduplication"
val LOG: Logger = Logger.getLogger(this.getClass.getCanonicalName)
def run(spark: SparkSession, kafkaBroker: String, kafkaTopic: String, fullOutputPath: String, deduplicatedOutputPath: String, windowSeconds: String): Unit = {
import spark.implicits._
spark.sparkContext.setLogLevel("ERROR")
val inputSchema: StructType = new StructType().add("ts", TimestampType).add("usr",StringType).add("tmp",IntegerType)
val df:DataFrame = spark.readStream
.format("kafka")
.option("kafka.bootstrap.servers",kafkaBroker)
.option("subscribe",kafkaTopic)
.load()
val data: Dataset[UserData] = df
.select(col("value").cast("STRING"))
.select(from_json(col("value"), inputSchema).as("jsonConverted"))
.select(col("jsonConverted.usr").as("user"), col("jsonConverted.tmp").as("temperature"), col("jsonConverted.ts").as("eventTime"))
.as[UserData]
val deduplicated: Dataset[UserData] = data
.groupByKey(_.user)
.flatMapGroupsWithState[UserData, UserData](OutputMode.Append(), GroupStateTimeout.NoTimeout)(StateOperations.deduplicate)
val deduplicatedQuery: StreamingQuery = deduplicated
.writeStream
.format("parquet")
.option("path", deduplicatedOutputPath)
.trigger(Trigger.ProcessingTime(s"$windowSeconds seconds"))
.outputMode(OutputMode.Append())
.option("checkpointLocation", s"${deduplicatedOutputPath}_checkpoint")
.start()
val fullQuery: StreamingQuery = data
.writeStream
.format("parquet")
.option("path", fullOutputPath)
.trigger(Trigger.ProcessingTime(s"$windowSeconds seconds"))
.outputMode(OutputMode.Append())
.option("checkpointLocation", s"${fullOutputPath}_checkpoint")
.start()
deduplicatedQuery.awaitTermination()
}
def main(args: Array[String]): Unit = {
if(args.length != 5) {
println("Invalid usage")
println("Usage: spark-submit --master <local|yarn> spark-structured-streaming-deduplication-1.0.jar <kafka_broker> <kafka_topic> <full_output_path> <deduplicated_output_path> <window_seconds>")
LOG.error(s"Invalid number of arguments, arguments given: [${args.mkString(",")}]")
System.exit(1)
}
val spark: SparkSession = SparkSession.builder().appName(JOB_NAME)
.config("spark.serializer", "org.apache.spark.serializer.KryoSerializer").getOrCreate()
val spark: SparkSession = SparkSession.builder().appName(JOB_NAME).getOrCreate()
spark.sql("SELECT 'hello' AS col1").show()
run(spark, args(0), args(1), args(2), args(3), args(4))
LOG.info("Dummy info message")
LOG.warn("Dummy warn message")
LOG.error("Dummy error message")
}
}

View File

@ -0,0 +1,32 @@
package com.barrelsofdata.sparkexamples
import java.sql.Timestamp
import org.apache.spark.sql.streaming.GroupState
import scala.math.Ordering
object StateOperations {
implicit def ordered: Ordering[Timestamp] = (x: Timestamp, y: Timestamp) => x compareTo y
def deduplicate(user: String,
currentBatchData: Iterator[UserData],
state: GroupState[UserData]): Iterator[UserData] = {
var currentState: Option[UserData] = state.getOption
val sortedBatch = currentBatchData.toSeq.sortBy(_.eventTime)
val results = for {
userData <- sortedBatch
if currentState.isEmpty || currentState.get.temperature != userData.temperature
} yield {
currentState = Some(userData)
userData
}
state.update(currentState.get)
results.toIterator
}
}

View File

@ -0,0 +1,5 @@
package com.barrelsofdata.sparkexamples
import java.sql.Timestamp
case class UserData (user: String, temperature: Int, eventTime: Timestamp)

View File

@ -0,0 +1,24 @@
#!/bin/bash
display_usage() {
echo "Usage: $0 <KAFKA_BROKER> <KAFKA_TOPIC>"
}
if [ "$#" -ne 2 ]; then
display_usage
exit 1
fi
KAFKA_BROKER=$1
KAFKA_TOPIC=$2
TEMP=(96 97 98)
USERS=("user1" "user2" "user3")
while sleep 1; do
tmp=${TEMP[$RANDOM % ${#TEMP[@]}]}
usr=${USERS[$RANDOM % ${#USERS[@]}]}
epochSeconds=$(date '+%s')
echo "{\"ts\":$epochSeconds,\"usr\":\"${usr}\",\"tmp\":$tmp}" | kafka-console-producer.sh --broker-list ${KAFKA_BROKER} --topic ${KAFKA_TOPIC}
echo "{\"ts\":$epochSeconds,\"usr\":\"${usr}\",\"tmp\":$tmp}" | kafka-console-producer.sh --broker-list ${KAFKA_BROKER} --topic ${KAFKA_TOPIC}
done

View File

@ -1,16 +0,0 @@
log4j.rootLogger=INFO, console
log4j.logger.com.barrelsofdata.sparkexamples=INFO, console
log4j.additivity.com.barrelsofdata.sparkexamples=false
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.out
log4j.appender.console.immediateFlush=true
log4j.appender.console.encoding=UTF-8
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %p %c: %m%n
log4j.logger.org.apache=ERROR
log4j.logger.org.spark_project=ERROR
log4j.logger.org.sparkproject=ERROR
log4j.logger.parquet=ERROR

View File

@ -1,41 +0,0 @@
package com.barrelsofdata.sparkexamples
import java.util.Properties
import org.apache.log4j.{LogManager, Logger, PropertyConfigurator}
import org.apache.spark.sql.SparkSession
import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite
class DriverTest extends AnyFunSuite with BeforeAndAfterAll {
val JOB_NAME: String = "Driver Test Job"
val LOGGER_PROPERTIES: String = "log4j-test.properties"
val LOG: Logger = Logger.getLogger(this.getClass.getCanonicalName)
var spark: SparkSession = _
def setupLogger(): Unit = {
val properties = new Properties
properties.load(getClass.getClassLoader.getResource(LOGGER_PROPERTIES).openStream())
LogManager.resetConfiguration()
PropertyConfigurator.configure(properties)
}
override def beforeAll: Unit = {
setupLogger()
LOG.info("Setting up spark session")
spark = SparkSession.builder().appName(JOB_NAME).master("local[*]").getOrCreate()
}
test("Check if spark session is working") {
LOG.info("Testing spark job")
assertResult("hello")(spark.sql("SELECT 'hello'").collect().head.get(0))
}
override def afterAll: Unit = {
LOG.info("Closing spark session")
spark.close()
}
}