barrelsofdata/spark-wordcount
1
0
Fork 0
generated from barrelsofdata/spark-boilerplate
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
karthik 2020-07-09 09:21:43 -04:00
parent 398391c920
commit 87b0462cb0
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/scala/com/barrelsofdata/sparkexamples/Driver.scala
View File

@ -23,7 +23,16 @@ object Driver {
.rdd.reduceByKey(_ + _) .rdd.reduceByKey(_ + _)
.toDF("word", "frequency") .toDF("word", "frequency")
wordFrequencies.write.option("header","true").csv(outputFilePath) // val wordFrequencies: DataFrame = words
// .groupBy(col("value")).count()
// .toDF("word", "frequency")
wordFrequencies
.coalesce(1)
.write
.option("header","true")
.csv(outputFilePath)
LOG.info(s"Result successfully written to $outputFilePath") LOG.info(s"Result successfully written to $outputFilePath")
} }