spark-wordcount | programme in scala | total: 1file

wordcount.scala

package spark

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf

object wordcount extends App{
  val logFile = "hdfs://hadoop:9000/README.md"
  val conf = new SparkConf().setAppName("wordcount").setMaster("local")
  val sc = new SparkContext(conf)
  val logData = sc.textFile(logFile, 2).cache()
  val numAs = logData.filter(line => line.contains("a")).count()
  val numBs = logData.filter(line => line.contains("b")).count()
  println("Lines with a: %s, Lines with b: %s".format(numAs, numBs))
}