Last active
March 1, 2018 17:58
-
-
Save dilipbobby/c447d5ab894d173888f875b372804d93 to your computer and use it in GitHub Desktop.
Integration code of pyspark with cassandra
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from pyspark import SparkContext, SparkConf | |
from pyspark.sql import SQLContext | |
conf = SparkConf() | |
conf.setMaster("local[4]") | |
conf.setAppName("Spark Cassandra") | |
conf.set("spark.cassandra.connection.host","127.0.0.1") | |
sc = SparkContext(conf=conf) | |
sqlContext = SQLContext(sc) | |
#sqlContext.read.format("org.apache.spark.sql.cassandra").options(table="payments", keyspace="deepa").load().show() | |
df=sqlContext.read\ | |
.format("org.apache.spark.sql.cassandra")\ | |
.options(table="payments", keyspace="deepa")\ | |
.load() | |
df.show() | |
#submission | |
#spark-submit --packages com.datastax.spark:spark-cassandra-connector_2.11:2.0.7 ./Main/Code/myPysparkFile.py |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment