create_spark_session_dataframe.py

spark = SparkSession.builder\
                    .master("local")\
                    .appName("Structured Streaming - Twitter Sentiment")\
                    .getOrCreate()

pythonSchema = StructType() \
          .add("id", StringType(), True) \
          .add("tweet", StringType(), True) \
          .add("ts", StringType(), True)

awsAccessKeyId = "" # update the access key
awsSecretKey = ""   # update the secret key
kinesisStreamName = ""  # update the kinesis stream name (need to set up the stream first and ingest data)
kinesisRegion = ""

kinesisDF = spark \
  .readStream \
  .format("kinesis") \
  .option("streamName", kinesisStreamName)\
  .option("region", kinesisRegion) \
  .option("initialPosition", "LATEST") \
  .option("format", "json") \
  .option("awsAccessKey", awsAccessKeyId)\
  .option("awsSecretKey", awsSecretKey) \
  .option("inferSchema", "true") \
  .load()