How to process Event Hub Archive's files using Spark

spark-avro-json-sample.py

from pyspark.sql import SparkSession
spark = SparkSession \
    .builder \
    .appName("spark-avro-json-sample") \
    .config('spark.hadoop.avro.mapred.ignore.inputs.without.extension', 'false') \
    .getOrCreate()

#storage->avro
avroDf = spark.read.format("com.databricks.spark.avro").load(in_path)

#avro->json
jsonRdd = avroDf.select(avroDf.Body.cast("string")).rdd.map(lambda x: x[0])
data = spark.read.json(jsonRdd) # in real world it's better to specify a schema for the JSON

#do whatever you want with `data`