pyspark

spark_job.py

from pyspark.sql import SparkSession

# config
INPUT = ""
OUTPUT = ""

def main():
    spark = ( SparkSession.builder
        .appName("Viet PySpark")
        .config("spark.dynamicAllocation.enabled","true")
        .config("spark.dynamicAllocation.maxExecutors","16")
        .config("spark.dynamicAllocation.minExecutors","1")
        .config("spark.shuffle.service.enabled","true")
        .config('spark.sql.warehouse.dir', '/apps/hive/warehouse')
        .config('spark.sql.catalogImplementation', 'hive')
        .config("spark.port.maxRetries","32")
        .getOrCreate() )
    
    # job
    
    # 
    spark.sparkContext().stop()
    spark.stop()

if __name__ == '__main__':
    main()