spark-python-version.py

from pyspark import SparkContext
from pyspark import SparkConf
import os
import sys

if __name__ == "__main__":

    conf = SparkConf()
    conf.setAppName("version-check")

    sc = SparkContext(conf=conf)

    print(sc.defaultParallelism)

    def noop(x):
       import socket
       import sys
       return socket.gethostname() + ' '.join(sys.path) + ' '.join(os.environ)

    rdd = sc.parallelize(range(1000), 100)
    hosts = rdd.map(noop).distinct().collect()
    print(hosts)