This script is a work-in-progress. It helps with installing dependencies and # packages for numerical analytics and simulation.

py_numerics_setup.sh

# This script is a work-in-progress. THe goal is to help others install a 
# numerical platform for analytics and simulation.
#
# It will install the libraries needed to work with mpich in python. It also 
# compiles OpenBLAS and uses it to build the latest release of numpy and scipy. 
# ------------------------------------------------------------------------------ #
# This version uses github and bitbucket sources, alternatively you may select
# non-git sources, supported via wget
# ------------------------------------------------------------------------------ #
# ------------------------------------------------------------------------------ #

env_prep() {
    sandbox=/home/nuburu/sandbox
    echo "sandbox=${sandbox}"
    builddir=${sandbox}/opt
    echo "builddir=${builddir}"
    logdir=${builddir}/logs
    echo "logdir=${logdir}"
    srcdir=${builddir}/src
    echo "srcdir=${srcdir}"
    profile=$HOME/.zshrc
    echo "profile=${profile}"

    mpich_ver=3.4.1
    blas_ver=0.3.13
    hdf5_ver=1.12
    fftw_ver=3.3.9
    conda_environment=numerics
    numpy_ver=1.21.1
    scipy_ver=1.7.1
    py_ver=3.9

    MPICH_DIR=${builddir}/mpich/${mpich_ver}
    BLAS_DIR=${builddir}/blas/${blas_ver}
    HDF5_DIR=${builddir}/hdf5/${hdf5_ver}
    FFTW_DIR=${builddir}/fftw/${fftw_ver}

    mkdir -p ${sandbox} ${builddir} ${logdir} ${srcdir}
}

echo "pre-installation libraries"
env_prep

_pynum_make(){
    make -j$(nproc)
    make install
}

_pynum_env_source(){
    source ${profile}
    conda activate ${conda_environment}
    cd ${srcdir}
}

pynum_install_dependencies(){
    cd ${srcdir}
    sudo apt install apt-transport-https git wget build-essential gnu-standards \
        linux-headers-$(uname -r) linux-image-$(uname -r) \
        libtinfo5 openssl libssl-dev libcupti-dev libnuma-dev libboost-all-dev \
        libtool autoconf automake autotools-dev valgrind valgrind-mpi
}

pynum_install_miniconda(){
    cd ${srcdir}
    echo "downloading miniconda"
    if [ ! -f ${srcdir}/Miniconda3-latest-Linux-x86_64.sh ]; then
        wget -P ${srcdir} https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
    fi

    echo "installing miniconda. Follow instructions"
    cd ${srcdir}
    sh Miniconda3-latest-Linux-x86_64.sh
}

pynum_setup_conda_env(){
    echo "create conda environment ${conda_environment} with python version ${py_ver}"
    connda install -c conda-forge -y jupyterlab nb_conda_kernels
    conda create -n ${conda_environment} -c conda-forge python=${py_ver} jupyterlab nb_conda_kernels \
    cython pybind11 pythran nose pytest hypothesis

}

pynum_install_mpich(){
    cd ${srcdir}
    if [ ! -d mpich ]; then
        git clone [email protected]:pmodels/mpich.git 
    fi
    cd mpich
    if [ ! 'git branch -a | egrep "^\*?[[:space:]]+v${mpich_ver}$"' ]; then
        git checkout v${mpich_ver} -b v${mpich_ver}
    fi
    git submodule init
    git submodule foreach git clean -xfd
    git submodule update --recursive --remote
    ./autogen.sh
    ./configure --enable-shared --with-threads --enable-shared-libs=gcc \
        --enable-fast=all --disable-multi-aliases --enable-fortran=all --enable-cxx \
        --enable-romio \
        CFLAGS='-fPIC -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64' \
        FFLAGS='-fPIC -fallow-argument-mismatch' \
        --with-device=ch3 \
        --prefix=${MPICH_DIR}
    
    _pynum_make

    echo "\n# MPICH\nexport MPICH_DIR=${MPICH_DIR}"\
        "\nexport PATH=\${MPICH_DIR}/bin:\$PATH"\
        "\nexport LD_LIBRARY_PATH=\${MPICH_DIR}/lib:\$LD_LIBRARY_PATH" | tee -a ${profile} > /dev/null
    
    sudo sh -c "echo '${MPICH_DIR}/lib' > /etc/ld.so.conf.d/mpich.conf"
    sudo ldconfig

    sudo ldconfig -p | grep mpich

}

pynum_install_openblas(){
    cd ${srcdir}

    if [ ! -d OpenBLAS ]; then
        git clone https://github.com/xianyi/OpenBLAS
    fi
    cd OpenBLAS
    git fetch --all --tags

    if [ ! 'git branch | egrep "^\*?[[:space:]]+v${blas_ver}$"' ]; then
        git checkout v${blas_ver} -b v${blas_ver}
    fi

    git clean -xfd

    make -j$(nproc) CC=mpicc FF=mpifort

    make PREFIX=${BLAS_DIR} install

    sudo sh -c "echo '${BLAS_DIR}/lib' > /etc/ld.so.conf.d/openblas.conf"
    sudo ldconfig

    sudo ldconfig -p | grep openblas
}

pynum_install_hdf(){
    cd ${srcdir}

    if [ ! -d hdf5 ]; then
        git clone https://github.com/HDFGroup/hdf5.git
    fi
    cd hdf5

    if [ 'git branch | egrep "^\*?[[:space:]]+${hdf5_ver}"' ]; then
        git fetch --all --tags
        git checkout ${hdf5_ver}/master
    fi

    ./configure --enable-fortran --enable-optimization=-O3 \
        --enable-parallel --enable-shared \
        --prefix=${HDF5_DIR}

    make -j$(nproc)
    make -j$(nproc) check
    make install

    echo "\n# HDF5 \nexport HDF5_DIR=${HDF5_DIR}"\
        "\nexport PATH=\${HDF5_DIR}/bin:\${PATH}"\
        "\nexport LD_LIBRARY_PATH=\${HDF5_DIR}/lib:\${LD_LIBRARY_PATH}" | tee -a $profile > /dev/null 
    
    sudo sh -c "echo '${HDF5_DIR}/lib' > /etc/ld.so.conf.d/hdf5.conf"
    sudo ldconfig

    sudo ldconfig -p | grep hdf5
}

pynum_install_fftw(){
    cd ${srcdir}
    wget "http://www.fftw.org/fftw-${fftw_ver}.tar.gz"

    tar -xf fftw-${fftw_ver}.tar.gz

    cd fftw-${fftw_ver}
    ./configure --enable-shared --enable-mpi --prefix=${FFTW_DIR}

    make -j$(nproc)
    make -j$(nproc) check
    make install

    echo "\n# fftw \nexport FFTW_DIR=${FFTW_DIR}"\
        "\nexport PATH=\${FFTW_DIR}/bin:\${PATH}"\
        "\nexport LD_LIBRARY_PATH=\${FFTW_DIR}/lib:\${LD_LIBRARY_PATH}" | tee -a $profile > /dev/null 
    
    sudo sh -c "echo '${FFTW_DIR}/lib' > /etc/ld.so.conf.d/hdf5.conf"
    sudo ldconfig

    sudo ldconfig -p | grep fft
}

_numpy_scipy_conf_and_install(){
    git submodule update --init
    if [ ! -f site.cfg ]; then
        echo "writing config file"
        cp site.cfg.example site.cfg

        echo "\n[openblas]"\
            "\nlibraries = openblas"\
            "\nlibrary_dirs = ${BLAS_DIR}/lib"\
            "\ninclude_dirs = ${BLAS_DIR}/include"\
            "\nruntime_library_dirs = ${BLAS_DIR}/lib"\
            "\n\n[fftw]"\
            "\nlibraries = fftw3"| tee -a site.cfg > /dev/null
    else
        echo "site.cfg file exists"
    fi

    python setup.py config
    python setup.py build

    pip install cd .
}

pynum_install_numpy(){
    _pynum_env_source
    conda install -y cython

    if [ ! -d numpy ]; then
        git clone https://github.com/numpy/numpy
    fi

    cd numpy
    if [ ! 'git branch | egrep ".*v${numpy_ver}$"' ]; then
        git fetch --all --tags
        git checkout v${numpy_ver} -b v${numpy_ver}
    else
        git checkout v${numpy_ver}
    fi

    _numpy_scipy_conf_and_install

    cd .. 
    python -c "import numpy as np; np.__config__.show()" 
}

pynum_install_scipy(){
    _pynum_env_source
    conda install -y pybind11 pythran

    if [ ! -d scipy ]; then
        git clone https://github.com/scipy/scipy.git
    fi

    cd scipy
    if [ ! 'git branch | egrep ".*v${scipy_ver}$"' ]; then
        git fetch --all --tags
        git checkout v${scipy_ver} -b v${scipy_ver}
    else
        git checkout v${scipy_ver}
    fi
    
    _numpy_scipy_conf_and_install

    cd .. 
    python -c "import scipy as np; np.__config__.show()" 
}

pynum_install_mpi4py(){
    _pynum_env_source

    if [ ! -d mpi4py ]; then
        git clone https://bitbucket.org/mpi4py/mpi4py.git
    fi

    cd mpi4py
    python setup.py build

    pip install .

    mpiexec -n $(nproc) python test/runtests.py
}

pynum_install_h5py(){
    _pynum_env_source
    if [ ! -d h5py ]; then
        git clone https://github.com/h5py/h5py.git
    fi

    cd h5py
    git fetch --all --tags
    git pull

    CC="mpicc" HDF5_MPI="ON" HDF5_DIR=${HDF5_DIR} python setup.py build
    CC="mpicc" HDF5_MPI="ON" HDF5_DIR=${HDF5_DIR} pip install .
}

pynum_install_datavispackages(){
    pip install matplotlib seaborn bokeh plotly

    pip install mpl-scatter-density mpl-tune mpl-sankey mpl-styles mpl-finance \
        mpl-font mpl-colors mpl-format mpl-tools mpl-sns-viz

}

pynum_install_useful(){
    pip install click joblib PyYAML six tqdm
}

pynum_install_data-wrangling(){
    pip install tables pandas dask deepdish
}

pynum_install_boosting_stats_learning(){
    
    # statsmodel
    pip install git+https://github.com/statsmodels/statsmodels

    # boosting
    pip install xgboost catboost

    # scikit learn
    pip install scikit-learn
}