Skip to content

Instantly share code, notes, and snippets.

@alvarovm

alvarovm/vasp-GPU notes.md

Last active October 23, 2025 21:16
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save alvarovm/cf8dd347b5d2b0c8a4c11792cdc85d43 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save alvarovm/cf8dd347b5d2b0c8a4c11792cdc85d43 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
vasp-GPU notes.md

VASP in ALCF-Polaris computer

Compilation

Modules used to compile vasp:

Currently Loaded Modules:
  1) nvidia/24.11    3) cray-dsmml/0.3.1    5) cray-pmi/6.1.15   7) cray-libpals/1.6.1   9) darshan/3.4.4            11) PrgEnv-nvidia/8.6.0  13) craype-network-ofi      15) cuda/12.6
  2) craype/2.7.34   4) cray-mpich/8.1.32   6) cray-pals/1.6.1   8) craype-x86-milan    10) xalt/3.1.4-202508192222  12) libfabric/1.22.0     14) perftools-base/25.03.0  16) cray-libsci/25.03.0

Current makefile.include used for compiling vasp 6.x.

# Precompiler options
CPP_OPTIONS= -DHOST=\"LinuxNV_CrayMPICH\" \
             -DMPI -DMPI_BLOCK=8000 -Duse_collective \
             -DscaLAPACK \
             -DCACHE_SIZE=4000 \
             -Davoidalloc \
             -Dvasp6 \
             -Dtbdyn \
             -Dqd_emulate \
             -Dfock_dblbuf \
             -DACC_OFFLOAD \
             -D_OPENMP \
             -D_OPENACC \
             -DNVCUDA \
             -DUSENCCL

CPP        = nvfortran -Mpreprocess -Mfree -Mextend -E $(CPP_OPTIONS) $*$(FUFFIX)  > $*$(SUFFIX)
FC         = ftn -acc -gpu=cc80 -mp -target-accel=nvidia80
FCL        = ftn -acc -gpu=cc80 -c++libs -target-accel=nvidia80
FREE       = -Mfree

FFLAGS     = -Mbackslash -Mlarge_arrays
OFLAG      = -fast
DEBUG      = -Mfree -O0 -traceback

# Specify your NV HPC-SDK installation, try to set NVROOT automatically
# ...or set NVROOT manually
#NVHPC      ?= /opt/nvidia/hpc_sdk
NVROOT     =$(shell which nvfortran | awk -F /compilers/bin/nvfortran '{ print $$1 }')

# Use NV HPC-SDK provided BLAS and LAPACK libraries
LIBAOCL=/soft/libraries/aocl/3.2.0
BLAS       = /soft/applications/vasp/aol-libs/3.2/amd-blis/lib/LP64/libblis-mt.a
LAPACK     = /soft/applications/vasp/aol-libs/3.2/amd-libflame/lib/LP64/libflame.a

BLACS      =
SCALAPACK  =

CUDA       = -cudalib=cublas,cusolver,cufft,nccl -cuda

LLIBS      = $(SCALAPACK) $(LAPACK) $(BLAS) $(CUDA)

# Software emulation of quadruple precsion
QD         ?= $(NVROOT)/compilers/extras/qd
LLIBS      += -L$(QD)/lib -lqdmod -lqd -Wl,-rpath=$(QD)/lib
INCS       += -I$(QD)/include/qd
LLIBS      += -L$(NVROOT)/math_libs/lib64 -Wl,-rpath=$(NVROOT)/math_libs/lib64


# Use the FFTs from fftw
FFTW       = /soft/applications/vasp/aol-libs/3.2/amd-fftw
LLIBS      += -L$(FFTW)/lib -lfftw3 -lfftw3_omp -lomp
#INCS       += -I/soft/libraries/aocl/3.2.0/include_LP64/
INCS       += -I$(FFTW)/include

#OBJECTS    = fftmpiw.o fftmpi_map.o fftw3d.o fft3dlib.o

# Redefine the standard list of O1 and O2 objects
SOURCE_O1  := pade_fit.o
SOURCE_O2  := pead.o

# For what used to be vasp.5.lib
CPP_LIB    = $(CPP)
FC_LIB     = nvfortran
CC_LIB     = nvc -w
CFLAGS_LIB = -O
FFLAGS_LIB = -O1 -Mfixed
FREE_LIB   = $(FREE)

OBJECTS_LIB= linpack_double.o

# For the parser library
CXX_PARS   = nvc++ --no_warnings

# Normally no need to change this
SRCDIR     = ../../src
BINDIR     = ../../bin

References

Submission script sample

#!/bin/sh
#PBS -l select=1:system=polaris
#PBS -l place=scatter
#PBS -l walltime=0:30:00
#PBS -l filesystems=home:grand:eagle
#PBS -q debug
#PBS -A myproject

unset LD_PRELOAD
module rm xalt

module load cray-libsci

NVROOT=/opt/nvidia/hpc_sdk/Linux_x86_64/24.11

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$NVROOT/compilers/extras/qd/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/applications/vasp/aol-libs/3.2/amd-blis/lib/ILP64/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/applications/vasp/aol-libs/3.2/amd-libflame/lib/ILP64/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/applications/vasp/aol-libs/3.2/amd-fftw/lib

export MPICH_GPU_SUPPORT_ENABLED=1

# uncomment for more than one node
#export NCCL_NET_GDR_LEVEL=PHB
#export NCCL_CROSS_NIC=1
#export NCCL_COLLNET_ENABLE=1
#export NCCL_NET="AWS Libfabric"
#export LD_LIBRARY_PATH=/soft/libraries/hwloc/lib/:$LD_LIBRARY_PATH
#export LD_LIBRARY_PATH=/lus/eagle/projects/catalyst/world-shared/avazquez/aws-ofi-install/lib:$LD_LIBRARY_PATH
#export NCCL_SOCKET_IFNAME=hsn

NNODES=`wc -l < $PBS_NODEFILE`
NRANKS=$(nvidia-smi -L | wc -l)
NDEPTH=8
NTHREADS=1

NTOTRANKS=$(( NNODES * NRANKS ))
bin=/soft/applications/vasp/vasp.6.4.3/bin/vasp_std
#IF you hold a license for >=6.5
bin=/soft/applications/vasp/vasp.6.5.1/bin/vasp_std


mpiexec -n ${NTOTRANKS} --ppn ${NRANKS} --depth ${NDEPTH} --cpu-bind depth --env OMP_NUM_THREADS=${NTHREADS} /lus/eagle/projects/catalyst/world-shared/avazquez/affinity.sh  $bin

VASP in nersc

# Precompiler options
CPP_OPTIONS= -DHOST=\"LinuxNV_CrayMPICH\" \
             -DMPI -DMPI_BLOCK=8000 -Duse_collective \
             -DscaLAPACK \
             -DCACHE_SIZE=4000 \
             -Davoidalloc \
             -Dvasp6 \
             -Duse_bse_te \
             -Dtbdyn \
             -Dqd_emulate \
             -Dfock_dblbuf \
             -D_OPENMP \
             -D_OPENACC \
             -DUSENCCL -DUSENCCLP2P \
             -DVASP_HDF5 \
             -DVASP2WANNIER90 \
             -DDFTD4 \
             -DPROFILING
### Disabled for GPU build:
#             -Dsysv \
#             -DUSELIBXC \
#             -Dlibbeef \

CPP        = nvfortran -Mpreprocess -Mfree -Mextend -E $(CPP_OPTIONS) $*$(FUFFIX)  > $*$(SUFFIX)
FC         = ftn -mp -acc=gpu -gpu=cc80
FCL        = ftn -v -mp -acc=gpu -gpu=cc80 -c++libs
FREE       = -Mfree

FFLAGS     = -Mbackslash -Mlarge_arrays
OFLAG      = -fast
DEBUG      = -Mfree -O0 -traceback
OBJECTS    = fftmpiw.o fftmpi_map.o fftw3d.o fft3dlib.o
LLIBS       = -cudalib=cublas,cusolver,cufft,nccl -cuda

# Redefine the standard list of O1 and O2 objects
SOURCE_O1  := pade_fit.o
SOURCE_O2  := pead.o

# For what used to be vasp.5.lib
CPP_LIB     = $(CPP)
FC_LIB      = nvfortran
CC_LIB      = nvc -w
CFLAGS_LIB  = -O
FFLAGS_LIB  = -O1 -Mfixed
FREE_LIB    = $(FREE)
OBJECTS_LIB = linpack_double.o

# For the parser library
CXX_PARS    = nvc++ --no_warnings

#######

# Specify your NV HPC-SDK installation (mandatory)
NVROOT      =$(shell which nvfortran | awk -F /compilers/bin/nvfortran '{ print $$1 }')

# Software emulation of quadruple precision (mandatory)
QD         ?= $(NVROOT)/compilers/extras/qd
LLIBS      += -L$(QD)/lib -lqdmod -lqd -Wl,-rpath=$(QD)/lib
INCS       += -I$(QD)/include/qd
LLIBS      += -L$(NVROOT)/math_libs/lib64 -Wl,-rpath=$(NVROOT)/math_libs/lib64

# mandatory
BLAS        = -lblas
LAPACK      = -llapack
SCALAPACK   = -L/global/cfs/cdirs/omp/local/scalapack-2.1.0/nvidia-22.5/milan -lscalapack
LLIBS      += $(SCALAPACK) $(LAPACK) $(BLAS)

# use cray-fftw module for FFTs
# optional packages:

# NCCL (GPU builds only)
LLIBS     += -L$(NCCL_DIR)/lib -Wl,-rpath=$(NCCL_DIR)/lib
INCS      += -I$(NCCL_DIR)/include

# HDF5 (vasp >6.2.0 only)
LLIBS     += -L$(HDF5_ROOT)/lib -lhdf5_fortran
INCS      += -I$(HDF5_ROOT)/include

# fftlib (OpenMP builds only)
FCL        += fftlib.o
CXX_FFTLIB  = nvc++ -mp --no_warnings -std=c++11 -DFFTLIB_THREADSAFE
INCS_FFTLIB = -I./include -I$(FFTW_ROOT)/include
LIBS       += fftlib
LLIBS      += -ldl

# PATH FOR PLUGIN BUILDS
EXTLIBDIR   = $(CFS)/omp/local

# wannier90
WANNIER90_ROOT = /global/common/software/nersc9/vasp/dependencies/wannier90/nvidia-23.9/3.1.0/lib
LLIBS      += -L$(WANNIER90_ROOT) -lwannier

# libbeef (CPU builds only)
LIBBEEF_ROOT = $(EXTLIBDIR)/libbeef/2020-11-01/nvidia-22.5/milan/lib
#LLIBS      += -L$(LIBBEEF_ROOT) -lbeef

# libxc (CPU builds only)
LIBXC_ROOT  = $(EXTLIBDIR)/libxc/6.0.0/nvidia-22.5/x86-milan
#LLIBS      += -L$(LIBXC_ROOT)/lib -lxcf03 -lxc
#INCS       += -I$(LIBXC_ROOT)/include

# DFTD4
DFTD4_ROOT  = $(EXTLIBDIR)/dft4/2022-10-13/nvidia-22.5/$(CRAY_CPU_TARGET)
LLIBS      += $(DFTD4_ROOT)/lib64/libdftd4.a \
              $(DFTD4_ROOT)/lib64/libmctc-lib.a \
              $(DFTD4_ROOT)/lib64/libmulticharge.a
INCS       += -I$(DFTD4_ROOT)/include \
              -I$(DFTD4_ROOT)/include/dftd4/NVHPC-22.5.0 \
              -I$(DFTD4_ROOT)/include/multicharge/NVHPC-22.5.0 \
              -I$(DFTD4_ROOT)/include/mctc-lib/NVHPC-22.5.0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment