A simple recipe to run PDNN (Theano backend) DNNs training on HPC-DTIC GPU nodes

pdnnAcousticModelTraining.sh

#!/bin/bash

# task name
#$ -N RongGongDNNAcousticModelTraining

# use default.q, let HPC automatically distribute the GPU resources
#$ -q default.q

# use specific GPU node, here we use node02
#$ -l h=node02

# the error and output in current working directory
#$ -cwd

# load cuda module
module load cuda/7.5

# load theano module
module load theano/0.8.2

# two variables you need to set
pdnndir=/homedtic/rgong/dnnAcousticTraining/pdnn  # pointer to PDNN
device=gpu0  # the device to be used. set it to "cpu" if you don't have GPUs

# export environment variables
export PYTHONPATH=$PYTHONPATH:$pdnndir

# lib.cnmem distribute GPU memory
export THEANO_FLAGS=mode=FAST_RUN,device=$device,floatX=float32,lib.cnmem=0.9

# add these to use cuDNN
export LD_LIBRARY_PATH=/soft/cuda/cudnn/cuda/lib64:$LD_LIBRARY_PATH
export CPATH=/soft/cuda/cudnn/cuda/include:$CPATH
export LIBRARY_PATH=/soft/cuda/cudnn/cuda/lib64:$LD_LIBRARY_PATH

# train DNN model
echo "Training laosheng DNN model ..."
python $pdnndir/cmds/run_DNN.py --train-data "/homedtic/rgong/dnnAcousticTraining/phraseMatching/train_set_laosheng_phraseMatching.pickle.gz" \
                                --valid-data "/homedtic/rgong/dnnAcousticTraining/phraseMatching/validation_set_laosheng_phraseMatching.pickle.gz" \
                                --nnet-spec "40:512:512:32" --wdir ./ \
                                --l2-reg 0.0001 --lrate "C:0.1:200" --model-save-step 20 \
                                --activation rectifier \
                                --dropout-factor 0.2,0.2 \
                                --param-output-file dnn_laosheng_phraseMatching_layers_2.param \
                                --cfg-output-file dnn_laosheng_phraseMatching_layers_2.cfg  >& dnn.training.log