runs a queue of shell commands in parallel

parallel_proc.sh

#!/bin/bash

# Parallelization functions

# Maximum number of processes to run in parallel.
# A good value for this is one fewer than the number of cores available
NP=14

# wait until fewer jobs are running than number of processors
function queue {
  while [ `jobs -r|wc -l` -ge $NP ]; do
    sleep 1
  done
}
# wait until all jobs have finished
function barrier {
  while [ `jobs -r|wc -l` -ge 1 ]; do
    sleep 1
  done
}
# a single lock, implemented as a directory (mkdir is like test-and-set)
function lock {
  while ! mkdir .lock 2> /dev/null; do
    sleep 0.01
  done
}
function unlock {
  rmdir .lock 2> /dev/null
}


##########################################

if [ "$1" == "" ]; then
  echo -e "usage is:\n  $ parallel_proc.sh commands.txt\nwhere commands.txt is a list of commands to execute in parallel, one per line.'"
  exit
fi

echo "Executing `wc -l $1` commands with $NP concurrent processes"
while read line; do
  queue
  $line &
done < $1
barrier
echo "Execution completed"