parallel-map.sh

parallel_map() {
    local func="$1"

    local num_cores
    num_cores=$(nproc 2>/dev/null || echo 4)
    local pipe_buf_size
    pipe_buf_size=$(getconf PIPE_BUF / 2>/dev/null || echo 4096)
    local tmpdir
    tmpdir=$(mktemp -d)
    trap 'rm -rf "$tmpdir"' EXIT

    local job_queue="$tmpdir/queue"
    mkfifo "$job_queue"

    # workers
    for ((i = 0; i < num_cores; i++)); do
        (
            while IFS= read -r -d '' job; do
                [[ "$job" == $'\x00' ]] && break
                $func "$job"
            done <"$job_queue"
        ) &
    done

    (
        while IFS= read -r -d '' item; do
            if ((${#item} >= pipe_buf_size - 1)); then
                # fallback to a subshell if the item is too long to fit in the pipe buffer
                ( $func "$item" ) &
            else
                printf '%s\0' "$item"
            fi
        done

        # Send termination
        for ((i = 0; i < num_cores; i++)); do
            printf '\x00\0'
        done
    ) >"$job_queue"

    wait
}

process_file() {
    local file="$1"
    # Simulate processing the file
    echo "Processing $file"
}

find . -type f -print0 | parallel_map process_file