Skip to content

Instantly share code, notes, and snippets.

@nimmneun
Created November 11, 2015 00:03
Show Gist options
  • Save nimmneun/10d7e2b65ff2ab1c0414 to your computer and use it in GitHub Desktop.
Save nimmneun/10d7e2b65ff2ab1c0414 to your computer and use it in GitHub Desktop.
simple multi_curl based function with ability to set a limit on the max number of concurrent connections
<?php
/**
* Process any number of cURL requests in parallel, but limit
* the number of simultaneous requests to $parallel.
*
* @param array $urls Array with URLs to process
* @param int $parallel Number of concurrent requests
* @param array $extraOptions User defined CURLOPTS
* @return array[]
*/
function paraCurl($urls = [], $parallel = 10, $extraOptions = []) {
// $extraOptions override the hardcoded ones.
$options = $extraOptions + [
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_FOLLOWLOCATION => 1,
CURLOPT_CONNECTTIMEOUT => 3,
CURLOPT_SSL_VERIFYHOST => 0,
CURLOPT_SSL_VERIFYPEER => 0,
CURLOPT_HEADER => 1
];
// The curl_multi handle.
$mh = curl_multi_init();
// Array with curl handles.
$chs = [];
// Create the individual curl handles and set options.
foreach ($urls as $key => $url) {
$chs[$key] = curl_init($url);
curl_setopt_array($chs[$key], $options);
}
$curls = $chs;
$open = null;
// Perform the requests requests & dynamically (re)fill available slots
// up to the specified limit ($parallel) until all urls are processed.
while (0 < $open || 0 < count($curls)) {
if ($open < $parallel && 0 < count($curls)) {
curl_multi_add_handle($mh, array_shift($curls));
}
curl_multi_exec($mh, $open);
usleep(11111);
}
// Extract downloaded data from curl handle.
foreach ($chs as $key => $ch) {
$res[$key]['info'] = curl_getinfo($ch);
$response = curl_multi_getcontent($ch);
// Separate response header & body.
$res[$key]['head'] = substr($response, 0, $res[$key]['info']['header_size']);
$res[$key]['body'] = substr($response, $res[$key]['info']['header_size']);
curl_multi_remove_handle($mh, $ch);
}
// Close the curl_multi handle.
curl_multi_close($mh);
// Finally return all results.
return isset($res) ? $res : [];
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment