Created
November 11, 2015 00:03
-
-
Save nimmneun/10d7e2b65ff2ab1c0414 to your computer and use it in GitHub Desktop.
simple multi_curl based function with ability to set a limit on the max number of concurrent connections
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
/** | |
* Process any number of cURL requests in parallel, but limit | |
* the number of simultaneous requests to $parallel. | |
* | |
* @param array $urls Array with URLs to process | |
* @param int $parallel Number of concurrent requests | |
* @param array $extraOptions User defined CURLOPTS | |
* @return array[] | |
*/ | |
function paraCurl($urls = [], $parallel = 10, $extraOptions = []) { | |
// $extraOptions override the hardcoded ones. | |
$options = $extraOptions + [ | |
CURLOPT_RETURNTRANSFER => 1, | |
CURLOPT_FOLLOWLOCATION => 1, | |
CURLOPT_CONNECTTIMEOUT => 3, | |
CURLOPT_SSL_VERIFYHOST => 0, | |
CURLOPT_SSL_VERIFYPEER => 0, | |
CURLOPT_HEADER => 1 | |
]; | |
// The curl_multi handle. | |
$mh = curl_multi_init(); | |
// Array with curl handles. | |
$chs = []; | |
// Create the individual curl handles and set options. | |
foreach ($urls as $key => $url) { | |
$chs[$key] = curl_init($url); | |
curl_setopt_array($chs[$key], $options); | |
} | |
$curls = $chs; | |
$open = null; | |
// Perform the requests requests & dynamically (re)fill available slots | |
// up to the specified limit ($parallel) until all urls are processed. | |
while (0 < $open || 0 < count($curls)) { | |
if ($open < $parallel && 0 < count($curls)) { | |
curl_multi_add_handle($mh, array_shift($curls)); | |
} | |
curl_multi_exec($mh, $open); | |
usleep(11111); | |
} | |
// Extract downloaded data from curl handle. | |
foreach ($chs as $key => $ch) { | |
$res[$key]['info'] = curl_getinfo($ch); | |
$response = curl_multi_getcontent($ch); | |
// Separate response header & body. | |
$res[$key]['head'] = substr($response, 0, $res[$key]['info']['header_size']); | |
$res[$key]['body'] = substr($response, $res[$key]['info']['header_size']); | |
curl_multi_remove_handle($mh, $ch); | |
} | |
// Close the curl_multi handle. | |
curl_multi_close($mh); | |
// Finally return all results. | |
return isset($res) ? $res : []; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment