thinkphp5.1怎么实现多线程爬虫
建设一个cli下令
php think make:command Thread thread
登录后复制
测试能否乐成执行
php think thread
登录后复制 登录后复制
装置Guzzle类库
文档地点:guzzle文档地点(https://guzzle-cn.readthedocs.io/zh_CN/latest/quickstart.html)
实现代码
<?php /** * Created by. * User: Jim * Date: 2020/9/29 * Time: 14:31 */ namespace app\command; use GuzzleHttp\Client; use GuzzleHttp\Pool; use think\console\Command; use think\console\Input; use think\console\Output; /** * Guzzle * Class Thread * @package app\command * 文档地点 https://guzzle-cn.readthedocs.io/zh_CN/latest/quickstart.html */ class Thread extends Command { /** * 请求的总次数 * @var int */ protected $totalPageCount = 50; /** * 目今请求的次数 * @var int */ protected static $counter = 1; /** * 线程的数目 * @var int */ protected $threads = 20; protected function configure() { // 指令设置 $this->setName('thread'); // 设置参数 } protected function execute(Input $input, Output $output) { $client = new Client(); $requests = function ($total) use ($client) { foreach (range(1, $total) as $r) { $uri = 'https://apinew.juejin.im/content_api/v1/short_msg/detail'; yield function () use ($client, $uri) { return $client->postAsync($uri, [ 'verify' => false, 'json' => [ 'msg_id' => '6845185452727599118' ] ]); }; } }; $pool = new Pool($client, $requests($this->totalPageCount), [ 'concurrency' => $this->threads, // 请求乐成 'fulfilled' => function ($response, $index) use ($output) { $res = $response->getBody()->getContents(); $output->writeln($res); $output->writeln("正在执行第{$index}个·····"); if ($this->checkThreadIsEnd() == true) { $output->writeln("------------请求竣事---------"); return false; } }, // 请求失败 'rejected' => function ($reason, $index) use ($output) { $output->writeln("执行失败,{$reason}"); }, ]); $promise = $pool->promise(); $promise->wait(); } /** * 检测使命是否竣事 * @return bool */ private function checkThreadIsEnd() { if (self::$counter < $this->totalPageCount) { self::$counter++; return false; } else { return true; } } }
登录后复制
执行下令
php think thread
登录后复制 登录后复制
以上就是thinkphp5.1怎么实现多线程爬虫的详细内容,更多请关注本网内其它相关文章!
免责说明:以上展示内容泉源于相助媒体、企业机构、网友提供或网络网络整理,版权争议与本站无关,文章涉及看法与看法不代表GA黄金甲滤油机网官方态度,请读者仅做参考。本文接待转载,转载请说明来由。若您以为本文侵占了您的版权信息,或您发明该内容有任何涉及有违公德、冒犯执法等违法信息,请您连忙联系GA黄金甲实时修正或删除。