google_serp_scrapy_p4.py

def parse(self, response):
       di = json.loads(response.text)
       pos = response.meta['pos']
       dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
       for result in di['organic_results']:
           title = result['title']
           snippet = result['snippet']
           link = result['link']
           item = {'title': title, 'snippet': snippet, 'link': link, 'position': pos, 'date': dt}
           pos += 1
           yield item
       next_page = di['pagination']['nextPageUrl']
       if next_page:
           yield scrapy.Request(get_url(next_page), callback=self.parse, meta={'pos': pos})