scrapy when login is required

Solutions on MaxInterview for scrapy when login is required by the best coders in the world

showing results for - "scrapy when login is required"
Carolina
06 Oct 2016
1class ScrapySpider(CrawlSpider):
2    name = 'login'
3    allowed_domains = ['quotes.toscrape.com']
4    start_urls = ['http://quotes.toscrape.com/login']
5 
6    def parse(self, response):
7        inputs = response.css('form input')
8        print(inputs)
9 
10        formdata = {}
11        for input in inputs:
12            name = input.css('::attr(type)').get()
13            value = input.css('::attr(value)').get()
14            formdata[name] = value
15 
16        formdata['username'] = 'YOUR_USERNAME'
17        formdata['password'] = 'YOUR_PASSWORD'
18 
19        return scrapy.FormRequest.from_response(
20            response,
21            formdata = formdata,
22            callback = self.parse_after_login
23        )
24 
25    def parse_after_login(self, response):
26        print(response.xpath('.//div[@class = "col-md-4"]/p/a/text()').get())
27