scrapy模拟登陆（手动输入验证码），显示验证码会话无效

Question

class ZhihuSpider(scrapy.Spider):
    name = "zhihu"
    allowed_domains = ["www.zhihu.com"]
    start_urls = ['https://www.zhihu.com/']
    headers = {
        "HOST": "www.zhihu.com",
        "Referer": "https://www.zhizhu.com",
        'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36"
    }
    def parse(self, response):
        pass

    def parse_detail(self,response):
        pass

    def start_requests(self):
        return [scrapy.Request('https://www.zhihu.com#signin/', headers=self.headers, callback=self.login) ]


    def login(self,response):
        t = str(int(time.time() * 1000))
        captcha_url = "https://www.zhihu.com/captcha.gif?r={0}&type=login".format(t)
        session = requests.session()
        t = session.get(captcha_url, headers=self.headers)
        with open("captcha.jpg", "wb") as f:
            f.write(t.content)
            f.close()
        captcha = input("输入验证码：")
        response_text = response.text
        pattern = 'name="_xsrf" value="(.*?)"'  # 这里的_xsrf 返回的是一个list
        _xsrf = re.findall(pattern, response_text)
        if _xsrf[0]:
            xsrf = _xsrf[0]
        if xsrf:
            post_url = "https://www.zhihu.com/login/phone_num"
            post_data = {
                "_xsrf": xsrf,
                "phone_num": "手机号",
                "password": "密码",
                "captcha": captcha
            }
            return [scrapy.FormRequest(
                url=post_url,
                formdata=post_data,
                headers=self.headers,
                callback=self.check_login,
                encoding="utf-8"
            )]

    def check_login(self,response):
        text_json = json.load(response.text)
        if "msg" in text_json and text_json["msg"] == "登陆成功":
            for url in self.start_urls:
                yield scrapy.Request(url,dont_filter=True,headers=self.headers)

在check_login之前设置断点都能正常

但是在check_login这里设置断点之后

应该是我代码这里callback=self.check_login回调的原因，但如何修改？ 百度了很多callback还是感觉讲的挺模糊

Answer 1

因为你获取验证码的会话和抓取网页的会话不是同一个
其实你完全可以自己手动登录好，把cookie信息放到headers里就行啦，不用搞得这么复杂

Answer 2

你好,请问你解决了嘛，我也遇到了这个问题