添加Kward和第二台珀斯克萨纸后停止工作

Question

添加Kward脚本停止以输出任何刮擦数据后，它仅输出了普通的蜘蛛调试数据。我完全不知道为什么它是这样做的， 看来整个牧师都在那儿坐在那里，什么也没做。

这是我的代码：

# -*- coding: utf-8 -*-
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy import Request, Spider


class RynekMainSpider(scrapy.Spider):
    name = "RynekMain"
    start_urls = [
        'https://rynekpierwotny.pl/deweloperzy/?page=1']
    def parse(self, response):
        websites = response.css('div.root')
        for websitep in websites:
            websiteurl = websitep.css('div.rp-l0pkv6 a::attr(href)').get()
            href = websitep.css('li.rp-np9kb1 a::attr(href)').get()
            url = response.urljoin(href)
            yield Request(url, cb_kwargs={'websiteurl': websiteurl}, callback=self.parseMain)
    
    def parseMain(self, response, websiteurl):
   # def parse(self, response):
        for quote in response.css('.rp-y89gny.eboilu01 ul li'):
                address = quote.css('address.rp-o9b83y::text').get(),
                name = quote.css('h2.rp-69f2r4::text').get(),
                href = quote.css('li.rp-np9kb1 a::attr(href)').get(),
                PAGETEST = response.css('a.rp-mmikj9::attr(href)').get()
        yield {
            'address' : address,
            'name' : name,
            'href' : href,
            'PAGETEST' : PAGETEST,
            'websiteurl' : websiteurl
            }
        next_page=response.css('a.rp-mmikj9::attr(href)').get()
        if next_page is not None:
            next_page_link=response.urljoin(next_page)
            yield scrapy.Request(url=next_page_link, callback= self.parse)
    
if __name__ == "__main__":
    process =CrawlerProcess()
    process.crawl(RynekMainSpider)
    process.start()

感谢提前的帮助。 编辑：哦，拍摄，我忘了告诉我的代码应该做什么。 基本解析是从“ https://rynekpierwotny.pl/deweloperzy/dom-development-sa-955/”中的子页面内部获取网站URL。 当Parsemain从主页“ https://rynekpierwotny.pl/deweloperzy/?page=1”中获取所有数据（例如地址，名称）。

# -*- coding: utf-8 -*-
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy import Request, Spider


class RynekMainSpider(scrapy.Spider):
    name = "RynekMain"
    start_urls = [
        'https://rynekpierwotny.pl/deweloperzy/?page=1']
   
    
 
    def parse(self, response):
        for quote in response.css('.rp-y89gny.eboilu01 ul li'):
            yield {
                'address' : quote.css('address.rp-o9b83y::text').get(),
                'name' : quote.css('h2.rp-69f2r4::text').get(),
                'href' : quote.css('li.rp-np9kb1 a::attr(href)').get(),
                'PAGETEST' : response.css('a.rp-mmikj9::attr(href)').get()
            }
        next_page=response.css('a.rp-mmikj9::attr(href)').get()
        if next_page is not None:
            next_page_link=response.urljoin(next_page)
            yield scrapy.Request(url=next_page_link, callback= self.parse)
    
if __name__ == "__main__":
    process =CrawlerProcess()
    process.crawl(RynekMainSpider)
    process.start()

这起作用了

Answer 1

编辑：

我根据您要编程要做的事情进行了一些进一步的调整。它应该按照您现在的期望工作。

而是尝试一下：

# -*- coding: utf-8 -*-
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy import Request, Spider


class RynekMainSpider(scrapy.Spider):
    name = "RynekMain"
    start_urls = [
        'https://rynekpierwotny.pl/deweloperzy/?page=1']
    def parse(self, response):
        websites = response.css('div#root')[0]
        PAGETEST = response.xpath('//a[contains(@class,"rp-173nt6g")]/../following-sibling::li').css('a::attr(href)').get()
        for website in websites.css('li.rp-np9kb1'):
            page = website.css('a::attr(href)').get()
            address = website.css('address.rp-o9b83y::text').get()
            name = website.css('h2.rp-69f2r4::text').get()
            params = {
            'address' : address,
            'name' : name,
            'href' : page,
            }
            url  = response.urljoin(page)
            yield Request(url=url, cb_kwargs={'params': params}, callback=self.parseMain)
        yield Request(url=response.urljoin(PAGETEST), callback=self.parse)

    def parseMain(self, response, params=None):
        # print(response.url)
        website = response.css('div.rp-l0pkv6 a::attr(href)').get()
        params['website'] = website
        yield params

if __name__ == "__main__":
    process =CrawlerProcess()
    process.crawl(RynekMainSpider)
    process.start()