pyspider任务重新启动但是结果显示none
求指教,终端的报错看不太明白为啥返回是none,也不知道和on_result有什么关系
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-05-22 15:22:51
# Project: demo
from pyspider.libs.base_handler import *
import pymongo
class Handler(BaseHandler):
crawl_config = {
}
client = pymongo.MongoClient('localhost')
db = client['trip']
@every(minutes=30)
def on_start(self):
self.crawl('https://cn.tripadvisor.com/Attractions-g186338-Activities-London_England.html', callback=self.index_page)
@config(age=60*20)
def index_page(self, response):
for each in response.doc('.attraction_type_group .listing_info div.listing_title a').items():
self.crawl(each.attr.href,callback=self.group_detail)
for each in response.doc('#ATTR_ENTRY_ > div.attraction_clarity_cell > div > div > div.listing_info > div.listing_title > a').items():
self.crawl(each.attr.href,callback=self.detail_page)
def group_detail(self,response):
for each in response.doc('.listing_title a').items():
self.crawl(each.attr.href, callback=self.detail_page)
print(response.doc('.current').text())
if response.doc('.current').text() == '1':
for next in response.doc('.pageNumbers a').items():
self.crawl(next.attr.href,callback=self.group_detail)
@config(priority=2)
def detail_page(self, response):
url = response.url
name = response.doc('.heading_title').text()
rating = response.doc('.autoResize').text()
address = response.doc('.address :nth-child(n+2)').text()
phone = response.doc('.blEntry.phone > span').text()
return {
'url':url,
"name": name,
"rating": rating,
'address':address,
'phone':phone,
}
def on_result(self,result):
if result:
self.save_to_mongo(result)
def save_to_mongo(self,result):
if self.db['shdu'].insert(result):
print('save to mongo',result)
如果你对这篇内容有疑问,欢迎到本站社区发帖提问 参与讨论,获取更多帮助,或者扫码二维码加入 Web 技术交流群。
绑定邮箱获取回复消息
由于您还没有绑定你的真实邮箱,如果其他用户或者作者回复了您的评论,将不能在第一时间通知您!
发布评论
评论(1)
遇到同样的问题,看终端log 是on start 直接跳到on finish 。怀疑中间哪里有报错。请问您解决了吗?