python web刮擦rightmove

Question

我有一个称为“数据”的数据集，看起来像这样：

postcode    location_id url_name
NE30-2BG    1159502     https://www.rightmove.co.uk/property-for-sale/find.html?locationIdentifier=POSTCODE^1159502

我正在使用下面的代码从上面的数据中获取URL并从RightMove检索属性详细信息。我希望能够在下面的输出旁边输出邮政编码（从数据）输出。当代码所处的情况下，我无法将从RightMovesCraper检索到的数据链接到原始邮政编码。 任何想法都赞赏！

class RightmoveScraper:
    results = []
    
    def fetch(self, url):
        print('HTTP GET request to URL: %s' % url, end ='')
        response = requests.get(url)
        print(' | Status code: %s' % response.status_code)
        
        return response
    
    def parse(self, html):
        content = BeautifulSoup(html, 'html.parser')   #lxml
        
        titles = [title.text.strip() for title in content.findAll('h2', {'class': 'propertyCard-title'})]
        bedrooms = [title.text.split('bedroom')[0].strip() for title in content.findAll('h2', {'class': 'propertyCard-title'})]
        addresses = [address['content'] for address in content.findAll('meta', {'itemprop': 'streetAddress'})]
        descriptions = [description.text for description in content.findAll('span', {'data-test': 'property-description'})]
        prices = [price.text.strip() for price in content.findAll('div', {'class': 'propertyCard-priceValue'})]
        under_over = [underover.text.strip() for underover in content.findAll('div', {'class': 'propertyCard-priceQualifier'})]
        #code1 = [price.text.strip() for price in content.findAll('type', {'hidden': 'value'})]
        dates = [date.text for date in content.findAll('span', {'class': 'propertyCard-branchSummary-addedOrReduced'})]
        sellers = [seller.text.split('by')[-1].strip() for seller in content.findAll('span',{'class': 'propertyCard-branchSummary-branchName'})]
        
        
        for index in range(0, len(titles)):
            self.results.append({
                'title': titles[index],
                'no_of_bedrooms' : bedrooms[index],
                'address': addresses[index],
                'description': descriptions[index],
                'price': prices[index],
                'under_over': under_over[index],
                #'code1': code1[index],
                'date': dates[index],
                'seller': sellers[index]})
            

    def to_csv(self):
        with open('rightmove_data.csv','w') as csv_file:
            writer = csv.DictWriter(csv_file,fieldnames=self.results[0].keys())
            writer.writeheader()
            
            for row in self.results:
                writer.writerow(row)
                
            print('Stored results to "rightmove_data.csv"')
            
                
    def run(self):
        for url_name in data['url_name']:
            #postcode = data['postcode']
            url = url_name
            response = self.fetch(url)
            self.parse(response.text)
            
        self.to_csv()
        
        
          
if __name__ == '__main__':
    scraper = RightmoveScraper() 
    scraper.run()

Answer 1

看来您使用dataFrame，因此可以使用.iterrows（）像这样的

import pandas as pd

data = {
    'postcode': ['A','B','C'], 
    'url_name': ['www1','www2','www3'], 
    'other':    ['X','Y','Z']
}

df = pd.DataFrame(data)

def run():
    for index, row in df.iterrows():
        print('index   :', index   
        print('postcode:', row['postcode'])
        print('url_name:', row['url_name'])
        print('other   :', row['other'])
        print('----')
        #response = self.fetch(row['url_name'])
        #self.parse(response.text, row['postcode'])

run()

结果：

index   : 0
postcode: A
url_name: www1
other   : X
----
index   : 1
postcode: B
url_name: www2
other   : Y
----
index   : 2
postcode: C
url_name: www3
other   : Z

---

或者可以使用.apply（）在所有行。

import pandas as pd

def process(row):
    print('postcode:', row['postcode'])
    print('url_name:', row['url_name'])
    print('other   :', row['other'])
    print('----')
    #response = self.fetch(row['url_name'])
    #self.parse(response.text, row['postcode'])
    
data = {
    'postcode': ['A','B','C'], 
    'url_name': ['www1','www2','www3'], 
    'other':    ['X','Y','Z']
}

df = pd.DataFrame(data)

def run():
    df.apply(process, axis=1)

run()

Answer 2

感谢 furas ，这可以很好地享用！谢谢你！

 import pandas as pd
 import requests
 from bs4 import BeautifulSoup
 import csv

data = pd.read_csv('postcode data.csv')
df = pd.DataFrame(data)

class RightmoveScraper:
    results = []
    
    def fetch(self, url):
        print('HTTP GET request to URL: %s' % url, end ='')
        
        response = requests.get(url)
        print(' | Status code: %s' % response.status_code)
        
        return response
    

    def parse(self, html, pp):
        content = BeautifulSoup(html, 'html.parser')   #lxml
        
        titles = [title.text.strip() for title in content.findAll('h2', {'class': 'propertyCard-title'})]
        bedrooms = [title.text.split('bedroom')[0].strip() for title in content.findAll('h2', {'class': 'propertyCard-title'})]
        addresses = [address['content'] for address in content.findAll('meta', {'itemprop': 'streetAddress'})]
        descriptions = [description.text for description in content.findAll('span', {'data-test': 'property-description'})]
        prices = [price.text.strip() for price in content.findAll('div', {'class': 'propertyCard-priceValue'})]
        under_over = [underover.text.strip() for underover in content.findAll('div', {'class': 'propertyCard-priceQualifier'})]
        dates = [date.text for date in content.findAll('span', {'class': 'propertyCard-branchSummary-addedOrReduced'})]
        sellers = [seller.text.split('by')[-1].strip() for seller in content.findAll('span',{'class': 'propertyCard-branchSummary-branchName'})]
        
        
        for index in range(0, len(titles)):
            self.results.append({
                'postcode': pp,
                'title': titles[index],
                'no_of_bedrooms' : bedrooms[index],
                'address': addresses[index],
                'description': descriptions[index],
                'price': prices[index],
                'under_over': under_over[index],
                'date': dates[index],
                'seller': sellers[index]})           
            

    def to_csv(self):
        with open('output.csv','w') as csv_file:
            writer = csv.DictWriter(csv_file,fieldnames=self.results[0].keys())
            writer.writeheader()
            
            for row in self.results:
                writer.writerow(row)
                
            print('Stored results to "output.csv"')
            
               
    def run(self):
        for index, row in df.iterrows():
            pp = row['postcode']
            url = row['url_name']
            response = self.fetch(url)
            self.parse(response.text, pp)
    
                
        self.to_csv()

        
        
          
if __name__ == '__main__':
    scraper = RightmoveScraper() 
    scraper.run()