刮擦网络信息

Question

我正在尝试从网站上刮擦信息，并将其保存到CSV格式中。但是，即使我从网站得到回复，我也无法将任何数据检索到Excel中。我回来的只是我的标题，上面有空列。我的代码有什么问题？任何帮助都将受到赞赏。

from csv import writer
from bs4 import BeautifulSoup
import requests

url = "https://www.sephora.com/beauty/new-beauty-products"
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, 
like Gecko) Chrome/102.0.5005.63 Safari/537.36'}
page = requests.get(url, headers=headers)
print(page.status_code)
soup = BeautifulSoup(page.content, 'html.parser')
lists = soup.find_all('div', class_="s-card-container")

with open('Competition Pricing Sepho.csv', 'w+', encoding='utf8', newline='') as f:
    thewriter = writer(f)
    header = ['Title', 'Price']
    thewriter.writerow(header)

    for list in lists:
        Title = list.find('span', class_="css-12vkztw").text
        Price = list.find('span', class_="css-0").text

        info = [Title, Price]
        thewriter.writerow(info)

Answer 1

页面是动态的，而不是静态HTML。

import requests
import pandas as pd
import re
import json


url = 'https://www.res-x.com/ws/r2/Resonance.aspx'
payload = {
'appid': 'sephora01',
'tk': '283272623958663',
'ss': '985647104696884',
'sg': '1',
'pg': 'res22061315349337742234660',
'vr': '5.10x',
'bx': 'true',
'sc': 'content1_rr',
'ev': '',
'ei': '',
'no': '20',
'language': 'ENGLISH',
'categoryid': '',
'page': '1',
'ccb': 'Sephora.certona',
'ur': 'https://www.sephora.com/beauty/new-beauty-products',
'plk': '',
'rf': ''}


rows = []
for page in range(1,11):
    payload.update({'page':f'{page}'})
    response = requests.get(url, params=payload).text.strip('Sephora.certona(').rsplit(');',1)[0]
    jsonData = json.loads(response)
    
    items = jsonData['resonance']['schemes'][0]['items']
    rows += items
    print(f'Page: {page}')
    
df = pd.DataFrame(rows)

输出：

print(df.head(5).to_string())
        id                                  display_name                      variation_type product_type                                                    product_url          brand_name default_sku_id  reviews  rating heroImageAltText certona_algorithm_id certona_experience_id certona_audience_id certona_strategy_id                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     skus
0  P500452        SoftSculpt® Transforming Skin Enhancer                               Color     standard         /product/softsculpt-transforming-skin-enhancer-P500452     MAKEUP BY MARIO        2578011      101   4.485                                    vn                  4264                 651               81501  [{'sku_number': '2578011', 'sku_size': '0.18 oz / 5 g', 'sku_type': 'Standard', 'list_price': 30.0, 'primary_product_id': 'P500452', 'additional_sku_desc': 'SoftSculpt® Transforming Skin Enhancer Light Medium', 'grid_images': '/productimages/sku/s2578011-main-zoom.jpg?imwidth=135', 'hero_images': '/productimages/sku/s2578011-main-zoom.jpg?imwidth=270', 'large_hero_images': '/productimages/sku/s2578011-main-zoom.jpg?imwidth=450', 'is_sephora_exclusive': True, 'is_online_only': False, 'is_limited_edition': False, 'is_app_exclusive': False, 'is_first_access': False, 'is_limited_time_offer': False, 'is_new': True, 'variation_value': 'Light Medium', 'deep_link': 'https://sephora.app.link/FZ6NdCyKjqb?fallbackurl=/product/softsculpt-transforming-skin-enhancer-P500452?skuId=2578011', 'badgeAltText': '', 'highlights': ['Light Coverage', 'Vegan', 'Radiant Finish', 'Cream Formula', 'Fragrance Free', 'Cruelty-Free']}]
1  P500423             Multi-Peptide Lash and Brow Serum                                None     standard                 /product/multi-peptide-lash-brow-serum-P500423        The Ordinary        2532588       20   4.700                                    vn                  4264                 651               81501                                                                  [{'sku_number': '2532588', 'sku_size': '.16 oz / 5 mL', 'sku_type': 'Standard', 'list_price': 14.5, 'primary_product_id': 'P500423', 'additional_sku_desc': 'Multi-Peptide Lash and Brow Serum', 'grid_images': '/productimages/sku/s2532588-main-zoom.jpg?imwidth=135', 'hero_images': '/productimages/sku/s2532588-main-zoom.jpg?imwidth=270', 'large_hero_images': '/productimages/sku/s2532588-main-zoom.jpg?imwidth=450', 'is_sephora_exclusive': True, 'is_online_only': False, 'is_limited_edition': False, 'is_app_exclusive': False, 'is_first_access': False, 'is_limited_time_offer': False, 'is_new': True, 'variation_value': '', 'deep_link': 'https://sephora.app.link/HgVj5WH5hqb?fallbackurl=/product/multi-peptide-lash-brow-serum-P500423?skuId=2532588', 'badgeAltText': '', 'highlights': ['Vegan', 'Good for: Hair Thinning', 'Fragrance Free', 'Cruelty-Free']}]
2  P500062                             Y Eau de Toilette  Size + Concentration + Formulation     standard                             /product/y-eau-de-toilette-P500062  Yves Saint Laurent        2546216     1072   4.738                                    vn                  4264                 651               81501           [{'sku_number': '2546216', 'sku_size': '3.4 oz / 100 mL', 'sku_type': 'Standard', 'list_price': 100.0, 'primary_product_id': 'P500062', 'additional_sku_desc': 'Y Eau de Toilette 3.4 oz / 100 mL eau de toilette spray', 'grid_images': '/productimages/sku/s2546216-main-zoom.jpg?imwidth=135', 'hero_images': '/productimages/sku/s2546216-main-zoom.jpg?imwidth=270', 'large_hero_images': '/productimages/sku/s2546216-main-zoom.jpg?imwidth=450', 'is_sephora_exclusive': False, 'is_online_only': False, 'is_limited_edition': False, 'is_app_exclusive': False, 'is_first_access': False, 'is_limited_time_offer': False, 'is_new': True, 'variation_value': 'size:3.4 oz / 100 mL concentration:eau de toilette formulation:spray', 'deep_link': 'https://sephora.app.link/ppre1pZZlpb?fallbackurl=/product/y-eau-de-toilette-P500062?skuId=2546216', 'badgeAltText': '', 'highlights': ['Layerable Scent', 'Woody & Earthy Scent']}]
3  P500403       SoftSculpt® Transforming Skin Perfector                               Color     standard        /product/softsculpt-transforming-skin-perfector-P500403     MAKEUP BY MARIO        2577963       37   4.216                                    14                  4264                 651               81470                             [{'sku_number': '2577963', 'sku_size': '0.31 oz / 8.8 g', 'sku_type': 'Standard', 'list_price': 34.0, 'primary_product_id': 'P500403', 'additional_sku_desc': 'SoftSculpt® Transforming Skin Perfector Light', 'grid_images': '/productimages/sku/s2577963-main-zoom.jpg?imwidth=135', 'hero_images': '/productimages/sku/s2577963-main-zoom.jpg?imwidth=270', 'large_hero_images': '/productimages/sku/s2577963-main-zoom.jpg?imwidth=450', 'is_sephora_exclusive': True, 'is_online_only': False, 'is_limited_edition': False, 'is_app_exclusive': False, 'is_first_access': False, 'is_limited_time_offer': False, 'is_new': True, 'variation_value': 'Light', 'deep_link': 'https://sephora.app.link/urAEdzxKjqb?fallbackurl=/product/softsculpt-transforming-skin-perfector-P500403?skuId=2577963', 'badgeAltText': '', 'highlights': ['Light Coverage', 'Vegan', 'Radiant Finish', 'Fragrance Free', 'Cruelty-Free']}]
4  P500285  Beautiful Skin Sun-Kissed Glow Cream Bronzer                               Color     standard  /product/beautiful-skin-sun-kissed-glow-cream-bronzer-P500285   Charlotte Tilbury        2569366       47   4.319                                    14                  4264                 651               81470        [{'sku_number': '2569366', 'sku_size': '0.74 oz / 21 g', 'sku_type': 'Standard', 'list_price': 56.0, 'primary_product_id': 'P500285', 'additional_sku_desc': 'Beautiful Skin Sun-Kissed Glow Cream Bronzer 1 Fair', 'grid_images': '/productimages/sku/s2569366-main-zoom.jpg?imwidth=135', 'hero_images': '/productimages/sku/s2569366-main-zoom.jpg?imwidth=270', 'large_hero_images': '/productimages/sku/s2569366-main-zoom.jpg?imwidth=450', 'is_sephora_exclusive': False, 'is_online_only': False, 'is_limited_edition': False, 'is_app_exclusive': False, 'is_first_access': False, 'is_limited_time_offer': False, 'is_new': True, 'variation_value': '1 Fair', 'deep_link': 'https://sephora.app.link/j1Xzw9D5hqb?fallbackurl=/product/beautiful-skin-sun-kissed-glow-cream-bronzer-P500285?skuId=2569366', 'badgeAltText': '', 'highlights': ['Hyaluronic Acid', 'Long-wearing', 'Best for Dry, Combo, Normal Skin', 'Cruelty-Free']}]
....

[131 rows x 15 columns]