带有返回空列表的Web Crapping

Question

from bs4 import BeautifulSoup

import pandas as pd

from selenium import webdriver

driver = webdriver.Chrome()

driver.get('https://www.flaconi.de/haare/kerastase/chronologiste/kerastase-chronologiste-bain-regenerant-haarshampoo.html?yoReviewsPage=2')

soup = BeautifulSoup(driver.page_source, 'lxml') 
soup.find_all('div',class_='content-review')
# it always return empty list
# I want to scrap all of review contents from e.g "<div class="content-review" id="325243269"> Super Shampoo, meine Haare glänzt und sind sehr weich. 

              

Answer 1

您需要等到页面完全加载直到完全加载：

driver.get(url)
timeout = 5
try:
    element_present = EC.presence_of_element_located((By.CLASS_NAME, 'content-review'))
    WebDriverWait(driver, timeout).until(element_present)
except TimeoutException:
    print("Timed out waiting for page to load")
soup = BeautifulSoup(driver.page_source, 'lxml')
for review in soup.find_all('div', class_='content-review'):
    print(review.getText().strip())

添加必要的libs：

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

输出：

Super Shampoo, meine Haare glänzt und sind sehr weich. 

Answer 2

这里的主要问题是，您需要关闭位于 shadow dom 。

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import ElementNotInteractableException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

driver = webdriver.Chrome()
# OR: driver = webdriver.Chrome(executable_path='D:\Downloads\chromedriver\chromedriver.exe')
url = 'https://www.flaconi.de/haare/kerastase/chronologiste/kerastase-chronologiste-bain-regenerant-haarshampoo.html?yoReviewsPage=2'
driver.get(url)

webdriverWaiter = WebDriverWait(driver, 20)

webdriverWaiter.until(EC.text_to_be_present_in_element_attribute((By.CSS_SELECTOR, "body"), "class" ,"overflowHidden"))
shadow_host = driver.find_element(By.CSS_SELECTOR, '#usercentrics-root')
shadow_root = shadow_host.shadow_root

accept_cookies_button_css = "button[data-testid='uc-accept-all-button']"

# wait for accept cookies button to appear
accept_cookies_button = None
while not accept_cookies_button:
    try:
        accept_cookies_button = shadow_root.find_element(By.CSS_SELECTOR, accept_cookies_button_css)
    except NoSuchElementException:
        time.sleep(1)

# click accept cookies button
clicked = False
while not clicked:
    try:
        accept_cookies_button.click()
        clicked = True
    except ElementNotInteractableException:
        time.sleep(1)

content_review_css = ".content-review"
webdriverWaiter.until(EC.visibility_of_element_located((By.CSS_SELECTOR, content_review_css)))
reviews = driver.find_elements(By.CSS_SELECTOR, content_review_css)
for rev in reviews:
    print(rev.text)

弹出图像：