从多个URL中提取P标签

Question

我已经为此挣扎了几天，不确定问题是什么 - 基本上，我正在尝试提取每个链接的配置文件框数据（图下图） - 通过检查员，我认为我可以拉P标签并这样做。

我是新手，并试图理解，但这是我到目前为止所拥有的：

- （某种程度上）成功地提取一个链接的信息：

import requests
from bs4 import BeautifulSoup


# getting html

url = 'https://basketball.realgm.com/player/Darius-Adams/Summary/28720'
req = requests.get(url)

soup = BeautifulSoup(req.text, 'html.parser')

container = soup.find('div', attrs={'class', 'main-container'})
playerinfo = container.find_all('p')


print(playerinfo)

然后，我也有一个代码，从多个链接中拉出所有HREF标签：

    from bs4 import BeautifulSoup
import requests


def get_links(url):

    links = []

    website = requests.get(url)
    website_text = website.text
    soup = BeautifulSoup(website_text)

    for link in soup.find_all('a'):

        links.append(link.get('href'))

    for link in links:
        print(link)

        print(len(links))


get_links('https://basketball.realgm.com/dleague/players/2022')
get_links('https://basketball.realgm.com/dleague/players/2021')
get_links('https://basketball.realgm.com/dleague/players/2020')

因此，基本上，我的目标是结合这两个标签，并获取一个代码，该代码将从多个URL中提取所有P标签。我一直在尝试做到这一点，而且我真的不确定为什么这在这里不起作用：

    from bs4 import BeautifulSoup
import requests


def get_profile(url):

    profiles = []

    req = requests.get(url)
    soup = BeautifulSoup(req.text, 'html.parser')
    container = soup.find('div', attrs={'class', 'main-container'})

    for profile in container.find_all('a'):

        profiles.append(profile.get('p'))

    for profile in profiles:
        print(profile)


get_profile('https://basketball.realgm.com/player/Darius-Adams/Summary/28720')
get_profile('https://basketball.realgm.com/player/Marial-Shayok/Summary/26697')

再次，我真的是与Python一起抓取网络的新手，但是任何建议都将不胜感激。最终，我的最终目标是拥有一个可以立即以干净的方式刮擦此数据的工具。

（球员名称，当前团队，出生，出生地等）。也许我做错了，但欢迎任何指导！

Answer 1

您需要将两个脚本组合在一起，并为每个玩家提出请求。尝试以下方法。这将搜索具有data-td = player属性：

import requests
from bs4 import BeautifulSoup

def get_links(url):
    data = []
    req_url = requests.get(url)
    soup = BeautifulSoup(req_url.content, "html.parser")

    for td in soup.find_all('td', {'data-th' : 'Player'}):
        a_tag = td.a
        name = a_tag.text
        player_url = a_tag['href']
        print(f"Getting {name}")

        req_player_url = requests.get(f"https://basketball.realgm.com{player_url}")
        soup_player = BeautifulSoup(req_player_url.content, "html.parser")
        div_profile_box = soup_player.find("div", class_="profile-box")
        row = {"Name" : name, "URL" : player_url}
        
        for p in div_profile_box.find_all("p"):
            try:
                key, value = p.get_text(strip=True).split(':', 1)
                row[key.strip()] = value.strip()
            except:     # not all entries have values
                pass

        data.append(row)

    return data

urls = [
    'https://basketball.realgm.com/dleague/players/2022',
    'https://basketball.realgm.com/dleague/players/2021',
    'https://basketball.realgm.com/dleague/players/2020',
]


for url in urls:
    print(f"Getting: {url}")
    data = get_links(url)
    
    for entry in data:
        print(entry)