使用Python从数据库的HTML代码刮除网络

Question

我是Python的新手，正在缓慢地学习东西。我早些时候已经从数据库中进行了API调用以提取indromation。但是，我正在处理一个特定的印度数据库。 HTML脚本似乎令人困惑地提取我要寻找的特定范围。基本上，我有一个像输入的草药名称链接列表，看起来像这样（只有ID更改）：

http://envis.frlht.org/plantdetails/3315/fd01bd598f0869d65fe5a2861845f9f8
http://envis.frlht.org/plantdetails/2133/fd01bd598f0869d65fe5a2861845f9f9
http://envis.frlht.org/plantdetails/845/fd01bd598f0869d65fe5a2861845f9f10
http://envis.frlht.org/plantdetails/363/fd01bd598f0869d65fe5a2861845f9f11

当我打开每一个时，我想从网页中提取这些草药链接的“分发”细节。就这样。但是，在HTML脚本中，我无法理解哪个标头具有细节。我在来到这里之前尝试了很多。有人可以帮我吗提前致谢。

代码：

import requests
import urllib.request
import time
from bs4 import BeautifulSoup
import json
import pandas as pd
import os
from pathlib import Path
from pprint import pprint

user_home = os.path.expanduser('~')
OUTPUT_DIR = os.path.join(user_home, 'vk_frlht')
Path(OUTPUT_DIR).mkdir(parents=True, exist_ok=True)

herb_url = 'http://envis.frlht.org/bot_search'
response = requests.get(herb_url)
soup = BeautifulSoup(response.text, "html.parser")
token = soup.find('Type Botanical Name', {'type': 'hidden', 'name': 'token'})
herb_query_url = 'http://envis.frlht.org/plantdetails/3315/fd01bd598f0869d65fe5a2861845f9f8'

response = requests.get('http://envis.frlht.org/plantdetails/3315/fd01bd598f0869d65fe5a2861845f9f8')

#optional code for many links at once

with open(Path, 'r') as f:
    frlhtinput = f.readlines()
    frlht = [x[:-1] for x in frlhtinput]

    for line in frlht:
        out = requests.get(f'http://envis.frlht.org/plantdetails/{line}')
#end of the optional code

herb_query_soup = BeautifulSoup(response.text, "html.parser")
text = herb_query_soup.find('div', {'id': 'result-details'})
pprint(text)

Answer 1

这就是此页面报废之后的外观：

中间加载符号意味着仅在JavaScript代码执行后才可以加载内容。这意味着有人用JS代码保护了此内容。您必须使用硒浏览器而不是BS4。

请参阅教程在这里关于如何使用它。

Answer 2

尝试一下。

import requests
from bs4 import BeautifulSoup
from pprint import pprint

plant_ids = ["3315", "2133", "845", "363"]
results = []
for plant_id in plant_ids:
    herb_query_url = f"http://envis.frlht.org/plantdetails/{plant_id}/fd01bd598f0869d65fe5a2861845f9f8"
    headers = {
        "Referer": herb_query_url,
    }
    response = requests.get(
        f"http://envis.frlht.org/bot_search/plantdetails/plantid/{plant_id}/nocache/0.7763327765552295/referredfrom/extplantdetails",
        headers=headers,
    )
    herb_query_soup = BeautifulSoup(response.text, "html.parser")
    result = herb_query_soup.findAll("div", {"class": "initbriefdescription"})
    for r in result:
        result_dict = {r.text.split(":", 1)[0].strip(): r.text.split(":", 1)[1].strip()}
        results.append(result_dict)
pprint(results)

Answer 3

enter code here

import requests
from bs4 import BeautifulSoup
import csv


fieldnames = ["ID", "Accepted Name", "Family", "Used in", "Distribution"]

with open('IDs.txt') as f_input, open('output.csv', 'w', newline='') as f_output:
csv_output = csv.DictWriter(f_output, fieldnames=fieldnames, extrasaction='ignore')
csv_output.writeheader()

for line in f_input:
    url = line.strip()  # Remove newline
    print(url)
    url_split = url.split('/')
    url_details = f"http://envis.frlht.org/bot_search/plantdetails/plantid/{url_split[4]}/nocache/{url_split[5]}/referredfrom/extplantdetails"
    
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36',
        'Referer' : url,
    }
    
    req = requests.get(url_details, headers=headers)
    soup = BeautifulSoup(req.content, "html.parser")
    row = {field : '' for field in fieldnames}      # default values
    row['ID'] = url_split[4]
    
    result = soup.findAll("div", {"class": "initbriefdescription"})
    for r in result:
        result_dict = r.get_text(strip=True).split(":" ,1)
        results.append(result_dict)
        
        row[entry] = results
        print(row)

with open('output.csv', 'w', newline='') as f_output:
    csv_output = csv.DictWriter(f_output, fieldnames=fieldnames, extrasaction='ignore')
    csv_output.writeheader()
    csv_output.writerow(row)

Answer 4

信息是根据您拥有的URL从另一个URL获得的。首先，您需要构建所需的URL（发现查看浏览器）并要求。

该信息可以按照以下方式写入CSV文件。它假设您有一个文本文件ids.txt如下：

http://envis.frlht.org/plantdetails/3315/fd01bd598f0869d65fe5a2861845f9f8
http://envis.frlht.org/plantdetails/2133/fd01bd598f0869d65fe5a2861845f9f9
http://envis.frlht.org/plantdetails/845/fd01bd598f0869d65fe5a2861845f9f10
http://envis.frlht.org/plantdetails/363/fd01bd598f0869d65fe5a2861845f9f11

import requests
from bs4 import BeautifulSoup
import csv


fieldnames = ["ID", "Accepted Name", "Family", "Used in", "Distribution"]

with open('IDs.txt') as f_input, open('output.csv', 'w', newline='') as f_output:
    csv_output = csv.DictWriter(f_output, fieldnames=fieldnames, extrasaction='ignore')
    csv_output.writeheader()

    for line in f_input:
        url = line.strip()  # Remove newline
        print(url)
        url_split = url.split('/')
        url_details = f"http://envis.frlht.org/bot_search/plantdetails/plantid/{url_split[4]}/nocache/{url_split[5]}/referredfrom/extplantdetails"
        
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36',
            'Referer' : url,
        }
        
        req = requests.get(url_details, headers=headers)
        soup = BeautifulSoup(req.content, "html.parser")
        row = {field : '' for field in fieldnames}      # default values
        row['ID'] = url_split[4]
            
        for div in soup.find_all('div', class_="initbriefdescription"):
            entry, value = div.get_text(strip=True).split(":" ,1)
            row[entry] = value

        csv_output.writerow(row)

给出输出启动：

ID,Accepted Name,Family,Used in,Distribution
3315,Amaranthus hybridusL. subsp.cruentusvar.paniculatusTHELL.,AMARANTHACEAE,"Ayurveda, Siddha, Folk","This species is globally distributed in Africa, Asia and India. It is said to be cultivated as a leafy vegetable in Maharashtra, Karnataka (Coorg) and on the Nilgiri hills of Tamil Nadu. It is also found as an escape."
2133,Triticum aestivumL.,POACEAE,"Ayurveda, Siddha, Unani, Folk, Chinese, Modern",
845,Dolichos biflorusL.,FABACEAE,"Ayurveda, Siddha, Unani, Folk, Sowa Rigpa","This species is native to India, globally distributed in the Paleotropics. Within India, it occurs all over up to an altitude of 1500 m. It is an important pulse crop particularly in Madras, Mysore, Bombay and Hyderabad."
363,Brassica oleraceaL.,BRASSICACEAE,"Ayurveda, Siddha",