刮擦时间，标题和内容不是来自新闻列表，而是从封面和专栏封面新闻和侧边栏

Question

我有一个带有一个按钮的窗口，该按钮将网站的新闻头条删除到列表框中，然后我还有另一个按钮刮擦了与所选标题相关的新闻内容，并将它们显示在

当前在页面上的Textobox中，我刮擦了所有标题，他们的时间表及其内容，但我也想添加页面封面上的标题（带有相对内容）（this），以及页面右栏上的所有标题（带有相对内容）（这个）。

轻松的是，右列上的标题总是具有相同的唯一HTML名称，但是时间不直接可见，因为您必须打开链接才能获得时间。

我已经尝试使用这些，但是如果没有成功，

title_cover = " ".join(span.text for span in div.select("title absolute serif"))
title_sidebar = " ".join(span.text for span in div.select("bold"))
time_cover_and_sidebar = div.find('span', attrs={'class': 'upper'}).text
news_cover = f" {time_cover_and_sidebar} {place.upper()}, {title_cover} (TMW)"
news_sidebar = f" {time_cover_and_sidebar} {place.upper()}, {title_sidebar} (TMW)"
results_cover.append( [number, time_cover_and_sidebar, place, title_cover, news, link] )
results_sidebar.append( [number, time_cover_and_sidebar, place, title_sidebar, news, link])

我想获得的输出与我已经提供的代码（时间，地点/名称团队，标题）相同，而我也想插入时间，标题和封面和侧栏新闻的内容

完成可执行代码：

import tkinter as tk   # PEP8: `import *` is not preferred
from tkinter import ttk
from tkinter.scrolledtext import ScrolledText 
import requests
import requests_cache 
from bs4 import BeautifulSoup
import pandas as pd

# PEP8: all imports at the beginning

# --- functions ---   # PEP8: all functions directly after imports

def get_data_for(place):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
    }

    results = []

    response = requests.get(f'https://www.tuttomercatoweb.com/{place}/', headers=headers)
    print('url:', response.url)
    print('status:', response.status_code)
    #print('html:', response.text[:1000])

    soup = BeautifulSoup(response.content, 'html.parser')

    news = soup.find_all('div', attrs={"class": "tcc-list-news"})

    for number, each in enumerate(news):
        for div in each.find_all("div"):
            time  = div.find('span', attrs={'class': 'hh serif'}).text
            title = " ".join(span.text for span in div.select("a > span"))
            news = f" {time} {place.upper()}, {title} (TMW)"
            link  = div.find('a')['href']
            results.append( [number, time, place, title, news, link] )

    return results

def all_titles():
    global df

    allnews = []  # local variable

    for place in ['atalanta',  'bologna']:
        print('search:', place)
        results = get_data_for(place)
        print('found:', len(results))
        allnews += results
        text_download.insert('end', f"search: {place}\nfound: {len(results)}\n")

    df = pd.DataFrame(allnews, columns=['number', 'time', 'place', 'title', 'news', 'link'])
    df = df.sort_values(by=['number', 'time', 'place', 'title'], ascending=[True, False, True, True])
    df = df.reset_index()

    listbox_title.delete('0', 'end')

    for index, row in df.iterrows():
        listbox_title.insert('end', row['news'])

def content(event=None):   # `command=` executes without `event`, but `bind` executes with `event` - so it needs default value
    # tuple
    selection = listbox_title.curselection()
    print('selection:', selection)

    if selection:

        item = df.iloc[selection[-1]]
        #print('item:', item)

        url = item['link']
        #print('url:', url)

        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
        }

        # keep page in database `SQLite` 
        # https://github.com/reclosedev/requests-cache
        # https://sqlite.org/index.html
        session = requests_cache.CachedSession('titles')
        response = session.get(url, headers=headers)
        #response = requests.get(url, headers=headers)
        soup = BeautifulSoup(response.content, 'html.parser')

        content_download = "\n".join(item.get_text() for item in soup.select("div.text.mbottom"))

        text_download.delete('1.0', 'end') # remove previous content)
        text_download.insert('end', content_download)

# --- main ---

df = None

window = tk.Tk()
window.geometry("800x800")

# ---
# [Tkinter: How to display Listbox with Scrollbar — furas.pl](https://blog.furas.pl/python-tkitner-how-to-display-listbox-with-scrollbar-gb.html)

frame_title = tk.Frame(window)
frame_title.pack(fill='both', expand=True, pady=5, padx=5)

listbox_title = tk.Listbox(frame_title, selectbackground="#960000", selectforeground="white", bg="white")
listbox_title.pack(side='left', fill='both', expand=True)

scrollbar_title = tk.Scrollbar(frame_title)
scrollbar_title.pack(side='left', fill='y')

scrollbar_title['command'] = listbox_title.yview
listbox_title.config(yscrollcommand=scrollbar_title.set)

listbox_title.bind('<Double-Button-1>', content)  # it executes `content(event)`

# ----

text_download = ScrolledText(window, bg="white")
text_download.pack(fill='both', expand=True, pady=0, padx=5)

# ----

buttons_frame = tk.Frame(window)
buttons_frame.pack(fill='x')

button1 = tk.Button(buttons_frame, text="View Titles", command=all_titles)  # don't use `[]` to execute functions
button1.pack(side='left', pady=5, padx=5)

button2 = tk.Button(buttons_frame, text="View Content", command=content)   # don't use `[]` to execute functions
button2.pack(side='left', pady=5, padx=(0,5))

window.mainloop()

Answer 1

将来，将您的帖子/问题调整为仅包括最少的代码来回答问题。 TKINTER并不是您想在此处修复的一部分。

话虽如此，我只更新了有关获取这些封面和附带新闻的代码。您是正确的，您需要转到链接以获取数据。您可以从HTML中解析，或者它们确实在网站上的＆lt; script＆gt;标签中提供了一个不错的JSON格式以获取该格式。

import tkinter as tk   # PEP8: `import *` is not preferred
from tkinter import ttk
from tkinter.scrolledtext import ScrolledText 
import requests
import requests_cache 
from bs4 import BeautifulSoup
import pandas as pd
import re
import json
from dateutil import parser
import datetime
import locale

# PEP8: all imports at the beginning

# --- functions ---   # PEP8: all functions directly after imports

def get_data_for(place):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
    }

    results = []

    response = requests.get(f'https://www.tuttomercatoweb.com/{place}/', headers=headers)
    print('url:', response.url)
    print('status:', response.status_code)
    #print('html:', response.text[:1000])

    soup = BeautifulSoup(response.content, 'html.parser')

    #Cover
    cover_news = soup.find('div', {'class':'box pp'})
    link = cover_news.find('a', href=True)['href']
    coverNewsResponse = requests.get(link, headers=headers)
    coverSoup = BeautifulSoup(coverNewsResponse.content, 'html.parser')
    jsonStr = str(coverSoup.find('script'))
    jsonStr = re.search('({.*})', jsonStr).group(1)
    jsonData = json.loads(jsonStr)
    
    timePublished = parser.parse(jsonData['datePublished']).strftime("%H:%M")
    datePublished = parser.parse(jsonData['datePublished']).strftime("%Y-%m-%d")
    title = jsonData['headline']
    news = f" {timePublished} {place.upper()}, {title} (TMW)"
    results.append( [datePublished, timePublished, place, title, news, link] )

    # Side panel
    side_news = soup.find_all('div', {'class':'box small'})
    for each in side_news:
        link = each.find('a', href=True)['href']
        sideNewsResponse = requests.get(link, headers=headers)
        sideSoup = BeautifulSoup(sideNewsResponse.content, 'html.parser')
        jsonStr = str(sideSoup.find('script'))
        jsonStr = re.search('({.*})', jsonStr).group(1)
        jsonData = json.loads(jsonStr)
        
        timePublished = parser.parse(jsonData['datePublished']).strftime("%H:%M")
        datePublished = parser.parse(jsonData['datePublished']).strftime("%Y-%m-%d")
        title = jsonData['headline']
        news = f" {timePublished} {place.upper()}, {title} (TMW)"
        results.append( [datePublished, timePublished, place, title, news, link] )
        
        

    news = soup.find_all('div', attrs={"class": "tcc-list-news"})

    for each in news:
        for div in each.find_all("div"):
            timePublished  = div.find('span', attrs={'class': 'hh serif'}).text
            datePublished = div.find_previous('div', {'class':'tcc-border upper date'})
            
            if datePublished:
                if datePublished.text in ['Ieri']:
                    yesterday = datetime.datetime.today() - datetime.timedelta(days = 1)
                    datePublished = yesterday.strftime("%Y-%m-%d")
                else:
                    locale.setlocale(locale.LC_ALL, 'it')
                    currentYear = datetime.datetime.today().strftime("%Y")
                    dateStr = datePublished.text
                    dateStr = datetime.datetime.strptime(dateStr + ' ' + currentYear, '%A %d %B %Y')
                    datePublished = dateStr.strftime("%Y-%m-%d")
            else:
                datePublished = datetime.datetime.today().strftime("%Y-%m-%d")
            
            title = " ".join(span.text for span in div.select("a > span"))
            news = f" {timePublished} {place.upper()}, {title} (TMW)"
            link  = div.find('a')['href']
            results.append( [datePublished, timePublished, place, title, news, link] )

    return results

def all_titles():
    global df

    allnews = []  # local variable

    for place in ['atalanta',  'bologna']:
        print('search:', place)
        try:
            results = get_data_for(place)
            print('found:', len(results))
            allnews += results
            text_download.insert('end', f"search: {place}\nfound: {len(results)}\n")
        except Exception as e:
            print(f'Error with {place}')
            continue
            

    df = pd.DataFrame(allnews, columns=['date', 'time', 'place', 'title', 'news', 'link'])
    df = df.sort_values(by=['date', 'time', 'place', 'title'], ascending=[False, False, True, True])
    df = df.drop_duplicates(subset=['date', 'time', 'place', 'title'])
    df = df.reset_index(drop=True)

    listbox_title.delete('0', 'end')

    for index, row in df.iterrows():
        listbox_title.insert('end', row['news'])

def content(event=None):   # `command=` executes without `event`, but `bind` executes with `event` - so it needs default value
    # tuple
    selection = listbox_title.curselection()
    print('selection:', selection)

    if selection:

        item = df.iloc[selection[-1]]
        #print('item:', item)

        url = item['link']
        #print('url:', url)

        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
        }

        # keep page in database `SQLite` 
        # https://github.com/reclosedev/requests-cache
        # https://sqlite.org/index.html
        session = requests_cache.CachedSession('titles')
        response = session.get(url, headers=headers)
        #response = requests.get(url, headers=headers)
        soup = BeautifulSoup(response.content, 'html.parser')

        content_download = "\n".join(item.get_text() for item in soup.select("div.text.mbottom"))

        text_download.delete('1.0', 'end') # remove previous content)
        text_download.insert('end', content_download)

# --- main ---

df = None

window = tk.Tk()
window.geometry("800x800")

# ---
# [Tkinter: How to display Listbox with Scrollbar — furas.pl](https://blog.furas.pl/python-tkitner-how-to-display-listbox-with-scrollbar-gb.html)

frame_title = tk.Frame(window)
frame_title.pack(fill='both', expand=True, pady=5, padx=5)

listbox_title = tk.Listbox(frame_title, selectbackground="#960000", selectforeground="white", bg="white")
listbox_title.pack(side='left', fill='both', expand=True)

scrollbar_title = tk.Scrollbar(frame_title)
scrollbar_title.pack(side='left', fill='y')

scrollbar_title['command'] = listbox_title.yview
listbox_title.config(yscrollcommand=scrollbar_title.set)

listbox_title.bind('<Double-Button-1>', content)  # it executes `content(event)`

# ----

text_download = ScrolledText(window, bg="white")
text_download.pack(fill='both', expand=True, pady=0, padx=5)

# ----

buttons_frame = tk.Frame(window)
buttons_frame.pack(fill='x')

button1 = tk.Button(buttons_frame, text="View Titles", command=all_titles)  # don't use `[]` to execute functions
button1.pack(side='left', pady=5, padx=5)

button2 = tk.Button(buttons_frame, text="View Content", command=content)   # don't use `[]` to execute functions
button2.pack(side='left', pady=5, padx=(0,5))

window.mainloop()