连接使用请求模块刮擦时中止错误

Question

我正在使用请求模块在Python中进行一些网络刮擦，但是每次我使用标题和代理发送请求时，我都会收到连接中止的错误，即使我被告知如果我确实放置了问题标题。

这是我的完整代码：

import requests 
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
import pickle 


session = requests.Session()
unique = {}
retry = Retry(connect=3,backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://',adapter)
headers= {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:101.0) Gecko/20100101 Firefox/101.0'}
proxies1 = {"http":"http://23.238.33.186:80"}


lstofwords = pickle.load(f)
dic={}
first = int(len(lstofwords)/3)
for x in range(0,first):
    r = session.get(f'https://www.merriam-webster.com/dictionary/{lstofwords[x]}/',headers=headers,proxies=proxies1)
    soup = BeautifulSoup(r.content,'html.parser')
    descriptions = soup.find_all('span', class_='unText') 
    descs = [i.text for i in descriptions]
    dic[lstofwords[x]]= descs

for x in range(first,second):
    r = session.get(f'https://www.merriam-webster.com/dictionary/{lstofwords[x]}/',headers=headers,proxies=proxies2)
    soup = BeautifulSoup(r.content,'html.parser')
    descriptions = soup.find_all('span', class_='unText') 
    descs = [i.text for i in descriptions]
    dic[lstofwords[x]]= descs

for x in range(second,third):
    r = session.get(f'https://www.merriam-webster.com/dictionary/{lstofwords[x]}/',headers=headers,proxies=proxies3)
    soup = BeautifulSoup(r.content,'html.parser')
    descriptions = soup.find_all('span', class_='unText') 
    descs = [i.text for i in descriptions]
    dic[lstofwords[x]]= descs

with open('dictionary.pkl','wb') as f:
    pickle.dump(dic, f)
print("Success")

以下是我收到的错误消息：

Traceback (most recent call last):
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 703, in urlopen
    httplib_response = self._make_request(
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 449, in _make_request
    six.raise_from(e, None)
  File "<string>", line 3, in raise_from
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 444, in _make_request
    httplib_response = conn.getresponse()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1374, in getresponse
    response.begin()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 318, in begin
    version, status, reason = self._read_status()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 287, in _read_status
    raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\adapters.py", line 440, in send
    resp = conn.urlopen(
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 785, in urlopen
    retries = retries.increment(
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\util\retry.py", line 550, in increment
    raise six.reraise(type(error), error, _stacktrace)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\packages\six.py", line 769, in reraise
    raise value.with_traceback(tb)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 703, in urlopen
    httplib_response = self._make_request(
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 449, in _make_request
    six.raise_from(e, None)
  File "<string>", line 3, in raise_from
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 444, in _make_request
    httplib_response = conn.getresponse()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1374, in getresponse
    response.begin()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 318, in begin
    version, status, reason = self._read_status()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 287, in _read_status
    raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\ehabb\Downloads\research.py", line 53, in <module>
    r = session.get(f'https://www.merriam-webster.com/dictionary/{lstofwords[x]}/',headers=headers,proxies=proxies1)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 542, in get
    return self.request('GET', url, **kwargs)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 529, in request
    resp = self.send(prep, **send_kwargs)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 667, in send
    history = [resp for resp in gen]
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 667, in <listcomp>
    history = [resp for resp in gen]
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 237, in resolve_redirects
    resp = self.send(
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 645, in send
    r = adapter.send(request, **kwargs)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\adapters.py", line 501, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

Answer 1

我想问题可能是您使用的代理协议。代理是http协议，当您查询https资源时。尝试使用https代理添加一行（如果可用）：

proxies1 = {
    "http": "http://23.238.33.186:80",
    "https": "https://23.238.33.186:443"
}