怎样使用aiohttp请求多个url

Question

这是aiohttp官方请求单个url的方式，如果请求多个url并获取内容应该怎么做？

import aiohttp
import asyncio

async def fetch(session, url):
    async with session.get(url) as response:
        return await response.text()

async def main():
    async with aiohttp.ClientSession() as session:
        html = await fetch(session, 'http://python.org')
        print(html)

loop = asyncio.get_event_loop()
loop.run_until_complete(main())

Answer 1

urls = ['http://baidu.com', 'http://qq.com']
async def get(url):
    async with aiohttp.ClientSession() as session:
        html = await fetch(session, url)
        print(html)
tasks = [get(x) for x in urls]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(*tasks))

Answer 2

import asyncio
import aiohttp
import time

from my_spider import urls

async def async_craw(url):

async with aiohttp.ClientSession() as session:
    async with session.get(url) as resp:
        res = await resp.json()
        print(f'craw_url: {url},', res.get('message'))

超级循环

loop = asyncio.get_event_loop()

tasks = [

loop.create_task(async_craw(url))
for url in urls

]

start_time = time.time()

等待任务完成

loop.run_until_complete(asyncio.wait(tasks))
end_time = time.time()
print(f'耗费时间:{end_time - start_time}')