Question

17.5 节的 flags2 系列示例都使用了 flags2_common.py 模块（见示例 A-10）里的函数。

示例 A-10　flags2_common.py

"""为后续flag示例提供实用函数。
"""

import os
import time
import sys
import string
import argparse
from collections import namedtuple
from enum import Enum


Result = namedtuple('Result', 'status data')

HTTPStatus = Enum('Status', 'ok not_found error')

POP20_CC = ('CN IN US ID BR PK NG BD RU JP '
      'MX PH VN ET EG DE IR TR CD FR').split()

DEFAULT_CONCUR_REQ = 1
MAX_CONCUR_REQ = 1

SERVERS = {
  'REMOTE': 'http://flupy.org/data/flags',
  'LOCAL':  'http://localhost:8001/flags',
  'DELAY':  'http://localhost:8002/flags',
  'ERROR':  'http://localhost:8003/flags',
}
DEFAULT_SERVER = 'LOCAL'

DEST_DIR = 'downloads/'
COUNTRY_CODES_FILE = 'country_codes.txt'


def save_flag(img, filename):
  path = os.path.join(DEST_DIR, filename)
  with open(path, 'wb') as fp:
    fp.write(img)


def initial_report(cc_list, actual_req, server_label):
  if len(cc_list) <= 10:
    cc_msg = ', '.join(cc_list)
  else:
    cc_msg = 'from {} to {}'.format(cc_list[0], cc_list[-1])
  print('{} site: {}'.format(server_label, SERVERS[server_label]))
  msg = 'Searching for {} flag{}: {}'
  plural = 's' if len(cc_list) != 1 else ''
  print(msg.format(len(cc_list), plural, cc_msg))
  plural = 's' if actual_req != 1 else ''
  msg = '{} concurrent connection{} will be used.'
  print(msg.format(actual_req, plural))


def final_report(cc_list, counter, start_time):
  elapsed = time.time() - start_time
  print('-' * 20)
  msg = '{} flag{} downloaded.'
  plural = 's' if counter[HTTPStatus.ok] != 1 else ''
  print(msg.format(counter[HTTPStatus.ok], plural))
  if counter[HTTPStatus.not_found]:
    print(counter[HTTPStatus.not_found], 'not found.')
  if counter[HTTPStatus.error]:
    plural = 's' if counter[HTTPStatus.error] != 1 else ''
    print('{} error{}.'.format(counter[HTTPStatus.error], plural))
  print('Elapsed time: {:.2f}s'.format(elapsed))


def expand_cc_args(every_cc, all_cc, cc_args, limit):
  codes = set()
  A_Z = string.ascii_uppercase
  if every_cc:
    codes.update(a+b for a in A_Z for b in A_Z)
  elif all_cc:
    with open(COUNTRY_CODES_FILE) as fp:
      text = fp.read()
    codes.update(text.split())
  else:
    for cc in (c.upper() for c in cc_args):
      if len(cc) == 1 and cc in A_Z:
        codes.update(cc+c for c in A_Z)
      elif len(cc) == 2 and all(c in A_Z for c in cc):
        codes.add(cc)
      else:
        msg = 'each CC argument must be A to Z or AA to ZZ.'
        raise ValueError('*** Usage error: '+msg)
  return sorted(codes)[:limit]


def process_args(default_concur_req):
  server_options = ', '.join(sorted(SERVERS))
  parser = argparse.ArgumentParser(
        description='Download flags for country codes. '
        'Default: top 20 countries by population.')
  parser.add_argument('cc', metavar='CC', nargs='*',
        help='country code or 1st letter (eg. B for BA...BZ)')
  parser.add_argument('-a', '--all', action='store_true',
        help='get all available flags (AD to ZW)')
  parser.add_argument('-e', '--every', action='store_true',
        help='get flags for every possible code (AA...ZZ)')
  parser.add_argument('-l', '--limit', metavar='N', type=int,
        help='limit to N first codes', default=sys.maxsize)
  parser.add_argument('-m', '--max_req', metavar='CONCURRENT', type=int,
        default=default_concur_req,
        help='maximum concurrent requests (default={})'
            .format(default_concur_req))
  parser.add_argument('-s', '--server', metavar='LABEL',
        default=DEFAULT_SERVER,
        help='Server to hit; one of {} (default={})'
            .format(server_options, DEFAULT_SERVER))
  parser.add_argument('-v', '--verbose', action='store_true',
        help='output detailed progress info')
  args = parser.parse_args()
  if args.max_req < 1:
    print('*** Usage error: --max_req CONCURRENT must be >= 1')
    parser.print_usage()
    sys.exit(1)
  if args.limit < 1:
    print('*** Usage error: --limit N must be >= 1')
    parser.print_usage()
    sys.exit(1)
  args.server = args.server.upper()
  if args.server not in SERVERS:
    print('*** Usage error: --server LABEL must be one of',
        server_options)
    parser.print_usage()
    sys.exit(1)
  try:
    cc_list = expand_cc_args(args.every, args.all, args.cc, args.limit)
  except ValueError as exc:
    print(exc.args[0])
    parser.print_usage()
    sys.exit(1)
  if not cc_list:
    cc_list = sorted(POP20_CC)
  return args, cc_list

def main(download_many, default_concur_req, max_concur_req):
  args, cc_list = process_args(default_concur_req)
  actual_req = min(args.max_req, max_concur_req, len(cc_list))
  initial_report(cc_list, actual_req, args.server)
  base_url = SERVERS[args.server]
  t0 = time.time()
  counter = download_many(cc_list, base_url, args.verbose, actual_req)
  assert sum(counter.values()) == len(cc_list), \
    'some downloads are unaccounted for'
  final_report(cc_list, counter, t0)

flags2_sequential.py 脚本（见示例 A-11）是对比两种并发实现的基准。flags2_threadpool.py 脚本（见示例 17-14）还使用了 flags2_sequential.py 脚本中的 get_flag 和 download_one 两个函数。

示例 A-11　flags2_sequential.py

"""下载多个国家的国旗（包含错误处理代码）。

依序下载版

运行示例::

  $ python3 flags2_sequential.py -s DELAY b
  DELAY site: http://localhost:8002/flags
  Searching for 26 flags: from BA to BZ
  1 concurrent connection will be used.
  --------------------
  17 flags downloaded.
  9 not found.
  Elapsed time: 13.36s

"""

import collections

import requests
import tqdm

from flags2_common import main, save_flag, HTTPStatus, Result


DEFAULT_CONCUR_REQ = 1
MAX_CONCUR_REQ = 1

# BEGIN FLAGS2_BASIC_HTTP_FUNCTIONS
def get_flag(base_url, cc):
  url = '{}/{cc}/{cc}.gif'.format(base_url, cc=cc.lower())
  resp = requests.get(url)

  if resp.status_code != 200:
    resp.raise_for_status()
  return resp.content

def download_one(cc, base_url, verbose=False):
  try:
    image = get_flag(base_url, cc)
  except requests.exceptions.HTTPError as exc:
    res = exc.response
    if res.status_code == 404:
      status = HTTPStatus.not_found
      msg = 'not found'
    else:
      raise
  else:
    save_flag(image, cc.lower() + '.gif')
    status = HTTPStatus.ok
    msg = 'OK'

  if verbose:
    print(cc, msg)

  return Result(status, cc)
# END FLAGS2_BASIC_HTTP_FUNCTIONS

# BEGIN FLAGS2_DOWNLOAD_MANY_SEQUENTIAL
def download_many(cc_list, base_url, verbose, max_req):
  counter = collections.Counter()
  cc_iter = sorted(cc_list)
  if not verbose:
    cc_iter = tqdm.tqdm(cc_iter)
  for cc in cc_iter:
    try:
      res = download_one(cc, base_url, verbose)
    except requests.exceptions.HTTPError as exc:
      error_msg = 'HTTP error {res.status_code} - {res.reason}'
      error_msg = error_msg.format(res=exc.response)
    except requests.exceptions.ConnectionError as exc:
      error_msg = 'Connection error'
    else:
      error_msg = ''
      status = res.status

    if error_msg:
      status = HTTPStatus.error
    counter[status] += 1
    if verbose and error_msg:
      print('*** Error for {}: {}'.format(cc, error_msg))

  return counter
# END FLAGS2_DOWNLOAD_MANY_SEQUENTIAL

if __name__ == '__main__':
  main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ)