压力测试下的 golang http client 造成的连接数过多

Question

看博客，自己运行验证

Tuning the Go HTTP Client Settings for Load Testing

loadtest.go

package main

import (
  "fmt"
  "io"
  "io/ioutil"
  "net/http"
)

func startWebserver() {
  http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Hello, %q", r.URL.Path)
  })

  http.ListenAndServe(":8080", nil)
}

func startLoadTest() {
  for count := 0; ; count++ {
    resp, err := http.Get("http://localhost:8080/")
    if err != nil {
      panic(fmt.Sprintf("Got error: %v", err))
    }

    // io.Copy(ioutil.Discard, resp.Body) // <-- v2 add this line
    resp.Body.Close()
    fmt.Printf("Finished GET request #%v\n", count)
  }
}

func main() {
  // start a webserver in a goroutine
  go startWebserver()
  startLoadTest()
}

[2020-08-28 11:32:39.459] ❯ go run loadtest.go
2020/08/28 11:32:39 Finished GET request #96105
panic: Got error: Get "http://localhost:8080/": dial tcp [::1]:8080: connect: resource temporarily unavailable

goroutine 1 [running]:
main.startLoadTest()
  /Users/bingoobjca/github/weeklyreport/2020/loadtest.go:22 +0x168
main.main()
  /Users/bingoobjca/github/weeklyreport/2020/loadtest.go:32 +0x3a
exit status 2

[2020-08-28 11:30:18.925] ❯ while true; do date "+%Y-%m-%d %H:%M:%S"; netstat -n | grep -i 8080 | grep -i time_wait | wc -l; sleep 3; done
2020-08-28 11:30:27
     576
2020-08-28 11:30:30
    7229
2020-08-28 11:30:33
   13068
2020-08-28 11:30:36
   17130
2020-08-28 11:30:39
   19069
2020-08-28 11:30:42
   20892
2020-08-28 11:30:45
   22642
2020-08-28 11:30:48
   24283
2020-08-28 11:30:51
   25677
2020-08-28 11:30:55
   27111
2020-08-28 11:30:58
   28496
2020-08-28 11:31:01
   29842
2020-08-28 11:31:04
   31131
2020-08-28 11:31:07
    6979
2020-08-28 11:31:26
   11667
2020-08-28 11:31:29
   15000
2020-08-28 11:31:33
   18616
2020-08-28 11:31:36
   20287
2020-08-28 11:31:39
   20926
2020-08-28 11:31:42
   22444
2020-08-28 11:31:45
   24109
2020-08-28 11:31:48
   25698
2020-08-28 11:31:51
   27123
2020-08-28 11:31:54
   28364
2020-08-28 11:31:57
   29710
2020-08-28 11:32:01
   31009
2020-08-28 11:32:04
    6434
2020-08-28 11:32:23
    9741
2020-08-28 11:32:26
   13645
2020-08-28 11:32:29
   18158
2020-08-28 11:32:32
   20768
2020-08-28 11:32:35
   26138
2020-08-28 11:32:38
   32443
2020-08-28 11:32:41
   32689
2020-08-28 11:32:45
   32689
2020-08-28 11:32:48
   32689
2020-08-28 11:32:51
   32689
2020-08-28 11:32:54
       0

loadtest.go

add io.Copy(ioutil.Discard, resp.Body) // <-- v2 add this line before resp.Body.Close()

[2020-08-28 11:33:12.700] ❯ while true; do date "+%Y-%m-%d %H:%M:%S"; netstat -n | grep -i 8080 | grep -i time_wait | wc -l; sleep 3; done
2020-08-28 11:36:09
       0
2020-08-28 11:36:12
       0
2020-08-28 11:36:15
       0
2020-08-28 11:36:18
       0
2020-08-28 11:36:21
       0

好了，都是 TIME_WATI 都是 0 了，为啥呢，参考 Is it necessary to consume response body before closing it (net/http client code)? 里面的说法：

the io.Copy() drains the body meaning it can be reused via keepalive.
if there's still data pending, the Close() will actually close it and it can't be reused

并发 client

package main

import (
  "fmt"
  "io"
  "io/ioutil"
  "net/http"
  "time"
)

func startWebserver() {
  http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    time.Sleep(time.Millisecond * 50)
    fmt.Fprintf(w, "Hello, %q", r.URL.Path)
  })

  http.ListenAndServe(":8080", nil)
}

func startLoadTest() {
  for count := 0; ; count++ {
    resp, err := http.Get("http://localhost:8080/")
    if err != nil {
      panic(fmt.Sprintf("Got error: %v", err))
    }

    io.Copy(ioutil.Discard, resp.Body) // <-- v2 add this line
    resp.Body.Close()
    fmt.Printf("Finished GET request #%v\n", count)
  }
}

func resetClient() {
  // Customize the Transport to have larger connection pool
  defaultRoundTripper := http.DefaultTransport
  defaultTransportPointer, ok := defaultRoundTripper.(*http.Transport)
  if !ok {
    panic(fmt.Sprintf("defaultRoundTripper not an *http.Transport"))
  }
  defaultTransport := *defaultTransportPointer // dereference it to get a copy of the struct that the pointer points to
  defaultTransport.MaxIdleConns = 100
  defaultTransport.MaxIdleConnsPerHost = 100

  http.DefaultClient = &http.Client{Transport: &defaultTransport}
}

func main() {
  // start a webserver in a goroutine
  go startWebserver()
  // startLoadTest()

  // resetClient()
  for i := 0; i < 100; i++ {
    go startLoadTest()
  }

  time.Sleep(time.Second * 2400)
}

很快就 panic: Got error: Get "http://localhost:8080/": dial tcp [::1]:8080: socket: too many open files

放开 // startLoadTest() 注释，修复。

因为默认 const DefaultMaxIdleConnsPerHost = 2，造成每个主机最大闲置连接数为 2100 个并发，98 个需要等待 time_wait。

更多资源

1. Drain Response.Body to enable TCP/TLS connection reuse (4x speedup) #317
2. Do i need to read the body before close it?

$ lsof -p 23069

COMMAND   PID  USER   FD   TYPE DEVICE SIZE/OFF     NODE NAME
go      23069 diego  cwd    DIR   8,18     4096 14686472 /home/diego/projects/go/1.8.1/src/sample
go      23069 diego  rtd    DIR   8,18     4096        2 /
go      23069 diego  txt    REG   8,18 10073055 13523309 /home/diego/programs/go/1.8.1/bin/go
go      23069 diego  mem    REG   8,18  1981712  8129743 /usr/lib/libc-2.25.so
go      23069 diego  mem    REG   8,18   146568  8129721 /usr/lib/libpthread-2.25.so
go      23069 diego  mem    REG   8,18   168656  8129742 /usr/lib/ld-2.25.so
go      23069 diego    0u   CHR  136,0      0t0        3 /dev/pts/0
go      23069 diego    1u   CHR  136,0      0t0        3 /dev/pts/0
go      23069 diego    2u   CHR  136,0      0t0        3 /dev/pts/0