Go 使用 http 连接 fd 泄露问题

Question

模拟一下，服务端：

// gohttpserver.go

package main

import (
  "fmt"
  "log"
  "net/http"
  "time"
)

func main() {
  http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Hi there, say I love you! current time  %v", time.Now())
  })

  log.Fatal(http.ListenAndServe(":9901", nil))
}

客户端：

// gohttpclient.go
package main

import (
  "fmt"
  "io/ioutil"
  "net"
  "net/http"
  "time"
)

func main() {
  url := "http://127.0.0.1:9901"
  for i := 0; i < 10000; i++ {
    func() {
      trans := &http.Transport{
        DialContext: (&net.Dialer{
          Timeout:   3 * time.Second,
          KeepAlive: 30 * time.Second,
        }).DialContext,
        ForceAttemptHTTP2:     true,
        MaxIdleConns:          100,
        IdleConnTimeout:       90 * time.Second,
        TLSHandshakeTimeout:   10 * time.Second,
        ExpectContinueTimeout: 1 * time.Second,
      }
      //defer trans.CloseIdleConnections()

      client := &http.Client{
        Timeout:   time.Duration(100) * time.Millisecond,
        Transport: trans,
      }

      req, err := http.NewRequest("GET", url, nil)
      if err != nil {
        return
      }

      //req.Close = true // fd leak without setting this
      resp, err := client.Do(req)
      if err != nil {
        // handle error
      }
      defer resp.Body.Close()
      haha, _ := ioutil.ReadAll(resp.Body)
      fmt.Println(string(haha))
    }()

    time.Sleep(500 * time.Millisecond)
  }
}

运行服务端和客户端

$ go run gohttpserver.go &
$ go run gohttpclient.go &

查看连接数：

$ netstat -an |grep 9901 | grep ESTABLISHED
tcp4       0      0  127.0.0.1.9901         127.0.0.1.54403        ESTABLISHED
tcp4       0      0  127.0.0.1.54403        127.0.0.1.9901         ESTABLISHED
tcp4       0      0  127.0.0.1.9901         127.0.0.1.54402        ESTABLISHED
tcp4       0      0  127.0.0.1.54402        127.0.0.1.9901         ESTABLISHED
tcp4       0      0  127.0.0.1.9901         127.0.0.1.54398        ESTABLISHED
tcp4       0      0  127.0.0.1.54398        127.0.0.1.9901         ESTABLISHED
tcp4       0      0  127.0.0.1.9901         127.0.0.1.54397        ESTABLISHED
tcp4       0      0  127.0.0.1.54397        127.0.0.1.9901         ESTABLISHED

然后放开客户端的defer trans.CloseIdleConnections()或者req.Close = true，重新运行，都可以修复此问题。

查看源代码 net/http/client.go，可以找到

func (c *Client) transport() RoundTripper {
  if c.Transport != nil {
    return c.Transport
  }
  return DefaultTransport
}

然后在 net/http/transport.go 中，继续看，连接池是挂在 Transport 上，所以每次新建 Transport，如果不关闭，就会导致连接泄露。

// DefaultTransport is the default implementation of Transport and is
// used by DefaultClient. It establishes network connections as needed
// and caches them for reuse by subsequent calls. It uses HTTP proxies
// as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and
// $no_proxy) environment variables.
var DefaultTransport RoundTripper = &Transport{
  Proxy: ProxyFromEnvironment,
  DialContext: (&net.Dialer{
    Timeout:   30 * time.Second,
    KeepAlive: 30 * time.Second,
    DualStack: true,
  }).DialContext,
  ForceAttemptHTTP2:     true,
  MaxIdleConns:          100,
  IdleConnTimeout:       90 * time.Second,
  TLSHandshakeTimeout:   10 * time.Second,
  ExpectContinueTimeout: 1 * time.Second,
}

// ...
type Transport struct {
  idleMu       sync.Mutex
  closeIdle    bool                                // user has requested to close all idle conns
  idleConn     map[connectMethodKey][]*persistConn // most recently used at end
  idleConnWait map[connectMethodKey]wantConnQueue  // waiting getConns
  idleLRU      connLRU

  reqMu       sync.Mutex
  reqCanceler map[*Request]func(error)

  altMu    sync.Mutex   // guards changing altProto only
  altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme

  connsPerHostMu   sync.Mutex
  connsPerHost     map[connectMethodKey]int
  connsPerHostWait map[connectMethodKey]wantConnQueue // waiting getConns
// ...
}

感谢：

1. golang http client 关闭重用连接两种方法
2. 如何关闭 Golang 中的 HTTP 连接 How to Close Golang's HTTP connection
3. google golang http.Request close

查看指定进程的连接信息

# ps -ef|grep rig
root      4865  4258  0 11:30 pts/0    00:00:00 grep --color=auto rig
footsto+ 14995     1  0 11月12 ?      00:07:29 ./rig_linux_amd64 -u
footsto+ 15459     1  0 10月18 ?      00:45:26 java -server -Xmx768m -Xms768m -Xmn384m -XX:PermSize=128m -Xss256k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -Dlogpath.base=/home/footstone/logs/bjca-app-rig-metrics-server -cp /home/footstone/YUQI_TEST/config:/home/footstone/YUQI_TEST/lib/* -jar /home/footstone/YUQI_TEST/lib/rig-metrics-server-1.0.0-SNAPSHOT.jar bjca-app-rig-metrics-server
rigaga   20730     1  0 9月25 ?       03:30:40 /usr/bin/rigaga -config /etc/rigaga/rigaga.conf -config-directory /etc/rigaga/rigaga.d
# lsof -p 14995
COMMAND     PID      USER   FD      TYPE     DEVICE SIZE/OFF       NODE NAME
rig_linux 14995 footstone  cwd       DIR      253,1     4096    2818080 /home/footstone/go-opads/ops-rig/20191112-1573526761
rig_linux 14995 footstone  rtd       DIR      253,1     4096          2 /
rig_linux 14995 footstone  txt       REG      253,1 22682823    2818082 /home/footstone/go-opads/ops-rig/20191112-1573526761/rig_linux_amd64
rig_linux 14995 footstone  mem       REG      253,1  2173512     328343 /usr/lib64/libc-2.17.so
rig_linux 14995 footstone  mem       REG      253,1   144792     328369 /usr/lib64/libpthread-2.17.so
rig_linux 14995 footstone  mem       REG      253,1   164240     328336 /usr/lib64/ld-2.17.so
rig_linux 14995 footstone    0r      CHR        1,3      0t0       1028 /dev/null
rig_linux 14995 footstone    1w      REG      253,1  1179657    2818128 /home/footstone/go-opads/ops-rig/20191112-1573526761/nohup.out
rig_linux 14995 footstone    2w      REG      253,1  1179657    2818128 /home/footstone/go-opads/ops-rig/20191112-1573526761/nohup.out
rig_linux 14995 footstone    3r  a_inode        0,9        0       5937 inotify
rig_linux 14995 footstone    4u  a_inode        0,9        0       5937 [eventpoll]
rig_linux 14995 footstone    5u  a_inode        0,9        0       5937 [eventpoll]
rig_linux 14995 footstone    6r     FIFO        0,8      0t0 1441054004 pipe
rig_linux 14995 footstone    7w     FIFO        0,8      0t0 1441054004 pipe
rig_linux 14995 footstone    8w      CHR        1,3      0t0       1028 /dev/null
rig_linux 14995 footstone   10u     IPv6 1441050975      0t0        TCP *:10099 (LISTEN)
rig_linux 14995 footstone   11w      REG      253,1 29608321    2818151 /home/footstone/go-opads/ops-rig/20191112-1573526761/var/rig_linux_amd64.log.201911260000