共计 14111 个字符，预计需要花费 36 分钟才能阅读完成。
咱们晓得个别在调用 http client 后都会 close Response.Body, 如下:
client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {return nil, err}
defer resp.Body.Close()
上面咱们来看下为什么 resp.Body 须要 Close, 肯定须要 Close 吗？
咱们先通过 ”net/http/httptrace” 来验证下:
1. 不应用 Close
代码:
package main
import (
"fmt"
"net/http"
"net/http/httptrace"
)
func main() {req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
if err != nil {panic(err)
}
trace := &httptrace.ClientTrace{GetConn: func(hostPort string) {fmt.Println("GetConn:", hostPort)
},
GotConn: func(connInfo httptrace.GotConnInfo) {fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
},
PutIdleConn: func(err error) {fmt.Printf("PutIdleConn err=%v\n", err)
},
}
req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
client := &http.Client{
Transport: &http.Transport{
MaxIdleConns: 1, // 最大 Idle
MaxConnsPerHost: 2, // 每个 host 最大 conns
}}
for i := 1; i <= 10; i++ {fmt.Printf("============== 第 [%d] 次申请 ================\n", i)
resp, err := client.Do(req)
if err != nil {fmt.Println(err)
return
}
fmt.Println(resp.Status)
}
}输入:
============== 第 [1] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:55131
GotConn remoteAddr: 183.232.231.172:443
200 OK
============== 第 [2] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:55132
GotConn remoteAddr: 183.232.231.172:443
200 OK
============== 第 [3] 次申请 ================
GetConn: www.baidu.com:443
论断:
咱们设置了 MaxConnsPerHost=2, 因为没有 close 导致没有开释连贯，执行两次申请后就卡住了，不能持续向下执行。并且第一次和第二次申请连贯没有复用
2. 只应用 Close, 不读取 resp.Body
代码:
package main
import (
"fmt"
"net/http"
"net/http/httptrace"
)
func main() {req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
if err != nil {panic(err)
}
trace := &httptrace.ClientTrace{GetConn: func(hostPort string) {fmt.Println("GetConn:", hostPort)
},
GotConn: func(connInfo httptrace.GotConnInfo) {fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
},
PutIdleConn: func(err error) {fmt.Printf("PutIdleConn err=%v\n", err)
},
}
req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
client := &http.Client{
Transport: &http.Transport{
MaxIdleConns: 1, // 最大 Idle
MaxConnsPerHost: 2, // 每个 host 最大 conns
}}
for i := 1; i <= 10; i++ {fmt.Printf("============== 第 [%d] 次申请 ================\n", i)
resp, err := client.Do(req)
if err != nil {fmt.Println(err)
return
}
resp.Body.Close() // 留神这里 close 了
fmt.Println(resp.Status)
}
}输入:
==============1================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54948
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============2================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54949
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============3================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54950
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============4================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54954
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============5================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54955
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============6================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54956
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============7================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54957
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============8================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54958
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============9================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54959
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============10================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54960
GotConn remoteAddr: 183.232.231.172:443
200 OK论断：
尽管能够继续执行，阐明连贯开释了。但 GotConn 信息 eused=false WasIdle=false 并且每次 localAddr 都是不同的, 咱们发现每次申请取得的连贯都是新申请的。
3. 只读取 resp.Body, 不应用 Close()
咱们晓得 resp.Body 实现了 io.Reader 接口办法, 咱们通常的做法就通过调用 Read()办法来读取内容。
代码:
package main
import (
"fmt"
"io"
"net/http"
"net/http/httptrace"
)
func main() {req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
if err != nil {panic(err)
}
trace := &httptrace.ClientTrace{GetConn: func(hostPort string) {fmt.Println("GetConn:", hostPort)
},
GotConn: func(connInfo httptrace.GotConnInfo) {fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
},
PutIdleConn: func(err error) {fmt.Printf("PutIdleConn err=%v\n", err)
},
}
req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
client := &http.Client{
Transport: &http.Transport{
MaxIdleConns: 1, // 最大 Idle
MaxConnsPerHost: 2, // 每个 host 最大 conns
}}
for i := 1; i <= 10; i++ {fmt.Printf("============== 第 [%d] 次申请 ================\n", i)
resp, err := client.Do(req)
if err != nil {fmt.Println(err)
return
}
io.ReadAll(resp.Body) // 读取 body 外面的内容
fmt.Println(resp.Status)
}
}输入:
============== 第 [1] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [2] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=116.037µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [3] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=72.154µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [4] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=60.102µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [5] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=71.807µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [6] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=74.616µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [7] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=47.205µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [8] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=74.207µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [9] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=52.414µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [10] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=81.137µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK论断:
咱们能够看下每次输入的 GotConn 信息 Reused=true WasIdle=true 并且 localAddr 都是雷同的。阐明每次执行完后都把连贯放回了 idleConn pool 外面，上来获取连贯时能够从 idleConn pool 外面获取。
源码剖析
(c Client) Do(req Request)–>(t Transport) roundTrip(req Request)–>t.getConn(treq, cm)–>t.queueForDial(w)->go t.dialConnFor(w)–>go readLoop()
咱们 readLoop()源码来剖析，readLoop()是循环读取连贯内容的办法:
go1.18 net/http/transport.go
(pc *persistConn) readLoop()是一个长久化连贯在不停的读取 conn 外面的内容，上面我追随源码看一下为什么要 close/readAll
func (pc *persistConn) readLoop() {
closeErr := errReadLoopExiting // default value, if not changed below
defer func() {pc.close(closeErr)
pc.t.removeIdleConn(pc)
}()
// tryPutIdleConn 就是把连贯放回 IdleConn 而后让连贯能够复用
tryPutIdleConn := func(trace *httptrace.ClientTrace) bool {if err := pc.t.tryPutIdleConn(pc); err != nil {
closeErr = err
if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled {trace.PutIdleConn(err)
}
return false
}
if trace != nil && trace.PutIdleConn != nil {trace.PutIdleConn(nil)
}
return true
}
// eofc is used to block caller goroutines reading from Response.Body
// at EOF until this goroutines has (potentially) added the connection
// back to the idle pool.
eofc := make(chan struct{})
defer close(eofc) // unblock reader on errors
// Read this once, before loop starts. (to avoid races in tests)
testHookMu.Lock()
testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
testHookMu.Unlock()
// 这里的 alive 用来判断是否持续读取连贯内容
alive := true
for alive {pc.readLimit = pc.maxHeaderResponseSize()
_, err := pc.br.Peek(1)
pc.mu.Lock()
if pc.numExpectedResponses == 0 {pc.readLoopPeekFailLocked(err)
pc.mu.Unlock()
return
}
pc.mu.Unlock()
rc := <-pc.reqch
trace := httptrace.ContextClientTrace(rc.req.Context())
var resp *Response
if err == nil {resp, err = pc.readResponse(rc, trace)
} else {err = transportReadFromServerError{err}
closeErr = err
}
if err != nil {
if pc.readLimit <= 0 {err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize())
}
select {case rc.ch <- responseAndError{err: err}:
case <-rc.callerGone:
return
}
return
}
pc.readLimit = maxInt64 // effectively no limit for response bodies
pc.mu.Lock()
pc.numExpectedResponses--
pc.mu.Unlock()
bodyWritable := resp.bodyIsWritable()
hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0
if resp.Close || rc.req.Close || resp.StatusCode <= 199 || bodyWritable {
// Don't do keep-alive on error if either party requested a close
// or we get an unexpected informational (1xx) response.
// StatusCode 100 is already handled above.
alive = false
}
......
waitForBodyRead := make(chan bool, 2)
// 最重要的就是这个把 resp.body 通过 bodyEOFSignal 封装生成新的 resp.Body，// 上面会讲到为什么通过 bodyEOFSignal 封装
body := &bodyEOFSignal{
body: resp.Body,
// 如果调用 earlyCloseFn 就执行 waitForBodyRead <- false
earlyCloseFn: func() error {
waitForBodyRead <- false
<-eofc // will be closed by deferred call at the end of the function
return nil
},
// 如果调用 fn()并且 err == io.EOF 就执行 waitForBodyRead <- true
fn: func(err error) error {
isEOF := err == io.EOF
waitForBodyRead <- isEOF
if isEOF {<-eofc // see comment above eofc declaration} else if err != nil {if cerr := pc.canceled(); cerr != nil {return cerr}
}
return err
},
}
// 从新把封装成 bodyEOFSignal 的 body 赋值给 resp.Body
resp.Body = body
if rc.addedGzip && ascii.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {resp.Body = &gzipReader{body: body}
resp.Header.Del("Content-Encoding")
resp.Header.Del("Content-Length")
resp.ContentLength = -1
resp.Uncompressed = true
}
select {// 把 resp 推送到 rc.ch 而后在 roundTrip()返回给内部
case rc.ch <- responseAndError{res: resp}:
case <-rc.callerGone:
return
}
// Before looping back to the top of this function and peeking on
// the bufio.Reader, wait for the caller goroutine to finish
// reading the response body. (or for cancellation or death)
select {
// 这里是最重要的, 从 waitForBodyRead 阻塞获取 bodyEof
case bodyEOF := <-waitForBodyRead:
replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) // before pc might return to idle pool
// 这里比拟奇妙的是如果 alive && bodyEOF && !pc.sawEOF && pc.wroteRequest()
// 才会执行 tryPutIdleConn(trace), 就是把连贯放回 idleConn, 这就是为什么应用 close()是敞开了连贯然而没有复用，而应用 ReadAll()确复用了 idle。// 因为应用应用 ReadAll 后返回的 bodyEOF=true 而间接应用 Close()返回的 bodyEOF=false 导致永远不会执行 tryPutIdleConn(trace)。// 留神 pc.sawEOF 和 body Close 没关系，这个是检测 conn 是否 Close
alive = alive &&
bodyEOF &&
!pc.sawEOF &&
pc.wroteRequest() &&
replaced && tryPutIdleConn(trace)
if bodyEOF {eofc <- struct{}{}}
case <-rc.req.Cancel:
alive = false
pc.t.CancelRequest(rc.req)
case <-rc.req.Context().Done():
alive = false
pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err())
case <-pc.closech:
alive = false
}
testHookReadLoopBeforeNextRead()}
}上面咱们来看下 resp.Body 的构造 bodyEOFSignal:
//readLoop()外面的 body
body := &bodyEOFSignal{
body: resp.Body,
// 如果调用 earlyCloseFn 就执行 waitForBodyRead <- false
earlyCloseFn: func() error {
waitForBodyRead <- false
<-eofc // will be closed by deferred call at the end of the function
return nil
},
// 如果调用 fn()并且 err == io.EOF 就执行 waitForBodyRead <- true
fn: func(err error) error {
isEOF := err == io.EOF
waitForBodyRead <- isEOF
if isEOF {<-eofc // see comment above eofc declaration} else if err != nil {if cerr := pc.canceled(); cerr != nil {return cerr}
}
return err
},
}
// bodyEOFSignal is used by the HTTP/1 transport when reading response
// bodies to make sure we see the end of a response body before
// proceeding and reading on the connection again.
//
// It wraps a ReadCloser but runs fn (if non-nil) at most
// once, right before its final (error-producing) Read or Close call
// returns. fn should return the new error to return from Read or Close.
//
// If earlyCloseFn is non-nil and Close is called before io.EOF is
// seen, earlyCloseFn is called instead of fn, and its return value is
// the return value from Close.
type bodyEOFSignal struct {
body io.ReadCloser
mu sync.Mutex // guards following 4 fields
closed bool // whether Close has been called
rerr error // sticky Read error
fn func(error) error // err will be nil on Read io.EOF
earlyCloseFn func() error // optional alt Close func used if io.EOF not seen 如果没 EOF 就会被调用}
func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {es.mu.Lock()
closed, rerr := es.closed, es.rerr
es.mu.Unlock()
if closed {return 0, errReadOnClosedResBody}
if rerr != nil {return 0, rerr}
n, err = es.body.Read(p)
if err != nil {es.mu.Lock()
defer es.mu.Unlock()
if es.rerr == nil {es.rerr = err}
//condfn 会去执行 es.fn
err = es.condfn(err)
}
return
}
func (es *bodyEOFSignal) Close() error {es.mu.Lock()
defer es.mu.Unlock()
if es.closed {return nil}
es.closed = true
// 如果调用 Close 的时候没有 EOF 就会调用 earlyCloseFn()
if es.earlyCloseFn != nil && es.rerr != io.EOF {return es.earlyCloseFn()
}
err := es.body.Close()
return es.condfn(err)
}
// caller must hold es.mu.
// 把 err 传给 es.fn, es.fn 会通过 err 做不同的操作，次要是判断 err 是不是 EOF
func (es *bodyEOFSignal) condfn(err error) error {
if es.fn == nil {return err}
err = es.fn(err)
es.fn = nil
return err
}总结
咱们收到的 resp.body 是 bodyEOFSignal, 如果不执行 Close()会导致 readLoop()阻塞。此时如果设置了最大连贯 MaxConnsPerHost 则达到连接数后不能再申请。然而如果只是 Close()而不执行 Read()则会导致连贯不会放回 idleConn
举荐用法:
不肯定要执行 Close(), 但肯定要执行 Read():
如不须要 body 外面的内容能够执行
client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {return nil, err}
defer io.Copy(io.Discard,resp.Body)如果须要 resp.Body 则执行:
client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {return nil, err}
// 这里的 close 是避免 Read 操作不能被正确执行到做兜底,
// 如果可能确保 resp.Body 被执行到也不须要 Close
defer resp.Body.Close()
// 读取 resp.Body, 如 io.ReadAll/io.Copy()...
io.ReadAll(resp.Body)
正文完
