乐趣区

关于go:golang-http-client一定要close-ResponseBody吗

咱们晓得个别在调用 http client 后都会 close Response.Body, 如下:

client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {return nil, err}
defer resp.Body.Close()

上面咱们来看下为什么 resp.Body 须要 Close, 肯定须要 Close 吗?

咱们先通过 ”net/http/httptrace” 来验证下:

1. 不应用 Close

代码:

package main

import (
    "fmt"
    "net/http"
    "net/http/httptrace"
)

func main() {req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
    if err != nil {panic(err)
    }

    trace := &httptrace.ClientTrace{GetConn: func(hostPort string) {fmt.Println("GetConn:", hostPort)
        },
        GotConn: func(connInfo httptrace.GotConnInfo) {fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
            fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
            fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
        },
        PutIdleConn: func(err error) {fmt.Printf("PutIdleConn err=%v\n", err)
        },
    }

    req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))

    client := &http.Client{
        Transport: &http.Transport{
            MaxIdleConns:    1, // 最大 Idle
            MaxConnsPerHost: 2, // 每个 host 最大 conns
        }}

    for i := 1; i <= 10; i++ {fmt.Printf("============== 第 [%d] 次申请 ================\n", i)
        resp, err := client.Do(req)
        if err != nil {fmt.Println(err)
            return
        }
        fmt.Println(resp.Status)
    }
}

输入:

============== 第 [1] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:55131
GotConn remoteAddr: 183.232.231.172:443
200 OK
============== 第 [2] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:55132
GotConn remoteAddr: 183.232.231.172:443
200 OK
============== 第 [3] 次申请 ================
GetConn: www.baidu.com:443

论断:
咱们设置了 MaxConnsPerHost=2, 因为没有 close 导致没有开释连贯,执行两次申请后就卡住了,不能持续向下执行。并且第一次和第二次申请连贯没有复用

2. 只应用 Close, 不读取 resp.Body

代码:

package main

import (
    "fmt"
    "net/http"
    "net/http/httptrace"
)

func main() {req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
    if err != nil {panic(err)
    }

    trace := &httptrace.ClientTrace{GetConn: func(hostPort string) {fmt.Println("GetConn:", hostPort)
        },
        GotConn: func(connInfo httptrace.GotConnInfo) {fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
            fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
            fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
        },
        PutIdleConn: func(err error) {fmt.Printf("PutIdleConn err=%v\n", err)
        },
    }

    req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))

    client := &http.Client{
        Transport: &http.Transport{
            MaxIdleConns:    1, // 最大 Idle
            MaxConnsPerHost: 2, // 每个 host 最大 conns
        }}

    for i := 1; i <= 10; i++ {fmt.Printf("============== 第 [%d] 次申请 ================\n", i)
        resp, err := client.Do(req)
        if err != nil {fmt.Println(err)
            return
        }
        resp.Body.Close() // 留神这里 close 了
        fmt.Println(resp.Status)
    }
}

输入:

==============1================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54948
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============2================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54949
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============3================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54950
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============4================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54954
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============5================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54955
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============6================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54956
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============7================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54957
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============8================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54958
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============9================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54959
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============10================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54960
GotConn remoteAddr: 183.232.231.172:443
200 OK

论断:
尽管能够继续执行,阐明连贯开释了。但 GotConn 信息 eused=false WasIdle=false 并且每次 localAddr 都是不同的, 咱们发现每次申请取得的连贯都是新申请的。

3. 只读取 resp.Body, 不应用 Close()

咱们晓得 resp.Body 实现了 io.Reader 接口办法, 咱们通常的做法就通过调用 Read()办法来读取内容。
代码:

package main

import (
    "fmt"
    "io"
    "net/http"
    "net/http/httptrace"
)

func main() {req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
    if err != nil {panic(err)
    }

    trace := &httptrace.ClientTrace{GetConn: func(hostPort string) {fmt.Println("GetConn:", hostPort)
        },
        GotConn: func(connInfo httptrace.GotConnInfo) {fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
            fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
            fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
        },
        PutIdleConn: func(err error) {fmt.Printf("PutIdleConn err=%v\n", err)
        },
    }

    req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))

    client := &http.Client{
        Transport: &http.Transport{
            MaxIdleConns:    1, // 最大 Idle
            MaxConnsPerHost: 2, // 每个 host 最大 conns
        }}

    for i := 1; i <= 10; i++ {fmt.Printf("============== 第 [%d] 次申请 ================\n", i)
        resp, err := client.Do(req)
        if err != nil {fmt.Println(err)
            return
        }
        io.ReadAll(resp.Body) // 读取 body 外面的内容
        fmt.Println(resp.Status)
    }
}

输入:

============== 第 [1] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [2] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=116.037µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [3] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=72.154µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [4] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=60.102µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [5] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=71.807µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [6] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=74.616µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [7] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=47.205µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [8] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=74.207µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [9] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=52.414µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
============== 第 [10] 次申请 ================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=81.137µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK

论断:
咱们能够看下每次输入的 GotConn 信息 Reused=true WasIdle=true 并且 localAddr 都是雷同的。阐明每次执行完后都把连贯放回了 idleConn pool 外面,上来获取连贯时能够从 idleConn pool 外面获取。

源码剖析

(c Client) Do(req Request)–>(t Transport) roundTrip(req Request)–>t.getConn(treq, cm)–>t.queueForDial(w)->go t.dialConnFor(w)–>go readLoop()
咱们 readLoop()源码来剖析,readLoop()是循环读取连贯内容的办法:

go1.18 net/http/transport.go
(pc *persistConn) readLoop()是一个长久化连贯在不停的读取 conn 外面的内容,上面我追随源码看一下为什么要 close/readAll

func (pc *persistConn) readLoop() {
    closeErr := errReadLoopExiting // default value, if not changed below
    defer func() {pc.close(closeErr)
        pc.t.removeIdleConn(pc)
    }()

    // tryPutIdleConn 就是把连贯放回 IdleConn 而后让连贯能够复用
    tryPutIdleConn := func(trace *httptrace.ClientTrace) bool {if err := pc.t.tryPutIdleConn(pc); err != nil {
            closeErr = err
            if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled {trace.PutIdleConn(err)
            }
            return false
        }
        if trace != nil && trace.PutIdleConn != nil {trace.PutIdleConn(nil)
        }
        return true
    }

    // eofc is used to block caller goroutines reading from Response.Body
    // at EOF until this goroutines has (potentially) added the connection
    // back to the idle pool.
    eofc := make(chan struct{})
    defer close(eofc) // unblock reader on errors

    // Read this once, before loop starts. (to avoid races in tests)
    testHookMu.Lock()
    testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
    testHookMu.Unlock()

     // 这里的 alive 用来判断是否持续读取连贯内容
    alive := true
    for alive {pc.readLimit = pc.maxHeaderResponseSize()
        _, err := pc.br.Peek(1)

        pc.mu.Lock()
        if pc.numExpectedResponses == 0 {pc.readLoopPeekFailLocked(err)
            pc.mu.Unlock()
            return
        }
        pc.mu.Unlock()

        rc := <-pc.reqch
        trace := httptrace.ContextClientTrace(rc.req.Context())

        var resp *Response
        if err == nil {resp, err = pc.readResponse(rc, trace)
        } else {err = transportReadFromServerError{err}
            closeErr = err
        }

        if err != nil {
            if pc.readLimit <= 0 {err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize())
            }

            select {case rc.ch <- responseAndError{err: err}:
            case <-rc.callerGone:
                return
            }
            return
        }
        pc.readLimit = maxInt64 // effectively no limit for response bodies

        pc.mu.Lock()
        pc.numExpectedResponses--
        pc.mu.Unlock()

        bodyWritable := resp.bodyIsWritable()
        hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0

        if resp.Close || rc.req.Close || resp.StatusCode <= 199 || bodyWritable {
            // Don't do keep-alive on error if either party requested a close
            // or we get an unexpected informational (1xx) response.
            // StatusCode 100 is already handled above.
            alive = false
        }

        ......

        waitForBodyRead := make(chan bool, 2)
        // 最重要的就是这个把 resp.body 通过 bodyEOFSignal 封装生成新的 resp.Body,// 上面会讲到为什么通过 bodyEOFSignal 封装
        body := &bodyEOFSignal{
            body: resp.Body,
            // 如果调用 earlyCloseFn 就执行 waitForBodyRead <- false
            earlyCloseFn: func() error {
                waitForBodyRead <- false
                <-eofc // will be closed by deferred call at the end of the function
                return nil

            },
            // 如果调用 fn()并且 err == io.EOF 就执行 waitForBodyRead <- true
            fn: func(err error) error {
                isEOF := err == io.EOF
                waitForBodyRead <- isEOF
                if isEOF {<-eofc // see comment above eofc declaration} else if err != nil {if cerr := pc.canceled(); cerr != nil {return cerr}
                }
                return err
            },
        }

        // 从新把封装成 bodyEOFSignal 的 body 赋值给 resp.Body
        resp.Body = body
        if rc.addedGzip && ascii.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {resp.Body = &gzipReader{body: body}
            resp.Header.Del("Content-Encoding")
            resp.Header.Del("Content-Length")
            resp.ContentLength = -1
            resp.Uncompressed = true
        }

        select {// 把 resp 推送到 rc.ch 而后在 roundTrip()返回给内部
        case rc.ch <- responseAndError{res: resp}:
        case <-rc.callerGone:
            return
        }

        // Before looping back to the top of this function and peeking on
        // the bufio.Reader, wait for the caller goroutine to finish
        // reading the response body. (or for cancellation or death)
        select {
        // 这里是最重要的, 从 waitForBodyRead 阻塞获取 bodyEof
        case bodyEOF := <-waitForBodyRead:
            replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) // before pc might return to idle pool
            // 这里比拟奇妙的是如果 alive && bodyEOF && !pc.sawEOF && pc.wroteRequest()
            // 才会执行 tryPutIdleConn(trace), 就是把连贯放回 idleConn, 这就是为什么应用 close()是敞开了连贯然而没有复用,而应用 ReadAll()确复用了 idle。// 因为应用应用 ReadAll 后返回的 bodyEOF=true 而间接应用 Close()返回的 bodyEOF=false 导致永远不会执行 tryPutIdleConn(trace)。// 留神 pc.sawEOF 和 body Close 没关系,这个是检测 conn 是否 Close
            alive = alive &&
                bodyEOF &&
                !pc.sawEOF &&
                pc.wroteRequest() &&
                replaced && tryPutIdleConn(trace)
            if bodyEOF {eofc <- struct{}{}}
        case <-rc.req.Cancel:
            alive = false
            pc.t.CancelRequest(rc.req)
        case <-rc.req.Context().Done():
            alive = false
            pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err())
        case <-pc.closech:
            alive = false
        }

        testHookReadLoopBeforeNextRead()}
}

上面咱们来看下 resp.Body 的构造 bodyEOFSignal:

//readLoop()外面的 body
        body := &bodyEOFSignal{
            body: resp.Body,
             // 如果调用 earlyCloseFn 就执行 waitForBodyRead <- false
            earlyCloseFn: func() error {
                waitForBodyRead <- false
                <-eofc // will be closed by deferred call at the end of the function
                return nil

            },
            // 如果调用 fn()并且 err == io.EOF 就执行 waitForBodyRead <- true
            fn: func(err error) error {
                isEOF := err == io.EOF
                waitForBodyRead <- isEOF
                if isEOF {<-eofc // see comment above eofc declaration} else if err != nil {if cerr := pc.canceled(); cerr != nil {return cerr}
                }
                return err
            },
        }

// bodyEOFSignal is used by the HTTP/1 transport when reading response
// bodies to make sure we see the end of a response body before
// proceeding and reading on the connection again.
//
// It wraps a ReadCloser but runs fn (if non-nil) at most
// once, right before its final (error-producing) Read or Close call
// returns. fn should return the new error to return from Read or Close.
//
// If earlyCloseFn is non-nil and Close is called before io.EOF is
// seen, earlyCloseFn is called instead of fn, and its return value is
// the return value from Close.
type bodyEOFSignal struct {
    body         io.ReadCloser
    mu           sync.Mutex        // guards following 4 fields
    closed       bool              // whether Close has been called
    rerr         error             // sticky Read error
    fn           func(error) error // err will be nil on Read io.EOF
    earlyCloseFn func() error      // optional alt Close func used if io.EOF not seen 如果没 EOF 就会被调用}

func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {es.mu.Lock()
    closed, rerr := es.closed, es.rerr
    es.mu.Unlock()
    if closed {return 0, errReadOnClosedResBody}
    if rerr != nil {return 0, rerr}

    n, err = es.body.Read(p)
    if err != nil {es.mu.Lock()
        defer es.mu.Unlock()
        if es.rerr == nil {es.rerr = err}
        //condfn 会去执行 es.fn
        err = es.condfn(err)
    }
    return
}

func (es *bodyEOFSignal) Close() error {es.mu.Lock()
    defer es.mu.Unlock()
    if es.closed {return nil}
    es.closed = true
    // 如果调用 Close 的时候没有 EOF 就会调用 earlyCloseFn() 
    if es.earlyCloseFn != nil && es.rerr != io.EOF {return es.earlyCloseFn() 
    }
    err := es.body.Close()
    return es.condfn(err)
}

// caller must hold es.mu.
// 把 err 传给 es.fn, es.fn 会通过 err 做不同的操作,次要是判断 err 是不是 EOF
func (es *bodyEOFSignal) condfn(err error) error {
    if es.fn == nil {return err}
    err = es.fn(err) 
    es.fn = nil
    return err
}

总结

咱们收到的 resp.body 是 bodyEOFSignal, 如果不执行 Close()会导致 readLoop()阻塞。此时如果设置了最大连贯 MaxConnsPerHost 则达到连接数后不能再申请。然而如果只是 Close()而不执行 Read()则会导致连贯不会放回 idleConn

举荐用法:
不肯定要执行 Close(), 但肯定要执行 Read():
如不须要 body 外面的内容能够执行

client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {return nil, err}
defer io.Copy(io.Discard,resp.Body)

如果须要 resp.Body 则执行:

client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {return nil, err}
// 这里的 close 是避免 Read 操作不能被正确执行到做兜底,
// 如果可能确保 resp.Body 被执行到也不须要 Close
defer resp.Body.Close() 
// 读取 resp.Body, 如 io.ReadAll/io.Copy()...
io.ReadAll(resp.Body)
退出移动版