乐趣区

关于go:golang-http-client一定要close-ResponseBody吗

作者：

jiezi

在

咱们晓得个别在调用http client后都会close Response.Body,如下:

client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {
    return nil, err
}
defer resp.Body.Close()

上面咱们来看下为什么resp.Body须要Close,肯定须要Close吗？

咱们先通过”net/http/httptrace”来验证下:

1.不应用Close

代码:

package main

import (
    "fmt"
    "net/http"
    "net/http/httptrace"
)

func main() {
    req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
    if err != nil {
        panic(err)
    }

    trace := &httptrace.ClientTrace{
        GetConn: func(hostPort string) {
            fmt.Println("GetConn:", hostPort)
        },
        GotConn: func(connInfo httptrace.GotConnInfo) {
            fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
            fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
            fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
        },
        PutIdleConn: func(err error) {
            fmt.Printf("PutIdleConn err=%v\n", err)
        },
    }

    req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))

    client := &http.Client{
        Transport: &http.Transport{
            MaxIdleConns:    1, //最大Idle
            MaxConnsPerHost: 2, //每个host最大conns
        }}

    for i := 1; i <= 10; i++ {
        fmt.Printf("==============第[%d]次申请================\n", i)
        resp, err := client.Do(req)
        if err != nil {
            fmt.Println(err)
            return
        }
        fmt.Println(resp.Status)
    }
}

输入:

==============第[1]次申请================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:55131
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============第[2]次申请================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:55132
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============第[3]次申请================
GetConn: www.baidu.com:443

论断:
咱们设置了MaxConnsPerHost=2,因为没有close导致没有开释连贯，执行两次申请后就卡住了，不能持续向下执行。并且第一次和第二次申请连贯没有复用

2.只应用Close,不读取resp.Body

代码:

package main

import (
    "fmt"
    "net/http"
    "net/http/httptrace"
)

func main() {
    req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
    if err != nil {
        panic(err)
    }

    trace := &httptrace.ClientTrace{
        GetConn: func(hostPort string) {
            fmt.Println("GetConn:", hostPort)
        },
        GotConn: func(connInfo httptrace.GotConnInfo) {
            fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
            fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
            fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
        },
        PutIdleConn: func(err error) {
            fmt.Printf("PutIdleConn err=%v\n", err)
        },
    }

    req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))

    client := &http.Client{
        Transport: &http.Transport{
            MaxIdleConns:    1, //最大Idle
            MaxConnsPerHost: 2, //每个host最大conns
        }}

    for i := 1; i <= 10; i++ {
        fmt.Printf("==============第[%d]次申请================\n", i)
        resp, err := client.Do(req)
        if err != nil {
            fmt.Println(err)
            return
        }
        resp.Body.Close() //留神这里close了
        fmt.Println(resp.Status)
    }
}

输入:

==============1================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54948
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============2================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54949
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============3================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54950
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============4================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54954
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============5================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54955
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============6================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54956
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============7================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54957
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============8================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54958
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============9================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54959
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============10================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54960
GotConn remoteAddr: 183.232.231.172:443
200 OK

论断：
尽管能够继续执行，阐明连贯开释了。但GotConn信息eused=false WasIdle=false并且每次localAddr都是不同的,咱们发现每次申请取得的连贯都是新申请的。

3.只读取resp.Body,不应用Close()

咱们晓得resp.Body实现了io.Reader接口办法,咱们通常的做法就通过调用Read()办法来读取内容。
代码:

package main

import (
    "fmt"
    "io"
    "net/http"
    "net/http/httptrace"
)

func main() {
    req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
    if err != nil {
        panic(err)
    }

    trace := &httptrace.ClientTrace{
        GetConn: func(hostPort string) {
            fmt.Println("GetConn:", hostPort)
        },
        GotConn: func(connInfo httptrace.GotConnInfo) {
            fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
            fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
            fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
        },
        PutIdleConn: func(err error) {
            fmt.Printf("PutIdleConn err=%v\n", err)
        },
    }

    req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))

    client := &http.Client{
        Transport: &http.Transport{
            MaxIdleConns:    1, //最大Idle
            MaxConnsPerHost: 2, //每个host最大conns
        }}

    for i := 1; i <= 10; i++ {
        fmt.Printf("==============第[%d]次申请================\n", i)
        resp, err := client.Do(req)
        if err != nil {
            fmt.Println(err)
            return
        }
        io.ReadAll(resp.Body) //读取body外面的内容
        fmt.Println(resp.Status)
    }
}

输入:

==============第[1]次申请================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[2]次申请================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=116.037µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[3]次申请================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=72.154µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[4]次申请================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=60.102µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[5]次申请================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=71.807µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[6]次申请================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=74.616µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[7]次申请================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=47.205µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[8]次申请================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=74.207µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[9]次申请================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=52.414µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[10]次申请================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=81.137µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK

论断:
咱们能够看下每次输入的GotConn信息Reused=true WasIdle=true并且localAddr都是雷同的。阐明每次执行完后都把连贯放回了idleConn pool外面，上来获取连贯时能够从idleConn pool外面获取。

源码剖析

(c Client) Do(req Request)–>(t Transport) roundTrip(req Request)–>t.getConn(treq, cm)–>t.queueForDial(w)->go t.dialConnFor(w)–>go readLoop()
咱们readLoop()源码来剖析，readLoop()是循环读取连贯内容的办法:

go1.18 net/http/transport.go
(pc *persistConn) readLoop()是一个长久化连贯在不停的读取conn外面的内容，上面我追随源码看一下为什么要close/readAll

func (pc *persistConn) readLoop() {
    closeErr := errReadLoopExiting // default value, if not changed below
    defer func() {
        pc.close(closeErr)
        pc.t.removeIdleConn(pc)
    }()

    // tryPutIdleConn就是把连贯放回IdleConn而后让连贯能够复用
    tryPutIdleConn := func(trace *httptrace.ClientTrace) bool {
        if err := pc.t.tryPutIdleConn(pc); err != nil {
            closeErr = err
            if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled {
                trace.PutIdleConn(err)
            }
            return false
        }
        if trace != nil && trace.PutIdleConn != nil {
            trace.PutIdleConn(nil)
        }
        return true
    }

    // eofc is used to block caller goroutines reading from Response.Body
    // at EOF until this goroutines has (potentially) added the connection
    // back to the idle pool.
    eofc := make(chan struct{})
    defer close(eofc) // unblock reader on errors

    // Read this once, before loop starts. (to avoid races in tests)
    testHookMu.Lock()
    testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
    testHookMu.Unlock()

     //这里的alive用来判断是否持续读取连贯内容
    alive := true
    for alive {
        pc.readLimit = pc.maxHeaderResponseSize()
        _, err := pc.br.Peek(1)

        pc.mu.Lock()
        if pc.numExpectedResponses == 0 {
            pc.readLoopPeekFailLocked(err)
            pc.mu.Unlock()
            return
        }
        pc.mu.Unlock()

        rc := <-pc.reqch
        trace := httptrace.ContextClientTrace(rc.req.Context())

        var resp *Response
        if err == nil {
            resp, err = pc.readResponse(rc, trace)
        } else {
            err = transportReadFromServerError{err}
            closeErr = err
        }

        if err != nil {
            if pc.readLimit <= 0 {
                err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize())
            }

            select {
            case rc.ch <- responseAndError{err: err}:
            case <-rc.callerGone:
                return
            }
            return
        }
        pc.readLimit = maxInt64 // effectively no limit for response bodies

        pc.mu.Lock()
        pc.numExpectedResponses--
        pc.mu.Unlock()

        bodyWritable := resp.bodyIsWritable()
        hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0

        if resp.Close || rc.req.Close || resp.StatusCode <= 199 || bodyWritable {
            // Don't do keep-alive on error if either party requested a close
            // or we get an unexpected informational (1xx) response.
            // StatusCode 100 is already handled above.
            alive = false
        }

        ......

        waitForBodyRead := make(chan bool, 2)
        //最重要的就是这个把resp.body通过bodyEOFSignal封装生成新的resp.Body，
        //上面会讲到为什么通过bodyEOFSignal封装
        body := &bodyEOFSignal{
            body: resp.Body,
            //如果调用earlyCloseFn就执行waitForBodyRead <- false
            earlyCloseFn: func() error {
                waitForBodyRead <- false
                <-eofc // will be closed by deferred call at the end of the function
                return nil

            },
            //如果调用fn()并且 err == io.EOF 就执行waitForBodyRead <- true
            fn: func(err error) error {
                isEOF := err == io.EOF
                waitForBodyRead <- isEOF
                if isEOF {
                    <-eofc // see comment above eofc declaration
                } else if err != nil {
                    if cerr := pc.canceled(); cerr != nil {
                        return cerr
                    }
                }
                return err
            },
        }

        // 从新把封装成bodyEOFSignal的body赋值给resp.Body
        resp.Body = body
        if rc.addedGzip && ascii.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
            resp.Body = &gzipReader{body: body}
            resp.Header.Del("Content-Encoding")
            resp.Header.Del("Content-Length")
            resp.ContentLength = -1
            resp.Uncompressed = true
        }

        select {
        // 把resp推送到rc.ch而后在roundTrip()返回给内部
        case rc.ch <- responseAndError{res: resp}:
        case <-rc.callerGone:
            return
        }

        // Before looping back to the top of this function and peeking on
        // the bufio.Reader, wait for the caller goroutine to finish
        // reading the response body. (or for cancellation or death)
        select {
        // 这里是最重要的,从waitForBodyRead阻塞获取bodyEof
        case bodyEOF := <-waitForBodyRead:
            replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) // before pc might return to idle pool
            //这里比拟奇妙的是如果alive && bodyEOF && !pc.sawEOF && pc.wroteRequest()
            //才会执行tryPutIdleConn(trace),就是把连贯放回idleConn,这就是为什么应用close()是敞开了连贯然而没有复用，而应用ReadAll()确复用了idle。
            //因为应用应用ReadAll后返回的bodyEOF=true而间接应用Close()返回的bodyEOF=false导致永远不会执行tryPutIdleConn(trace)。
            //留神pc.sawEOF和body Close没关系，这个是检测conn是否Close
            alive = alive &&
                bodyEOF &&
                !pc.sawEOF &&
                pc.wroteRequest() &&
                replaced && tryPutIdleConn(trace)
            if bodyEOF {
                eofc <- struct{}{}
            }
        case <-rc.req.Cancel:
            alive = false
            pc.t.CancelRequest(rc.req)
        case <-rc.req.Context().Done():
            alive = false
            pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err())
        case <-pc.closech:
            alive = false
        }

        testHookReadLoopBeforeNextRead()
    }
}

上面咱们来看下resp.Body的构造bodyEOFSignal:

//readLoop()外面的body
        body := &bodyEOFSignal{
            body: resp.Body,
             //如果调用earlyCloseFn就执行waitForBodyRead <- false
            earlyCloseFn: func() error {
                waitForBodyRead <- false
                <-eofc // will be closed by deferred call at the end of the function
                return nil

            },
            //如果调用fn()并且 err == io.EOF 就执行waitForBodyRead <- true
            fn: func(err error) error {
                isEOF := err == io.EOF
                waitForBodyRead <- isEOF
                if isEOF {
                    <-eofc // see comment above eofc declaration
                } else if err != nil {
                    if cerr := pc.canceled(); cerr != nil {
                        return cerr
                    }
                }
                return err
            },
        }

// bodyEOFSignal is used by the HTTP/1 transport when reading response
// bodies to make sure we see the end of a response body before
// proceeding and reading on the connection again.
//
// It wraps a ReadCloser but runs fn (if non-nil) at most
// once, right before its final (error-producing) Read or Close call
// returns. fn should return the new error to return from Read or Close.
//
// If earlyCloseFn is non-nil and Close is called before io.EOF is
// seen, earlyCloseFn is called instead of fn, and its return value is
// the return value from Close.
type bodyEOFSignal struct {
    body         io.ReadCloser
    mu           sync.Mutex        // guards following 4 fields
    closed       bool              // whether Close has been called
    rerr         error             // sticky Read error
    fn           func(error) error // err will be nil on Read io.EOF
    earlyCloseFn func() error      // optional alt Close func used if io.EOF not seen 如果没EOF就会被调用
}

func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
    es.mu.Lock()
    closed, rerr := es.closed, es.rerr
    es.mu.Unlock()
    if closed {
        return 0, errReadOnClosedResBody
    }
    if rerr != nil {
        return 0, rerr
    }

    n, err = es.body.Read(p)
    if err != nil {
        es.mu.Lock()
        defer es.mu.Unlock()
        if es.rerr == nil {
            es.rerr = err
        }
        //condfn会去执行es.fn
        err = es.condfn(err)
    }
    return
}

func (es *bodyEOFSignal) Close() error {
    es.mu.Lock()
    defer es.mu.Unlock()
    if es.closed {
        return nil
    }
    es.closed = true
    // 如果调用Close的时候没有EOF就会调用earlyCloseFn() 
    if es.earlyCloseFn != nil && es.rerr != io.EOF {
        return es.earlyCloseFn() 
    }
    err := es.body.Close()
    return es.condfn(err)
}

// caller must hold es.mu.
// 把err传给es.fn, es.fn会通过err做不同的操作，次要是判断err是不是EOF
func (es *bodyEOFSignal) condfn(err error) error {
    if es.fn == nil {
        return err
    }
    err = es.fn(err) 
    es.fn = nil
    return err
}

总结

咱们收到的resp.body是bodyEOFSignal,如果不执行Close()会导致readLoop()阻塞。此时如果设置了最大连贯MaxConnsPerHost则达到连接数后不能再申请。然而如果只是Close()而不执行Read()则会导致连贯不会放回idleConn

举荐用法:
不肯定要执行Close(),但肯定要执行Read():
如不须要body外面的内容能够执行

client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {
    return nil, err
}
defer io.Copy(io.Discard,resp.Body)

如果须要resp.Body则执行:

client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {
    return nil, err
}
//这里的close是避免Read操作不能被正确执行到做兜底,
//如果可能确保resp.Body被执行到也不须要Close
defer resp.Body.Close() 
//读取resp.Body,如io.ReadAll/io.Copy()...
io.ReadAll(resp.Body)

评论

发表回复

您的邮箱地址不会被公开。 必填项已用 * 标注

这个站点使用 Akismet 来减少垃圾评论。了解你的评论数据如何被处理。

更多文章