golang gin框架源码分析（三）---- 问其根本 net/http包需要深究才能有始有终
文章目录
全系列总结博客链接
前引
本意是没有打算写这一篇的 但是这两天在刷面经的时候发现 还是有问这个net/http包的
这个看样子是逃不掉了 还是需要深挖一下
发现要做的事情太多 太忙 但是想到如果能回到自己的hometown 今年9月份就要定正式工作了
这个offer起码决定了后面好几年的日子 也不得不这几个月再硬着头皮冲一下了
golang gin框架源码分析（三）---- 问其根本 net/http包需要深究才能有始有终
1、从示例代码入手
刚刚也才基本把net/http的框架给梳理了一遍 还是很有收获的^^
这里还是简单推荐一下 一篇对于net/http解析写的挺好的帖子 我下面贴一下链接 大家可以先去看看这篇
Introduction 万字手撕Go http源码server.go
还是从示例代码入手 下面是非常常见的用net/http写一个高并发的web服务器的代码 我们注册url相对应的处理函数就是以下面的方式来注册的 那我们就可以顺藤摸瓜的去看一下 ListenAndServe函数 和 HandleFunc函数是怎么运行的
package main
import (
"net/http"
"strconv"
"github.com/gin-gonic/gin"
)
func main() {
http.HandleFunc("/hello", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`hello world`))
})
http.HandleFunc("/go/", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`go`))
})
http.ListenAndServe(":8002", nil)
}
访问url的实际效果
2、ListenAndServe(addr string, handler Handler) error
下面可以仔细看看 我贴出来了Server的结构体 如我们的示例代码所示 其实结构体中只赋值了addr 对于handler部分我们是填写的nil 那么对于该Server的话 之后找我们的handler只会去我们的默认多路复用器 也就是DefaultServerMux里面去找我们所注册的handler了 也就是我们调用了两个http.HandleFunc 其实也就是往我们默认的DefaultServerMux注册处理函数了 当然这是后话了 我们可以顺藤摸瓜下去
// A Server defines parameters for running an HTTP server.
// The zero value for Server is a valid configuration.
type Server struct {
Addr string
Handler Handler // handler to invoke, http.DefaultServeMux if nil
DisableGeneralOptionsHandler bool
TLSConfig *tls.Config
ReadTimeout time.Duration
ReadHeaderTimeout time.Duration
WriteTimeout time.Duration
IdleTimeout time.Duration
MaxHeaderBytes int
TLSNextProto map[string]func(*Server, *tls.Conn, Handler)
ConnState func(net.Conn, ConnState)
ErrorLog *log.Logger
BaseContext func(net.Listener) context.Context
ConnContext func(ctx context.Context, c net.Conn) context.Context
inShutdown atomic.Bool // true when server is in shutdown
disableKeepAlives atomic.Bool
nextProtoOnce sync.Once // guards setupHTTP2_* init
nextProtoErr error // result of http2.ConfigureServer if used
mu sync.Mutex
listeners map[*net.Listener]struct{}
activeConn map[*conn]struct{}
onShutdown []func()
listenerGroup sync.WaitGroup
}
func ListenAndServe(addr string, handler Handler) error {
server := &Server{Addr: addr, Handler: handler}
return server.ListenAndServe()
}
1、server.ListenAndServe()
这里前面可以省略 处理shutdown情况 取出来addr
而后面重点就是这两句话 ln, err := net.Listen("tcp", addr) return srv.Serve(ln)
对于服务器 常见的处理框架也就是 监听端口->取出新连接->工作线程/子线程处理/多路复用处理新连接
而这里的ln其实就是一个接口 当然递归的去看看listen工作其实也就是包装了一下相关资源 对于不同类型返回不同的结构体而已 也就可以当作是监听结构体
然后我们得到了后 就去调用serve 开始服务了 我们往下继续看看
ln, err := net.Listen("tcp", addr)
return srv.Serve(ln)
-----------
// A Listener is a generic network listener for stream-oriented protocols.
//
// Multiple goroutines may invoke methods on a Listener simultaneously.
type Listener interface {
// Accept waits for and returns the next connection to the listener.
Accept() (Conn, error)
// Close closes the listener.
// Any blocked Accept operations will be unblocked and return errors.
Close() error
// Addr returns the listener's network address.
Addr() Addr
}
// Listen announces on the local network address.
//
// See func Listen for a description of the network and address
// parameters.
func (lc *ListenConfig) Listen(ctx context.Context, network, address string) (Listener, error) {
addrs, err := DefaultResolver.resolveAddrList(ctx, "listen", network, address, nil)
if err != nil {
return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: nil, Err: err}
}
sl := &sysListener{
ListenConfig: *lc,
network: network,
address: address,
}
var l Listener
la := addrs.first(isIPv4)
switch la := la.(type) {
case *TCPAddr:
l, err = sl.listenTCP(ctx, la)
case *UnixAddr:
l, err = sl.listenUnix(ctx, la)
default:
return nil, &OpError{Op: "listen", Net: sl.network, Source: nil, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: address}}
}
if err != nil {
return nil, &OpError{Op: "listen", Net: sl.network, Source: nil, Addr: la, Err: err} // l is non-nil interface containing nil pointer
}
return l, nil
}
------------------
// ListenAndServe listens on the TCP network address srv.Addr and then
// calls Serve to handle requests on incoming connections.
// Accepted connections are configured to enable TCP keep-alives.
//
// If srv.Addr is blank, ":http" is used.
//
// ListenAndServe always returns a non-nil error. After Shutdown or Close,
// the returned error is ErrServerClosed.
func (srv *Server) ListenAndServe() error {
if srv.shuttingDown() {
return ErrServerClosed
}
addr := srv.Addr
if addr == "" {
addr = ":http"
}
ln, err := net.Listen("tcp", addr)
if err != nil {
return err
}
return srv.Serve(ln)
}
2、server.Serve(l net.Listener) error
前面其实也是不是很重要的 可以抓一下关键 直接看到for循环内部
我已经把最核心的地方节省出来了 Accept获取新连接 然后获取新连接结构体server + net.Conn 返回
启用go协程去处理新连接
这个可以说是就是 把例如c++的网络库中 例如Epoll/Select 处理服务器的多路复用 改用为用go协程去处理新连接
相当于高并发核心应该就是来自于这里 则go协程的快捷以及切换快速 也让go web高并发得到了实现
我们可以继续往下看看
// Create new connection from rwc.
func (srv *Server) newConn(rwc net.Conn) *conn {
c := &conn{
server: srv,
rwc: rwc,
}
if debugServerConnections {
c.rwc = newLoggingConn("server", c.rwc)
}
return c
}
for {
rw, err := l.Accept()
.....
.....
c := srv.newConn(rw)
c.setState(c.rwc, StateNew, runHooks) // before Serve can return
go c.serve(connCtx)
}
----
// Serve accepts incoming connections on the Listener l, creating a
// new service goroutine for each. The service goroutines read requests and
// then call srv.Handler to reply to them.
//
// HTTP/2 support is only enabled if the Listener returns *tls.Conn
// connections and they were configured with "h2" in the TLS
// Config.NextProtos.
//
// Serve always returns a non-nil error and closes l.
// After Shutdown or Close, the returned error is ErrServerClosed.
func (srv *Server) Serve(l net.Listener) error {
if fn := testHookServerServe; fn != nil {
fn(srv, l) // call hook with unwrapped listener
}
origListener := l
l = &onceCloseListener{Listener: l}
defer l.Close()
if err := srv.setupHTTP2_Serve(); err != nil {
return err
}
if !srv.trackListener(&l, true) {
return ErrServerClosed
}
defer srv.trackListener(&l, false)
baseCtx := context.Background()
if srv.BaseContext != nil {
baseCtx = srv.BaseContext(origListener)
if baseCtx == nil {
panic("BaseContext returned a nil context")
}
}
var tempDelay time.Duration // how long to sleep on accept failure
ctx := context.WithValue(baseCtx, ServerContextKey, srv)
for {
rw, err := l.Accept()
if err != nil {
if srv.shuttingDown() {
return ErrServerClosed
}
if ne, ok := err.(net.Error); ok && ne.Temporary() {
if tempDelay == 0 {
tempDelay = 5 * time.Millisecond
} else {
tempDelay *= 2
}
if max := 1 * time.Second; tempDelay > max {
tempDelay = max
}
srv.logf("http: Accept error: %v; retrying in %v", err, tempDelay)
time.Sleep(tempDelay)
continue
}
return err
}
connCtx := ctx
if cc := srv.ConnContext; cc != nil {
connCtx = cc(connCtx, rw)
if connCtx == nil {
panic("ConnContext returned nil")
}
}
tempDelay = 0
c := srv.newConn(rw)
c.setState(c.rwc, StateNew, runHooks) // before Serve can return
go c.serve(connCtx)
}
}
3、conn.serve(ctx context.Context) {
我对下面的代码进行了抛肥捡瘦 for循环我理解的应该是长连接 需要多次request response 一旦请求终止即return
而这里面最核心的一句话就是serverHandler{c.server}.ServeHTTP(w, w.req)
serveHTTP 这个是内部的函数 我贴在了下面 是最重要的语句 我们详细来看看
首先先赋值handler 这个handler 对于我们的示例代码还记得吗 我们填写的是nil 如果我们自己写了类 实现了ServeHTTP的话 则响应函数只会触发我们自己写的了 而不是我们用HandlerFunc默认的多路复用器我们添加的那部分函数了
最后这个net/http 最核心的一句话 处理函数就是在handler.ServeHTTP(rw, req) 调用我们的handler
那现在明了了 逻辑基本理清了 那这个时候我们可以最后再去看看我们的DefaultServeMux是如何存储我们的多个url对应的响应函数的
func (sh serverHandler) ServeHTTP(rw ResponseWriter, req *Request) {
handler := sh.srv.Handler
if handler == nil {
handler = DefaultServeMux
}
if !sh.srv.DisableGeneralOptionsHandler && req.RequestURI == "*" && req.Method == "OPTIONS" {
handler = globalOptionsHandler{}
}
if req.URL != nil && strings.Contains(req.URL.RawQuery, ";") {
var allowQuerySemicolonsInUse atomic.Bool
req = req.WithContext(context.WithValue(req.Context(), silenceSemWarnContextKey, func() {
allowQuerySemicolonsInUse.Store(true)
}))
defer func() {
if !allowQuerySemicolonsInUse.Load() {
sh.srv.logf("http: URL query contains semicolon, which is no longer a supported separator; parts of the query may be stripped when parsed; see golang.org/issue/25192")
}
}()
}
handler.ServeHTTP(rw, req)
}
------------------------------------------------
// Serve a new connection.
func (c *conn) serve(ctx context.Context) {
......
......
// HTTP/1.x from here on.
ctx, cancelCtx := context.WithCancel(ctx)
c.cancelCtx = cancelCtx
defer cancelCtx()
c.r = &connReader{conn: c}
c.bufr = newBufioReader(c.r)
c.bufw = newBufioWriterSize(checkConnErrorWriter{c}, 4<<10)
for {
w, err := c.readRequest(ctx)
.....
.....
.....
// Expect 100 Continue support
req := w.req
.......
// HTTP cannot have multiple simultaneous active requests.[*]
// Until the server replies to this request, it can't read another,
// so we might as well run the handler in this goroutine.
// [*] Not strictly true: HTTP pipelining. We could let them all process
// in parallel even if their responses need to be serialized.
// But we're not going to implement HTTP pipelining because it
// was never deployed in the wild and the answer is HTTP/2.
inFlightResponse = w
serverHandler{c.server}.ServeHTTP(w, w.req)
inFlightResponse = nil
w.cancelCtx()
if c.hijacked() {
return
}
w.finishRequest()
........
}
}
3、HandleFunc(pattern string, handler func(ResponseWriter, *Request))
下面我把相关的结构体贴出来了
我可以这里就简单介绍一下
mu是读写锁控制竞态环境的 m则是存相关url的map 我们示例代码/go /hello也就被存储在了这里面
而es则是 我们存储path末尾是/的匹配模式串 而且会按照长度从长到短按照顺序来存储 这个都会后面的讲
我们可以递归的去看看mux.HandleFunc 我们是如何写入map的
type muxEntry struct {
h Handler
pattern string
}
type ServeMux struct {
mu sync.RWMutex
m map[string]muxEntry
es []muxEntry // slice of entries sorted from longest to shortest.
hosts bool // whether any patterns contain hostnames
}
// DefaultServeMux is the default ServeMux used by Serve.
var DefaultServeMux = &defaultServeMux
var defaultServeMux ServeMux
// HandleFunc registers the handler function for the given pattern
// in the DefaultServeMux.
// The documentation for ServeMux explains how patterns are matched.
func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) {
DefaultServeMux.HandleFunc(pattern, handler)
}
1、mtx.HandleFunc(pattern string, handler func(ResponseWriter, *Request))
最核心的两个函数我已经贴在了下方
这里其实之前贴的那个帖子说的挺好的 这里有点像是语法糖 传入 其实本质上做了类型转换 我们只需要传入这个类型的函数 而不需要自己去定义新的类然后实现ServeHTTP这个函数
然后我们仔细看看Handle 函数 其实很简单
没有m 初始化一个 然后把我们的url对应的函数写入结构体 对于末尾是/的模式串单独写入es
那最后我们看看ServeMux的ServeHTTP 默认的多路复用器是怎么复用的
type HandlerFunc func(ResponseWriter, *Request)
// ServeHTTP calls f(w, r).
func (f HandlerFunc) ServeHTTP(w ResponseWriter, r *Request) {
f(w, r)
}
// Handle registers the handler for the given pattern.
// If a handler already exists for pattern, Handle panics.
func (mux *ServeMux) Handle(pattern string, handler Handler) {
mux.mu.Lock()
defer mux.mu.Unlock()
if pattern == "" {
panic("http: invalid pattern")
}
if handler == nil {
panic("http: nil handler")
}
if _, exist := mux.m[pattern]; exist {
panic("http: multiple registrations for " + pattern)
}
if mux.m == nil {
mux.m = make(map[string]muxEntry)
}
e := muxEntry{h: handler, pattern: pattern}
mux.m[pattern] = e
if pattern[len(pattern)-1] == '/' {
mux.es = appendSorted(mux.es, e)
}
if pattern[0] != '/' {
mux.hosts = true
}
}
// HandleFunc registers the handler function for the given pattern.
func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Request)) {
if handler == nil {
panic("http: nil handler")
}
mux.Handle(pattern, HandlerFunc(handler))
}
2、mux.ServeHTTP(w ResponseWriter, r *Request)
长话短说 h.ServeHTTP(w, r) 是真正的执行函数 而找到handler是在mux.Handler(r)内部 递归的看看 贴在了下面
最后找handler 又是递归到了mux.handler(host, path string)
然后就发现 我们如何找到handler 就是mutex.match来找 正常匹配的话 则是内部mux.match 递归再看看
ok 显然易见了 直接从map中取 如果没有找到 则是去es 去模糊匹配 而且es内部的字符串是从长到短进行排序 这个可以从上一篇mtx.HandleFunc()去找 mux.es = appendSorted(mux.es, e)
基本上 这个net/http已经被我们梳理了出来了 文章基本到这里就差不多了
// Find a handler on a handler map given a path string.
// Most-specific (longest) pattern wins.
func (mux *ServeMux) match(path string) (h Handler, pattern string) {
// Check for exact match first.
v, ok := mux.m[path]
if ok {
return v.h, v.pattern
}
// Check for longest valid match. mux.es contains all patterns
// that end in / sorted from longest to shortest.
for _, e := range mux.es {
if strings.HasPrefix(path, e.pattern) {
return e.h, e.pattern
}
}
return nil, ""
}
-------------------------------
func (mux *ServeMux) handler(host, path string) (h Handler, pattern string) {
mux.mu.RLock()
defer mux.mu.RUnlock()
// Host-specific pattern takes precedence over generic ones
if mux.hosts {
h, pattern = mux.match(host + path)
}
if h == nil {
h, pattern = mux.match(path)
}
if h == nil {
h, pattern = NotFoundHandler(), ""
}
return
}
--------------------------
func (mux *ServeMux) Handler(r *Request) (h Handler, pattern string) {
if r.Method == "CONNECT" {
if u, ok := mux.redirectToPathSlash(r.URL.Host, r.URL.Path, r.URL); ok {
return RedirectHandler(u.String(), StatusMovedPermanently), u.Path
}
return mux.handler(r.Host, r.URL.Path)
}
host := stripHostPort(r.Host)
path := cleanPath(r.URL.Path)
if u, ok := mux.redirectToPathSlash(host, path, r.URL); ok {
return RedirectHandler(u.String(), StatusMovedPermanently), u.Path
}
if path != r.URL.Path {
_, pattern = mux.handler(host, path)
u := &url.URL{Path: path, RawQuery: r.URL.RawQuery}
return RedirectHandler(u.String(), StatusMovedPermanently), pattern
}
return mux.handler(host, r.URL.Path)
}
------------------------
// ServeHTTP dispatches the request to the handler whose
// pattern most closely matches the request URL.
func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) {
if r.RequestURI == "*" {
if r.ProtoAtLeast(1, 1) {
w.Header().Set("Connection", "close")
}
w.WriteHeader(StatusBadRequest)
return
}
h, _ := mux.Handler(r)
h.ServeHTTP(w, r)
}
4、简单总结一下
这里面博客写的挺全的 想下楼转一转了
后面面试的时候 再想想咋总结了 = = 晚上回来还要再学一学GRPC 那就这样啦
想看点更相信的可以烦请跳转到下面链接啦～
Introduction 万字手撕Go http源码server.go