Upgrade k8s package verison (#5358)

* upgrade k8s package version Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io> * Script upgrade and code formatting. Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io> Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io>
2022-11-15 14:56:38 +08:00
parent 5f91c1663a
commit 44167aa47a
3106 changed files with 321340 additions and 172080 deletions
--- a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go
+++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go
@@ -24,6 +24,7 @@ import (
 	"math/rand"
 	"net"
 	"sync"
+	"sync/atomic"
 	"time"

 	"google.golang.org/grpc"
@@ -35,25 +36,101 @@ import (
 type Tunnel interface {
 	// Dial connects to the address on the named network, similar to
 	// what net.Dial does. The only supported protocol is tcp.
-	Dial(protocol, address string) (net.Conn, error)
+	DialContext(requestCtx context.Context, protocol, address string) (net.Conn, error)
+	// Done returns a channel that is closed when the tunnel is no longer serving any connections,
+	// and can no longer be used.
+	Done() <-chan struct{}
 }

 type dialResult struct {
-	err    string
+	err    *dialFailure
 	connid int64
 }

+type pendingDial struct {
+	// resultCh is the channel to send the dial result to
+	resultCh chan<- dialResult
+	// cancelCh is the channel closed when resultCh no longer has a receiver
+	cancelCh <-chan struct{}
+}
+
+// TODO: Replace with a generic implementation once it is safe to assume the client is built with go1.18+
+type pendingDialManager struct {
+	pendingDials map[int64]pendingDial
+	mutex        sync.RWMutex
+}
+
+func (p *pendingDialManager) add(dialID int64, pd pendingDial) {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+	p.pendingDials[dialID] = pd
+}
+
+func (p *pendingDialManager) remove(dialID int64) {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+	delete(p.pendingDials, dialID)
+}
+
+func (p *pendingDialManager) get(dialID int64) (pendingDial, bool) {
+	p.mutex.RLock()
+	defer p.mutex.RUnlock()
+	pd, ok := p.pendingDials[dialID]
+	return pd, ok
+}
+
+// TODO: Replace with a generic implementation once it is safe to assume the client is built with go1.18+
+type connectionManager struct {
+	conns map[int64]*conn
+	mutex sync.RWMutex
+}
+
+func (cm *connectionManager) add(connID int64, c *conn) {
+	cm.mutex.Lock()
+	defer cm.mutex.Unlock()
+	cm.conns[connID] = c
+}
+
+func (cm *connectionManager) remove(connID int64) {
+	cm.mutex.Lock()
+	defer cm.mutex.Unlock()
+	delete(cm.conns, connID)
+}
+
+func (cm *connectionManager) get(connID int64) (*conn, bool) {
+	cm.mutex.RLock()
+	defer cm.mutex.RUnlock()
+	c, ok := cm.conns[connID]
+	return c, ok
+}
+
+func (cm *connectionManager) closeAll() {
+	cm.mutex.Lock()
+	defer cm.mutex.Unlock()
+	for _, conn := range cm.conns {
+		close(conn.readCh)
+	}
+}
+
 // grpcTunnel implements Tunnel
 type grpcTunnel struct {
-	stream          client.ProxyService_ProxyClient
-	pendingDial     map[int64]chan<- dialResult
-	conns           map[int64]*conn
-	pendingDialLock sync.RWMutex
-	connsLock       sync.RWMutex
+	stream      client.ProxyService_ProxyClient
+	clientConn  clientConn
+	pendingDial pendingDialManager
+	conns       connectionManager

 	// The tunnel will be closed if the caller fails to read via conn.Read()
 	// more than readTimeoutSeconds after a packet has been received.
 	readTimeoutSeconds int
+
+	// The done channel is closed after the tunnel has cleaned up all connections and is no longer
+	// serving.
+	done chan struct{}
+
+	// closing is an atomic bool represented as a 0 or 1, and set to true when the tunnel is being closed.
+	// closing should only be accessed through atomic methods.
+	// TODO: switch this to an atomic.Bool once the client is exclusively buit with go1.19+
+	closing uint32
 }

 type clientConn interface {
@@ -66,37 +143,65 @@ var _ clientConn = &grpc.ClientConn{}
 // gRPC based proxy service.
 // Currently, a single tunnel supports a single connection, and the tunnel is closed when the connection is terminated
 // The Dial() method of the returned tunnel should only be called once
-func CreateSingleUseGrpcTunnel(address string, opts ...grpc.DialOption) (Tunnel, error) {
-	c, err := grpc.Dial(address, opts...)
+// Deprecated 2022-06-07: use CreateSingleUseGrpcTunnelWithContext
+func CreateSingleUseGrpcTunnel(tunnelCtx context.Context, address string, opts ...grpc.DialOption) (Tunnel, error) {
+	return CreateSingleUseGrpcTunnelWithContext(context.TODO(), tunnelCtx, address, opts...)
+}
+
+// CreateSingleUseGrpcTunnelWithContext creates a Tunnel to dial to a remote server through a
+// gRPC based proxy service.
+// Currently, a single tunnel supports a single connection.
+// The tunnel is normally closed when the connection is terminated.
+// If createCtx is cancelled before tunnel creation, an error will be returned.
+// If tunnelCtx is cancelled while the tunnel is still in use, the tunnel (and any in flight connections) will be closed.
+// The Dial() method of the returned tunnel should only be called once
+func CreateSingleUseGrpcTunnelWithContext(createCtx, tunnelCtx context.Context, address string, opts ...grpc.DialOption) (Tunnel, error) {
+	c, err := grpc.DialContext(createCtx, address, opts...)
 	if err != nil {
 		return nil, err
 	}

 	grpcClient := client.NewProxyServiceClient(c)

-	stream, err := grpcClient.Proxy(context.Background())
+	stream, err := grpcClient.Proxy(tunnelCtx)
 	if err != nil {
+		c.Close()
 		return nil, err
 	}

-	tunnel := &grpcTunnel{
-		stream:             stream,
-		pendingDial:        make(map[int64]chan<- dialResult),
-		conns:              make(map[int64]*conn),
-		readTimeoutSeconds: 10,
-	}
+	tunnel := newUnstartedTunnel(stream, c)

-	go tunnel.serve(c)
+	go tunnel.serve(tunnelCtx)

 	return tunnel, nil
 }

-func (t *grpcTunnel) serve(c clientConn) {
-	defer c.Close()
+func newUnstartedTunnel(stream client.ProxyService_ProxyClient, c clientConn) *grpcTunnel {
+	return &grpcTunnel{
+		stream:             stream,
+		clientConn:         c,
+		pendingDial:        pendingDialManager{pendingDials: make(map[int64]pendingDial)},
+		conns:              connectionManager{conns: make(map[int64]*conn)},
+		readTimeoutSeconds: 10,
+		done:               make(chan struct{}),
+	}
+}
+
+func (t *grpcTunnel) serve(tunnelCtx context.Context) {
+	defer func() {
+		t.clientConn.Close()
+
+		// A connection in t.conns after serve() returns means
+		// we never received a CLOSE_RSP for it, so we need to
+		// close any channels remaining for these connections.
+		t.conns.closeAll()
+
+		close(t.done)
+	}()

 	for {
 		pkt, err := t.stream.Recv()
-		if err == io.EOF {
+		if err == io.EOF || t.isClosing() {
 			return
 		}
 		if err != nil || pkt == nil {
@@ -109,24 +214,35 @@ func (t *grpcTunnel) serve(c clientConn) {
 		switch pkt.Type {
 		case client.PacketType_DIAL_RSP:
 			resp := pkt.GetDialResponse()
-			t.pendingDialLock.RLock()
-			ch, ok := t.pendingDial[resp.Random]
-			t.pendingDialLock.RUnlock()
+			pendingDial, ok := t.pendingDial.get(resp.Random)

 			if !ok {
-				klog.V(1).Infoln("DialResp not recognized; dropped")
-			} else {
-				result := dialResult{
-					err:    resp.Error,
-					connid: resp.ConnectID,
-				}
-				select  {
-				case ch <- result:
-				default:
-					klog.ErrorS(fmt.Errorf("blocked pending channel"), "Received second dial response for connection request", "connectionID", resp.ConnectID, "dialID", resp.Random)
-					// On multiple dial responses, avoid leaking serve goroutine.
-					return
-				}
+				// If the DIAL_RSP does not match a pending dial, it means one of two things:
+				//   1. There was a second DIAL_RSP for the connection request (this is very unlikely but possible)
+				//   2. grpcTunnel.DialContext() returned early due to a dial timeout or the client canceling the context
+				//
+				// In either scenario, we should return here and close the tunnel as it is no longer needed.
+				klog.V(1).InfoS("DialResp not recognized; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random)
+				return
+			}
+
+			result := dialResult{connid: resp.ConnectID}
+			if resp.Error != "" {
+				result.err = &dialFailure{resp.Error, DialFailureEndpoint}
+			}
+			select {
+			// try to send to the result channel
+			case pendingDial.resultCh <- result:
+			// unblock if the cancel channel is closed
+			case <-pendingDial.cancelCh:
+				// Note: this condition can only be hit by a race condition where the
+				// DialContext() returns early (timeout) after the pendingDial is already
+				// fetched here, but before the result is sent.
+				klog.V(1).InfoS("Pending dial has been cancelled; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random)
+				return
+			case <-tunnelCtx.Done():
+				klog.V(1).InfoS("Tunnel has been closed; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random)
+				return
 			}

 			if resp.Error != "" {
@@ -134,62 +250,93 @@ func (t *grpcTunnel) serve(c clientConn) {
 				return
 			}

+		case client.PacketType_DIAL_CLS:
+			resp := pkt.GetCloseDial()
+			pendingDial, ok := t.pendingDial.get(resp.Random)
+
+			if !ok {
+				// If the DIAL_CLS does not match a pending dial, it means one of two things:
+				//   1. There was a DIAL_CLS receieved after a DIAL_RSP (unlikely but possible)
+				//   2. grpcTunnel.DialContext() returned early due to a dial timeout or the client canceling the context
+				//
+				// In either scenario, we should return here and close the tunnel as it is no longer needed.
+				klog.V(1).InfoS("DIAL_CLS after dial finished", "dialID", resp.Random)
+			} else {
+				result := dialResult{
+					err: &dialFailure{"dial closed", DialFailureDialClosed},
+				}
+				select {
+				case pendingDial.resultCh <- result:
+				case <-pendingDial.cancelCh:
+					// Note: this condition can only be hit by a race condition where the
+					// DialContext() returns early (timeout) after the pendingDial is already
+					// fetched here, but before the result is sent.
+				case <-tunnelCtx.Done():
+				}
+			}
+			return // Stop serving & close the tunnel.
+
 		case client.PacketType_DATA:
 			resp := pkt.GetData()
 			// TODO: flow control
-			t.connsLock.RLock()
-			conn, ok := t.conns[resp.ConnectID]
-			t.connsLock.RUnlock()
+			conn, ok := t.conns.get(resp.ConnectID)

-			if ok {
-				timer := time.NewTimer((time.Duration)(t.readTimeoutSeconds) * time.Second)
-				select {
-				case conn.readCh <- resp.Data:
-					timer.Stop()
-				case <-timer.C:
-					klog.ErrorS(fmt.Errorf("timeout"), "readTimeout has been reached, the grpc connection to the proxy server will be closed", "connectionID", conn.connID, "readTimeoutSeconds", t.readTimeoutSeconds)
-					return
-				}
-			} else {
-				klog.V(1).InfoS("connection not recognized", "connectionID", resp.ConnectID)
+			if !ok {
+				klog.V(1).InfoS("Connection not recognized", "connectionID", resp.ConnectID)
+				continue
 			}
+			timer := time.NewTimer((time.Duration)(t.readTimeoutSeconds) * time.Second)
+			select {
+			case conn.readCh <- resp.Data:
+				timer.Stop()
+			case <-timer.C:
+				klog.ErrorS(fmt.Errorf("timeout"), "readTimeout has been reached, the grpc connection to the proxy server will be closed", "connectionID", conn.connID, "readTimeoutSeconds", t.readTimeoutSeconds)
+				return
+			case <-tunnelCtx.Done():
+				klog.V(1).InfoS("Tunnel has been closed, the grpc connection to the proxy server will be closed", "connectionID", conn.connID)
+			}
+
 		case client.PacketType_CLOSE_RSP:
 			resp := pkt.GetCloseResponse()
-			t.connsLock.RLock()
-			conn, ok := t.conns[resp.ConnectID]
-			t.connsLock.RUnlock()
+			conn, ok := t.conns.get(resp.ConnectID)

-			if ok {
-				close(conn.readCh)
-				conn.closeCh <- resp.Error
-				close(conn.closeCh)
-				t.connsLock.Lock()
-				delete(t.conns, resp.ConnectID)
-				t.connsLock.Unlock()
-				return
+			if !ok {
+				klog.V(1).InfoS("Connection not recognized", "connectionID", resp.ConnectID)
+				continue
 			}
-			klog.V(1).InfoS("connection not recognized", "connectionID", resp.ConnectID)
+			close(conn.readCh)
+			conn.closeCh <- resp.Error
+			close(conn.closeCh)
+			t.conns.remove(resp.ConnectID)
+			return
 		}
 	}
 }

 // Dial connects to the address on the named network, similar to
 // what net.Dial does. The only supported protocol is tcp.
-func (t *grpcTunnel) Dial(protocol, address string) (net.Conn, error) {
+func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) {
+	select {
+	case <-t.done:
+		return nil, errors.New("tunnel is closed")
+	default: // Tunnel is open, carry on.
+	}
+
 	if protocol != "tcp" {
 		return nil, errors.New("protocol not supported")
 	}

 	random := rand.Int63() /* #nosec G404 */
-	resCh := make(chan dialResult, 1)
-	t.pendingDialLock.Lock()
-	t.pendingDial[random] = resCh
-	t.pendingDialLock.Unlock()
-	defer func() {
-		t.pendingDialLock.Lock()
-		delete(t.pendingDial, random)
-		t.pendingDialLock.Unlock()
-	}()
+
+	// This channel is closed once we're returning and no longer waiting on resultCh
+	cancelCh := make(chan struct{})
+	defer close(cancelCh)
+
+	// This channel MUST NOT be buffered. The sender needs to know when we are not receiving things, so they can abort.
+	resCh := make(chan dialResult)
+
+	t.pendingDial.add(random, pendingDial{resultCh: resCh, cancelCh: cancelCh})
+	defer t.pendingDial.remove(random)

 	req := &client.Packet{
 		Type: client.PacketType_DIAL_REQ,
@@ -210,22 +357,98 @@ func (t *grpcTunnel) Dial(protocol, address string) (net.Conn, error) {

 	klog.V(5).Infoln("DIAL_REQ sent to proxy server")

-	c := &conn{stream: t.stream}
+	c := &conn{
+		stream:      t.stream,
+		random:      random,
+		closeTunnel: t.closeTunnel,
+	}

 	select {
 	case res := <-resCh:
-		if res.err != "" {
-			return nil, errors.New(res.err)
+		if res.err != nil {
+			return nil, res.err
 		}
 		c.connID = res.connid
 		c.readCh = make(chan []byte, 10)
 		c.closeCh = make(chan string, 1)
-		t.connsLock.Lock()
-		t.conns[res.connid] = c
-		t.connsLock.Unlock()
+		t.conns.add(res.connid, c)
 	case <-time.After(30 * time.Second):
-		return nil, errors.New("dial timeout")
+		klog.V(5).InfoS("Timed out waiting for DialResp", "dialID", random)
+		go t.closeDial(random)
+		return nil, &dialFailure{"dial timeout, backstop", DialFailureTimeout}
+	case <-requestCtx.Done():
+		klog.V(5).InfoS("Context canceled waiting for DialResp", "ctxErr", requestCtx.Err(), "dialID", random)
+		go t.closeDial(random)
+		return nil, &dialFailure{"dial timeout, context", DialFailureContext}
+	case <-t.done:
+		klog.V(5).InfoS("Tunnel closed while waiting for DialResp", "dialID", random)
+		return nil, &dialFailure{"tunnel closed", DialFailureTunnelClosed}
 	}

 	return c, nil
 }
+
+func (t *grpcTunnel) Done() <-chan struct{} {
+	return t.done
+}
+
+// Send a best-effort DIAL_CLS request for the given dial ID.
+func (t *grpcTunnel) closeDial(dialID int64) {
+	req := &client.Packet{
+		Type: client.PacketType_DIAL_CLS,
+		Payload: &client.Packet_CloseDial{
+			CloseDial: &client.CloseDial{
+				Random: dialID,
+			},
+		},
+	}
+	if err := t.stream.Send(req); err != nil {
+		klog.V(5).InfoS("Failed to send DIAL_CLS", "err", err, "dialID", dialID)
+	}
+	t.closeTunnel()
+}
+
+func (t *grpcTunnel) closeTunnel() {
+	atomic.StoreUint32(&t.closing, 1)
+	t.clientConn.Close()
+}
+
+func (t *grpcTunnel) isClosing() bool {
+	return atomic.LoadUint32(&t.closing) != 0
+}
+
+func GetDialFailureReason(err error) (isDialFailure bool, reason DialFailureReason) {
+	var df *dialFailure
+	if errors.As(err, &df) {
+		return true, df.reason
+	}
+	return false, DialFailureUnknown
+}
+
+type dialFailure struct {
+	msg    string
+	reason DialFailureReason
+}
+
+func (df *dialFailure) Error() string {
+	return df.msg
+}
+
+type DialFailureReason string
+
+const (
+	DialFailureUnknown DialFailureReason = "unknown"
+	// DialFailureTimeout indicates the hard 30 second timeout was hit.
+	DialFailureTimeout DialFailureReason = "timeout"
+	// DialFailureContext indicates that the context was cancelled or reached it's deadline before
+	// the dial response was returned.
+	DialFailureContext DialFailureReason = "context"
+	// DialFailureEndpoint indicates that the konnectivity-agent was unable to reach the backend endpoint.
+	DialFailureEndpoint DialFailureReason = "endpoint"
+	// DialFailureDialClosed indicates that the client received a CloseDial response, indicating the
+	// connection was closed before the dial could complete.
+	DialFailureDialClosed DialFailureReason = "dialclosed"
+	// DialFailureTunnelClosed indicates that the client connection was closed before the dial could
+	// complete.
+	DialFailureTunnelClosed DialFailureReason = "tunnelclosed"
+)