update vendor

Signed-off-by: Roland.Ma <rolandma@yunify.com>
This commit is contained in:
Roland.Ma
2021-08-11 07:10:14 +00:00
parent a18f72b565
commit ea8f47c73a
2901 changed files with 269317 additions and 43103 deletions

View File

@@ -28,7 +28,7 @@ import (
"io/ioutil"
"math/big"
"net"
"path"
"path/filepath"
"strings"
"time"
@@ -96,8 +96,8 @@ func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, a
maxAge := time.Hour * 24 * 365 // one year self-signed certs
baseName := fmt.Sprintf("%s_%s_%s", host, strings.Join(ipsToStrings(alternateIPs), "-"), strings.Join(alternateDNS, "-"))
certFixturePath := path.Join(fixtureDirectory, baseName+".crt")
keyFixturePath := path.Join(fixtureDirectory, baseName+".key")
certFixturePath := filepath.Join(fixtureDirectory, baseName+".crt")
keyFixturePath := filepath.Join(fixtureDirectory, baseName+".key")
if len(fixtureDirectory) > 0 {
cert, err := ioutil.ReadFile(certFixturePath)
if err == nil {

View File

@@ -33,18 +33,40 @@ type DialFunc func(ctx context.Context, network, address string) (net.Conn, erro
// Dialer opens connections through Dial and tracks them.
type Dialer struct {
dial DialFunc
*ConnectionTracker
}
// NewDialer creates a new Dialer instance.
// Equivalent to NewDialerWithTracker(dial, nil).
func NewDialer(dial DialFunc) *Dialer {
return NewDialerWithTracker(dial, nil)
}
// NewDialerWithTracker creates a new Dialer instance.
//
// If dial is not nil, it will be used to create new underlying connections.
// Otherwise net.DialContext is used.
// If tracker is not nil, it will be used to track new underlying connections.
// Otherwise NewConnectionTracker() is used.
func NewDialerWithTracker(dial DialFunc, tracker *ConnectionTracker) *Dialer {
if tracker == nil {
tracker = NewConnectionTracker()
}
return &Dialer{
dial: dial,
ConnectionTracker: tracker,
}
}
// ConnectionTracker keeps track of opened connections
type ConnectionTracker struct {
mu sync.Mutex
conns map[*closableConn]struct{}
}
// NewDialer creates a new Dialer instance.
//
// If dial is not nil, it will be used to create new underlying connections.
// Otherwise net.DialContext is used.
func NewDialer(dial DialFunc) *Dialer {
return &Dialer{
dial: dial,
// NewConnectionTracker returns a connection tracker for use with NewDialerWithTracker
func NewConnectionTracker() *ConnectionTracker {
return &ConnectionTracker{
conns: make(map[*closableConn]struct{}),
}
}
@@ -52,17 +74,40 @@ func NewDialer(dial DialFunc) *Dialer {
// CloseAll forcibly closes all tracked connections.
//
// Note: new connections may get created before CloseAll returns.
func (d *Dialer) CloseAll() {
d.mu.Lock()
conns := d.conns
d.conns = make(map[*closableConn]struct{})
d.mu.Unlock()
func (c *ConnectionTracker) CloseAll() {
c.mu.Lock()
conns := c.conns
c.conns = make(map[*closableConn]struct{})
c.mu.Unlock()
for conn := range conns {
conn.Close()
}
}
// Track adds the connection to the list of tracked connections,
// and returns a wrapped copy of the connection that stops tracking the connection
// when it is closed.
func (c *ConnectionTracker) Track(conn net.Conn) net.Conn {
closable := &closableConn{Conn: conn}
// When the connection is closed, remove it from the map. This will
// be no-op if the connection isn't in the map, e.g. if CloseAll()
// is called.
closable.onClose = func() {
c.mu.Lock()
delete(c.conns, closable)
c.mu.Unlock()
}
// Start tracking the connection
c.mu.Lock()
c.conns[closable] = struct{}{}
c.mu.Unlock()
return closable
}
// Dial creates a new tracked connection.
func (d *Dialer) Dial(network, address string) (net.Conn, error) {
return d.DialContext(context.Background(), network, address)
@@ -74,24 +119,7 @@ func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.
if err != nil {
return nil, err
}
closable := &closableConn{Conn: conn}
// When the connection is closed, remove it from the map. This will
// be no-op if the connection isn't in the map, e.g. if CloseAll()
// is called.
closable.onClose = func() {
d.mu.Lock()
delete(d.conns, closable)
d.mu.Unlock()
}
// Start tracking the connection
d.mu.Lock()
d.conns[closable] = struct{}{}
d.mu.Unlock()
return closable, nil
return d.ConnectionTracker.Track(conn), nil
}
type closableConn struct {

View File

@@ -18,6 +18,7 @@ package jsonpath
import (
"bytes"
"encoding/json"
"fmt"
"io"
"reflect"
@@ -29,13 +30,14 @@ import (
type JSONPath struct {
name string
parser *Parser
stack [][]reflect.Value // push and pop values in different scopes
cur []reflect.Value // current scope values
beginRange int
inRange int
endRange int
lastEndNode *Node
allowMissingKeys bool
outputJSON bool
}
// New creates a new JSONPath with the given name.
@@ -81,12 +83,12 @@ func (j *JSONPath) FindResults(data interface{}) ([][]reflect.Value, error) {
return nil, fmt.Errorf("%s is an incomplete jsonpath template", j.name)
}
j.cur = []reflect.Value{reflect.ValueOf(data)}
cur := []reflect.Value{reflect.ValueOf(data)}
nodes := j.parser.Root.Nodes
fullResult := [][]reflect.Value{}
for i := 0; i < len(nodes); i++ {
node := nodes[i]
results, err := j.walk(j.cur, node)
results, err := j.walk(cur, node)
if err != nil {
return nil, err
}
@@ -94,34 +96,90 @@ func (j *JSONPath) FindResults(data interface{}) ([][]reflect.Value, error) {
// encounter an end node, break the current block
if j.endRange > 0 && j.endRange <= j.inRange {
j.endRange--
j.lastEndNode = &nodes[i]
break
}
// encounter a range node, start a range loop
if j.beginRange > 0 {
j.beginRange--
j.inRange++
for k, value := range results {
j.parser.Root.Nodes = nodes[i+1:]
if k == len(results)-1 {
j.inRange--
if len(results) > 0 {
for _, value := range results {
j.parser.Root.Nodes = nodes[i+1:]
nextResults, err := j.FindResults(value.Interface())
if err != nil {
return nil, err
}
fullResult = append(fullResult, nextResults...)
}
nextResults, err := j.FindResults(value.Interface())
} else {
// If the range has no results, we still need to process the nodes within the range
// so the position will advance to the end node
j.parser.Root.Nodes = nodes[i+1:]
_, err := j.FindResults(nil)
if err != nil {
return nil, err
}
fullResult = append(fullResult, nextResults...)
}
break
j.inRange--
// Fast forward to resume processing after the most recent end node that was encountered
for k := i + 1; k < len(nodes); k++ {
if &nodes[k] == j.lastEndNode {
i = k
break
}
}
continue
}
fullResult = append(fullResult, results)
}
return fullResult, nil
}
// EnableJSONOutput changes the PrintResults behavior to return a JSON array of results
func (j *JSONPath) EnableJSONOutput(v bool) {
j.outputJSON = v
}
// PrintResults writes the results into writer
func (j *JSONPath) PrintResults(wr io.Writer, results []reflect.Value) error {
if j.outputJSON {
// convert the []reflect.Value to something that json
// will be able to marshal
r := make([]interface{}, 0, len(results))
for i := range results {
r = append(r, results[i].Interface())
}
results = []reflect.Value{reflect.ValueOf(r)}
}
for i, r := range results {
text, err := j.evalToText(r)
var text []byte
var err error
outputJSON := true
kind := r.Kind()
if kind == reflect.Interface {
kind = r.Elem().Kind()
}
switch kind {
case reflect.Map:
case reflect.Array:
case reflect.Slice:
case reflect.Struct:
default:
outputJSON = false
}
switch {
case outputJSON || j.outputJSON:
if j.outputJSON {
text, err = json.MarshalIndent(r.Interface(), "", " ")
text = append(text, '\n')
} else {
text, err = json.Marshal(r.Interface())
}
default:
text, err = j.evalToText(r)
}
if err != nil {
return err
}
@@ -132,7 +190,9 @@ func (j *JSONPath) PrintResults(wr io.Writer, results []reflect.Value) error {
return err
}
}
return nil
}
// walk visits tree rooted at the given node in DFS order
@@ -212,17 +272,11 @@ func (j *JSONPath) evalIdentifier(input []reflect.Value, node *IdentifierNode) (
results := []reflect.Value{}
switch node.Name {
case "range":
j.stack = append(j.stack, j.cur)
j.beginRange++
results = input
case "end":
if j.endRange < j.inRange { // inside a loop, break the current block
if j.inRange > 0 {
j.endRange++
break
}
// the loop is about to end, pop value and continue the following execution
if len(j.stack) > 0 {
j.cur, j.stack = j.stack[len(j.stack)-1], j.stack[:len(j.stack)-1]
} else {
return results, fmt.Errorf("not in range, nothing to end")
}

View File

@@ -214,8 +214,11 @@ func (p *Parser) parseIdentifier(cur *ListNode) error {
return p.parseInsideAction(cur)
}
// parseRecursive scans the recursive desent operator ..
// parseRecursive scans the recursive descent operator ..
func (p *Parser) parseRecursive(cur *ListNode) error {
if lastIndex := len(cur.Nodes) - 1; lastIndex >= 0 && cur.Nodes[lastIndex].Type() == NodeRecursive {
return fmt.Errorf("invalid multiple recursive descent")
}
p.pos += len("..")
p.consumeText()
cur.append(newRecursive())

View File

@@ -62,54 +62,6 @@ func (r *BucketRateLimiter) NumRequeues(item interface{}) int {
func (r *BucketRateLimiter) Forget(item interface{}) {
}
// ItemBucketRateLimiter implements a workqueue ratelimiter API using standard rate.Limiter.
// Each key is using a separate limiter.
type ItemBucketRateLimiter struct {
r rate.Limit
burst int
limitersLock sync.Mutex
limiters map[interface{}]*rate.Limiter
}
var _ RateLimiter = &ItemBucketRateLimiter{}
// NewItemBucketRateLimiter creates new ItemBucketRateLimiter instance.
func NewItemBucketRateLimiter(r rate.Limit, burst int) *ItemBucketRateLimiter {
return &ItemBucketRateLimiter{
r: r,
burst: burst,
limiters: make(map[interface{}]*rate.Limiter),
}
}
// When returns a time.Duration which we need to wait before item is processed.
func (r *ItemBucketRateLimiter) When(item interface{}) time.Duration {
r.limitersLock.Lock()
defer r.limitersLock.Unlock()
limiter, ok := r.limiters[item]
if !ok {
limiter = rate.NewLimiter(r.r, r.burst)
r.limiters[item] = limiter
}
return limiter.Reserve().Delay()
}
// NumRequeues returns always 0 (doesn't apply to ItemBucketRateLimiter).
func (r *ItemBucketRateLimiter) NumRequeues(item interface{}) int {
return 0
}
// Forget removes item from the internal state.
func (r *ItemBucketRateLimiter) Forget(item interface{}) {
r.limitersLock.Lock()
defer r.limitersLock.Unlock()
delete(r.limiters, item)
}
// ItemExponentialFailureRateLimiter does a simple baseDelay*2^<num-failures> limit
// dealing with max failures and expiration are up to the caller
type ItemExponentialFailureRateLimiter struct {

View File

@@ -38,6 +38,12 @@ func NewDelayingQueue() DelayingInterface {
return NewDelayingQueueWithCustomClock(clock.RealClock{}, "")
}
// NewDelayingQueueWithCustomQueue constructs a new workqueue with ability to
// inject custom queue Interface instead of the default one
func NewDelayingQueueWithCustomQueue(q Interface, name string) DelayingInterface {
return newDelayingQueue(clock.RealClock{}, q, name)
}
// NewNamedDelayingQueue constructs a new named workqueue with delayed queuing ability
func NewNamedDelayingQueue(name string) DelayingInterface {
return NewDelayingQueueWithCustomClock(clock.RealClock{}, name)
@@ -46,8 +52,12 @@ func NewNamedDelayingQueue(name string) DelayingInterface {
// NewDelayingQueueWithCustomClock constructs a new named workqueue
// with ability to inject real or fake clock for testing purposes
func NewDelayingQueueWithCustomClock(clock clock.Clock, name string) DelayingInterface {
return newDelayingQueue(clock, NewNamed(name), name)
}
func newDelayingQueue(clock clock.Clock, q Interface, name string) *delayingType {
ret := &delayingType{
Interface: NewNamed(name),
Interface: q,
clock: clock,
heartbeat: clock.NewTicker(maxWait),
stopCh: make(chan struct{}),
@@ -56,7 +66,6 @@ func NewDelayingQueueWithCustomClock(clock clock.Clock, name string) DelayingInt
}
go ret.waitingLoop()
return ret
}

View File

@@ -25,39 +25,77 @@ import (
type DoWorkPieceFunc func(piece int)
type options struct {
chunkSize int
}
type Options func(*options)
// WithChunkSize allows to set chunks of work items to the workers, rather than
// processing one by one.
// It is recommended to use this option if the number of pieces significantly
// higher than the number of workers and the work done for each item is small.
func WithChunkSize(c int) func(*options) {
return func(o *options) {
o.chunkSize = c
}
}
// ParallelizeUntil is a framework that allows for parallelizing N
// independent pieces of work until done or the context is canceled.
func ParallelizeUntil(ctx context.Context, workers, pieces int, doWorkPiece DoWorkPieceFunc) {
var stop <-chan struct{}
if ctx != nil {
stop = ctx.Done()
func ParallelizeUntil(ctx context.Context, workers, pieces int, doWorkPiece DoWorkPieceFunc, opts ...Options) {
if pieces == 0 {
return
}
o := options{}
for _, opt := range opts {
opt(&o)
}
chunkSize := o.chunkSize
if chunkSize < 1 {
chunkSize = 1
}
toProcess := make(chan int, pieces)
for i := 0; i < pieces; i++ {
chunks := ceilDiv(pieces, chunkSize)
toProcess := make(chan int, chunks)
for i := 0; i < chunks; i++ {
toProcess <- i
}
close(toProcess)
if pieces < workers {
workers = pieces
var stop <-chan struct{}
if ctx != nil {
stop = ctx.Done()
}
if chunks < workers {
workers = chunks
}
wg := sync.WaitGroup{}
wg.Add(workers)
for i := 0; i < workers; i++ {
go func() {
defer utilruntime.HandleCrash()
defer wg.Done()
for piece := range toProcess {
select {
case <-stop:
return
default:
doWorkPiece(piece)
for chunk := range toProcess {
start := chunk * chunkSize
end := start + chunkSize
if end > pieces {
end = pieces
}
for p := start; p < end; p++ {
select {
case <-stop:
return
default:
doWorkPiece(p)
}
}
}
}()
}
wg.Wait()
}
func ceilDiv(a, b int) int {
return (a + b - 1) / b
}

View File

@@ -55,7 +55,13 @@ func newQueue(c clock.Clock, metrics queueMetrics, updatePeriod time.Duration) *
metrics: metrics,
unfinishedWorkUpdatePeriod: updatePeriod,
}
go t.updateUnfinishedWorkLoop()
// Don't start the goroutine for a type of noMetrics so we don't consume
// resources unnecessarily
if _, ok := metrics.(noMetrics); !ok {
go t.updateUnfinishedWorkLoop()
}
return t
}